cpython.git - https://github.com/python/cpython.git

diff options

Diffstat (limited to 'Python/compile.c')

0 files changed, 0 insertions, 0 deletions

yping.py +++ b/Lib/test/test_typing.py @@ -41,11 +41,9 @@ class ManagingFounder(Manager, Founder): class AnyTests(TestCase): - def test_any_instance(self): - self.assertIsInstance(Employee(), Any) - self.assertIsInstance(42, Any) - self.assertIsInstance(None, Any) - self.assertIsInstance(object(), Any) + def test_any_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Any) def test_any_subclass(self): self.assertTrue(issubclass(Employee, Any)) @@ -109,9 +107,6 @@ class TypeVarTests(TestCase): def test_basic_plain(self): T = TypeVar('T') - # Nothing is an instance if T. - with self.assertRaises(TypeError): - isinstance('', T) # Every class is a subclass of T. assert issubclass(int, T) assert issubclass(str, T) @@ -119,12 +114,16 @@ class TypeVarTests(TestCase): assert T == T # T is a subclass of itself. assert issubclass(T, T) + # T is an instance of TypeVar + assert isinstance(T, TypeVar) + + def test_typevar_instance_type_error(self): + T = TypeVar('T') + with self.assertRaises(TypeError): + isinstance(42, T) def test_basic_constrained(self): A = TypeVar('A', str, bytes) - # Nothing is an instance of A. - with self.assertRaises(TypeError): - isinstance('', A) # Only str and bytes are subclasses of A. assert issubclass(str, A) assert issubclass(bytes, A) @@ -213,8 +212,6 @@ class UnionTests(TestCase): def test_basics(self): u = Union[int, float] self.assertNotEqual(u, Union) - self.assertIsInstance(42, u) - self.assertIsInstance(3.14, u) self.assertTrue(issubclass(int, u)) self.assertTrue(issubclass(float, u)) @@ -247,7 +244,6 @@ class UnionTests(TestCase): def test_subclass(self): u = Union[int, Employee] - self.assertIsInstance(Manager(), u) self.assertTrue(issubclass(Manager, u)) def test_self_subclass(self): @@ -256,7 +252,6 @@ class UnionTests(TestCase): def test_multiple_inheritance(self): u = Union[int, Employee] - self.assertIsInstance(ManagingFounder(), u) self.assertTrue(issubclass(ManagingFounder, u)) def test_single_class_disappears(self): @@ -309,9 +304,6 @@ class UnionTests(TestCase): o = Optional[int] u = Union[int, None] self.assertEqual(o, u) - self.assertIsInstance(42, o) - self.assertIsInstance(None, o) - self.assertNotIsInstance(3.14, o) def test_empty(self): with self.assertRaises(TypeError): @@ -321,11 +313,9 @@ class UnionTests(TestCase): assert issubclass(Union[int, str], Union) assert not issubclass(int, Union) - def test_isinstance_union(self): - # Nothing is an instance of bare Union. - assert not isinstance(42, Union) - assert not isinstance(int, Union) - assert not isinstance(Union[int, str], Union) + def test_union_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Union[int, str]) class TypeVarUnionTests(TestCase): @@ -352,22 +342,11 @@ class TypeVarUnionTests(TestCase): TU = TypeVar('TU', Union[int, float], None) assert issubclass(int, TU) assert issubclass(float, TU) - with self.assertRaises(TypeError): - isinstance(42, TU) - with self.assertRaises(TypeError): - isinstance('', TU) class TupleTests(TestCase): def test_basics(self): - self.assertIsInstance((42, 3.14, ''), Tuple) - self.assertIsInstance((42, 3.14, ''), Tuple[int, float, str]) - self.assertIsInstance((42,), Tuple[int]) - self.assertNotIsInstance((3.14,), Tuple[int]) - self.assertNotIsInstance((42, 3.14), Tuple[int, float, str]) - self.assertNotIsInstance((42, 3.14, 100), Tuple[int, float, str]) - self.assertNotIsInstance((42, 3.14, 100), Tuple[int, float]) self.assertTrue(issubclass(Tuple[int, str], Tuple)) self.assertTrue(issubclass(Tuple[int, str], Tuple[int, str])) self.assertFalse(issubclass(int, Tuple)) @@ -382,14 +361,11 @@ class TupleTests(TestCase): pass self.assertTrue(issubclass(MyTuple, Tuple)) - def test_tuple_ellipsis(self): - t = Tuple[int, ...] - assert isinstance((), t) - assert isinstance((1,), t) - assert isinstance((1, 2), t) - assert isinstance((1, 2, 3), t) - assert not isinstance((3.14,), t) - assert not isinstance((1, 2, 3.14,), t) + def test_tuple_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance((0, 0), Tuple[int, int]) + with self.assertRaises(TypeError): + isinstance((0, 0), Tuple) def test_tuple_ellipsis_subclass(self): @@ -419,18 +395,6 @@ class TupleTests(TestCase): class CallableTests(TestCase): - def test_basics(self): - c = Callable[[int, float], str] - - def flub(a: int, b: float) -> str: - return str(a * b) - - def flob(a: int, b: int) -> str: - return str(a * b) - - self.assertIsInstance(flub, c) - self.assertNotIsInstance(flob, c) - def test_self_subclass(self): self.assertTrue(issubclass(Callable[[int], int], Callable)) self.assertFalse(issubclass(Callable, Callable[[int], int])) @@ -453,91 +417,6 @@ class CallableTests(TestCase): self.assertNotEqual(Callable[[int], int], Callable[[], int]) self.assertNotEqual(Callable[[int], int], Callable) - def test_with_none(self): - c = Callable[[None], None] - - def flub(self: None) -> None: - pass - - def flab(self: Any) -> None: - pass - - def flob(self: None) -> Any: - pass - - self.assertIsInstance(flub, c) - self.assertIsInstance(flab, c) - self.assertNotIsInstance(flob, c) # Test contravariance. - - def test_with_subclasses(self): - c = Callable[[Employee, Manager], Employee] - - def flub(a: Employee, b: Employee) -> Manager: - return Manager() - - def flob(a: Manager, b: Manager) -> Employee: - return Employee() - - self.assertIsInstance(flub, c) - self.assertNotIsInstance(flob, c) - - def test_with_default_args(self): - c = Callable[[int], int] - - def flub(a: int, b: float = 3.14) -> int: - return a - - def flab(a: int, *, b: float = 3.14) -> int: - return a - - def flob(a: int = 42) -> int: - return a - - self.assertIsInstance(flub, c) - self.assertIsInstance(flab, c) - self.assertIsInstance(flob, c) - - def test_with_varargs(self): - c = Callable[[int], int] - - def flub(*args) -> int: - return 42 - - def flab(*args: int) -> int: - return 42 - - def flob(*args: float) -> int: - return 42 - - self.assertIsInstance(flub, c) - self.assertIsInstance(flab, c) - self.assertNotIsInstance(flob, c) - - def test_with_method(self): - - class C: - - def imethod(self, arg: int) -> int: - self.last_arg = arg - return arg + 1 - - @classmethod - def cmethod(cls, arg: int) -> int: - cls.last_cls_arg = arg - return arg + 1 - - @staticmethod - def smethod(arg: int) -> int: - return arg + 1 - - ct = Callable[[int], int] - self.assertIsInstance(C().imethod, ct) - self.assertIsInstance(C().cmethod, ct) - self.assertIsInstance(C.cmethod, ct) - self.assertIsInstance(C().smethod, ct) - self.assertIsInstance(C.smethod, ct) - self.assertIsInstance(C.imethod, Callable[[Any, int], int]) - def test_cannot_subclass(self): with self.assertRaises(TypeError): @@ -556,21 +435,21 @@ class CallableTests(TestCase): with self.assertRaises(TypeError): c() - def test_varargs(self): - ct = Callable[..., int] + def test_callable_instance_works(self): + f = lambda: None + assert isinstance(f, Callable) + assert not isinstance(None, Callable) - def foo(a, b) -> int: - return 42 - - def bar(a=42) -> int: - return a - - def baz(*, x, y, z) -> int: - return 100 - - self.assertIsInstance(foo, ct) - self.assertIsInstance(bar, ct) - self.assertIsInstance(baz, ct) + def test_callable_instance_type_error(self): + f = lambda: None + with self.assertRaises(TypeError): + assert isinstance(f, Callable[[], None]) + with self.assertRaises(TypeError): + assert isinstance(f, Callable[[], Any]) + with self.assertRaises(TypeError): + assert not isinstance(None, Callable[[], None]) + with self.assertRaises(TypeError): + assert not isinstance(None, Callable[[], Any]) def test_repr(self): ct0 = Callable[[], bool] @@ -659,6 +538,10 @@ class ProtocolTests(TestCase): assert issubclass(list, typing.Reversible) assert not issubclass(int, typing.Reversible) + def test_protocol_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance([], typing.Reversible) + class GenericTests(TestCase): @@ -889,6 +772,11 @@ class ForwardRefTests(TestCase): right_hints = get_type_hints(t.add_right, globals(), locals()) assert right_hints['node'] == Optional[Node[T]] + def test_forwardref_instance_type_error(self): + fr = typing._ForwardRef('int') + with self.assertRaises(TypeError): + isinstance(42, fr) + def test_union_forward(self): def foo(a: Union['T']): @@ -1069,50 +957,17 @@ class CollectionsAbcTests(TestCase): def test_list(self): assert issubclass(list, typing.List) - assert isinstance([], typing.List) - assert not isinstance((), typing.List) - t = typing.List[int] - assert isinstance([], t) - assert isinstance([42], t) - assert not isinstance([''], t) def test_set(self): assert issubclass(set, typing.Set) assert not issubclass(frozenset, typing.Set) - assert isinstance(set(), typing.Set) - assert not isinstance({}, typing.Set) - t = typing.Set[int] - assert isinstance(set(), t) - assert isinstance({42}, t) - assert not isinstance({''}, t) def test_frozenset(self): assert issubclass(frozenset, typing.FrozenSet) assert not issubclass(set, typing.FrozenSet) - assert isinstance(frozenset(), typing.FrozenSet) - assert not isinstance({}, typing.FrozenSet) - t = typing.FrozenSet[int] - assert isinstance(frozenset(), t) - assert isinstance(frozenset({42}), t) - assert not isinstance(frozenset({''}), t) - assert not isinstance({42}, t) - - def test_mapping_views(self): - # TODO: These tests are kind of lame. - assert isinstance({}.keys(), typing.KeysView) - assert isinstance({}.items(), typing.ItemsView) - assert isinstance({}.values(), typing.ValuesView) def test_dict(self): assert issubclass(dict, typing.Dict) - assert isinstance({}, typing.Dict) - assert not isinstance([], typing.Dict) - t = typing.Dict[int, str] - assert isinstance({}, t) - assert isinstance({42: ''}, t) - assert not isinstance({42: 42}, t) - assert not isinstance({'': 42}, t) - assert not isinstance({'': ''}, t) def test_no_list_instantiation(self): with self.assertRaises(TypeError): @@ -1191,8 +1046,6 @@ class CollectionsAbcTests(TestCase): yield 42 g = foo() assert issubclass(type(g), typing.Generator) - assert isinstance(g, typing.Generator) - assert not isinstance(foo, typing.Generator) assert issubclass(typing.Generator[Manager, Employee, Manager], typing.Generator[Employee, Manager, Employee]) assert not issubclass(typing.Generator[Manager, Manager, Manager], @@ -1228,12 +1081,6 @@ class CollectionsAbcTests(TestCase): assert len(MMB[str, str]()) == 0 assert len(MMB[KT, VT]()) == 0 - def test_recursive_dict(self): - D = typing.Dict[int, 'D'] # Uses a _ForwardRef - assert isinstance({}, D) # Easy - assert isinstance({0: {}}, D) # Touches _ForwardRef - assert isinstance({0: {0: {}}}, D) # Etc... - class NamedTupleTests(TestCase): @@ -1294,8 +1141,6 @@ class RETests(TestCase): def test_basics(self): pat = re.compile('[a-z]+', re.I) assert issubclass(pat.__class__, Pattern) - assert isinstance(pat, Pattern[str]) - assert not isinstance(pat, Pattern[bytes]) assert issubclass(type(pat), Pattern) assert issubclass(type(pat), Pattern[str]) @@ -1307,12 +1152,10 @@ class RETests(TestCase): assert issubclass(type(mat), Match[str]) p = Pattern[Union[str, bytes]] - assert isinstance(pat, p) assert issubclass(Pattern[str], Pattern) assert issubclass(Pattern[str], p) m = Match[Union[bytes, str]] - assert isinstance(mat, m) assert issubclass(Match[bytes], Match) assert issubclass(Match[bytes], m) @@ -1327,6 +1170,12 @@ class RETests(TestCase): with self.assertRaises(TypeError): # Too complicated? m[str] + with self.assertRaises(TypeError): + # We don't support isinstance(). + isinstance(42, Pattern) + with self.assertRaises(TypeError): + # We don't support isinstance(). + isinstance(42, Pattern[str]) def test_repr(self): assert repr(Pattern) == 'Pattern[~AnyStr]' diff --git a/Lib/typing.py b/Lib/typing.py index 38e07ad..66bee91 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -176,6 +176,9 @@ class _ForwardRef(TypingMeta): self.__forward_evaluated__ = True return self.__forward_value__ + def __instancecheck__(self, obj): + raise TypeError("Forward references cannot be used with isinstance().") + def __subclasscheck__(self, cls): if not self.__forward_evaluated__: globalns = self.__forward_frame__.f_globals @@ -186,16 +189,6 @@ class _ForwardRef(TypingMeta): return False # Too early. return issubclass(cls, self.__forward_value__) - def __instancecheck__(self, obj): - if not self.__forward_evaluated__: - globalns = self.__forward_frame__.f_globals - localns = self.__forward_frame__.f_locals - try: - self._eval_type(globalns, localns) - except NameError: - return False # Too early. - return isinstance(obj, self.__forward_value__) - def __repr__(self): return '_ForwardRef(%r)' % (self.__forward_arg__,) @@ -259,8 +252,7 @@ class _TypeAlias: self.impl_type, self.type_checker) def __instancecheck__(self, obj): - return (isinstance(obj, self.impl_type) and - isinstance(self.type_checker(obj), self.type_var)) + raise TypeError("Type aliases cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -332,8 +324,8 @@ class AnyMeta(TypingMeta): self = super().__new__(cls, name, bases, namespace, _root=_root) return self - def __instancecheck__(self, instance): - return True + def __instancecheck__(self, obj): + raise TypeError("Any cannot be used with isinstance().") def __subclasscheck__(self, cls): if not isinstance(cls, type): @@ -548,9 +540,8 @@ class UnionMeta(TypingMeta): def __hash__(self): return hash(self.__union_set_params__) - def __instancecheck__(self, instance): - return (self.__union_set_params__ is not None and - any(isinstance(instance, t) for t in self.__union_params__)) + def __instancecheck__(self, obj): + raise TypeError("Unions cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -709,18 +700,8 @@ class TupleMeta(TypingMeta): def __hash__(self): return hash(self.__tuple_params__) - def __instancecheck__(self, t): - if not isinstance(t, tuple): - return False - if self.__tuple_params__ is None: - return True - if self.__tuple_use_ellipsis__: - p = self.__tuple_params__[0] - return all(isinstance(x, p) for x in t) - else: - return (len(t) == len(self.__tuple_params__) and - all(isinstance(x, p) - for x, p in zip(t, self.__tuple_params__))) + def __instancecheck__(self, obj): + raise TypeError("Tuples cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -826,57 +807,14 @@ class CallableMeta(TypingMeta): def __hash__(self): return hash(self.__args__) ^ hash(self.__result__) - def __instancecheck__(self, instance): - if not callable(instance): - return False + def __instancecheck__(self, obj): + # For unparametrized Callable we allow this, because + # typing.Callable should be equivalent to + # collections.abc.Callable. if self.__args__ is None and self.__result__ is None: - return True - assert self.__args__ is not None - assert self.__result__ is not None - my_args, my_result = self.__args__, self.__result__ - import inspect # TODO: Avoid this import. - # Would it be better to use Signature objects? - try: - (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, - annotations) = inspect.getfullargspec(instance) - except TypeError: - return False # We can't find the signature. Give up. - msg = ("When testing isinstance(, Callable[...], " - "'s annotations must be types.") - if my_args is not Ellipsis: - if kwonlyargs and (not kwonlydefaults or - len(kwonlydefaults) < len(kwonlyargs)): - return False - if isinstance(instance, types.MethodType): - # For methods, getfullargspec() includes self/cls, - # but it's not part of the call signature, so drop it. - del args[0] - min_call_args = len(args) - if defaults: - min_call_args -= len(defaults) - if varargs: - max_call_args = 999999999 - if len(args) < len(my_args): - args += [varargs] * (len(my_args) - len(args)) - else: - max_call_args = len(args) - if not min_call_args <= len(my_args) <= max_call_args: - return False - for my_arg_type, name in zip(my_args, args): - if name in annotations: - annot_type = _type_check(annotations[name], msg) - else: - annot_type = Any - if not issubclass(my_arg_type, annot_type): - return False - # TODO: If mutable type, check invariance? - if 'return' in annotations: - annot_return_type = _type_check(annotations['return'], msg) - # Note contravariance here! - if not issubclass(annot_return_type, my_result): - return False - # Can't find anything wrong... - return True + return isinstance(obj, collections_abc.Callable) + else: + raise TypeError("Callable[] cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -1073,13 +1011,6 @@ class GenericMeta(TypingMeta, abc.ABCMeta): return False return issubclass(cls, self.__extra__) - def __instancecheck__(self, obj): - if super().__instancecheck__(obj): - return True - if self.__extra__ is None: - return False - return isinstance(obj, self.__extra__) - class Generic(metaclass=GenericMeta): """Abstract base class for generic types. @@ -1234,6 +1165,9 @@ class _ProtocolMeta(GenericMeta): from Generic. """ + def __instancecheck__(self, obj): + raise TypeError("Protocols cannot be used with isinstance().") + def __subclasscheck__(self, cls): if not self._is_protocol: # No structural checks since this isn't a protocol. @@ -1399,19 +1333,7 @@ class ByteString(Sequence[int], extra=collections_abc.ByteString): ByteString.register(type(memoryview(b''))) -class _ListMeta(GenericMeta): - - def __instancecheck__(self, obj): - if not super().__instancecheck__(obj): - return False - itemtype = self.__parameters__[0] - for x in obj: - if not isinstance(x, itemtype): - return False - return True - - -class List(list, MutableSequence[T], metaclass=_ListMeta): +class List(list, MutableSequence[T]): def __new__(cls, *args, **kwds): if _geqv(cls, List): @@ -1420,19 +1342,7 @@ class List(list, MutableSequence[T], metaclass=_ListMeta): return list.__new__(cls, *args, **kwds) -class _SetMeta(GenericMeta): - - def __instancecheck__(self, obj): - if not super().__instancecheck__(obj): - return False - itemtype = self.__parameters__[0] - for x in obj: - if not isinstance(x, itemtype): - return False - return True - - -class Set(set, MutableSet[T], metaclass=_SetMeta): +class Set(set, MutableSet[T]): def __new__(cls, *args, **kwds): if _geqv(cls, Set): @@ -1441,7 +1351,7 @@ class Set(set, MutableSet[T], metaclass=_SetMeta): return set.__new__(cls, *args, **kwds) -class _FrozenSetMeta(_SetMeta): +class _FrozenSetMeta(GenericMeta): """This metaclass ensures set is not a subclass of FrozenSet. Without this metaclass, set would be considered a subclass of @@ -1454,11 +1364,6 @@ class _FrozenSetMeta(_SetMeta): return False return super().__subclasscheck__(cls) - def __instancecheck__(self, obj): - if issubclass(obj.__class__, Set): - return False - return super().__instancecheck__(obj) - class FrozenSet(frozenset, AbstractSet[T_co], metaclass=_FrozenSetMeta): @@ -1488,20 +1393,7 @@ class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): pass -class _DictMeta(GenericMeta): - - def __instancecheck__(self, obj): - if not super().__instancecheck__(obj): - return False - keytype, valuetype = self.__parameters__ - for key, value in obj.items(): - if not (isinstance(key, keytype) and - isinstance(value, valuetype)): - return False - return True - - -class Dict(dict, MutableMapping[KT, VT], metaclass=_DictMeta): +class Dict(dict, MutableMapping[KT, VT]): def __new__(cls, *args, **kwds): if _geqv(cls, Dict): -- cgit v0.12 From 79e434d9e583c9a7fcea9f8d7813e39764326667 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 7 Jun 2015 13:36:19 -0700 Subject: Mapping key type is invariant. --- Lib/test/test_typing.py | 6 +++--- Lib/typing.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9740425..2bb21ed 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -707,11 +707,11 @@ class VarianceTests(TestCase): typing.Sequence[Manager]) def test_covariance_mapping(self): - # Ditto for Mapping (a generic class with two parameters). + # Ditto for Mapping (covariant in the value, invariant in the key). assert issubclass(typing.Mapping[Employee, Manager], typing.Mapping[Employee, Employee]) - assert issubclass(typing.Mapping[Manager, Employee], - typing.Mapping[Employee, Employee]) + assert not issubclass(typing.Mapping[Manager, Employee], + typing.Mapping[Employee, Employee]) assert not issubclass(typing.Mapping[Employee, Manager], typing.Mapping[Manager, Manager]) assert not issubclass(typing.Mapping[Manager, Employee], diff --git a/Lib/typing.py b/Lib/typing.py index 66bee91..bc6fcdd 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -439,7 +439,6 @@ KT = TypeVar('KT') # Key type. VT = TypeVar('VT') # Value type. T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. -KT_co = TypeVar('KT_co', covariant=True) # Key type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. @@ -1308,7 +1307,8 @@ class MutableSet(AbstractSet[T], extra=collections_abc.MutableSet): pass -class Mapping(Sized, Iterable[KT_co], Container[KT_co], Generic[KT_co, VT_co], +# NOTE: Only the value type is covariant. +class Mapping(Sized, Iterable[KT], Container[KT], Generic[KT, VT_co], extra=collections_abc.Mapping): pass @@ -1378,13 +1378,13 @@ class MappingView(Sized, Iterable[T_co], extra=collections_abc.MappingView): pass -class KeysView(MappingView[KT_co], AbstractSet[KT_co], +class KeysView(MappingView[KT], AbstractSet[KT], extra=collections_abc.KeysView): pass -# TODO: Enable Set[Tuple[KT_co, VT_co]] instead of Generic[KT_co, VT_co]. -class ItemsView(MappingView, Generic[KT_co, VT_co], +# TODO: Enable Set[Tuple[KT, VT_co]] instead of Generic[KT, VT_co]. +class ItemsView(MappingView, Generic[KT, VT_co], extra=collections_abc.ItemsView): pass -- cgit v0.12 From 47aacc8f695c4253a8fd085adaff1e552d57da48 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 12 Jun 2015 17:26:23 +0200 Subject: Issue #15745: Rewrite os.utime() tests in test_os * Don't use the timestamp of an existing file anymore, only use fixed timestamp * Enhance the code checking the resolution of the filesystem timestamps. * Check timestamps with a resolution of 1 microsecond instead of 1 millisecond * When os.utime() uses the current system clock, tolerate a delta of 20 ms. Before some os.utime() tolerated a different of 10 seconds. * Merge duplicated tests and simplify the code --- Lib/test/test_os.py | 431 ++++++++++++++++++++++++++-------------------------- 1 file changed, 219 insertions(+), 212 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d2dfcaf..987b4af8 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2,33 +2,34 @@ # does add tests for a few functions which have been determined to be more # portable than they had been thought to be. -import os +import asynchat +import asyncore +import codecs +import contextlib +import decimal import errno +import fractions import getpass -import unittest -import warnings -import sys -import signal -import subprocess -import time -import shutil -from test import support -import contextlib +import itertools +import locale +import math import mmap +import os +import pickle import platform import re -import uuid -import asyncore -import asynchat +import shutil +import signal import socket -import itertools import stat -import locale -import codecs -import decimal -import fractions -import pickle +import subprocess +import sys import sysconfig +import time +import unittest +import uuid +import warnings +from test import support try: import threading except ImportError: @@ -70,16 +71,6 @@ root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - os.stat_float_times(True) -st = os.stat(__file__) -stat_supports_subsecond = ( - # check if float and int timestamps are different - (st.st_atime != st[7]) - or (st.st_mtime != st[8]) - or (st.st_ctime != st[9])) - # Detect whether we're on a Linux system that uses the (now outdated # and unmaintained) linuxthreads threading library. There's an issue # when combining linuxthreads with a failed execv call: see @@ -223,15 +214,10 @@ class FileTests(unittest.TestCase): # Test attributes on return values from os.*stat* family. class StatAttributeTests(unittest.TestCase): def setUp(self): - os.mkdir(support.TESTFN) - self.fname = os.path.join(support.TESTFN, "f1") - f = open(self.fname, 'wb') - f.write(b"ABC") - f.close() - - def tearDown(self): - os.unlink(self.fname) - os.rmdir(support.TESTFN) + self.fname = support.TESTFN + self.addCleanup(support.unlink, self.fname) + with open(self.fname, 'wb') as fp: + fp.write(b"ABC") @unittest.skipUnless(hasattr(os, 'stat'), 'test needs os.stat()') def check_stat_attributes(self, fname): @@ -383,179 +369,6 @@ class StatAttributeTests(unittest.TestCase): unpickled = pickle.loads(p) self.assertEqual(result, unpickled) - def test_utime_dir(self): - delta = 1000000 - st = os.stat(support.TESTFN) - # round to int, because some systems may support sub-second - # time stamps in stat, but not in utime. - os.utime(support.TESTFN, (st.st_atime, int(st.st_mtime-delta))) - st2 = os.stat(support.TESTFN) - self.assertEqual(st2.st_mtime, int(st.st_mtime-delta)) - - def _test_utime(self, filename, attr, utime, delta): - # Issue #13327 removed the requirement to pass None as the - # second argument. Check that the previous methods of passing - # a time tuple or None work in addition to no argument. - st0 = os.stat(filename) - # Doesn't set anything new, but sets the time tuple way - utime(filename, (attr(st0, "st_atime"), attr(st0, "st_mtime"))) - # Setting the time to the time you just read, then reading again, - # should always return exactly the same times. - st1 = os.stat(filename) - self.assertEqual(attr(st0, "st_mtime"), attr(st1, "st_mtime")) - self.assertEqual(attr(st0, "st_atime"), attr(st1, "st_atime")) - # Set to the current time in the old explicit way. - os.utime(filename, None) - st2 = os.stat(support.TESTFN) - # Set to the current time in the new way - os.utime(filename) - st3 = os.stat(filename) - self.assertAlmostEqual(attr(st2, "st_mtime"), attr(st3, "st_mtime"), delta=delta) - - def test_utime(self): - def utime(file, times): - return os.utime(file, times) - self._test_utime(self.fname, getattr, utime, 10) - self._test_utime(support.TESTFN, getattr, utime, 10) - - - def _test_utime_ns(self, set_times_ns, test_dir=True): - def getattr_ns(o, attr): - return getattr(o, attr + "_ns") - ten_s = 10 * 1000 * 1000 * 1000 - self._test_utime(self.fname, getattr_ns, set_times_ns, ten_s) - if test_dir: - self._test_utime(support.TESTFN, getattr_ns, set_times_ns, ten_s) - - def test_utime_ns(self): - def utime_ns(file, times): - return os.utime(file, ns=times) - self._test_utime_ns(utime_ns) - - requires_utime_dir_fd = unittest.skipUnless( - os.utime in os.supports_dir_fd, - "dir_fd support for utime required for this test.") - requires_utime_fd = unittest.skipUnless( - os.utime in os.supports_fd, - "fd support for utime required for this test.") - requires_utime_nofollow_symlinks = unittest.skipUnless( - os.utime in os.supports_follow_symlinks, - "follow_symlinks support for utime required for this test.") - - @requires_utime_nofollow_symlinks - def test_lutimes_ns(self): - def lutimes_ns(file, times): - return os.utime(file, ns=times, follow_symlinks=False) - self._test_utime_ns(lutimes_ns) - - @requires_utime_fd - def test_futimes_ns(self): - def futimes_ns(file, times): - with open(file, "wb") as f: - os.utime(f.fileno(), ns=times) - self._test_utime_ns(futimes_ns, test_dir=False) - - def _utime_invalid_arguments(self, name, arg): - with self.assertRaises(ValueError): - getattr(os, name)(arg, (5, 5), ns=(5, 5)) - - def test_utime_invalid_arguments(self): - self._utime_invalid_arguments('utime', self.fname) - - - @unittest.skipUnless(stat_supports_subsecond, - "os.stat() doesn't has a subsecond resolution") - def _test_utime_subsecond(self, set_time_func): - asec, amsec = 1, 901 - atime = asec + amsec * 1e-3 - msec, mmsec = 2, 901 - mtime = msec + mmsec * 1e-3 - filename = self.fname - os.utime(filename, (0, 0)) - set_time_func(filename, atime, mtime) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - os.stat_float_times(True) - st = os.stat(filename) - self.assertAlmostEqual(st.st_atime, atime, places=3) - self.assertAlmostEqual(st.st_mtime, mtime, places=3) - - def test_utime_subsecond(self): - def set_time(filename, atime, mtime): - os.utime(filename, (atime, mtime)) - self._test_utime_subsecond(set_time) - - @requires_utime_fd - def test_futimes_subsecond(self): - def set_time(filename, atime, mtime): - with open(filename, "wb") as f: - os.utime(f.fileno(), times=(atime, mtime)) - self._test_utime_subsecond(set_time) - - @requires_utime_fd - def test_futimens_subsecond(self): - def set_time(filename, atime, mtime): - with open(filename, "wb") as f: - os.utime(f.fileno(), times=(atime, mtime)) - self._test_utime_subsecond(set_time) - - @requires_utime_dir_fd - def test_futimesat_subsecond(self): - def set_time(filename, atime, mtime): - dirname = os.path.dirname(filename) - dirfd = os.open(dirname, os.O_RDONLY) - try: - os.utime(os.path.basename(filename), dir_fd=dirfd, - times=(atime, mtime)) - finally: - os.close(dirfd) - self._test_utime_subsecond(set_time) - - @requires_utime_nofollow_symlinks - def test_lutimes_subsecond(self): - def set_time(filename, atime, mtime): - os.utime(filename, (atime, mtime), follow_symlinks=False) - self._test_utime_subsecond(set_time) - - @requires_utime_dir_fd - def test_utimensat_subsecond(self): - def set_time(filename, atime, mtime): - dirname = os.path.dirname(filename) - dirfd = os.open(dirname, os.O_RDONLY) - try: - os.utime(os.path.basename(filename), dir_fd=dirfd, - times=(atime, mtime)) - finally: - os.close(dirfd) - self._test_utime_subsecond(set_time) - - # Restrict tests to Win32, since there is no guarantee other - # systems support centiseconds - def get_file_system(path): - if sys.platform == 'win32': - root = os.path.splitdrive(os.path.abspath(path))[0] + '\\' - import ctypes - kernel32 = ctypes.windll.kernel32 - buf = ctypes.create_unicode_buffer("", 100) - if kernel32.GetVolumeInformationW(root, None, 0, None, None, None, buf, len(buf)): - return buf.value - - @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") - @unittest.skipUnless(get_file_system(support.TESTFN) == "NTFS", - "requires NTFS") - def test_1565150(self): - t1 = 1159195039.25 - os.utime(self.fname, (t1, t1)) - self.assertEqual(os.stat(self.fname).st_mtime, t1) - - @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") - @unittest.skipUnless(get_file_system(support.TESTFN) == "NTFS", - "requires NTFS") - def test_large_time(self): - t1 = 5000000000 # some day in 2128 - os.utime(self.fname, (t1, t1)) - self.assertEqual(os.stat(self.fname).st_mtime, t1) - @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_1686475(self): # Verify that an open file can be stat'ed @@ -596,12 +409,206 @@ class StatAttributeTests(unittest.TestCase): 0) # test directory st_file_attributes (FILE_ATTRIBUTE_DIRECTORY set) - result = os.stat(support.TESTFN) + dirname = support.TESTFN + "dir" + os.mkdir(dirname) + self.addCleanup(os.rmdir, dirname) + + result = os.stat(dirname) self.check_file_attributes(result) self.assertEqual( result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) + +class UtimeTests(unittest.TestCase): + def setUp(self): + self.dirname = support.TESTFN + self.fname = os.path.join(self.dirname, "f1") + + self.addCleanup(support.rmtree, self.dirname) + os.mkdir(self.dirname) + with open(self.fname, 'wb') as fp: + fp.write(b"ABC") + + # ensure that st_atime and st_mtime are float + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + + old_state = os.stat_float_times(-1) + self.addCleanup(os.stat_float_times, old_state) + + os.stat_float_times(True) + + def support_subsecond(self, filename): + # Heuristic to check if the filesystem supports timestamp with + # subsecond resolution: check if float and int timestamps are different + st = os.stat(filename) + return ((st.st_atime != st[7]) + or (st.st_mtime != st[8]) + or (st.st_ctime != st[9])) + + def _test_utime(self, set_time, filename=None): + if not filename: + filename = self.fname + + support_subsecond = self.support_subsecond(filename) + if support_subsecond: + # Timestamp with a resolution of 1 microsecond (10^-6). + # + # The resolution of the C internal function used by os.utime() + # depends on the platform: 1 sec, 1 us, 1 ns. Writing a portable + # test with a resolution of 1 ns requires more work: + # see the issue #15745. + atime_ns = 1002003000 # 1.002003 seconds + mtime_ns = 4005006000 # 4.005006 seconds + else: + # use a resolution of 1 second + atime_ns = 5 * 10**9 + mtime_ns = 8 * 10**9 + + set_time(filename, (atime_ns, mtime_ns)) + st = os.stat(filename) + + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) + + def test_utime(self): + def set_time(filename, ns): + # test the ns keyword parameter + os.utime(filename, ns=ns) + self._test_utime(set_time) + + @staticmethod + def ns_to_sec(ns): + # Convert a number of nanosecond (int) to a number of seconds (float). + # Round towards infinity by adding 0.5 nanosecond to avoid rounding + # issue, os.utime() rounds towards minus infinity. + return (ns * 1e-9) + 0.5e-9 + + def test_utime_by_indexed(self): + # pass times as floating point seconds as the second indexed parameter + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec(atime_ns) + mtime = self.ns_to_sec(mtime_ns) + # test utimensat(timespec), utimes(timeval), utime(utimbuf) + # or utime(time_t) + os.utime(filename, (atime, mtime)) + self._test_utime(set_time) + + def test_utime_by_times(self): + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec(atime_ns) + mtime = self.ns_to_sec(mtime_ns) + # test the times keyword parameter + os.utime(filename, times=(atime, mtime)) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_follow_symlinks, + "follow_symlinks support for utime required " + "for this test.") + def test_utime_nofollow_symlinks(self): + def set_time(filename, ns): + # use follow_symlinks=False to test utimensat(timespec) + # or lutimes(timeval) + os.utime(filename, ns=ns, follow_symlinks=False) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_fd, + "fd support for utime required for this test.") + def test_utime_fd(self): + def set_time(filename, ns): + with open(filename, 'wb') as fp: + # use a file descriptor to test futimens(timespec) + # or futimes(timeval) + os.utime(fp.fileno(), ns=ns) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_dir_fd, + "dir_fd support for utime required for this test.") + def test_utime_dir_fd(self): + def set_time(filename, ns): + dirname, name = os.path.split(filename) + dirfd = os.open(dirname, os.O_RDONLY) + try: + # pass dir_fd to test utimensat(timespec) or futimesat(timeval) + os.utime(name, dir_fd=dirfd, ns=ns) + finally: + os.close(dirfd) + self._test_utime(set_time) + + def test_utime_directory(self): + def set_time(filename, ns): + # test calling os.utime() on a directory + os.utime(filename, ns=ns) + self._test_utime(set_time, filename=self.dirname) + + def _test_utime_current(self, set_time): + # Get the system clock + current = time.time() + + # Call os.utime() to set the timestamp to the current system clock + set_time(self.fname) + + if not self.support_subsecond(self.fname): + delta = 1.0 + else: + # On Windows, the usual resolution of time.time() is 15.6 ms + delta = 0.020 + st = os.stat(self.fname) + msg = ("st_time=%r, current=%r, dt=%r" + % (st.st_mtime, current, st.st_mtime - current)) + self.assertAlmostEqual(st.st_mtime, current, + delta=delta, msg=msg) + + def test_utime_current(self): + def set_time(filename): + # Set to the current time in the new way + os.utime(self.fname) + self._test_utime_current(set_time) + + def test_utime_current_old(self): + def set_time(filename): + # Set to the current time in the old explicit way. + os.utime(self.fname, None) + self._test_utime_current(set_time) + + def get_file_system(self, path): + if sys.platform == 'win32': + root = os.path.splitdrive(os.path.abspath(path))[0] + '\\' + import ctypes + kernel32 = ctypes.windll.kernel32 + buf = ctypes.create_unicode_buffer("", 100) + ok = kernel32.GetVolumeInformationW(root, None, 0, + None, None, None, + buf, len(buf)) + if ok: + return buf.value + # return None if the filesystem is unknown + + def test_large_time(self): + # Many filesystems are limited to the year 2038. At least, the test + # pass with NTFS filesystem. + if self.get_file_system(self.dirname) != "NTFS": + self.skipTest("requires NTFS") + + large = 5000000000 # some day in 2128 + os.utime(self.fname, (large, large)) + self.assertEqual(os.stat(self.fname).st_mtime, large) + + def test_utime_invalid_arguments(self): + # seconds and nanoseconds parameters are mutually exclusive + with self.assertRaises(ValueError): + os.utime(self.fname, (5, 5), ns=(5, 5)) + + from test import mapping_tests class EnvironTests(mapping_tests.BasicTestMappingProtocol): -- cgit v0.12 From cf05970307da87c242fd22af0441e6b98427ffda Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 12 Jun 2015 21:57:50 +0200 Subject: Remove unused import on test_os --- Lib/test/test_os.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 987b4af8..fd8269d 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -12,7 +12,6 @@ import fractions import getpass import itertools import locale -import math import mmap import os import pickle -- cgit v0.12 From 72dd5227614746d5965c302d901dbb3384947fbd Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 17 Jun 2015 10:09:24 -0500 Subject: Remove deprecated buildbot scripts --- Tools/buildbot/build-amd64.bat | 5 ----- Tools/buildbot/clean-amd64.bat | 5 ----- Tools/buildbot/external-amd64.bat | 3 --- Tools/buildbot/external.bat | 3 --- Tools/buildbot/test-amd64.bat | 6 ------ 5 files changed, 22 deletions(-) delete mode 100644 Tools/buildbot/build-amd64.bat delete mode 100644 Tools/buildbot/clean-amd64.bat delete mode 100644 Tools/buildbot/external-amd64.bat delete mode 100644 Tools/buildbot/external.bat delete mode 100644 Tools/buildbot/test-amd64.bat diff --git a/Tools/buildbot/build-amd64.bat b/Tools/buildbot/build-amd64.bat deleted file mode 100644 index f77407b..0000000 --- a/Tools/buildbot/build-amd64.bat +++ /dev/null @@ -1,5 +0,0 @@ -@rem Formerly used by the buildbot "compile" step. -@echo This script is no longer used and may be removed in the future. -@echo To get the same effect as this script, use -@echo PCbuild\build.bat -d -e -k -p x64 -call "%~dp0build.bat" -p x64 %* diff --git a/Tools/buildbot/clean-amd64.bat b/Tools/buildbot/clean-amd64.bat deleted file mode 100644 index b53c7c1..0000000 --- a/Tools/buildbot/clean-amd64.bat +++ /dev/null @@ -1,5 +0,0 @@ -@rem Formerly used by the buildbot "clean" step. -@echo This script is no longer used and may be removed in the future. -@echo To get the same effect as this script, use `clean.bat` from this -@echo directory and pass `-p x64` as two arguments. -call "%~dp0clean.bat" -p x64 %* diff --git a/Tools/buildbot/external-amd64.bat b/Tools/buildbot/external-amd64.bat deleted file mode 100644 index bfaef05..0000000 --- a/Tools/buildbot/external-amd64.bat +++ /dev/null @@ -1,3 +0,0 @@ -@echo This script is no longer used and may be removed in the future. -@echo Please use PCbuild\get_externals.bat instead. -@"%~dp0..\..\PCbuild\get_externals.bat" %* diff --git a/Tools/buildbot/external.bat b/Tools/buildbot/external.bat deleted file mode 100644 index bfaef05..0000000 --- a/Tools/buildbot/external.bat +++ /dev/null @@ -1,3 +0,0 @@ -@echo This script is no longer used and may be removed in the future. -@echo Please use PCbuild\get_externals.bat instead. -@"%~dp0..\..\PCbuild\get_externals.bat" %* diff --git a/Tools/buildbot/test-amd64.bat b/Tools/buildbot/test-amd64.bat deleted file mode 100644 index e48329c..0000000 --- a/Tools/buildbot/test-amd64.bat +++ /dev/null @@ -1,6 +0,0 @@ -@rem Formerly used by the buildbot "test" step. -@echo This script is no longer used and may be removed in the future. -@echo To get the same effect as this script, use -@echo PCbuild\rt.bat -q -d -x64 -uall -rwW -@echo or use `test.bat` in this directory and pass `-x64` as an argument. -call "%~dp0test.bat" -x64 %* -- cgit v0.12 From 6ee588f14e23206db3c927653956fd35f6ca857a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 20 Jun 2015 21:39:51 -0700 Subject: Restore quick exit (no freeslot check) for common case (found null on first probe). --- Objects/setobject.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 707ab95..d621647 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -142,7 +142,10 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) entry = &table[i]; if (entry->key == NULL) - goto found_null; + goto found_null_first; + + freeslot = NULL; + perturb = hash; while (1) { if (entry->hash == hash) { @@ -207,6 +210,13 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_null; } + found_null_first: + so->fill++; + so->used++; + entry->key = key; + entry->hash = hash; + return 0; + found_null: if (freeslot == NULL) { /* UNUSED */ -- cgit v0.12 From 66dc4648fcca725bc48b0c8d7030c107dfeda709 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 21 Jun 2015 14:06:55 +0300 Subject: Issue #24426: Fast searching optimization in regular expressions now works for patterns that starts with capturing groups. Fast searching optimization now can't be disabled at compile time. --- Lib/sre_compile.py | 117 +++++++++++++++++++++++++++++++---------------------- Misc/NEWS | 4 ++ Modules/_sre.c | 3 -- Modules/sre_lib.h | 53 ++++++++++++------------ 4 files changed, 99 insertions(+), 78 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index 502b061..4edb03f 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -409,57 +409,39 @@ def _generate_overlap_table(prefix): table[i] = idx + 1 return table -def _compile_info(code, pattern, flags): - # internal: compile an info block. in the current version, - # this contains min/max pattern width, and an optional literal - # prefix or a character map - lo, hi = pattern.getwidth() - if hi > MAXCODE: - hi = MAXCODE - if lo == 0: - code.extend([INFO, 4, 0, lo, hi]) - return - # look for a literal prefix +def _get_literal_prefix(pattern): + # look for literal prefix prefix = [] prefixappend = prefix.append - prefix_skip = 0 + prefix_skip = None + got_all = True + for op, av in pattern.data: + if op is LITERAL: + prefixappend(av) + elif op is SUBPATTERN: + prefix1, prefix_skip1, got_all = _get_literal_prefix(av[1]) + if prefix_skip is None: + if av[0] is not None: + prefix_skip = len(prefix) + elif prefix_skip1 is not None: + prefix_skip = len(prefix) + prefix_skip1 + prefix.extend(prefix1) + if not got_all: + break + else: + got_all = False + break + return prefix, prefix_skip, got_all + +def _get_charset_prefix(pattern): charset = [] # not used charsetappend = charset.append - if not (flags & SRE_FLAG_IGNORECASE): - # look for literal prefix - for op, av in pattern.data: + if pattern.data: + op, av = pattern.data[0] + if op is SUBPATTERN and av[1]: + op, av = av[1][0] if op is LITERAL: - if len(prefix) == prefix_skip: - prefix_skip = prefix_skip + 1 - prefixappend(av) - elif op is SUBPATTERN and len(av[1]) == 1: - op, av = av[1][0] - if op is LITERAL: - prefixappend(av) - else: - break - else: - break - # if no prefix, look for charset prefix - if not prefix and pattern.data: - op, av = pattern.data[0] - if op is SUBPATTERN and av[1]: - op, av = av[1][0] - if op is LITERAL: - charsetappend((op, av)) - elif op is BRANCH: - c = [] - cappend = c.append - for p in av[1]: - if not p: - break - op, av = p[0] - if op is LITERAL: - cappend((op, av)) - else: - break - else: - charset = c + charsetappend((op, av)) elif op is BRANCH: c = [] cappend = c.append @@ -473,8 +455,43 @@ def _compile_info(code, pattern, flags): break else: charset = c - elif op is IN: - charset = av + elif op is BRANCH: + c = [] + cappend = c.append + for p in av[1]: + if not p: + break + op, av = p[0] + if op is LITERAL: + cappend((op, av)) + else: + break + else: + charset = c + elif op is IN: + charset = av + return charset + +def _compile_info(code, pattern, flags): + # internal: compile an info block. in the current version, + # this contains min/max pattern width, and an optional literal + # prefix or a character map + lo, hi = pattern.getwidth() + if hi > MAXCODE: + hi = MAXCODE + if lo == 0: + code.extend([INFO, 4, 0, lo, hi]) + return + # look for a literal prefix + prefix = [] + prefix_skip = 0 + charset = [] # not used + if not (flags & SRE_FLAG_IGNORECASE): + # look for literal prefix + prefix, prefix_skip, got_all = _get_literal_prefix(pattern) + # if no prefix, look for charset prefix + if not prefix: + charset = _get_charset_prefix(pattern) ## if prefix: ## print("*** PREFIX", prefix, prefix_skip) ## if charset: @@ -487,7 +504,7 @@ def _compile_info(code, pattern, flags): mask = 0 if prefix: mask = SRE_INFO_PREFIX - if len(prefix) == prefix_skip == len(pattern.data): + if prefix_skip is None and got_all: mask = mask | SRE_INFO_LITERAL elif charset: mask = mask | SRE_INFO_CHARSET @@ -502,6 +519,8 @@ def _compile_info(code, pattern, flags): # add literal prefix if prefix: emit(len(prefix)) # length + if prefix_skip is None: + prefix_skip = len(prefix) emit(prefix_skip) # skip code.extend(prefix) # generate overlap table diff --git a/Misc/NEWS b/Misc/NEWS index 3fdda95..dda1286 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,10 @@ Core and Builtins Library ------- +- Issue #24426: Fast searching optimization in regular expressions now works + for patterns that starts with capturing groups. Fast searching optimization + now can't be disabled at compile time. + Documentation ------------- diff --git a/Modules/_sre.c b/Modules/_sre.c index 4016a45..1d90281 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -62,9 +62,6 @@ static char copyright[] = /* -------------------------------------------------------------------- */ /* optional features */ -/* enables fast searching */ -#define USE_FAST_SEARCH - /* enables copy/deepcopy handling (work in progress) */ #undef USE_BUILTIN_COPY diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 463a908..422f168 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -1248,7 +1248,32 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) prefix, prefix_len, prefix_skip)); TRACE(("charset = %p\n", charset)); -#if defined(USE_FAST_SEARCH) + if (prefix_len == 1) { + /* pattern starts with a literal character */ + SRE_CHAR c = (SRE_CHAR) prefix[0]; +#if SIZEOF_SRE_CHAR < 4 + if ((SRE_CODE) c != prefix[0]) + return 0; /* literal can't match: doesn't fit in char width */ +#endif + end = (SRE_CHAR *)state->end; + while (ptr < end) { + while (*ptr != c) { + if (++ptr >= end) + return 0; + } + TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr)); + state->start = ptr; + state->ptr = ptr + prefix_skip; + if (flags & SRE_INFO_LITERAL) + return 1; /* we got all of it */ + status = SRE(match)(state, pattern + 2*prefix_skip, 0); + if (status != 0) + return status; + ++ptr; + } + return 0; + } + if (prefix_len > 1) { /* pattern starts with a known prefix. use the overlap table to skip forward as fast as we possibly can */ @@ -1297,32 +1322,8 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) } return 0; } -#endif - if (pattern[0] == SRE_OP_LITERAL) { - /* pattern starts with a literal character. this is used - for short prefixes, and if fast search is disabled */ - SRE_CHAR c = (SRE_CHAR) pattern[1]; -#if SIZEOF_SRE_CHAR < 4 - if ((SRE_CODE) c != pattern[1]) - return 0; /* literal can't match: doesn't fit in char width */ -#endif - end = (SRE_CHAR *)state->end; - while (ptr < end) { - while (*ptr != c) { - if (++ptr >= end) - return 0; - } - TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr)); - state->start = ptr; - state->ptr = ++ptr; - if (flags & SRE_INFO_LITERAL) - return 1; /* we got all of it */ - status = SRE(match)(state, pattern + 2, 0); - if (status != 0) - break; - } - } else if (charset) { + if (charset) { /* pattern starts with a character from a known set */ end = (SRE_CHAR *)state->end; for (;;) { -- cgit v0.12 From 7e3592dca6e9d1f991ef500e9c66c271474907b7 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 21 Jun 2015 10:47:20 -0700 Subject: Harmonize the bottom of the outer loop with its entry point giving a small simplification. Timings show that hash pre-check seems only benefit the inner-loop (the linear probes). --- Objects/setobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index d621647..70ec644 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -118,7 +118,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) i = (i * 5 + 1 + perturb) & mask; entry = &table[i]; - if (entry->hash == 0 && entry->key == NULL) + if (entry->key == NULL) return entry; } } @@ -206,7 +206,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) i = (i * 5 + 1 + perturb) & mask; entry = &table[i]; - if (entry->hash == 0 && entry->key == NULL) + if (entry->key == NULL) goto found_null; } -- cgit v0.12 From 38bb95e49d36754b9e9a45c0b1d6d7e1f9a82f03 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 24 Jun 2015 01:22:19 -0700 Subject: Minor code cleanup. --- Objects/setobject.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 70ec644..5b430b3 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -53,7 +53,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *table = so->table; setentry *entry; - size_t perturb = hash; + size_t perturb; size_t mask = so->mask; size_t i = (size_t)hash & mask; /* Unsigned for defined overflow behavior */ size_t j; @@ -63,6 +63,8 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) if (entry->key == NULL) return entry; + perturb = hash; + while (1) { if (entry->hash == hash) { PyObject *startkey = entry->key; @@ -132,9 +134,9 @@ static int set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *table = so->table; - setentry *freeslot = NULL; + setentry *freeslot; setentry *entry; - size_t perturb = hash; + size_t perturb; size_t mask = so->mask; size_t i = (size_t)hash & mask; /* Unsigned for defined overflow behavior */ size_t j; -- cgit v0.12 From 91672617d5189e3593415a2a3e7df759ff3aca6b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 26 Jun 2015 02:50:21 -0700 Subject: Minor tweeak to tighten the inner-loop. --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 5b430b3..85cb4bc 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -199,7 +199,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_active; mask = so->mask; } - if (entry->hash == -1 && freeslot == NULL) + else if (entry->hash == -1 && freeslot == NULL) freeslot = entry; } } -- cgit v0.12 From 2eff9e9441d76904792141b4b38b0eb649d9eb86 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 27 Jun 2015 22:03:35 -0700 Subject: Minor refactoring. Move reference count logic into function that adds entry. --- Objects/setobject.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 85cb4bc..09d1129 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -125,11 +125,6 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) } } -/* -Internal routine to insert a new key into the table. -Used by the public insert routine. -Eats a reference to key. -*/ static int set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) { @@ -213,6 +208,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) } found_null_first: + Py_INCREF(key); so->fill++; so->used++; entry->key = key; @@ -220,6 +216,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) return 0; found_null: + Py_INCREF(key); if (freeslot == NULL) { /* UNUSED */ so->fill++; @@ -233,7 +230,6 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) return 0; found_active: - Py_DECREF(key); return 0; } @@ -381,11 +377,8 @@ set_add_entry(PySetObject *so, setentry *entry) assert(so->fill <= so->mask); /* at least one empty slot */ n_used = so->used; - Py_INCREF(key); - if (set_insert_key(so, key, hash)) { - Py_DECREF(key); + if (set_insert_key(so, key, hash)) return -1; - } if (!(so->used > n_used && so->fill*3 >= (so->mask+1)*2)) return 0; return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); @@ -678,11 +671,8 @@ set_merge(PySetObject *so, PyObject *otherset) for (i = 0; i <= other->mask; i++, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { - Py_INCREF(key); - if (set_insert_key(so, key, other_entry->hash)) { - Py_DECREF(key); + if (set_insert_key(so, key, other_entry->hash)) return -1; - } } } return 0; -- cgit v0.12 From 15f08696096a80d026ed21164f892a661fc72e98 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 3 Jul 2015 17:21:17 -0700 Subject: Move insertion resize logic into set_insert_key(). Simplifies the code a little bit and does the resize check only when a new key is added (giving a small speed up in the case where the key already exists). Fixes possible bug in set_merge() where the set_insert_key() call relies on a big resize at the start to make enough room for the keys but is vulnerable to a comparision callback that could cause the table to shrink in the middle of the merge. Also, changed the resize threshold from two-thirds of the mask+1 to just two-thirds. The plus one offset gave no real benefit (afterall, the two-thirds mark is just a heuristic and isn't a precise cut-off). --- Objects/setobject.c | 69 +++++++++++++++++++---------------------------------- 1 file changed, 24 insertions(+), 45 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 09d1129..56084c1 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -125,6 +125,8 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) } } +static int set_table_resize(PySetObject *, Py_ssize_t); + static int set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) { @@ -139,7 +141,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) entry = &table[i]; if (entry->key == NULL) - goto found_null_first; + goto found_unused; freeslot = NULL; perturb = hash; @@ -173,7 +175,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) for (j = 0 ; j < LINEAR_PROBES ; j++) { entry++; if (entry->hash == 0 && entry->key == NULL) - goto found_null; + goto found_unused_or_dummy; if (entry->hash == hash) { PyObject *startkey = entry->key; assert(startkey != dummy); @@ -204,30 +206,27 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) entry = &table[i]; if (entry->key == NULL) - goto found_null; + goto found_unused_or_dummy; } - found_null_first: + found_unused_or_dummy: + if (freeslot == NULL) + goto found_unused; Py_INCREF(key); - so->fill++; so->used++; - entry->key = key; - entry->hash = hash; + freeslot->key = key; + freeslot->hash = hash; return 0; - found_null: + found_unused: Py_INCREF(key); - if (freeslot == NULL) { - /* UNUSED */ - so->fill++; - } else { - /* DUMMY */ - entry = freeslot; - } + so->fill++; so->used++; entry->key = key; entry->hash = hash; - return 0; + if ((size_t)so->fill*3 < mask*2) + return 0; + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); found_active: return 0; @@ -366,28 +365,15 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) return 0; } -/* CAUTION: set_add_key/entry() must guarantee it won't resize the table */ - static int set_add_entry(PySetObject *so, setentry *entry) { - Py_ssize_t n_used; - PyObject *key = entry->key; - Py_hash_t hash = entry->hash; - - assert(so->fill <= so->mask); /* at least one empty slot */ - n_used = so->used; - if (set_insert_key(so, key, hash)) - return -1; - if (!(so->used > n_used && so->fill*3 >= (so->mask+1)*2)) - return 0; - return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); + return set_insert_key(so, entry->key, entry->hash); } static int set_add_key(PySetObject *so, PyObject *key) { - setentry entry; Py_hash_t hash; if (!PyUnicode_CheckExact(key) || @@ -396,9 +382,7 @@ set_add_key(PySetObject *so, PyObject *key) if (hash == -1) return -1; } - entry.key = key; - entry.hash = hash; - return set_add_entry(so, &entry); + return set_insert_key(so, key, hash); } #define DISCARD_NOTFOUND 0 @@ -631,7 +615,7 @@ set_merge(PySetObject *so, PyObject *otherset) * incrementally resizing as we insert new keys. Expect * that there will be no (or few) overlapping keys. */ - if ((so->fill + other->used)*3 >= (so->mask+1)*2) { + if ((so->fill + other->used)*3 >= so->mask*2) { if (set_table_resize(so, (so->used + other->used)*2) != 0) return -1; } @@ -979,16 +963,12 @@ set_update_internal(PySetObject *so, PyObject *other) */ if (dictsize == -1) return -1; - if ((so->fill + dictsize)*3 >= (so->mask+1)*2) { + if ((so->fill + dictsize)*3 >= so->mask*2) { if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - setentry an_entry; - - an_entry.hash = hash; - an_entry.key = key; - if (set_add_entry(so, &an_entry)) + if (set_insert_key(so, key, hash)) return -1; } return 0; @@ -1583,17 +1563,16 @@ set_difference(PySetObject *so, PyObject *other) if (PyDict_CheckExact(other)) { while (set_next(so, &pos, &entry)) { - setentry entrycopy; + PyObject *key = entry->key; + Py_hash_t hash = entry->hash; int rv; - entrycopy.hash = entry->hash; - entrycopy.key = entry->key; - rv = _PyDict_Contains(other, entry->key, entry->hash); + rv = _PyDict_Contains(other, key, hash); if (rv < 0) { Py_DECREF(result); return NULL; } if (!rv) { - if (set_add_entry((PySetObject *)result, &entrycopy)) { + if (set_insert_key((PySetObject *)result, key, hash)) { Py_DECREF(result); return NULL; } -- cgit v0.12 From 3c1f52e829e9877225cb171d779aacc4c6353daf Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 3 Jul 2015 18:31:09 -0700 Subject: Call set_lookkey() directly to avoid unnecessary memory spills and reloads. --- Objects/setobject.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 56084c1..bf9718e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -678,7 +678,7 @@ set_contains_entry(PySetObject *so, setentry *entry) static int set_contains_key(PySetObject *so, PyObject *key) { - setentry entry; + setentry *entry; Py_hash_t hash; if (!PyUnicode_CheckExact(key) || @@ -687,9 +687,10 @@ set_contains_key(PySetObject *so, PyObject *key) if (hash == -1) return -1; } - entry.key = key; - entry.hash = hash; - return set_contains_entry(so, &entry); + entry = set_lookkey(so, key, hash); + if (entry == NULL) + return -1; + return entry->key != NULL; } static PyObject * -- cgit v0.12 From 4897300276d870f99459c82b937f0ac22450f0b6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 3 Jul 2015 20:00:03 -0700 Subject: Minor factoring: move redundant resize scaling logic into the resize function. --- Objects/setobject.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index bf9718e..9228e0c 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -226,7 +226,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); + return set_table_resize(so, so->used); found_active: return 0; @@ -290,6 +290,7 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry small_copy[PySet_MINSIZE]; assert(minused >= 0); + minused = (minused > 50000) ? minused * 2 : minused * 4; /* Find the smallest table size > minused. */ /* XXX speed-up with intrinsics */ @@ -616,7 +617,7 @@ set_merge(PySetObject *so, PyObject *otherset) * that there will be no (or few) overlapping keys. */ if ((so->fill + other->used)*3 >= so->mask*2) { - if (set_table_resize(so, (so->used + other->used)*2) != 0) + if (set_table_resize(so, so->used + other->used) != 0) return -1; } so_entry = so->table; @@ -965,7 +966,7 @@ set_update_internal(PySetObject *so, PyObject *other) if (dictsize == -1) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { - if (set_table_resize(so, (so->used + dictsize)*2) != 0) + if (set_table_resize(so, so->used + dictsize) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { @@ -1508,7 +1509,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* If more than 1/5 are dummies, then resize them away. */ if ((so->fill - so->used) * 5 < so->mask) return 0; - return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); + return set_table_resize(so, so->used); } static PyObject * -- cgit v0.12 From b322326f48b03acfcd1bafe7b37a7061e7207058 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 3 Jul 2015 23:37:16 -0700 Subject: Minor nit: Make the style of checking error return values more consistent. --- Objects/setobject.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 9228e0c..ed588ed 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1270,7 +1270,7 @@ set_intersection(PySetObject *so, PyObject *other) while (set_next((PySetObject *)other, &pos, &entry)) { int rv = set_contains_entry(so, entry); - if (rv == -1) { + if (rv < 0) { Py_DECREF(result); return NULL; } @@ -1304,7 +1304,7 @@ set_intersection(PySetObject *so, PyObject *other) entry.hash = hash; entry.key = key; rv = set_contains_entry(so, &entry); - if (rv == -1) { + if (rv < 0) { Py_DECREF(it); Py_DECREF(result); Py_DECREF(key); @@ -1431,7 +1431,7 @@ set_isdisjoint(PySetObject *so, PyObject *other) } while (set_next((PySetObject *)other, &pos, &entry)) { int rv = set_contains_entry(so, entry); - if (rv == -1) + if (rv < 0) return NULL; if (rv) Py_RETURN_FALSE; @@ -1486,7 +1486,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) Py_ssize_t pos = 0; while (set_next((PySetObject *)other, &pos, &entry)) - if (set_discard_entry(so, entry) == -1) + if (set_discard_entry(so, entry) < 0) return -1; } else { PyObject *key, *it; @@ -1495,7 +1495,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) return -1; while ((key = PyIter_Next(it)) != NULL) { - if (set_discard_key(so, key) == -1) { + if (set_discard_key(so, key) < 0) { Py_DECREF(it); Py_DECREF(key); return -1; @@ -1536,7 +1536,7 @@ set_copy_and_difference(PySetObject *so, PyObject *other) result = set_copy(so); if (result == NULL) return NULL; - if (set_difference_update_internal((PySetObject *) result, other) != -1) + if (set_difference_update_internal((PySetObject *) result, other) == 0) return result; Py_DECREF(result); return NULL; @@ -1586,7 +1586,7 @@ set_difference(PySetObject *so, PyObject *other) /* Iterate over so, checking for common elements in other. */ while (set_next(so, &pos, &entry)) { int rv = set_contains_entry((PySetObject *)other, entry); - if (rv == -1) { + if (rv < 0) { Py_DECREF(result); return NULL; } @@ -1670,7 +1670,7 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) an_entry.key = key; rv = set_discard_entry(so, &an_entry); - if (rv == -1) { + if (rv < 0) { Py_DECREF(key); return NULL; } @@ -1696,7 +1696,7 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) while (set_next(otherset, &pos, &entry)) { int rv = set_discard_entry(so, entry); - if (rv == -1) { + if (rv < 0) { Py_DECREF(otherset); return NULL; } @@ -1778,7 +1778,7 @@ set_issubset(PySetObject *so, PyObject *other) while (set_next(so, &pos, &entry)) { int rv = set_contains_entry((PySetObject *)other, entry); - if (rv == -1) + if (rv < 0) return NULL; if (!rv) Py_RETURN_FALSE; @@ -1869,7 +1869,7 @@ set_contains(PySetObject *so, PyObject *key) int rv; rv = set_contains_key(so, key); - if (rv == -1) { + if (rv < 0) { if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return -1; PyErr_Clear(); @@ -1888,7 +1888,7 @@ set_direct_contains(PySetObject *so, PyObject *key) long result; result = set_contains(so, key); - if (result == -1) + if (result < 0) return NULL; return PyBool_FromLong(result); } @@ -1902,7 +1902,7 @@ set_remove(PySetObject *so, PyObject *key) int rv; rv = set_discard_key(so, key); - if (rv == -1) { + if (rv < 0) { if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); @@ -1911,7 +1911,7 @@ set_remove(PySetObject *so, PyObject *key) return NULL; rv = set_discard_key(so, tmpkey); Py_DECREF(tmpkey); - if (rv == -1) + if (rv < 0) return NULL; } @@ -1934,7 +1934,7 @@ set_discard(PySetObject *so, PyObject *key) int rv; rv = set_discard_key(so, key); - if (rv == -1) { + if (rv < 0) { if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); @@ -1943,7 +1943,7 @@ set_discard(PySetObject *so, PyObject *key) return NULL; rv = set_discard_key(so, tmpkey); Py_DECREF(tmpkey); - if (rv == -1) + if (rv < 0) return NULL; } Py_RETURN_NONE; -- cgit v0.12 From c2480dc0c4cf13719c5aa514db9f579990a1603f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 4 Jul 2015 08:46:31 -0700 Subject: Minor cleanup. --- Objects/setobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index ed588ed..8e56f72 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -963,7 +963,7 @@ set_update_internal(PySetObject *so, PyObject *other) * incrementally resizing as we insert new keys. Expect * that there will be no (or few) overlapping keys. */ - if (dictsize == -1) + if (dictsize < 0) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { if (set_table_resize(so, so->used + dictsize) != 0) @@ -1457,7 +1457,7 @@ set_isdisjoint(PySetObject *so, PyObject *other) entry.key = key; rv = set_contains_entry(so, &entry); Py_DECREF(key); - if (rv == -1) { + if (rv < 0) { Py_DECREF(it); return NULL; } -- cgit v0.12 From e186c7674cbc8d2796a7971b53daa98e60d7f601 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 4 Jul 2015 11:28:35 -0700 Subject: Make sure the dummy percentage calculation won't overflow. --- Objects/setobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 8e56f72..cc87f28 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1506,8 +1506,8 @@ set_difference_update_internal(PySetObject *so, PyObject *other) if (PyErr_Occurred()) return -1; } - /* If more than 1/5 are dummies, then resize them away. */ - if ((so->fill - so->used) * 5 < so->mask) + /* If more than 1/4th are dummies, then resize them away. */ + if ((size_t)(so->fill - so->used) <= (size_t)so->mask / 4) return 0; return set_table_resize(so, so->used); } -- cgit v0.12 From ac2ef65c320606e30132ca58bbd6b5d6861ce644 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 4 Jul 2015 16:04:44 -0700 Subject: Make the unicode equality test an external function rather than in-lining it. The real benefit of the unicode specialized function comes from bypassing the overhead of PyObject_RichCompareBool() and not from being in-lined (especially since there was almost no shared data between the caller and callee). Also, the in-lining was having a negative effect on code generation for the callee. --- Include/unicodeobject.h | 4 ++++ Objects/setobject.c | 9 ++++----- Objects/unicodeobject.c | 7 +++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 4ba6328..33e8f19 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -2261,6 +2261,10 @@ PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); /* Clear all static strings. */ PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void); +/* Fast equality check when the inputs are known to be exact unicode types + and where the hash values are equal (i.e. a very probable match) */ +PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *); + #ifdef __cplusplus } #endif diff --git a/Objects/setobject.c b/Objects/setobject.c index cc87f28..d0fb4f1 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -29,7 +29,6 @@ #include "Python.h" #include "structmember.h" -#include "stringlib/eq.h" /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -74,7 +73,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) return entry; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && unicode_eq(startkey, key)) + && _PyUnicode_EQ(startkey, key)) return entry; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); @@ -100,7 +99,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) return entry; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && unicode_eq(startkey, key)) + && _PyUnicode_EQ(startkey, key)) return entry; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); @@ -155,7 +154,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_active; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && unicode_eq(startkey, key)) + && _PyUnicode_EQ(startkey, key)) goto found_active; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); @@ -183,7 +182,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_active; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && unicode_eq(startkey, key)) + && _PyUnicode_EQ(startkey, key)) goto found_active; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1eaf2e9..796e0b4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -42,6 +42,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "Python.h" #include "ucnhash.h" #include "bytes_methods.h" +#include "stringlib/eq.h" #ifdef MS_WINDOWS #include @@ -10887,6 +10888,12 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } int +_PyUnicode_EQ(PyObject *aa, PyObject *bb) +{ + return unicode_eq(aa, bb); +} + +int PyUnicode_Contains(PyObject *container, PyObject *element) { PyObject *str, *sub; -- cgit v0.12 From dc28d5a1986e88057bb44bfeff34296cf4c01f54 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 5 Jul 2015 10:03:20 -0700 Subject: Clean-up call patterns for add/contains/discard to better match the caller's needs. --- Objects/setobject.c | 69 +++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index d0fb4f1..1387812 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -127,7 +127,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_table_resize(PySetObject *, Py_ssize_t); static int -set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) +set_add_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *table = so->table; setentry *freeslot; @@ -162,7 +162,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) if (cmp < 0) /* unlikely */ return -1; if (table != so->table || entry->key != startkey) /* unlikely */ - return set_insert_key(so, key, hash); + return set_add_key_hash(so, key, hash); if (cmp > 0) /* likely */ goto found_active; mask = so->mask; /* help avoid a register spill */ @@ -190,7 +190,7 @@ set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) if (cmp < 0) return -1; if (table != so->table || entry->key != startkey) - return set_insert_key(so, key, hash); + return set_add_key_hash(so, key, hash); if (cmp > 0) goto found_active; mask = so->mask; @@ -368,7 +368,7 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) static int set_add_entry(PySetObject *so, setentry *entry) { - return set_insert_key(so, entry->key, entry->hash); + return set_add_key_hash(so, entry->key, entry->hash); } static int @@ -382,19 +382,19 @@ set_add_key(PySetObject *so, PyObject *key) if (hash == -1) return -1; } - return set_insert_key(so, key, hash); + return set_add_key_hash(so, key, hash); } #define DISCARD_NOTFOUND 0 #define DISCARD_FOUND 1 static int -set_discard_entry(PySetObject *so, setentry *oldentry) +set_discard_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *entry; PyObject *old_key; - entry = set_lookkey(so, oldentry->key, oldentry->hash); + entry = set_lookkey(so, key, hash); if (entry == NULL) return -1; if (entry->key == NULL) @@ -408,9 +408,14 @@ set_discard_entry(PySetObject *so, setentry *oldentry) } static int +set_discard_entry(PySetObject *so, setentry *entry) +{ + return set_discard_key_hash(so, entry->key, entry->hash); +} + +static int set_discard_key(PySetObject *so, PyObject *key) { - setentry entry; Py_hash_t hash; assert (PyAnySet_Check(so)); @@ -421,9 +426,7 @@ set_discard_key(PySetObject *so, PyObject *key) if (hash == -1) return -1; } - entry.key = key; - entry.hash = hash; - return set_discard_entry(so, &entry); + return set_discard_key_hash(so, key, hash); } static void @@ -655,7 +658,7 @@ set_merge(PySetObject *so, PyObject *otherset) for (i = 0; i <= other->mask; i++, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { - if (set_insert_key(so, key, other_entry->hash)) + if (set_add_key_hash(so, key, other_entry->hash)) return -1; } } @@ -663,16 +666,20 @@ set_merge(PySetObject *so, PyObject *otherset) } static int -set_contains_entry(PySetObject *so, setentry *entry) +set_contains_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) { - PyObject *key; setentry *lu_entry; - lu_entry = set_lookkey(so, entry->key, entry->hash); - if (lu_entry == NULL) - return -1; - key = lu_entry->key; - return key != NULL; + lu_entry = set_lookkey(so, key, hash); + if (lu_entry != NULL) + return lu_entry->key != NULL; + return -1; +} + +static int +set_contains_entry(PySetObject *so, setentry *entry) +{ + return set_contains_key_hash(so, entry->key, entry->hash); } static int @@ -969,7 +976,7 @@ set_update_internal(PySetObject *so, PyObject *other) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_insert_key(so, key, hash)) + if (set_add_key_hash(so, key, hash)) return -1; } return 0; @@ -1291,7 +1298,6 @@ set_intersection(PySetObject *so, PyObject *other) while ((key = PyIter_Next(it)) != NULL) { int rv; - setentry entry; Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { @@ -1300,9 +1306,7 @@ set_intersection(PySetObject *so, PyObject *other) Py_DECREF(key); return NULL; } - entry.hash = hash; - entry.key = key; - rv = set_contains_entry(so, &entry); + rv = set_contains_key_hash(so, key, hash); if (rv < 0) { Py_DECREF(it); Py_DECREF(result); @@ -1310,7 +1314,7 @@ set_intersection(PySetObject *so, PyObject *other) return NULL; } if (rv) { - if (set_add_entry(result, &entry)) { + if (set_add_key_hash(result, key, hash)) { Py_DECREF(it); Py_DECREF(result); Py_DECREF(key); @@ -1444,7 +1448,6 @@ set_isdisjoint(PySetObject *so, PyObject *other) while ((key = PyIter_Next(it)) != NULL) { int rv; - setentry entry; Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { @@ -1452,9 +1455,7 @@ set_isdisjoint(PySetObject *so, PyObject *other) Py_DECREF(it); return NULL; } - entry.hash = hash; - entry.key = key; - rv = set_contains_entry(so, &entry); + rv = set_contains_key_hash(so, key, hash); Py_DECREF(key); if (rv < 0) { Py_DECREF(it); @@ -1573,7 +1574,7 @@ set_difference(PySetObject *so, PyObject *other) return NULL; } if (!rv) { - if (set_insert_key((PySetObject *)result, key, hash)) { + if (set_add_key_hash((PySetObject *)result, key, hash)) { Py_DECREF(result); return NULL; } @@ -1662,19 +1663,15 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) int rv; Py_hash_t hash; while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - setentry an_entry; - Py_INCREF(key); - an_entry.hash = hash; - an_entry.key = key; - rv = set_discard_entry(so, &an_entry); + rv = set_discard_key_hash(so, key, hash); if (rv < 0) { Py_DECREF(key); return NULL; } if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, &an_entry)) { + if (set_add_key_hash(so, key, hash)) { Py_DECREF(key); return NULL; } -- cgit v0.12 From 73799b181ea13949be2e008f431fdd0fab15d7f1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 5 Jul 2015 16:06:10 -0700 Subject: Change add/contains/discard calls to pass the key and hash instead of an entry struct. --- Objects/setobject.c | 94 ++++++++++++++++++++++++----------------------------- 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 1387812..fd10008 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -127,7 +127,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_table_resize(PySetObject *, Py_ssize_t); static int -set_add_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) +set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *table = so->table; setentry *freeslot; @@ -162,7 +162,7 @@ set_add_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) if (cmp < 0) /* unlikely */ return -1; if (table != so->table || entry->key != startkey) /* unlikely */ - return set_add_key_hash(so, key, hash); + return set_add_entry(so, key, hash); if (cmp > 0) /* likely */ goto found_active; mask = so->mask; /* help avoid a register spill */ @@ -190,7 +190,7 @@ set_add_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) if (cmp < 0) return -1; if (table != so->table || entry->key != startkey) - return set_add_key_hash(so, key, hash); + return set_add_entry(so, key, hash); if (cmp > 0) goto found_active; mask = so->mask; @@ -366,12 +366,6 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) } static int -set_add_entry(PySetObject *so, setentry *entry) -{ - return set_add_key_hash(so, entry->key, entry->hash); -} - -static int set_add_key(PySetObject *so, PyObject *key) { Py_hash_t hash; @@ -382,14 +376,14 @@ set_add_key(PySetObject *so, PyObject *key) if (hash == -1) return -1; } - return set_add_key_hash(so, key, hash); + return set_add_entry(so, key, hash); } #define DISCARD_NOTFOUND 0 #define DISCARD_FOUND 1 static int -set_discard_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) +set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *entry; PyObject *old_key; @@ -408,12 +402,6 @@ set_discard_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) } static int -set_discard_entry(PySetObject *so, setentry *entry) -{ - return set_discard_key_hash(so, entry->key, entry->hash); -} - -static int set_discard_key(PySetObject *so, PyObject *key) { Py_hash_t hash; @@ -426,7 +414,7 @@ set_discard_key(PySetObject *so, PyObject *key) if (hash == -1) return -1; } - return set_discard_key_hash(so, key, hash); + return set_discard_entry(so, key, hash); } static void @@ -658,7 +646,7 @@ set_merge(PySetObject *so, PyObject *otherset) for (i = 0; i <= other->mask; i++, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { - if (set_add_key_hash(so, key, other_entry->hash)) + if (set_add_entry(so, key, other_entry->hash)) return -1; } } @@ -666,7 +654,7 @@ set_merge(PySetObject *so, PyObject *otherset) } static int -set_contains_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) +set_contains_entry(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *lu_entry; @@ -677,12 +665,6 @@ set_contains_key_hash(PySetObject *so, PyObject *key, Py_hash_t hash) } static int -set_contains_entry(PySetObject *so, setentry *entry) -{ - return set_contains_key_hash(so, entry->key, entry->hash); -} - -static int set_contains_key(PySetObject *so, PyObject *key) { setentry *entry; @@ -976,7 +958,7 @@ set_update_internal(PySetObject *so, PyObject *other) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_key_hash(so, key, hash)) + if (set_add_entry(so, key, hash)) return -1; } return 0; @@ -1256,6 +1238,8 @@ set_intersection(PySetObject *so, PyObject *other) { PySetObject *result; PyObject *key, *it, *tmp; + Py_hash_t hash; + int rv; if ((PyObject *)so == other) return set_copy(so); @@ -1275,13 +1259,15 @@ set_intersection(PySetObject *so, PyObject *other) } while (set_next((PySetObject *)other, &pos, &entry)) { - int rv = set_contains_entry(so, entry); + key = entry->key; + hash = entry->hash; + rv = set_contains_entry(so, key, hash); if (rv < 0) { Py_DECREF(result); return NULL; } if (rv) { - if (set_add_entry(result, entry)) { + if (set_add_entry(result, key, hash)) { Py_DECREF(result); return NULL; } @@ -1297,16 +1283,14 @@ set_intersection(PySetObject *so, PyObject *other) } while ((key = PyIter_Next(it)) != NULL) { - int rv; - Py_hash_t hash = PyObject_Hash(key); - + hash = PyObject_Hash(key); if (hash == -1) { Py_DECREF(it); Py_DECREF(result); Py_DECREF(key); return NULL; } - rv = set_contains_key_hash(so, key, hash); + rv = set_contains_entry(so, key, hash); if (rv < 0) { Py_DECREF(it); Py_DECREF(result); @@ -1314,7 +1298,7 @@ set_intersection(PySetObject *so, PyObject *other) return NULL; } if (rv) { - if (set_add_key_hash(result, key, hash)) { + if (set_add_entry(result, key, hash)) { Py_DECREF(it); Py_DECREF(result); Py_DECREF(key); @@ -1415,6 +1399,7 @@ static PyObject * set_isdisjoint(PySetObject *so, PyObject *other) { PyObject *key, *it, *tmp; + int rv; if ((PyObject *)so == other) { if (PySet_GET_SIZE(so) == 0) @@ -1433,7 +1418,7 @@ set_isdisjoint(PySetObject *so, PyObject *other) other = tmp; } while (set_next((PySetObject *)other, &pos, &entry)) { - int rv = set_contains_entry(so, entry); + rv = set_contains_entry(so, entry->key, entry->hash); if (rv < 0) return NULL; if (rv) @@ -1447,7 +1432,6 @@ set_isdisjoint(PySetObject *so, PyObject *other) return NULL; while ((key = PyIter_Next(it)) != NULL) { - int rv; Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { @@ -1455,7 +1439,7 @@ set_isdisjoint(PySetObject *so, PyObject *other) Py_DECREF(it); return NULL; } - rv = set_contains_key_hash(so, key, hash); + rv = set_contains_entry(so, key, hash); Py_DECREF(key); if (rv < 0) { Py_DECREF(it); @@ -1486,7 +1470,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) Py_ssize_t pos = 0; while (set_next((PySetObject *)other, &pos, &entry)) - if (set_discard_entry(so, entry) < 0) + if (set_discard_entry(so, entry->key, entry->hash) < 0) return -1; } else { PyObject *key, *it; @@ -1546,8 +1530,11 @@ static PyObject * set_difference(PySetObject *so, PyObject *other) { PyObject *result; + PyObject *key; + Py_hash_t hash; setentry *entry; Py_ssize_t pos = 0; + int rv; if (!PyAnySet_Check(other) && !PyDict_CheckExact(other)) { return set_copy_and_difference(so, other); @@ -1565,16 +1552,15 @@ set_difference(PySetObject *so, PyObject *other) if (PyDict_CheckExact(other)) { while (set_next(so, &pos, &entry)) { - PyObject *key = entry->key; - Py_hash_t hash = entry->hash; - int rv; + key = entry->key; + hash = entry->hash; rv = _PyDict_Contains(other, key, hash); if (rv < 0) { Py_DECREF(result); return NULL; } if (!rv) { - if (set_add_key_hash((PySetObject *)result, key, hash)) { + if (set_add_entry((PySetObject *)result, key, hash)) { Py_DECREF(result); return NULL; } @@ -1585,13 +1571,15 @@ set_difference(PySetObject *so, PyObject *other) /* Iterate over so, checking for common elements in other. */ while (set_next(so, &pos, &entry)) { - int rv = set_contains_entry((PySetObject *)other, entry); + key = entry->key; + hash = entry->hash; + rv = set_contains_entry((PySetObject *)other, key, hash); if (rv < 0) { Py_DECREF(result); return NULL; } if (!rv) { - if (set_add_entry((PySetObject *)result, entry)) { + if (set_add_entry((PySetObject *)result, key, hash)) { Py_DECREF(result); return NULL; } @@ -1653,25 +1641,24 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) PySetObject *otherset; PyObject *key; Py_ssize_t pos = 0; + Py_hash_t hash; setentry *entry; + int rv; if ((PyObject *)so == other) return set_clear(so); if (PyDict_CheckExact(other)) { PyObject *value; - int rv; - Py_hash_t hash; while (_PyDict_Next(other, &pos, &key, &value, &hash)) { Py_INCREF(key); - - rv = set_discard_key_hash(so, key, hash); + rv = set_discard_entry(so, key, hash); if (rv < 0) { Py_DECREF(key); return NULL; } if (rv == DISCARD_NOTFOUND) { - if (set_add_key_hash(so, key, hash)) { + if (set_add_entry(so, key, hash)) { Py_DECREF(key); return NULL; } @@ -1691,13 +1678,15 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) } while (set_next(otherset, &pos, &entry)) { - int rv = set_discard_entry(so, entry); + key = entry->key; + hash = entry->hash; + rv = set_discard_entry(so, key, hash); if (rv < 0) { Py_DECREF(otherset); return NULL; } if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, entry)) { + if (set_add_entry(so, key, hash)) { Py_DECREF(otherset); return NULL; } @@ -1759,6 +1748,7 @@ set_issubset(PySetObject *so, PyObject *other) { setentry *entry; Py_ssize_t pos = 0; + int rv; if (!PyAnySet_Check(other)) { PyObject *tmp, *result; @@ -1773,7 +1763,7 @@ set_issubset(PySetObject *so, PyObject *other) Py_RETURN_FALSE; while (set_next(so, &pos, &entry)) { - int rv = set_contains_entry((PySetObject *)other, entry); + rv = set_contains_entry((PySetObject *)other, entry->key, entry->hash); if (rv < 0) return NULL; if (!rv) -- cgit v0.12 From b48d6a63ffdba18a0fab80f3458632dcf01e2bea Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 5 Jul 2015 16:27:44 -0700 Subject: Bring related functions add/contains/discard together in the code. --- Objects/setobject.c | 74 ++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index fd10008..307f19e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -366,17 +366,14 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) } static int -set_add_key(PySetObject *so, PyObject *key) +set_contains_entry(PySetObject *so, PyObject *key, Py_hash_t hash) { - Py_hash_t hash; + setentry *entry; - if (!PyUnicode_CheckExact(key) || - (hash = ((PyASCIIObject *) key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; - } - return set_add_entry(so, key, hash); + entry = set_lookkey(so, key, hash); + if (entry != NULL) + return entry->key != NULL; + return -1; } #define DISCARD_NOTFOUND 0 @@ -402,11 +399,37 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) } static int -set_discard_key(PySetObject *so, PyObject *key) +set_add_key(PySetObject *so, PyObject *key) { Py_hash_t hash; - assert (PyAnySet_Check(so)); + if (!PyUnicode_CheckExact(key) || + (hash = ((PyASCIIObject *) key)->hash) == -1) { + hash = PyObject_Hash(key); + if (hash == -1) + return -1; + } + return set_add_entry(so, key, hash); +} + +static int +set_contains_key(PySetObject *so, PyObject *key) +{ + Py_hash_t hash; + + if (!PyUnicode_CheckExact(key) || + (hash = ((PyASCIIObject *) key)->hash) == -1) { + hash = PyObject_Hash(key); + if (hash == -1) + return -1; + } + return set_contains_entry(so, key, hash); +} + +static int +set_discard_key(PySetObject *so, PyObject *key) +{ + Py_hash_t hash; if (!PyUnicode_CheckExact(key) || (hash = ((PyASCIIObject *) key)->hash) == -1) { @@ -653,35 +676,6 @@ set_merge(PySetObject *so, PyObject *otherset) return 0; } -static int -set_contains_entry(PySetObject *so, PyObject *key, Py_hash_t hash) -{ - setentry *lu_entry; - - lu_entry = set_lookkey(so, key, hash); - if (lu_entry != NULL) - return lu_entry->key != NULL; - return -1; -} - -static int -set_contains_key(PySetObject *so, PyObject *key) -{ - setentry *entry; - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = ((PyASCIIObject *) key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; - } - entry = set_lookkey(so, key, hash); - if (entry == NULL) - return -1; - return entry->key != NULL; -} - static PyObject * set_pop(PySetObject *so) { -- cgit v0.12 From ef6bd7d96336426f951c4636e232cdb855ef3d04 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 6 Jul 2015 08:43:37 -0700 Subject: Tighten-up code in set_next() to use an entry pointer rather than indexing. --- Objects/setobject.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 307f19e..9753c8d 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -518,20 +518,22 @@ set_next(PySetObject *so, Py_ssize_t *pos_ptr, setentry **entry_ptr) { Py_ssize_t i; Py_ssize_t mask; - setentry *table; + setentry *entry; assert (PyAnySet_Check(so)); i = *pos_ptr; assert(i >= 0); - table = so->table; mask = so->mask; - while (i <= mask && (table[i].key == NULL || table[i].key == dummy)) + entry = &so->table[i]; + while (i <= mask && (entry->key == NULL || entry->key == dummy)) { i++; + entry++; + } *pos_ptr = i+1; if (i > mask) return 0; - assert(table[i].key != NULL); - *entry_ptr = &table[i]; + assert(entry != NULL); + *entry_ptr = entry; return 1; } -- cgit v0.12 From 3dbc11cadd0aead3ee6d4d06fc3bf7309c6ce6a7 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 6 Jul 2015 19:03:01 -0700 Subject: Tighten-up code in the set iterator to use an entry pointer rather than indexing. --- Objects/setobject.c | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 9753c8d..383e7a4 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -766,8 +766,8 @@ typedef struct { PyObject_HEAD PySetObject *si_set; /* Set to NULL when iterator is exhausted */ Py_ssize_t si_used; - Py_ssize_t si_pos; Py_ssize_t len; + setentry *entry; } setiterobject; static void @@ -845,8 +845,6 @@ static PyMethodDef setiter_methods[] = { static PyObject *setiter_iternext(setiterobject *si) { - PyObject *key; - Py_ssize_t i, mask; setentry *entry; PySetObject *so = si->si_set; @@ -860,25 +858,18 @@ static PyObject *setiter_iternext(setiterobject *si) si->si_used = -1; /* Make this state sticky */ return NULL; } - - i = si->si_pos; - assert(i>=0); - entry = so->table; - mask = so->mask; - while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) - i++; - si->si_pos = i+1; - if (i > mask) - goto fail; + if (si->len <= 0) { + Py_DECREF(so); + si->si_set = NULL; + return NULL; + } + entry = si->entry; + while (entry->key == NULL || entry->key == dummy) + entry++; si->len--; - key = entry[i].key; - Py_INCREF(key); - return key; - -fail: - Py_DECREF(so); - si->si_set = NULL; - return NULL; + si->entry = entry + 1; + Py_INCREF(entry->key); + return entry->key; } PyTypeObject PySetIter_Type = { @@ -923,8 +914,8 @@ set_iter(PySetObject *so) Py_INCREF(so); si->si_set = so; si->si_used = so->used; - si->si_pos = 0; si->len = so->used; + si->entry = so->table; _PyObject_GC_TRACK(si); return (PyObject *)si; } -- cgit v0.12 From 11ce8e6c37c9f6b14ccac8abfe26d4c5e8313f5c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 6 Jul 2015 19:08:49 -0700 Subject: Minor bit of factoring-out common code. --- Objects/setobject.c | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 383e7a4..12f82f3 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1271,26 +1271,14 @@ set_intersection(PySetObject *so, PyObject *other) while ((key = PyIter_Next(it)) != NULL) { hash = PyObject_Hash(key); - if (hash == -1) { - Py_DECREF(it); - Py_DECREF(result); - Py_DECREF(key); - return NULL; - } + if (hash == -1) + goto error; rv = set_contains_entry(so, key, hash); - if (rv < 0) { - Py_DECREF(it); - Py_DECREF(result); - Py_DECREF(key); - return NULL; - } + if (rv < 0) + goto error; if (rv) { - if (set_add_entry(result, key, hash)) { - Py_DECREF(it); - Py_DECREF(result); - Py_DECREF(key); - return NULL; - } + if (set_add_entry(result, key, hash)) + goto error; } Py_DECREF(key); } @@ -1300,6 +1288,11 @@ set_intersection(PySetObject *so, PyObject *other) return NULL; } return (PyObject *)result; + error: + Py_DECREF(it); + Py_DECREF(result); + Py_DECREF(key); + return NULL; } static PyObject * -- cgit v0.12 From 9632a7d735071b56f2361f09fde559b3b5edef69 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 7 Jul 2015 15:29:24 -0700 Subject: Issue 24581: Revert c9782a9ac031 pending a stronger test for mutation during iteration. --- Objects/setobject.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 12f82f3..7c91c38 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -766,8 +766,8 @@ typedef struct { PyObject_HEAD PySetObject *si_set; /* Set to NULL when iterator is exhausted */ Py_ssize_t si_used; + Py_ssize_t si_pos; Py_ssize_t len; - setentry *entry; } setiterobject; static void @@ -845,6 +845,8 @@ static PyMethodDef setiter_methods[] = { static PyObject *setiter_iternext(setiterobject *si) { + PyObject *key; + Py_ssize_t i, mask; setentry *entry; PySetObject *so = si->si_set; @@ -858,18 +860,25 @@ static PyObject *setiter_iternext(setiterobject *si) si->si_used = -1; /* Make this state sticky */ return NULL; } - if (si->len <= 0) { - Py_DECREF(so); - si->si_set = NULL; - return NULL; - } - entry = si->entry; - while (entry->key == NULL || entry->key == dummy) - entry++; + + i = si->si_pos; + assert(i>=0); + entry = so->table; + mask = so->mask; + while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) + i++; + si->si_pos = i+1; + if (i > mask) + goto fail; si->len--; - si->entry = entry + 1; - Py_INCREF(entry->key); - return entry->key; + key = entry[i].key; + Py_INCREF(key); + return key; + +fail: + Py_DECREF(so); + si->si_set = NULL; + return NULL; } PyTypeObject PySetIter_Type = { @@ -914,8 +923,8 @@ set_iter(PySetObject *so) Py_INCREF(so); si->si_set = so; si->si_used = so->used; + si->si_pos = 0; si->len = so->used; - si->entry = so->table; _PyObject_GC_TRACK(si); return (PyObject *)si; } -- cgit v0.12 From 5d2385ff6f761c9c9a6b68afdcc88322d739e7d1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 8 Jul 2015 11:52:27 -0700 Subject: Neaten-up a little bit. --- Objects/setobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 7c91c38..e6fb46e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -902,7 +902,7 @@ PyTypeObject PySetIter_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ (traverseproc)setiter_traverse, /* tp_traverse */ 0, /* tp_clear */ @@ -2145,7 +2145,7 @@ static PyMethodDef frozenset_methods[] = { copy_doc}, {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, difference_doc}, - {"intersection",(PyCFunction)set_intersection_multi, METH_VARARGS, + {"intersection", (PyCFunction)set_intersection_multi, METH_VARARGS, intersection_doc}, {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, isdisjoint_doc}, @@ -2216,7 +2216,7 @@ PyTypeObject PyFrozenSet_Type = { (traverseproc)set_traverse, /* tp_traverse */ (inquiry)set_clear_internal, /* tp_clear */ (richcmpfunc)set_richcompare, /* tp_richcompare */ - offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ + offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ (getiterfunc)set_iter, /* tp_iter */ 0, /* tp_iternext */ frozenset_methods, /* tp_methods */ -- cgit v0.12 From dc87e4b88551f64cb4053d8ea72dffc0c3675413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Mon, 13 Jul 2015 21:01:39 +0100 Subject: Issue #23530: Improve os.cpu_count() description. Patch by Julian Taylor. --- Doc/library/multiprocessing.rst | 9 +++++++-- Doc/library/os.rst | 5 +++++ Misc/ACKS | 1 + Modules/posixmodule.c | 6 ++++-- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 8703f8f..e74430d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -869,8 +869,13 @@ Miscellaneous .. function:: cpu_count() - Return the number of CPUs in the system. May raise - :exc:`NotImplementedError`. + Return the number of CPUs in the system. + + This number is not equivalent to the number of CPUs the current process can + use. The number of usable CPUs can be obtained with + ``len(os.sched_getaffinity(0))`` + + May raise :exc:`NotImplementedError`. .. seealso:: :func:`os.cpu_count` diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 16e5019..0acb8a5 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3596,6 +3596,11 @@ Miscellaneous System Information Return the number of CPUs in the system. Returns None if undetermined. + This number is not equivalent to the number of CPUs the current process can + use. The number of usable CPUs can be obtained with + ``len(os.sched_getaffinity(0))`` + + .. versionadded:: 3.4 diff --git a/Misc/ACKS b/Misc/ACKS index 8a007ea..e4ba783 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1385,6 +1385,7 @@ William Tanksley Christian Tanzer Steven Taschuk Amy Taylor +Julian Taylor Monty Taylor Anatoly Techtonik Gustavo Temple diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ec8c526..23d74a3 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5760,7 +5760,7 @@ os.sched_getaffinity pid: pid_t / -Return the affinity of the process identified by pid. +Return the affinity of the process identified by pid (or the current process if zero). The affinity is returned as a set of CPU identifiers. [clinic start generated code]*/ @@ -11201,7 +11201,9 @@ get_terminal_size(PyObject *self, PyObject *args) /*[clinic input] os.cpu_count -Return the number of CPUs in the system; return None if indeterminable. +Return the number of CPUs in the system; return None if indeterminable. This +number is not equivalent to the number of CPUs the current process can use. +The number of usable CPUs can be obtained with ``len(os.sched_getaffinity(0))`` [clinic start generated code]*/ static PyObject * -- cgit v0.12 From a3fffb0539010a73dc9fb808aa504712a63b7bc4 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 14 Jul 2015 13:51:40 +1200 Subject: Issue #23661: unittest.mock side_effects can now be exceptions again. This was a regression vs Python 3.4. Patch from Ignacio Rossi --- Lib/unittest/mock.py | 3 ++- Lib/unittest/test/testmock/testmock.py | 9 +++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index c3ab4e8..191a175 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -506,7 +506,8 @@ class NonCallableMock(Base): if delegated is None: return self._mock_side_effect sf = delegated.side_effect - if sf is not None and not callable(sf) and not isinstance(sf, _MockIter): + if (sf is not None and not callable(sf) + and not isinstance(sf, _MockIter) and not _is_exception(sf)): sf = _MockIter(sf) delegated.side_effect = sf return sf diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py index 3a104cb..f4a723d 100644 --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/unittest/test/testmock/testmock.py @@ -173,6 +173,15 @@ class MockTest(unittest.TestCase): self.assertEqual([mock(), mock(), mock()], [3, 2, 1], "callable side effect not used correctly") + def test_autospec_side_effect_exception(self): + # Test for issue 23661 + def f(): + pass + + mock = create_autospec(f) + mock.side_effect = ValueError('Bazinga!') + self.assertRaisesRegex(ValueError, 'Bazinga!', mock) + @unittest.skipUnless('java' in sys.platform, 'This test only applies to Jython') def test_java_exception_side_effect(self): diff --git a/Misc/ACKS b/Misc/ACKS index e4ba783..36a7472 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1582,3 +1582,4 @@ Tarek Ziadé Gennadiy Zlobin Doug Zongker Peter Åstrand +Ignacio Rossi diff --git a/Misc/NEWS b/Misc/NEWS index 554d9aa..e737c46 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Library for patterns that starts with capturing groups. Fast searching optimization now can't be disabled at compile time. +- Issue #23661: unittest.mock side_effects can now be exceptions again. This + was a regression vs Python 3.4. Patch from Ignacio Rossi + What's New in Python 3.5.0 beta 4? ================================== -- cgit v0.12 From 061091a7c505403fbb95e8f8e16e1b7b6ed5dd1b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 15 Jul 2015 23:54:02 -0700 Subject: Issue #24583: Fix crash when set is mutated while being updated. --- Objects/setobject.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 922c619..fbac596 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -127,7 +127,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_table_resize(PySetObject *, Py_ssize_t); static int -set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) +_set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *table = so->table; setentry *freeslot; @@ -162,7 +162,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) if (cmp < 0) /* unlikely */ return -1; if (table != so->table || entry->key != startkey) /* unlikely */ - return set_add_entry(so, key, hash); + return _set_add_entry(so, key, hash); if (cmp > 0) /* likely */ goto found_active; mask = so->mask; /* help avoid a register spill */ @@ -190,7 +190,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) if (cmp < 0) return -1; if (table != so->table || entry->key != startkey) - return set_add_entry(so, key, hash); + return _set_add_entry(so, key, hash); if (cmp > 0) goto found_active; mask = so->mask; @@ -211,14 +211,12 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) found_unused_or_dummy: if (freeslot == NULL) goto found_unused; - Py_INCREF(key); so->used++; freeslot->key = key; freeslot->hash = hash; return 0; found_unused: - Py_INCREF(key); so->fill++; so->used++; entry->key = key; @@ -231,6 +229,16 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) return 0; } +static int +set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) +{ + Py_INCREF(key); + if (!_set_add_entry(so, key, hash)) + return 0; + Py_DECREF(key); + return -1; +} + /* Internal routine used by set_table_resize() to insert an item which is known to be absent from the set. This routine also assumes that -- cgit v0.12 From cfe5b6ca04a14ef1ba42e78bf6da05cfa9c0b66b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 20 Jul 2015 00:25:50 -0400 Subject: Divisions-by-two for a positive Py_ssize_t compile more cleanly with >>1 than /2. --- Modules/_collectionsmodule.c | 2 +- Modules/_heapqmodule.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c1fd8b9..47db46f 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -788,7 +788,7 @@ deque_reverse(dequeobject *deque, PyObject *unused) block *rightblock = deque->rightblock; Py_ssize_t leftindex = deque->leftindex; Py_ssize_t rightindex = deque->rightindex; - Py_ssize_t n = Py_SIZE(deque) / 2; + Py_ssize_t n = Py_SIZE(deque) >> 1; Py_ssize_t i; PyObject *tmp; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index c343862..28604af 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -66,7 +66,7 @@ siftup(PyListObject *heap, Py_ssize_t pos) /* Bubble up the smaller child until hitting a leaf. */ arr = _PyList_ITEMS(heap); - limit = endpos / 2; /* smallest pos that has no child */ + limit = endpos >> 1; /* smallest pos that has no child */ while (pos < limit) { /* Set childpos to index of smaller child. */ childpos = 2*pos + 1; /* leftmost child position */ @@ -347,7 +347,7 @@ heapify_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) n is odd = 2*j+1, this is (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. */ - for (i = n/2 - 1 ; i >= 0 ; i--) + for (i = (n >> 1) - 1 ; i >= 0 ; i--) if (siftup_func((PyListObject *)heap, i)) return NULL; Py_RETURN_NONE; @@ -420,7 +420,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) /* Bubble up the smaller child until hitting a leaf. */ arr = _PyList_ITEMS(heap); - limit = endpos / 2; /* smallest pos that has no child */ + limit = endpos >> 1; /* smallest pos that has no child */ while (pos < limit) { /* Set childpos to index of smaller child. */ childpos = 2*pos + 1; /* leftmost child position */ -- cgit v0.12 From 482c05cbb5506a5332e435ffdd2c32ae62e7b9a1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 20 Jul 2015 01:23:32 -0400 Subject: Issue #24583: Fix refcount leak. --- Objects/setobject.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index fbac596..83bff81 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -223,9 +223,13 @@ _set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used); + if (!set_table_resize(so, so->used)) + return 0; + Py_INCREF(key); + return -1; found_active: + Py_DECREF(key); return 0; } -- cgit v0.12 From ff9e18a863aa9f7efd894f660d413cd02ccb1834 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 20 Jul 2015 07:34:05 -0400 Subject: Issue #24583: Consolidate previous set object updates into a single function with a single entry point, named exit points at the bottom, more self-evident refcount adjustments, and a comment describing why the pre-increment was necessary at all. --- Objects/setobject.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 83bff81..3c4c484 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -127,17 +127,27 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_table_resize(PySetObject *, Py_ssize_t); static int -_set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) +set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) { - setentry *table = so->table; + setentry *table; setentry *freeslot; setentry *entry; size_t perturb; - size_t mask = so->mask; - size_t i = (size_t)hash & mask; /* Unsigned for defined overflow behavior */ + size_t mask; + size_t i; /* Unsigned for defined overflow behavior */ size_t j; int cmp; + /* Pre-increment is necessary to prevent arbitrary code in the rich + comparison from deallocating the key just before the insertion. */ + Py_INCREF(key); + + restart: + + table = so->table; + mask = so->mask; + i = (size_t)hash & mask; + entry = &table[i]; if (entry->key == NULL) goto found_unused; @@ -160,9 +170,9 @@ _set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); if (cmp < 0) /* unlikely */ - return -1; + goto comparison_error; if (table != so->table || entry->key != startkey) /* unlikely */ - return _set_add_entry(so, key, hash); + goto restart; if (cmp > 0) /* likely */ goto found_active; mask = so->mask; /* help avoid a register spill */ @@ -188,9 +198,9 @@ _set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); if (cmp < 0) - return -1; + goto comparison_error; if (table != so->table || entry->key != startkey) - return _set_add_entry(so, key, hash); + goto restart; if (cmp > 0) goto found_active; mask = so->mask; @@ -223,22 +233,13 @@ _set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - if (!set_table_resize(so, so->used)) - return 0; - Py_INCREF(key); - return -1; + return set_table_resize(so, so->used); found_active: Py_DECREF(key); return 0; -} -static int -set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) -{ - Py_INCREF(key); - if (!_set_add_entry(so, key, hash)) - return 0; + comparison_error: Py_DECREF(key); return -1; } -- cgit v0.12 From ad1a34197e0d4b890776b8490853d6b9ee38934a Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Tue, 21 Jul 2015 00:54:19 -0700 Subject: Close issue6549: minor ttk.Style fixes --- Lib/tkinter/ttk.py | 10 +++++++--- Misc/NEWS | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index b9c57ad..bad9596 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -381,7 +381,9 @@ class Style(object): a sequence identifying the value for that option.""" if query_opt is not None: kw[query_opt] = None - return _val_or_dict(self.tk, kw, self._name, "configure", style) + result = _val_or_dict(self.tk, kw, self._name, "configure", style) + if result or query_opt: + return result def map(self, style, query_opt=None, **kw): @@ -466,12 +468,14 @@ class Style(object): def element_names(self): """Returns the list of elements defined in the current theme.""" - return self.tk.splitlist(self.tk.call(self._name, "element", "names")) + return tuple(n.lstrip('-') for n in self.tk.splitlist( + self.tk.call(self._name, "element", "names"))) def element_options(self, elementname): """Return the list of elementname's options.""" - return self.tk.splitlist(self.tk.call(self._name, "element", "options", elementname)) + return tuple(o.lstrip('-') for o in self.tk.splitlist( + self.tk.call(self._name, "element", "options", elementname))) def theme_create(self, themename, parent=None, settings=None): diff --git a/Misc/NEWS b/Misc/NEWS index 9c5c65b..960798e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -61,6 +61,9 @@ Library - Issue #15014: SMTP.auth() and SMTP.login() now support RFC 4954's optional initial-response argument to the SMTP AUTH command. +- Issue #6549: Remove hyphen from ttk.Style().element options. Only return result + from ttk.Style().configure if a result was generated or a query submitted. + What's New in Python 3.5.0 beta 3? ================================== -- cgit v0.12 From 9e1bf4968f0e122c7e1b86dc74ec9a59c717c5c3 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 23 Jul 2015 02:57:56 +1200 Subject: Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. --- Lib/test/test_zipimport.py | 21 ++++++++++++++++++++- Misc/NEWS | 2 ++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index a97a778..4f19535 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -214,7 +214,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): packdir2 = packdir + TESTPACK2 + os.sep files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), + "spam" + pyc_ext: (NOW, test_pyc)} z = ZipFile(TEMP_ZIP, "w") try: @@ -228,6 +229,14 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) self.assertEqual(zi.is_package(TESTPACK), True) + + find_mod = zi.find_module('spam') + self.assertIsNotNone(find_mod) + self.assertIsInstance(find_mod, zipimport.zipimporter) + self.assertFalse(find_mod.is_package('spam')) + load_mod = find_mod.load_module('spam') + self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__) + mod = zi.load_module(TESTPACK) self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) @@ -287,6 +296,16 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEqual( zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) + pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2 + zi2 = zipimport.zipimporter(pkg_path) + find_mod_dotted = zi2.find_module(TESTMOD) + self.assertIsNotNone(find_mod_dotted) + self.assertIsInstance(find_mod_dotted, zipimport.zipimporter) + self.assertFalse(zi2.is_package(TESTMOD)) + load_mod = find_mod_dotted.load_module(TESTMOD) + self.assertEqual( + find_mod_dotted.get_filename(TESTMOD), load_mod.__file__) + mod_path = TESTPACK2 + os.sep + TESTMOD mod_name = module_path_to_dotted_name(mod_path) __import__(mod_name) diff --git a/Misc/NEWS b/Misc/NEWS index 08dcefd..ce0bd56 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,8 @@ Core and Builtins Library ------- +- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. + - Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. Patch from Nicola Palumbo and Laurent De Buyst. -- cgit v0.12 From d4f4a6dddb91de101fa98a4e1c12e06a3ff058a6 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 23 Jul 2015 06:19:18 +1200 Subject: Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. --- Lib/lib2to3/fixes/fix_types.py | 2 +- Lib/lib2to3/tests/test_fixers.py | 4 ++++ Misc/NEWS | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/lib2to3/fixes/fix_types.py b/Lib/lib2to3/fixes/fix_types.py index db34104..00327a7 100644 --- a/Lib/lib2to3/fixes/fix_types.py +++ b/Lib/lib2to3/fixes/fix_types.py @@ -42,7 +42,7 @@ _TYPE_MAPPING = { 'NotImplementedType' : 'type(NotImplemented)', 'SliceType' : 'slice', 'StringType': 'bytes', # XXX ? - 'StringTypes' : 'str', # XXX ? + 'StringTypes' : '(str,)', # XXX ? 'TupleType': 'tuple', 'TypeType' : 'type', 'UnicodeType': 'str', diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 06b0033..def9b0e 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -3322,6 +3322,10 @@ class Test_types(FixerTestCase): a = """type(None)""" self.check(b, a) + b = "types.StringTypes" + a = "(str,)" + self.check(b, a) + class Test_idioms(FixerTestCase): fixer = "idioms" diff --git a/Misc/NEWS b/Misc/NEWS index ce0bd56..f2f7599 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,8 @@ Core and Builtins Library ------- +- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. + - Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. -- cgit v0.12 From 70559b5c2026ad7fc76996ef05571ac8e977ef26 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 23 Jul 2015 07:42:23 -0400 Subject: Issue #24681: Move the store of so->table to the code block where it is used. --- Objects/setobject.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 3c4c484..0e443c4 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -50,7 +50,7 @@ static PyObject _dummy_struct; static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { - setentry *table = so->table; + setentry *table; setentry *entry; size_t perturb; size_t mask = so->mask; @@ -58,7 +58,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) size_t j; int cmp; - entry = &table[i]; + entry = &so->table[i]; if (entry->key == NULL) return entry; @@ -75,6 +75,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) && PyUnicode_CheckExact(key) && _PyUnicode_EQ(startkey, key)) return entry; + table = so->table; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); @@ -101,6 +102,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) && PyUnicode_CheckExact(key) && _PyUnicode_EQ(startkey, key)) return entry; + table = so->table; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); @@ -118,7 +120,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) perturb >>= PERTURB_SHIFT; i = (i * 5 + 1 + perturb) & mask; - entry = &table[i]; + entry = &so->table[i]; if (entry->key == NULL) return entry; } @@ -144,11 +146,10 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) restart: - table = so->table; mask = so->mask; i = (size_t)hash & mask; - entry = &table[i]; + entry = &so->table[i]; if (entry->key == NULL) goto found_unused; @@ -166,6 +167,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) && PyUnicode_CheckExact(key) && _PyUnicode_EQ(startkey, key)) goto found_active; + table = so->table; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); @@ -177,7 +179,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_active; mask = so->mask; /* help avoid a register spill */ } - if (entry->hash == -1 && freeslot == NULL) + else if (entry->hash == -1 && freeslot == NULL) freeslot = entry; if (i + LINEAR_PROBES <= mask) { @@ -194,6 +196,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) && PyUnicode_CheckExact(key) && _PyUnicode_EQ(startkey, key)) goto found_active; + table = so->table; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); @@ -213,7 +216,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) perturb >>= PERTURB_SHIFT; i = (i * 5 + 1 + perturb) & mask; - entry = &table[i]; + entry = &so->table[i]; if (entry->key == NULL) goto found_unused_or_dummy; } -- cgit v0.12 From f1b5ccb9937b8336078f023bf7deb6615a5e28c8 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 23 Jul 2015 17:36:02 +0300 Subject: Issue #13248: Remove inspect.getargspec from 3.6 (deprecated from 3.0) --- Doc/library/inspect.rst | 18 ------------------ Doc/whatsnew/3.6.rst | 3 ++- Lib/inspect.py | 25 ------------------------- Lib/test/test_inspect.py | 40 +++++----------------------------------- 4 files changed, 7 insertions(+), 79 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 66b9238..b5c4a7a 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -805,24 +805,6 @@ Classes and functions classes using multiple inheritance and their descendants will appear multiple times. - -.. function:: getargspec(func) - - Get the names and default values of a Python function's arguments. A - :term:`named tuple` ``ArgSpec(args, varargs, keywords, defaults)`` is - returned. *args* is a list of the argument names. *varargs* and *keywords* - are the names of the ``*`` and ``**`` arguments or ``None``. *defaults* is a - tuple of default argument values or ``None`` if there are no default - arguments; if this tuple has *n* elements, they correspond to the last - *n* elements listed in *args*. - - .. deprecated:: 3.0 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. This function will be removed - in Python 3.6. - - .. function:: getfullargspec(func) Get the names and default values of a Python function's arguments. A diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 375d94e..b6388d5 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -145,7 +145,8 @@ Removed API and Feature Removals ------------------------ -* None yet. +* ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). + :func:`inspect.getfullargspec` is an almost drop in replacement. Porting to Python 3.6 diff --git a/Lib/inspect.py b/Lib/inspect.py index 24c8df7..4e78d80 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1002,31 +1002,6 @@ def _getfullargs(co): varkw = co.co_varnames[nargs] return args, varargs, kwonlyargs, varkw - -ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def getargspec(func): - """Get the names and default values of a function's arguments. - - A tuple of four things is returned: (args, varargs, keywords, defaults). - 'args' is a list of the argument names, including keyword-only argument names. - 'varargs' and 'keywords' are the names of the * and ** arguments or None. - 'defaults' is an n-tuple of the default values of the last n arguments. - - Use the getfullargspec() API for Python 3 code, as annotations - and keyword arguments are supported. getargspec() will raise ValueError - if the func has either annotations or keyword arguments. - """ - warnings.warn("inspect.getargspec() is deprecated, " - "use inspect.signature() instead", DeprecationWarning, - stacklevel=2) - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - getfullargspec(func) - if kwonlyargs or ann: - raise ValueError("Function has keyword-only arguments or annotations" - ", use getfullargspec() API which can support them") - return ArgSpec(args, varargs, varkw, defaults) - FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 042617b..bd0605e 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -629,18 +629,6 @@ class TestClassesAndFunctions(unittest.TestCase): got = inspect.getmro(D) self.assertEqual(expected, got) - def assertArgSpecEquals(self, routine, args_e, varargs_e=None, - varkw_e=None, defaults_e=None, formatted=None): - with self.assertWarns(DeprecationWarning): - args, varargs, varkw, defaults = inspect.getargspec(routine) - self.assertEqual(args, args_e) - self.assertEqual(varargs, varargs_e) - self.assertEqual(varkw, varkw_e) - self.assertEqual(defaults, defaults_e) - if formatted is not None: - self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults), - formatted) - def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, kwonlyargs_e=[], kwonlydefaults_e=None, @@ -659,23 +647,6 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs, kwonlydefaults, ann), formatted) - def test_getargspec(self): - self.assertArgSpecEquals(mod.eggs, ['x', 'y'], formatted='(x, y)') - - self.assertArgSpecEquals(mod.spam, - ['a', 'b', 'c', 'd', 'e', 'f'], - 'g', 'h', (3, 4, 5), - '(a, b, c, d=3, e=4, f=5, *g, **h)') - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyworded, []) - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.annotated, []) - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyword_only_arg, []) - - def test_getfullargspec(self): self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', kwonlyargs_e=['arg2'], @@ -689,20 +660,19 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs_e=['arg'], formatted='(*, arg)') - def test_argspec_api_ignores_wrapped(self): + def test_fullargspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) def ham(x, y): pass # Basic check - self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(functools.partial(ham), ['x', 'y'], formatted='(x, y)') # Other variants def check_method(f): - self.assertArgSpecEquals(f, ['self', 'x', 'y'], - formatted='(self, x, y)') + self.assertFullArgSpecEquals(f, ['self', 'x', 'y'], + formatted='(self, x, y)') class C: @functools.wraps(mod.spam) def ham(self, x, y): @@ -780,11 +750,11 @@ class TestClassesAndFunctions(unittest.TestCase): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) - def test_getargspec_method(self): + def test_getfullargspec_method(self): class A(object): def m(self): pass - self.assertArgSpecEquals(A.m, ['self']) + self.assertFullArgSpecEquals(A.m, ['self']) def test_classify_newstyle(self): class A(object): -- cgit v0.12 From 6dfbc5d98eee400fa7b775b53c11a4e37fca3547 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 23 Jul 2015 17:49:00 +0300 Subject: Issue #13248: Remove inspect.getmoduleinfo() from 3.6 (deprecated in 3.3) --- Doc/library/inspect.rst | 21 +-------------------- Doc/whatsnew/3.6.rst | 4 ++++ Lib/inspect.py | 17 ----------------- 3 files changed, 5 insertions(+), 37 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index b5c4a7a..1d0edea 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -227,24 +227,6 @@ attributes: listed in the metaclass' custom :meth:`__dir__`. -.. function:: getmoduleinfo(path) - - Returns a :term:`named tuple` ``ModuleInfo(name, suffix, mode, module_type)`` - of values that describe how Python will interpret the file identified by - *path* if it is a module, or ``None`` if it would not be identified as a - module. In that tuple, *name* is the name of the module without the name of - any enclosing package, *suffix* is the trailing part of the file name (which - may not be a dot-delimited extension), *mode* is the :func:`open` mode that - would be used (``'r'`` or ``'rb'``), and *module_type* is an integer giving - the type of the module. *module_type* will have a value which can be - compared to the constants defined in the :mod:`imp` module; see the - documentation for that module for more information on module types. - - .. deprecated:: 3.3 - You may check the file path's suffix against the supported suffixes - listed in :mod:`importlib.machinery` to infer the same information. - - .. function:: getmodulename(path) Return the name of the module named by the file *path*, without including the @@ -258,8 +240,7 @@ attributes: still return ``None``. .. versionchanged:: 3.3 - This function is now based directly on :mod:`importlib` rather than the - deprecated :func:`getmoduleinfo`. + The function is based directly on :mod:`importlib`. .. function:: ismodule(object) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b6388d5..c881d30 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -148,6 +148,10 @@ API and Feature Removals * ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). :func:`inspect.getfullargspec` is an almost drop in replacement. +* ``inspect.getmoduleinfo`` was removed (was deprecated since CPython 3.3). + :func:`inspect.getmodulename` should be used for obtaining the module + name for a given path. + Porting to Python 3.6 ===================== diff --git a/Lib/inspect.py b/Lib/inspect.py index 4e78d80..305aafe 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -623,23 +623,6 @@ def getfile(object): raise TypeError('{!r} is not a module, class, method, ' 'function, traceback, frame, or code object'.format(object)) -ModuleInfo = namedtuple('ModuleInfo', 'name suffix mode module_type') - -def getmoduleinfo(path): - """Get the module name, suffix, mode, and module type for a given file.""" - warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning, - 2) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) - import imp - filename = os.path.basename(path) - suffixes = [(-len(suffix), suffix, mode, mtype) - for suffix, mode, mtype in imp.get_suffixes()] - suffixes.sort() # try longest suffixes first, in case they overlap - for neglen, suffix, mode, mtype in suffixes: - if filename[neglen:] == suffix: - return ModuleInfo(filename[:neglen], suffix, mode, mtype) - def getmodulename(path): """Return the module name for a given file, or None.""" fname = os.path.basename(path) -- cgit v0.12 From 5661316e4d50a27666f206ef6483118f801a854e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 23 Jul 2015 17:51:34 +0300 Subject: Sync NEWS (re removed inspect.getargspec and inspect.getmoduleinfo) --- Doc/whatsnew/3.6.rst | 2 +- Misc/NEWS | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c881d30..3f83f19 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -148,7 +148,7 @@ API and Feature Removals * ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). :func:`inspect.getfullargspec` is an almost drop in replacement. -* ``inspect.getmoduleinfo`` was removed (was deprecated since CPython 3.3). +* ``inspect.getmoduleinfo()`` was removed (was deprecated since CPython 3.3). :func:`inspect.getmodulename` should be used for obtaining the module name for a given path. diff --git a/Misc/NEWS b/Misc/NEWS index b68f8a8..d433b95 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Library - Issue #23661: unittest.mock side_effects can now be exceptions again. This was a regression vs Python 3.4. Patch from Ignacio Rossi +- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo + functions. + What's New in Python 3.5.0 beta 4? ================================== -- cgit v0.12 From 9f4b8605e624c249ad3d0f31d03f42c38d7ccdb2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 24 Jul 2015 00:43:44 -0400 Subject: Issue #24300: Minor refactoring. --- Modules/nismodule.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/nismodule.c b/Modules/nismodule.c index 64eb5db..b6a855c 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -76,11 +76,7 @@ nis_mapname (char *map, int *pfix) *pfix = 0; for (i=0; aliases[i].alias != 0L; i++) { - if (!strcmp (aliases[i].alias, map)) { - *pfix = aliases[i].fix; - return aliases[i].map; - } - if (!strcmp (aliases[i].map, map)) { + if (!strcmp (aliases[i].alias, map) || !strcmp (aliases[i].map, map)) { *pfix = aliases[i].fix; return aliases[i].map; } -- cgit v0.12 From c94a1dc4c9a5d23fdb21f6928dd737a59ef3dfb2 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sun, 26 Jul 2015 06:43:13 +1200 Subject: - Issue #2091: error correctly on open() with mode 'U' and '+' open() accepted a 'U' mode string containing '+', but 'U' can only be used with 'r'. Patch from Jeff Balogh and John O'Connor. --- Lib/_pyio.py | 4 ++-- Lib/test/test_file.py | 2 +- Misc/NEWS | 3 +++ Modules/_io/_iomodule.c | 16 ++++++++-------- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 50ad9ff..33d8a3f 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -181,8 +181,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, text = "t" in modes binary = "b" in modes if "U" in modes: - if creating or writing or appending: - raise ValueError("can't use U and writing mode at once") + if creating or writing or appending or updating: + raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index d54e976..94f189a 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -139,7 +139,7 @@ class OtherFileTests: def testModeStrings(self): # check invalid mode strings - for mode in ("", "aU", "wU+"): + for mode in ("", "aU", "wU+", "U+", "+U", "rU+"): try: f = self.open(TESTFN, mode) except ValueError: diff --git a/Misc/NEWS b/Misc/NEWS index 81d5f30..f85eb16 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,6 +15,9 @@ Library - Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. +- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can + only be used with 'r'. Patch from Jeff Balogh and John O'Connor. + - Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 528bcd4..1c2d3a0 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -248,8 +248,8 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, _Py_IDENTIFIER(close); if (!PyUnicode_Check(file) && - !PyBytes_Check(file) && - !PyNumber_Check(file)) { + !PyBytes_Check(file) && + !PyNumber_Check(file)) { PyErr_Format(PyExc_TypeError, "invalid file: %R", file); return NULL; } @@ -307,9 +307,9 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, /* Parameters validation */ if (universal) { - if (writing || appending) { + if (creating || writing || appending || updating) { PyErr_SetString(PyExc_ValueError, - "can't use U and writing mode at once"); + "mode U cannot be combined with x', 'w', 'a', or '+'"); return NULL; } if (PyErr_WarnEx(PyExc_DeprecationWarning, @@ -437,10 +437,10 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, /* wraps into a TextIOWrapper */ wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type, - "Osssi", - buffer, - encoding, errors, newline, - line_buffering); + "Osssi", + buffer, + encoding, errors, newline, + line_buffering); if (wrapper == NULL) goto error; result = wrapper; -- cgit v0.12 From aece8248b6257cd8c0840963a794537b9e1d9d14 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sun, 26 Jul 2015 06:50:51 +1200 Subject: Issue #24710: Use cls in TracebackException.from_exception. Minor cleanup patch from Berker Peksag. --- Lib/traceback.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 02edeb6..3d2e5e0 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -477,10 +477,9 @@ class TracebackException: self._load_lines() @classmethod - def from_exception(self, exc, *args, **kwargs): + def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" - return TracebackException( - type(exc), exc, exc.__traceback__, *args, **kwargs) + return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" -- cgit v0.12 From 870db9025f79ea47b1f93260b8714bc25fa7cc47 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 26 Jul 2015 13:11:49 +0200 Subject: Closes #20544: use specific asserts in operator tests. Patch by Serhiy. --- Lib/test/test_operator.py | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index da9c8ef..54fd1f4 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -120,63 +120,63 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.add) self.assertRaises(TypeError, operator.add, None, None) - self.assertTrue(operator.add(3, 4) == 7) + self.assertEqual(operator.add(3, 4), 7) def test_bitwise_and(self): operator = self.module self.assertRaises(TypeError, operator.and_) self.assertRaises(TypeError, operator.and_, None, None) - self.assertTrue(operator.and_(0xf, 0xa) == 0xa) + self.assertEqual(operator.and_(0xf, 0xa), 0xa) def test_concat(self): operator = self.module self.assertRaises(TypeError, operator.concat) self.assertRaises(TypeError, operator.concat, None, None) - self.assertTrue(operator.concat('py', 'thon') == 'python') - self.assertTrue(operator.concat([1, 2], [3, 4]) == [1, 2, 3, 4]) - self.assertTrue(operator.concat(Seq1([5, 6]), Seq1([7])) == [5, 6, 7]) - self.assertTrue(operator.concat(Seq2([5, 6]), Seq2([7])) == [5, 6, 7]) + self.assertEqual(operator.concat('py', 'thon'), 'python') + self.assertEqual(operator.concat([1, 2], [3, 4]), [1, 2, 3, 4]) + self.assertEqual(operator.concat(Seq1([5, 6]), Seq1([7])), [5, 6, 7]) + self.assertEqual(operator.concat(Seq2([5, 6]), Seq2([7])), [5, 6, 7]) self.assertRaises(TypeError, operator.concat, 13, 29) def test_countOf(self): operator = self.module self.assertRaises(TypeError, operator.countOf) self.assertRaises(TypeError, operator.countOf, None, None) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 3) == 1) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 5) == 0) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 3), 1) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 5), 0) def test_delitem(self): operator = self.module a = [4, 3, 2, 1] self.assertRaises(TypeError, operator.delitem, a) self.assertRaises(TypeError, operator.delitem, a, None) - self.assertTrue(operator.delitem(a, 1) is None) - self.assertTrue(a == [4, 2, 1]) + self.assertIsNone(operator.delitem(a, 1)) + self.assertEqual(a, [4, 2, 1]) def test_floordiv(self): operator = self.module self.assertRaises(TypeError, operator.floordiv, 5) self.assertRaises(TypeError, operator.floordiv, None, None) - self.assertTrue(operator.floordiv(5, 2) == 2) + self.assertEqual(operator.floordiv(5, 2), 2) def test_truediv(self): operator = self.module self.assertRaises(TypeError, operator.truediv, 5) self.assertRaises(TypeError, operator.truediv, None, None) - self.assertTrue(operator.truediv(5, 2) == 2.5) + self.assertEqual(operator.truediv(5, 2), 2.5) def test_getitem(self): operator = self.module a = range(10) self.assertRaises(TypeError, operator.getitem) self.assertRaises(TypeError, operator.getitem, a, None) - self.assertTrue(operator.getitem(a, 2) == 2) + self.assertEqual(operator.getitem(a, 2), 2) def test_indexOf(self): operator = self.module self.assertRaises(TypeError, operator.indexOf) self.assertRaises(TypeError, operator.indexOf, None, None) - self.assertTrue(operator.indexOf([4, 3, 2, 1], 3) == 1) + self.assertEqual(operator.indexOf([4, 3, 2, 1], 3), 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) def test_invert(self): @@ -189,21 +189,21 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.lshift) self.assertRaises(TypeError, operator.lshift, None, 42) - self.assertTrue(operator.lshift(5, 1) == 10) - self.assertTrue(operator.lshift(5, 0) == 5) + self.assertEqual(operator.lshift(5, 1), 10) + self.assertEqual(operator.lshift(5, 0), 5) self.assertRaises(ValueError, operator.lshift, 2, -1) def test_mod(self): operator = self.module self.assertRaises(TypeError, operator.mod) self.assertRaises(TypeError, operator.mod, None, 42) - self.assertTrue(operator.mod(5, 2) == 1) + self.assertEqual(operator.mod(5, 2), 1) def test_mul(self): operator = self.module self.assertRaises(TypeError, operator.mul) self.assertRaises(TypeError, operator.mul, None, None) - self.assertTrue(operator.mul(5, 2) == 10) + self.assertEqual(operator.mul(5, 2), 10) def test_matmul(self): operator = self.module @@ -227,7 +227,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.or_) self.assertRaises(TypeError, operator.or_, None, None) - self.assertTrue(operator.or_(0xa, 0x5) == 0xf) + self.assertEqual(operator.or_(0xa, 0x5), 0xf) def test_pos(self): operator = self.module @@ -250,8 +250,8 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.rshift) self.assertRaises(TypeError, operator.rshift, None, 42) - self.assertTrue(operator.rshift(5, 1) == 2) - self.assertTrue(operator.rshift(5, 0) == 5) + self.assertEqual(operator.rshift(5, 1), 2) + self.assertEqual(operator.rshift(5, 0), 5) self.assertRaises(ValueError, operator.rshift, 2, -1) def test_contains(self): @@ -266,15 +266,15 @@ class OperatorTestCase: a = list(range(3)) self.assertRaises(TypeError, operator.setitem, a) self.assertRaises(TypeError, operator.setitem, a, None, None) - self.assertTrue(operator.setitem(a, 0, 2) is None) - self.assertTrue(a == [2, 1, 2]) + self.assertIsNone(operator.setitem(a, 0, 2)) + self.assertEqual(a, [2, 1, 2]) self.assertRaises(IndexError, operator.setitem, a, 4, 2) def test_sub(self): operator = self.module self.assertRaises(TypeError, operator.sub) self.assertRaises(TypeError, operator.sub, None, None) - self.assertTrue(operator.sub(5, 2) == 3) + self.assertEqual(operator.sub(5, 2), 3) def test_truth(self): operator = self.module @@ -292,7 +292,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.xor) self.assertRaises(TypeError, operator.xor, None, None) - self.assertTrue(operator.xor(0xb, 0xc) == 0x7) + self.assertEqual(operator.xor(0xb, 0xc), 0x7) def test_is(self): operator = self.module -- cgit v0.12 From c6d9228290c1635588d8edddc04f2b2278705867 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 28 Jul 2015 15:55:07 +1200 Subject: Issue #23426: run_setup was broken in distutils. Patch from Alexander Belopolsky. --- Lib/distutils/core.py | 5 ++--- Lib/distutils/tests/test_core.py | 30 ++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py index f05b34b..d603d4a 100644 --- a/Lib/distutils/core.py +++ b/Lib/distutils/core.py @@ -204,16 +204,15 @@ def run_setup (script_name, script_args=None, stop_after="run"): global _setup_stop_after, _setup_distribution _setup_stop_after = stop_after - save_argv = sys.argv + save_argv = sys.argv.copy() g = {'__file__': script_name} - l = {} try: try: sys.argv[0] = script_name if script_args is not None: sys.argv[1:] = script_args with open(script_name, 'rb') as f: - exec(f.read(), g, l) + exec(f.read(), g) finally: sys.argv = save_argv _setup_stop_after = None diff --git a/Lib/distutils/tests/test_core.py b/Lib/distutils/tests/test_core.py index 41321f7..57856f1 100644 --- a/Lib/distutils/tests/test_core.py +++ b/Lib/distutils/tests/test_core.py @@ -28,6 +28,21 @@ from distutils.core import setup setup() """ +setup_does_nothing = """\ +from distutils.core import setup +setup() +""" + + +setup_defines_subclass = """\ +from distutils.core import setup +from distutils.command.install import install as _install + +class install(_install): + sub_commands = _install.sub_commands + ['cmd'] + +setup(cmdclass={'install': install}) +""" class CoreTestCase(support.EnvironGuard, unittest.TestCase): @@ -65,6 +80,21 @@ class CoreTestCase(support.EnvironGuard, unittest.TestCase): distutils.core.run_setup( self.write_setup(setup_using___file__)) + def test_run_setup_preserves_sys_argv(self): + # Make sure run_setup does not clobber sys.argv + argv_copy = sys.argv.copy() + distutils.core.run_setup( + self.write_setup(setup_does_nothing)) + self.assertEqual(sys.argv, argv_copy) + + def test_run_setup_defines_subclass(self): + # Make sure the script can use __file__; if that's missing, the test + # setup.py script will raise NameError. + dist = distutils.core.run_setup( + self.write_setup(setup_defines_subclass)) + install = dist.get_command_obj('install') + self.assertIn('cmd', install.sub_commands) + def test_run_setup_uses_current_dir(self): # This tests that the setup script is run with the current directory # as its own current directory; this was temporarily broken by a diff --git a/Misc/NEWS b/Misc/NEWS index 2232357..3004539 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- Issue #23426: run_setup was broken in distutils. + Patch from Alexander Belopolsky. + - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella. - Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. -- cgit v0.12 From 76b1714be84ab6e73b9b0716cbb233f044318516 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 29 Jul 2015 23:51:47 +0300 Subject: Issue #24360: Improve __repr__ of argparse.Namespace() for invalid identifiers. Patch by Matthias Bussonnier. --- Lib/argparse.py | 8 +++++++- Lib/test/test_argparse.py | 15 +++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index 9a06719..cc53841 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -118,10 +118,16 @@ class _AttributeHolder(object): def __repr__(self): type_name = type(self).__name__ arg_strings = [] + star_args = {} for arg in self._get_args(): arg_strings.append(repr(arg)) for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) + if name.isidentifier(): + arg_strings.append('%s=%r' % (name, value)) + else: + star_args[name] = value + if star_args: + arg_strings.append('**%s' % repr(star_args)) return '%s(%s)' % (type_name, ', '.join(arg_strings)) def _get_kwargs(self): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 27bfad5..893ec39 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4512,6 +4512,21 @@ class TestStrings(TestCase): string = "Namespace(bar='spam', foo=42)" self.assertStringEqual(ns, string) + def test_namespace_starkwargs_notidentifier(self): + ns = argparse.Namespace(**{'"': 'quote'}) + string = """Namespace(**{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_kwargs_and_starkwargs_notidentifier(self): + ns = argparse.Namespace(a=1, **{'"': 'quote'}) + string = """Namespace(a=1, **{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_starkwargs_identifier(self): + ns = argparse.Namespace(**{'valid': True}) + string = "Namespace(valid=True)" + self.assertStringEqual(ns, string) + def test_parser(self): parser = argparse.ArgumentParser(prog='PROG') string = ( diff --git a/Misc/NEWS b/Misc/NEWS index 2c5d17a..a3ada97 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid + identifiers. Patch by Matthias Bussonnier. + - Issue #23319: Fix ctypes.BigEndianStructure, swap correctly bytes. Patch written by Matthieu Gautier. -- cgit v0.12 From fa3922cfd0697bc3286c301c5d9458160c80cc89 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 31 Jul 2015 04:11:29 +0300 Subject: Issue #13248: Delete remaining references of inspect.getargspec(). Noticed by Yaroslav Halchenko. --- Doc/library/inspect.rst | 4 +--- Lib/inspect.py | 7 ++----- Lib/test/test_inspect.py | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 1e0996c..4357090 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -805,8 +805,6 @@ Classes and functions from kwonlyargs to defaults. *annotations* is a dictionary mapping argument names to annotations. - The first four items in the tuple correspond to :func:`getargspec`. - .. versionchanged:: 3.4 This function is now based on :func:`signature`, but still ignores ``__wrapped__`` attributes and includes the already bound first @@ -835,7 +833,7 @@ Classes and functions .. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) Format a pretty argument spec from the values returned by - :func:`getargspec` or :func:`getfullargspec`. + :func:`getfullargspec`. The first seven arguments are (``args``, ``varargs``, ``varkw``, ``defaults``, ``kwonlyargs``, ``kwonlydefaults``, ``annotations``). diff --git a/Lib/inspect.py b/Lib/inspect.py index 1e94a9c..a089be6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -16,7 +16,7 @@ Here are some of the useful functions provided by this module: getmodule() - determine the module that an object came from getclasstree() - arrange classes so as to represent their hierarchy - getargspec(), getargvalues(), getcallargs() - get info about function arguments + getargvalues(), getcallargs() - get info about function arguments getfullargspec() - same, with support for Python 3 features formatargspec(), formatargvalues() - format an argument spec getouterframes(), getinnerframes() - get info about frames @@ -1018,8 +1018,6 @@ def getfullargspec(func): 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping argument names to annotations. - The first four items in the tuple correspond to getargspec(). - This function is deprecated, use inspect.signature() instead. """ @@ -1130,8 +1128,7 @@ def formatargspec(args, varargs=None, varkw=None, defaults=None, formatvalue=lambda value: '=' + repr(value), formatreturns=lambda text: ' -> ' + text, formatannotation=formatannotation): - """Format an argument spec from the values returned by getargspec - or getfullargspec. + """Format an argument spec from the values returned by getfullargspec. The first seven arguments are (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). The other five arguments diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 309ee2a..db15b39 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -38,7 +38,7 @@ from test.test_import import _ready_to_import # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, # isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargspec, getargvalues, formatargspec, formatargvalues, +# getclasstree, getargvalues, formatargspec, formatargvalues, # currentframe, stack, trace, isdatadescriptor # NOTE: There are some additional tests relating to interaction with -- cgit v0.12 From daffc916aabb8424b6708039e64054f792fb660d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 31 Jul 2015 07:58:56 -0700 Subject: Issue #24681: Move the most likely test first in set_add_entry(). --- Objects/setobject.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 0e443c4..c590bbf 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -171,12 +171,15 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); - if (cmp < 0) /* unlikely */ - goto comparison_error; - if (table != so->table || entry->key != startkey) /* unlikely */ - goto restart; if (cmp > 0) /* likely */ goto found_active; + if (cmp < 0) + goto comparison_error; + /* Continuing the search from the current entry only makes + sense if the table and entry are unchanged; otherwise, + we have to restart from the beginning */ + if (table != so->table || entry->key != startkey) + goto restart; mask = so->mask; /* help avoid a register spill */ } else if (entry->hash == -1 && freeslot == NULL) @@ -200,12 +203,12 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); + if (cmp > 0) + goto found_active; if (cmp < 0) goto comparison_error; if (table != so->table || entry->key != startkey) goto restart; - if (cmp > 0) - goto found_active; mask = so->mask; } else if (entry->hash == -1 && freeslot == NULL) -- cgit v0.12 From 12f896c52128178b8b220ba63084cd68a4c15574 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 31 Jul 2015 12:03:20 -0700 Subject: Fix minor typo in a comment --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index d2c0448..749cf6b 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1047,7 +1047,7 @@ deque_clear(dequeobject *deque) static int valid_index(Py_ssize_t i, Py_ssize_t limit) { - /* The cast to size_t let us use just a single comparison + /* The cast to size_t lets us use just a single comparison to check whether i is in the range: 0 <= i < limit */ return (size_t) i < (size_t) limit; } -- cgit v0.12 From fbffdef47dd711d2441ff415f00c72ae7102e80f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 09:53:00 -0700 Subject: Issue #24762: Speed-up frozenset_hash() and greatly beef-up the comments. --- Objects/setobject.c | 70 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index c590bbf..2d4d8cd 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -739,41 +739,57 @@ set_traverse(PySetObject *so, visitproc visit, void *arg) return 0; } -static Py_hash_t -frozenset_hash(PyObject *self) +/* Work to increase the bit dispersion for closely spaced hash values. + This is important because some use cases have many combinations of a + small number of elements with nearby hashes so that many distinct + combinations collapse to only a handful of distinct hash values. */ + +static Py_uhash_t +_shuffle_bits(Py_uhash_t h) { - /* Most of the constants in this hash algorithm are randomly choosen - large primes with "interesting bit patterns" and that passed - tests for good collision statistics on a variety of problematic - datasets such as: + return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; +} - ps = [] - for r in range(21): - ps += itertools.combinations(range(20), r) - num_distinct_hashes = len({hash(frozenset(s)) for s in ps}) +/* Most of the constants in this hash algorithm are randomly chosen + large primes with "interesting bit patterns" and that passed tests + for good collision statistics on a variety of problematic datasets + including powersets and graph structures (such as David Eppstein's + graph recipes in Lib/test/test_set.py) */ - */ +static Py_hash_t +frozenset_hash(PyObject *self) +{ PySetObject *so = (PySetObject *)self; - Py_uhash_t h, hash = 1927868237UL; + Py_uhash_t hash = 1927868237UL; setentry *entry; - Py_ssize_t pos = 0; - - if (so->hash != -1) - return so->hash; + /* Make hash(frozenset({0})) distinct from hash(frozenset()) */ hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; - while (set_next(so, &pos, &entry)) { - /* Work to increase the bit dispersion for closely spaced hash - values. This is important because some use cases have many - combinations of a small number of elements with nearby - hashes so that many distinct combinations collapse to only - a handful of distinct hash values. */ - h = entry->hash; - hash ^= ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; - } - /* Make the final result spread-out in a different pattern - than the algorithm for tuples or other python objects. */ + + /* Xor-in shuffled bits from every entry's hash field because xor is + commutative and a frozenset hash should be independent of order. + + For speed, include null entries and dummy entries and then + subtract out their effect afterwards so that the final hash + depends only on active entries. This allows the code to be + vectorized by the compiler and it saves the unpredictable + branches that would arise when trying to exclude null and dummy + entries on every iteration. */ + + for (entry = so->table; entry <= &so->table[so->mask]; entry++) + hash ^= _shuffle_bits(entry->hash); + + /* Remove the effect of an odd number NULL entries */ + if ((so->mask + 1 - so->fill) & 1) + hash ^= _shuffle_bits(0); + + /* Remove the effect of an odd number of dummy entries */ + if ((so->fill - so->used) & 1) + hash ^= _shuffle_bits(-1); + + /* Disperse patterns arising in nested frozensets */ hash = hash * 69069U + 907133923UL; + if (hash == (Py_uhash_t)-1) hash = 590923713UL; so->hash = hash; -- cgit v0.12 From 36c0500990a383a47228855bae712595f7f5e1d5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 10:57:42 -0700 Subject: Tweak the comments --- Objects/setobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 2d4d8cd..d638c13 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -763,7 +763,7 @@ frozenset_hash(PyObject *self) Py_uhash_t hash = 1927868237UL; setentry *entry; - /* Make hash(frozenset({0})) distinct from hash(frozenset()) */ + /* Initial dispersion based on the number of active entries */ hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; /* Xor-in shuffled bits from every entry's hash field because xor is @@ -790,8 +790,10 @@ frozenset_hash(PyObject *self) /* Disperse patterns arising in nested frozensets */ hash = hash * 69069U + 907133923UL; + /* -1 is reserved as an error code */ if (hash == (Py_uhash_t)-1) hash = 590923713UL; + so->hash = hash; return hash; } -- cgit v0.12 From a286a51ae1a94cd85f7aee0bb55e44a363761325 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 11:07:11 -0700 Subject: Fix comment typo --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index d638c13..24424ad 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -779,7 +779,7 @@ frozenset_hash(PyObject *self) for (entry = so->table; entry <= &so->table[so->mask]; entry++) hash ^= _shuffle_bits(entry->hash); - /* Remove the effect of an odd number NULL entries */ + /* Remove the effect of an odd number of NULL entries */ if ((so->mask + 1 - so->fill) & 1) hash ^= _shuffle_bits(0); -- cgit v0.12 From 9344bd828cd391c25dc5ee6b4699b0025791a8f8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 15:21:41 -0700 Subject: Clarify comments on setentry invariants. --- Include/setobject.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Include/setobject.h b/Include/setobject.h index f17bc1b..87ec1c8 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -10,12 +10,13 @@ extern "C" { /* There are three kinds of entries in the table: -1. Unused: key == NULL -2. Active: key != NULL and key != dummy -3. Dummy: key == dummy +1. Unused: key == NULL and hash == 0 +2. Dummy: key == dummy and hash == -1 +3. Active: key != NULL and key != dummy and hash != -1 -The hash field of Unused slots have no meaning. -The hash field of Dummny slots are set to -1 +The hash field of Unused slots is always zero. + +The hash field of Dummy slots are set to -1 meaning that dummy entries can be detected by either entry->key==dummy or by entry->hash==-1. */ -- cgit v0.12 From b501a27ad8ec4b531a9c1057952d8f7c6ef8cb77 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 6 Aug 2015 22:15:22 -0700 Subject: Restore frozenset hash caching removed in cf707dd190a9 --- Objects/setobject.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Objects/setobject.c b/Objects/setobject.c index 24424ad..0a065cc 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -763,6 +763,9 @@ frozenset_hash(PyObject *self) Py_uhash_t hash = 1927868237UL; setentry *entry; + if (so->hash != -1) + return so->hash; + /* Initial dispersion based on the number of active entries */ hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; -- cgit v0.12 From 4148195c45862fb62df97943591e9771b5638156 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 7 Aug 2015 00:43:39 -0700 Subject: Move the active entry multiplication to later in the hash calculation --- Objects/setobject.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 0a065cc..03bb230 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -760,15 +760,12 @@ static Py_hash_t frozenset_hash(PyObject *self) { PySetObject *so = (PySetObject *)self; - Py_uhash_t hash = 1927868237UL; + Py_uhash_t hash = 0; setentry *entry; if (so->hash != -1) return so->hash; - /* Initial dispersion based on the number of active entries */ - hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; - /* Xor-in shuffled bits from every entry's hash field because xor is commutative and a frozenset hash should be independent of order. @@ -790,6 +787,9 @@ frozenset_hash(PyObject *self) if ((so->fill - so->used) & 1) hash ^= _shuffle_bits(-1); + /* Factor in the number of active entries */ + hash ^= ((Py_uhash_t)PySet_GET_SIZE(self) + 1) * 1927868237UL; + /* Disperse patterns arising in nested frozensets */ hash = hash * 69069U + 907133923UL; -- cgit v0.12 From 577d2069da6d73fa8ce86613fa7088d5e2a9229f Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 7 Aug 2015 18:50:24 -0400 Subject: Fix typo in comment. --- Objects/abstract.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 31cc0a8..039a4ca 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2131,7 +2131,7 @@ PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) /* PyObject_Call() must not be called with an exception set, because it may clear it (directly or indirectly) and so the - caller looses its exception */ + caller loses its exception */ assert(!PyErr_Occurred()); call = func->ob_type->tp_call; -- cgit v0.12 From 455b5092a1b60cc6d14c4d48729ec5c16e274df1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 9 Aug 2015 00:35:00 -0700 Subject: Add more tests of hash effectiveness. --- Lib/test/test_set.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 54de508..ade39fb 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -10,6 +10,8 @@ import sys import warnings import collections import collections.abc +import itertools +import string class PassThru(Exception): pass @@ -711,6 +713,28 @@ class TestFrozenSet(TestJointOps, unittest.TestCase): addhashvalue(hash(frozenset([e for e, m in elemmasks if m&i]))) self.assertEqual(len(hashvalues), 2**n) + def letter_range(n): + return string.ascii_letters[:n] + + def zf_range(n): + # https://en.wikipedia.org/wiki/Set-theoretic_definition_of_natural_numbers + nums = [frozenset()] + for i in range(n-1): + num = frozenset(nums) + nums.append(num) + return nums[:n] + + def powerset(s): + for i in range(len(s)+1): + yield from map(frozenset, itertools.combinations(s, i)) + + for n in range(18): + t = 2 ** n + mask = t - 1 + for nums in (range, letter_range, zf_range): + u = len({h & mask for h in map(hash, powerset(nums(n)))}) + self.assertGreater(4*u, t) + class FrozenSetSubclass(frozenset): pass -- cgit v0.12 From 57fbec8c7c6fa2de7e23161aa1c41764a6290925 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 9 Aug 2015 13:33:51 +0300 Subject: Added section for 3.5.0rc1 in Misc/NEWS and moved relevant entities to it. --- Misc/NEWS | 77 ++++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 8ec1f01..3c358b2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,50 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +Library +------- + +- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid + identifiers. Patch by Matthias Bussonnier. + +- Issue #23426: run_setup was broken in distutils. + Patch from Alexander Belopolsky. + +- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. + +- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can + only be used with 'r'. Patch from Jeff Balogh and John O'Connor. + +- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. + +- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. + Patch from Nicola Palumbo and Laurent De Buyst. + +- Issue #24426: Fast searching optimization in regular expressions now works + for patterns that starts with capturing groups. Fast searching optimization + now can't be disabled at compile time. + +- Issue #23661: unittest.mock side_effects can now be exceptions again. This + was a regression vs Python 3.4. Patch from Ignacio Rossi + +- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo + functions. + +Documentation +------------- + +Tests +----- + + +What's New in Python 3.5.0 release candidate 1? +=============================================== + +Release date: 2015-08-09 + +Core and Builtins +----------------- + - Issue #24667: Resize odict in all cases that the underlying dict resizes. Library @@ -18,6 +62,10 @@ Library - Issue #24824: Signatures of codecs.encode() and codecs.decode() now are compatible with pydoc. +- Issue #24634: Importing uuid should not try to load libc on Windows + +- Issue #24798: _msvccompiler.py doesn't properly support manifests + - Issue #4395: Better testing and documentation of binary operators. Patch by Martin Panter. @@ -38,41 +86,20 @@ Library - Issue #23779: imaplib raises TypeError if authenticator tries to abort. Patch from Craig Holmquist. -- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid - identifiers. Patch by Matthias Bussonnier. - - Issue #23319: Fix ctypes.BigEndianStructure, swap correctly bytes. Patch written by Matthieu Gautier. -- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. - - Issue #23254: Document how to close the TCPServer listening socket. Patch from Martin Panter. -- Issue #23426: run_setup was broken in distutils. - Patch from Alexander Belopolsky. +- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella. -- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. +- Issue #23812: Fix asyncio.Queue.get() to avoid loosing items on cancellation. + Patch by Gustavo J. A. M. Carneiro. -- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can - only be used with 'r'. Patch from Jeff Balogh and John O'Connor. - -- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - -- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. - Patch from Nicola Palumbo and Laurent De Buyst. - -- Issue #24426: Fast searching optimization in regular expressions now works - for patterns that starts with capturing groups. Fast searching optimization - now can't be disabled at compile time. - -- Issue #23661: unittest.mock side_effects can now be exceptions again. This - was a regression vs Python 3.4. Patch from Ignacio Rossi - -- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo - functions. +- Issue #24791: Fix grammar regression for call syntax: 'g(*a or b)'. Documentation ------------- -- cgit v0.12 From 846a1487cb2dea74dcb858b35fa406ac38641fe6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 9 Aug 2015 13:44:04 +0300 Subject: Backed out changeset 42e2e67b8e6f --- Misc/NEWS | 77 +++++++++++++++++++++------------------------------------------ 1 file changed, 25 insertions(+), 52 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 3c358b2..8ec1f01 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,50 +10,6 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- -Library -------- - -- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid - identifiers. Patch by Matthias Bussonnier. - -- Issue #23426: run_setup was broken in distutils. - Patch from Alexander Belopolsky. - -- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. - -- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can - only be used with 'r'. Patch from Jeff Balogh and John O'Connor. - -- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - -- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. - Patch from Nicola Palumbo and Laurent De Buyst. - -- Issue #24426: Fast searching optimization in regular expressions now works - for patterns that starts with capturing groups. Fast searching optimization - now can't be disabled at compile time. - -- Issue #23661: unittest.mock side_effects can now be exceptions again. This - was a regression vs Python 3.4. Patch from Ignacio Rossi - -- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo - functions. - -Documentation -------------- - -Tests ------ - - -What's New in Python 3.5.0 release candidate 1? -=============================================== - -Release date: 2015-08-09 - -Core and Builtins ------------------ - - Issue #24667: Resize odict in all cases that the underlying dict resizes. Library @@ -62,10 +18,6 @@ Library - Issue #24824: Signatures of codecs.encode() and codecs.decode() now are compatible with pydoc. -- Issue #24634: Importing uuid should not try to load libc on Windows - -- Issue #24798: _msvccompiler.py doesn't properly support manifests - - Issue #4395: Better testing and documentation of binary operators. Patch by Martin Panter. @@ -86,20 +38,41 @@ Library - Issue #23779: imaplib raises TypeError if authenticator tries to abort. Patch from Craig Holmquist. +- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid + identifiers. Patch by Matthias Bussonnier. + - Issue #23319: Fix ctypes.BigEndianStructure, swap correctly bytes. Patch written by Matthieu Gautier. +- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. + - Issue #23254: Document how to close the TCPServer listening socket. Patch from Martin Panter. -- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. +- Issue #23426: run_setup was broken in distutils. + Patch from Alexander Belopolsky. - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella. -- Issue #23812: Fix asyncio.Queue.get() to avoid loosing items on cancellation. - Patch by Gustavo J. A. M. Carneiro. +- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. -- Issue #24791: Fix grammar regression for call syntax: 'g(*a or b)'. +- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can + only be used with 'r'. Patch from Jeff Balogh and John O'Connor. + +- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. + +- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. + Patch from Nicola Palumbo and Laurent De Buyst. + +- Issue #24426: Fast searching optimization in regular expressions now works + for patterns that starts with capturing groups. Fast searching optimization + now can't be disabled at compile time. + +- Issue #23661: unittest.mock side_effects can now be exceptions again. This + was a regression vs Python 3.4. Patch from Ignacio Rossi + +- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo + functions. Documentation ------------- -- cgit v0.12 From dfa95c9a8f2772d1e8e54aa5aa14c91d4971964f Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 10 Aug 2015 09:53:30 +1200 Subject: Issue #20059: urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. --- Doc/library/urllib.parse.rst | 18 ++++++++++++++---- Doc/whatsnew/3.6.rst | 5 ++++- Lib/test/test_urlparse.py | 36 +++++++++++++++++------------------- Lib/urllib/parse.py | 3 +-- Misc/NEWS | 3 +++ 5 files changed, 39 insertions(+), 26 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 40098d0..7c075ad 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -115,8 +115,9 @@ or on combining URL components into a URL string. | | | if present | | +------------------+-------+--------------------------+----------------------+ - See section :ref:`urlparse-result-object` for more information on the result - object. + Reading the :attr:`port` attribute will raise a :exc:`ValueError` if + an invalid port is specified in the URL. See section + :ref:`urlparse-result-object` for more information on the result object. .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -126,6 +127,10 @@ or on combining URL components into a URL string. false), in accordance with :rfc:`3986`. Previously, a whitelist of schemes that support fragments existed. + .. versionchanged:: 3.6 + Out-of-range port numbers now raise :exc:`ValueError`, instead of + returning :const:`None`. + .. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace') @@ -228,8 +233,13 @@ or on combining URL components into a URL string. | | | if present | | +------------------+-------+-------------------------+----------------------+ - See section :ref:`urlparse-result-object` for more information on the result - object. + Reading the :attr:`port` attribute will raise a :exc:`ValueError` if + an invalid port is specified in the URL. See section + :ref:`urlparse-result-object` for more information on the result object. + + .. versionchanged:: 3.6 + Out-of-range port numbers now raise :exc:`ValueError`, instead of + returning :const:`None`. .. function:: urlunsplit(parts) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 4e686ad..e2aa1e2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -162,7 +162,10 @@ that may require changes to your code. Changes in the Python API ------------------------- -* None yet. +* Reading the :attr:`~urllib.parse.SplitResult.port` attribute of + :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results + now raises :exc:`ValueError` for out-of-range values, rather than + returning :const:`None`. See :issue:`20059`. Changes in the C API diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 0552f90..fcf5082 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -554,29 +554,27 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p.port, 80) self.assertEqual(p.geturl(), url) - # Verify an illegal port is returned as None + # Verify an illegal port raises ValueError url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag" p = urllib.parse.urlsplit(url) - self.assertEqual(p.port, None) + with self.assertRaisesRegex(ValueError, "out of range"): + p.port def test_attributes_bad_port(self): - """Check handling of non-integer ports.""" - p = urllib.parse.urlsplit("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - # Once again, repeat ourselves to test bytes - p = urllib.parse.urlsplit(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) + """Check handling of invalid ports.""" + for bytes in (False, True): + for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): + for port in ("foo", "1.5", "-1", "0x10"): + with self.subTest(bytes=bytes, parse=parse, port=port): + netloc = "www.example.net:" + port + url = "http://" + netloc + if bytes: + netloc = netloc.encode("ascii") + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 01c9e58..5e2155c 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -156,9 +156,8 @@ class _NetlocResultMixinBase(object): port = self._hostinfo[1] if port is not None: port = int(port, 10) - # Return None on an illegal port if not ( 0 <= port <= 65535): - return None + raise ValueError("Port out of range 0-65535") return port diff --git a/Misc/NEWS b/Misc/NEWS index 8ec1f01..1aa6b09 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,6 +15,9 @@ Core and Builtins Library ------- +- Issue #20059: urllib.parse raises ValueError on all invalid ports. + Patch by Martin Panter. + - Issue #24824: Signatures of codecs.encode() and codecs.decode() now are compatible with pydoc. -- cgit v0.12 From df395991f6341be037a5c6df9b14d924d5ac9e95 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 12 Aug 2015 08:00:06 +1200 Subject: Issue #9232: Support trailing commas in function declarations. For example, "def f(*, a = 3,): pass" is now legal. Patch from Mark Dickinson. --- Doc/reference/compound_stmts.rst | 8 +-- Grammar/Grammar | 17 ++++--- Lib/test/test_grammar.py | 38 ++++++++++++++ Misc/NEWS | 4 ++ Python/ast.c | 23 +++++---- Python/graminit.c | 106 ++++++++++++++++++++++----------------- 6 files changed, 131 insertions(+), 65 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 01cfd6d..00e6476 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -471,10 +471,10 @@ A function definition defines a user-defined function object (see section decorators: `decorator`+ decorator: "@" `dotted_name` ["(" [`parameter_list` [","]] ")"] NEWLINE dotted_name: `identifier` ("." `identifier`)* - parameter_list: (`defparameter` ",")* - : | "*" [`parameter`] ("," `defparameter`)* ["," "**" `parameter`] - : | "**" `parameter` - : | `defparameter` [","] ) + parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] + : | `parameter_list_starargs` + parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] + : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` diff --git a/Grammar/Grammar b/Grammar/Grammar index 99fcea0..14bdecd 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -27,13 +27,18 @@ async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' -typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' - ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] - | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) +typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ + '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']) tfpdef: NAME [':' test] -varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' - ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] - | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) +varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ + '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [',']]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [','] +) vfpdef: NAME stmt: simple_stmt | compound_stmt diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ec3d783..8f8d71c 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -295,6 +295,10 @@ class GrammarTests(unittest.TestCase): pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) + self.assertRaises(SyntaxError, eval, "def f(*): pass") + self.assertRaises(SyntaxError, eval, "def f(*,): pass") + self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") + # keyword arguments after *arglist def f(*args, **kwargs): return args, kwargs @@ -352,6 +356,23 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "f(*g(1=2))") check_syntax_error(self, "f(**g(1=2))") + # Check trailing commas are permitted in funcdef argument list + def f(a,): pass + def f(*args,): pass + def f(**kwds,): pass + def f(a, *args,): pass + def f(a, **kwds,): pass + def f(*args, b,): pass + def f(*, b,): pass + def f(*args, **kwds,): pass + def f(a, *args, b,): pass + def f(a, *, b,): pass + def f(a, *args, **kwds,): pass + def f(*args, b, **kwds,): pass + def f(*, b, **kwds,): pass + def f(a, *args, b, **kwds,): pass + def f(a, *, b, **kwds,): pass + def test_lambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 @@ -370,6 +391,23 @@ class GrammarTests(unittest.TestCase): self.assertEqual(l6(1,2), 1+2+20) self.assertEqual(l6(1,2,k=10), 1+2+10) + # check that trailing commas are permitted + l10 = lambda a,: 0 + l11 = lambda *args,: 0 + l12 = lambda **kwds,: 0 + l13 = lambda a, *args,: 0 + l14 = lambda a, **kwds,: 0 + l15 = lambda *args, b,: 0 + l16 = lambda *, b,: 0 + l17 = lambda *args, **kwds,: 0 + l18 = lambda a, *args, b,: 0 + l19 = lambda a, *, b,: 0 + l20 = lambda a, *args, **kwds,: 0 + l21 = lambda *args, b, **kwds,: 0 + l22 = lambda *, b, **kwds,: 0 + l23 = lambda a, *args, b, **kwds,: 0 + l24 = lambda a, *, b, **kwds,: 0 + ### stmt: simple_stmt | compound_stmt # Tested below diff --git a/Misc/NEWS b/Misc/NEWS index 1aa6b09..c2ea6d6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #9232: Modify Python's grammar to allow trailing commas in the + argument list of a function declaration. For example, "def f(*, a = + 3,): pass" is now legal. Patch from Mark Dickinson. + - Issue #24667: Resize odict in all cases that the underlying dict resizes. Library diff --git a/Python/ast.c b/Python/ast.c index b572088..c1c3907 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1260,16 +1260,20 @@ ast_for_arguments(struct compiling *c, const node *n) and varargslist (lambda definition). parameters: '(' [typedargslist] ')' - typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] - | '**' tfpdef) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ + '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']) tfpdef: NAME [':' test] - varargslist: ((vfpdef ['=' test] ',')* - ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] - | '**' vfpdef) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ + '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [',']]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [','] + ) vfpdef: NAME + */ int i, j, k, nposargs = 0, nkwonlyargs = 0; int nposdefaults = 0, found_default = 0; @@ -1371,7 +1375,8 @@ ast_for_arguments(struct compiling *c, const node *n) i += 2; /* the name and the comma */ break; case STAR: - if (i+1 >= NCH(n)) { + if (i+1 >= NCH(n) || + (i+2 == NCH(n) && TYPE(CHILD(n, i+1)) == COMMA)) { ast_error(c, CHILD(n, i), "named arguments must follow bare *"); return NULL; diff --git a/Python/graminit.c b/Python/graminit.c index 8212b2a..354dc12 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -204,11 +204,13 @@ static arc arcs_9_6[2] = { {32, 7}, {0, 6}, }; -static arc arcs_9_7[2] = { +static arc arcs_9_7[3] = { {30, 12}, {34, 3}, + {0, 7}, }; -static arc arcs_9_8[1] = { +static arc arcs_9_8[2] = { + {32, 13}, {0, 8}, }; static arc arcs_9_9[2] = { @@ -221,35 +223,39 @@ static arc arcs_9_10[3] = { {0, 10}, }; static arc arcs_9_11[3] = { - {30, 13}, - {32, 14}, + {30, 14}, + {32, 15}, {0, 11}, }; static arc arcs_9_12[3] = { {32, 7}, - {31, 15}, + {31, 16}, {0, 12}, }; -static arc arcs_9_13[2] = { - {32, 14}, +static arc arcs_9_13[1] = { {0, 13}, }; static arc arcs_9_14[2] = { - {30, 16}, + {32, 15}, + {0, 14}, +}; +static arc arcs_9_15[3] = { + {30, 17}, {34, 3}, + {0, 15}, }; -static arc arcs_9_15[1] = { +static arc arcs_9_16[1] = { {26, 6}, }; -static arc arcs_9_16[3] = { - {32, 14}, - {31, 17}, - {0, 16}, +static arc arcs_9_17[3] = { + {32, 15}, + {31, 18}, + {0, 17}, }; -static arc arcs_9_17[1] = { - {26, 13}, +static arc arcs_9_18[1] = { + {26, 14}, }; -static state states_9[18] = { +static state states_9[19] = { {3, arcs_9_0}, {3, arcs_9_1}, {3, arcs_9_2}, @@ -257,17 +263,18 @@ static state states_9[18] = { {1, arcs_9_4}, {4, arcs_9_5}, {2, arcs_9_6}, - {2, arcs_9_7}, - {1, arcs_9_8}, + {3, arcs_9_7}, + {2, arcs_9_8}, {2, arcs_9_9}, {3, arcs_9_10}, {3, arcs_9_11}, {3, arcs_9_12}, - {2, arcs_9_13}, + {1, arcs_9_13}, {2, arcs_9_14}, - {1, arcs_9_15}, - {3, arcs_9_16}, - {1, arcs_9_17}, + {3, arcs_9_15}, + {1, arcs_9_16}, + {3, arcs_9_17}, + {1, arcs_9_18}, }; static arc arcs_10_0[1] = { {23, 1}, @@ -319,11 +326,13 @@ static arc arcs_11_6[2] = { {32, 7}, {0, 6}, }; -static arc arcs_11_7[2] = { +static arc arcs_11_7[3] = { {36, 12}, {34, 3}, + {0, 7}, }; -static arc arcs_11_8[1] = { +static arc arcs_11_8[2] = { + {32, 13}, {0, 8}, }; static arc arcs_11_9[2] = { @@ -336,35 +345,39 @@ static arc arcs_11_10[3] = { {0, 10}, }; static arc arcs_11_11[3] = { - {36, 13}, - {32, 14}, + {36, 14}, + {32, 15}, {0, 11}, }; static arc arcs_11_12[3] = { {32, 7}, - {31, 15}, + {31, 16}, {0, 12}, }; -static arc arcs_11_13[2] = { - {32, 14}, +static arc arcs_11_13[1] = { {0, 13}, }; static arc arcs_11_14[2] = { - {36, 16}, + {32, 15}, + {0, 14}, +}; +static arc arcs_11_15[3] = { + {36, 17}, {34, 3}, + {0, 15}, }; -static arc arcs_11_15[1] = { +static arc arcs_11_16[1] = { {26, 6}, }; -static arc arcs_11_16[3] = { - {32, 14}, - {31, 17}, - {0, 16}, +static arc arcs_11_17[3] = { + {32, 15}, + {31, 18}, + {0, 17}, }; -static arc arcs_11_17[1] = { - {26, 13}, +static arc arcs_11_18[1] = { + {26, 14}, }; -static state states_11[18] = { +static state states_11[19] = { {3, arcs_11_0}, {3, arcs_11_1}, {3, arcs_11_2}, @@ -372,17 +385,18 @@ static state states_11[18] = { {1, arcs_11_4}, {4, arcs_11_5}, {2, arcs_11_6}, - {2, arcs_11_7}, - {1, arcs_11_8}, + {3, arcs_11_7}, + {2, arcs_11_8}, {2, arcs_11_9}, {3, arcs_11_10}, {3, arcs_11_11}, {3, arcs_11_12}, - {2, arcs_11_13}, + {1, arcs_11_13}, {2, arcs_11_14}, - {1, arcs_11_15}, - {3, arcs_11_16}, - {1, arcs_11_17}, + {3, arcs_11_15}, + {1, arcs_11_16}, + {3, arcs_11_17}, + {1, arcs_11_18}, }; static arc arcs_12_0[1] = { {23, 1}, @@ -1879,11 +1893,11 @@ static dfa dfas[85] = { "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "parameters", 0, 4, states_8, "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {265, "typedargslist", 0, 18, states_9, + {265, "typedargslist", 0, 19, states_9, "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {266, "tfpdef", 0, 4, states_10, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {267, "varargslist", 0, 18, states_11, + {267, "varargslist", 0, 19, states_11, "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {268, "vfpdef", 0, 2, states_12, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, -- cgit v0.12 From 80d62e628b25502239b5a3dc7c5aaca3deaa7e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Thu, 13 Aug 2015 20:37:08 +0100 Subject: Issue #23530: fix clinic comment. --- Modules/clinic/posixmodule.c.h | 10 +++++++--- Modules/posixmodule.c | 12 +++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 98eeae4..a7045a8 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2116,7 +2116,7 @@ PyDoc_STRVAR(os_sched_getaffinity__doc__, "sched_getaffinity($module, pid, /)\n" "--\n" "\n" -"Return the affinity of the process identified by pid.\n" +"Return the affinity of the process identified by pid (or the current process if zero).\n" "\n" "The affinity is returned as a set of CPU identifiers."); @@ -5178,7 +5178,11 @@ PyDoc_STRVAR(os_cpu_count__doc__, "cpu_count($module, /)\n" "--\n" "\n" -"Return the number of CPUs in the system; return None if indeterminable."); +"Return the number of CPUs in the system; return None if indeterminable.\n" +"\n" +"This number is not equivalent to the number of CPUs the current process can\n" +"use. The number of usable CPUs can be obtained with\n" +"``len(os.sched_getaffinity(0))``"); #define OS_CPU_COUNT_METHODDEF \ {"cpu_count", (PyCFunction)os_cpu_count, METH_NOARGS, os_cpu_count__doc__}, @@ -5788,4 +5792,4 @@ exit: #ifndef OS_SET_HANDLE_INHERITABLE_METHODDEF #define OS_SET_HANDLE_INHERITABLE_METHODDEF #endif /* !defined(OS_SET_HANDLE_INHERITABLE_METHODDEF) */ -/*[clinic end generated code: output=f3f92b2d2e2c3fe3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=35b50461dbecd65d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 23d74a3..4d25ed1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5767,7 +5767,7 @@ The affinity is returned as a set of CPU identifiers. static PyObject * os_sched_getaffinity_impl(PyModuleDef *module, pid_t pid) -/*[clinic end generated code: output=b431a8f310e369e7 input=eaf161936874b8a1]*/ +/*[clinic end generated code: output=b431a8f310e369e7 input=983ce7cb4a565980]*/ { int cpu, ncpus, count; size_t setsize; @@ -11201,14 +11201,16 @@ get_terminal_size(PyObject *self, PyObject *args) /*[clinic input] os.cpu_count -Return the number of CPUs in the system; return None if indeterminable. This -number is not equivalent to the number of CPUs the current process can use. -The number of usable CPUs can be obtained with ``len(os.sched_getaffinity(0))`` +Return the number of CPUs in the system; return None if indeterminable. + +This number is not equivalent to the number of CPUs the current process can +use. The number of usable CPUs can be obtained with +``len(os.sched_getaffinity(0))`` [clinic start generated code]*/ static PyObject * os_cpu_count_impl(PyModuleDef *module) -/*[clinic end generated code: output=c59ee7f6bce832b8 input=d55e2f8f3823a628]*/ +/*[clinic end generated code: output=c59ee7f6bce832b8 input=e7c8f4ba6dbbadd3]*/ { int ncpu = 0; #ifdef MS_WINDOWS -- cgit v0.12 From 7126eaa2d1185065a6d43d90185a6a56c7d5df18 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 14 Aug 2015 11:50:14 +1200 Subject: Issue #21159: Improve message in configparser.InterpolationMissingOptionError. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch from Łukasz Langa. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index a8bc453..9e823c0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,9 @@ Core and Builtins Library ------- +- Issue #21159: Improve message in configparser.InterpolationMissingOptionError. + Patch from Łukasz Langa. + - Issue #20059: urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. -- cgit v0.12 From 28c995d03b6e2f5a77c42a97e95368767846f14f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Aug 2015 02:07:41 -0700 Subject: Minor cleanup. --- Modules/_collectionsmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 749cf6b..35034a5 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2167,13 +2167,13 @@ _count_elements(PyObject *self, PyObject *args) oldval = _PyDict_GetItem_KnownHash(mapping, key, hash); if (oldval == NULL) { - if (_PyDict_SetItem_KnownHash(mapping, key, one, hash) == -1) + if (_PyDict_SetItem_KnownHash(mapping, key, one, hash) < 0) goto done; } else { newval = PyNumber_Add(oldval, one); if (newval == NULL) goto done; - if (_PyDict_SetItem_KnownHash(mapping, key, newval, hash) == -1) + if (_PyDict_SetItem_KnownHash(mapping, key, newval, hash) < 0) goto done; Py_CLEAR(newval); } @@ -2199,7 +2199,7 @@ _count_elements(PyObject *self, PyObject *args) Py_DECREF(oldval); if (newval == NULL) break; - if (PyObject_SetItem(mapping, key, newval) == -1) + if (PyObject_SetItem(mapping, key, newval) < 0) break; Py_CLEAR(newval); Py_DECREF(key); -- cgit v0.12 From 721b9aab10a4c4f95c3fba08af24337efe63384c Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Fri, 14 Aug 2015 08:03:30 -0700 Subject: One-character typo fix. os.path.commonprefix always returns a valid *path*, not a valid *patch*. --- Doc/whatsnew/3.5.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index a974130..11f871b 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -729,7 +729,7 @@ os.path * New :func:`~os.path.commonpath` function that extracts common path prefix. Unlike the :func:`~os.path.commonprefix` function, it always returns a valid - patch. (Contributed by Rafik Draoui and Serhiy Storchaka in :issue:`10395`.) + path. (Contributed by Rafik Draoui and Serhiy Storchaka in :issue:`10395`.) pickle ------ -- cgit v0.12 From b468e1f59533fd296df7d899e9a71650871f30ce Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Aug 2015 14:10:49 -0700 Subject: Neaten-up vertical alignment of comments --- Modules/itertoolsmodule.c | 124 +++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ac31e2f..349da0f 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -178,7 +178,7 @@ static PyMethodDef groupby_methods[] = { reduce_doc}, {"__setstate__", (PyCFunction)groupby_setstate, METH_O, setstate_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(groupby_doc, @@ -364,7 +364,7 @@ static PyTypeObject _grouper_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)_grouper_traverse,/* tp_traverse */ + (traverseproc)_grouper_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -582,7 +582,7 @@ static PyMethodDef teedataobject_methods[] = { PyDoc_STRVAR(teedataobject_doc, "Data container common to multiple tee objects."); static PyTypeObject teedataobject_type = { - PyVarObject_HEAD_INIT(0, 0) /* Must fill in type value later */ + PyVarObject_HEAD_INIT(0, 0) /* Must fill in type value later */ "itertools._tee_dataobject", /* tp_name */ sizeof(teedataobject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -602,7 +602,7 @@ static PyTypeObject teedataobject_type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ teedataobject_doc, /* tp_doc */ (traverseproc)teedataobject_traverse, /* tp_traverse */ (inquiry)teedataobject_clear, /* tp_clear */ @@ -791,7 +791,7 @@ static PyTypeObject tee_type = { (traverseproc)tee_traverse, /* tp_traverse */ (inquiry)tee_clear, /* tp_clear */ 0, /* tp_richcompare */ - offsetof(teeobject, weakreflist), /* tp_weaklistoffset */ + offsetof(teeobject, weakreflist), /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)tee_next, /* tp_iternext */ tee_methods, /* tp_methods */ @@ -1189,13 +1189,13 @@ static PyTypeObject dropwhile_type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ dropwhile_doc, /* tp_doc */ - (traverseproc)dropwhile_traverse, /* tp_traverse */ + (traverseproc)dropwhile_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)dropwhile_next, /* tp_iternext */ - dropwhile_methods, /* tp_methods */ + dropwhile_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -1353,7 +1353,7 @@ static PyTypeObject takewhile_type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ takewhile_doc, /* tp_doc */ - (traverseproc)takewhile_traverse, /* tp_traverse */ + (traverseproc)takewhile_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -2287,7 +2287,7 @@ product((0,1), (0,1), (0,1)) --> (0,0,0) (0,0,1) (0,1,0) (0,1,1) (1,0,0) ..."); static PyTypeObject product_type = { PyVarObject_HEAD_INIT(NULL, 0) "itertools.product", /* tp_name */ - sizeof(productobject), /* tp_basicsize */ + sizeof(productobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)product_dealloc, /* tp_dealloc */ @@ -2334,8 +2334,8 @@ static PyTypeObject product_type = { typedef struct { PyObject_HEAD PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ Py_ssize_t r; /* size of result tuple */ int stopped; /* set to 1 when the combinations iterator is exhausted */ } combinationsobject; @@ -2675,8 +2675,8 @@ static PyTypeObject combinations_type = { typedef struct { PyObject_HEAD PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ Py_ssize_t r; /* size of result tuple */ int stopped; /* set to 1 when the cwr iterator is exhausted */ } cwrobject; @@ -3326,11 +3326,11 @@ permutations(range(3), 2) --> (0,1), (0,2), (1,0), (1,2), (2,0), (2,1)"); static PyTypeObject permutations_type = { PyVarObject_HEAD_INIT(NULL, 0) - "itertools.permutations", /* tp_name */ + "itertools.permutations", /* tp_name */ sizeof(permutationsobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)permutations_dealloc, /* tp_dealloc */ + (destructor)permutations_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3347,13 +3347,13 @@ static PyTypeObject permutations_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ - permutations_doc, /* tp_doc */ - (traverseproc)permutations_traverse, /* tp_traverse */ + permutations_doc, /* tp_doc */ + (traverseproc)permutations_traverse,/* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)permutations_next, /* tp_iternext */ + (iternextfunc)permutations_next, /* tp_iternext */ permuations_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -3364,7 +3364,7 @@ static PyTypeObject permutations_type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - permutations_new, /* tp_new */ + permutations_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; @@ -3664,44 +3664,44 @@ static PyTypeObject compress_type = { PyVarObject_HEAD_INIT(NULL, 0) "itertools.compress", /* tp_name */ sizeof(compressobject), /* tp_basicsize */ - 0, /* tp_itemsize */ + 0, /* tp_itemsize */ /* methods */ (destructor)compress_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - compress_doc, /* tp_doc */ - (traverseproc)compress_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ + Py_TPFLAGS_BASETYPE, /* tp_flags */ + compress_doc, /* tp_doc */ + (traverseproc)compress_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ (iternextfunc)compress_next, /* tp_iternext */ - compress_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - compress_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ + compress_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + compress_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ }; @@ -3824,7 +3824,7 @@ static PyTypeObject filterfalse_type = { sizeof(filterfalseobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)filterfalse_dealloc, /* tp_dealloc */ + (destructor)filterfalse_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3842,7 +3842,7 @@ static PyTypeObject filterfalse_type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ filterfalse_doc, /* tp_doc */ - (traverseproc)filterfalse_traverse, /* tp_traverse */ + (traverseproc)filterfalse_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -4081,15 +4081,15 @@ static PyTypeObject count_type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ + Py_TPFLAGS_BASETYPE, /* tp_flags */ count_doc, /* tp_doc */ - (traverseproc)count_traverse, /* tp_traverse */ + (traverseproc)count_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)count_next, /* tp_iternext */ - count_methods, /* tp_methods */ + count_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -4488,7 +4488,7 @@ static PyTypeObject ziplongest_type = { sizeof(ziplongestobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)zip_longest_dealloc, /* tp_dealloc */ + (destructor)zip_longest_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4505,8 +4505,8 @@ static PyTypeObject ziplongest_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ - zip_longest_doc, /* tp_doc */ - (traverseproc)zip_longest_traverse, /* tp_traverse */ + zip_longest_doc, /* tp_doc */ + (traverseproc)zip_longest_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -4522,7 +4522,7 @@ static PyTypeObject ziplongest_type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - zip_longest_new, /* tp_new */ + zip_longest_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; -- cgit v0.12 From 79c878d5f24d58fe09e842f9a1cc13d6cf560aab Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 13:51:59 -0700 Subject: Fix crash in itertools.cycle.__setstate__() caused by lack of type checking. Will backport after the 3.6 release is done. --- Lib/test/test_itertools.py | 33 +++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/itertoolsmodule.c | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index fcd8869..53d6564 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -613,6 +613,39 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + def test_cycle_setstate(self): + # Verify both modes for restoring state + + # Mode 0 is efficient. It uses an incompletely consumed input + # iterator to build a cycle object and then passes in state with + # a list of previously consumed values. There is no data + # overlap bewteen the two. + c = cycle('defg') + c.__setstate__((list('abc'), 0)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # Mode 1 is inefficient. It starts with a cycle object built + # from an iterator over the remaining elements in a partial + # cycle and then passes in state with all of the previously + # seen values (this overlaps values included in the iterator). + c = cycle('defg') + c.__setstate__((list('abcdefg'), 1)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # The first argument to setstate needs to be a tuple + with self.assertRaises(SystemError): + cycle('defg').__setstate__([list('abcdefg'), 0]) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + c = cycle('defg') + c.__setstate__((dict.fromkeys('defg'), 0)) + take(20, c) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + cycle('defg').__setstate__((list('abcdefg'), 'x')) + def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) diff --git a/Misc/NEWS b/Misc/NEWS index 574e279..a8b93e6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Library - Issue #21159: Improve message in configparser.InterpolationMissingOptionError. Patch from Łukasz Langa. +- Fix crash in itertools.cycle.__setstate__() when the first argument wasn't + a list. + - Issue #20059: urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 349da0f..2cca6b3 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -973,7 +973,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) { PyObject *saved=NULL; int firstpass; - if (!PyArg_ParseTuple(state, "Oi", &saved, &firstpass)) + if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) return NULL; Py_CLEAR(lz->saved); lz->saved = saved; -- cgit v0.12 From a166ce561c8c80dc892deab491ff5625cc0409c8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 14:45:49 -0700 Subject: Add more tests for pickling itertools.cycle --- Lib/test/test_itertools.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 53d6564..c012efd 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -613,6 +613,23 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # test with partial consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(2)] # consume to 2 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + + # test with completely consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(7)] # consume to 7 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_setstate(self): # Verify both modes for restoring state -- cgit v0.12 From 1cadf769b3d453777a116a1d11831ef625ebbe99 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 14:47:27 -0700 Subject: Improve comment --- Lib/test/test_itertools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index c012efd..81e1711 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -617,7 +617,7 @@ class TestBasicOps(unittest.TestCase): # test with partial consumed input iterable it = iter('abcde') c = cycle(it) - _ = [next(c) for i in range(2)] # consume to 2 of 5 inputs + _ = [next(c) for i in range(2)] # consume 2 of 5 inputs p = pickle.dumps(c, proto) d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) @@ -625,7 +625,7 @@ class TestBasicOps(unittest.TestCase): # test with completely consumed input iterable it = iter('abcde') c = cycle(it) - _ = [next(c) for i in range(7)] # consume to 7 of 5 inputs + _ = [next(c) for i in range(7)] # consume 7 of 5 inputs p = pickle.dumps(c, proto) d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) -- cgit v0.12 From 98958fedb6b8f6f8f9bc8587f7c9884ea7c47e69 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 15:09:30 -0700 Subject: Remove dead code (unreachable) --- Modules/itertoolsmodule.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 2cca6b3..0d1a15b 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -940,12 +940,9 @@ cycle_next(cycleobject *lz) } return item; } - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else + /* Note: StopIteration is already cleared by PyIter_Next() */ + if (PyErr_Occurred()) return NULL; - } if (PyList_Size(lz->saved) == 0) return NULL; it = PyObject_GetIter(lz->saved); -- cgit v0.12 From c39786dc08d68ebb832c3b249c42dcdaba38b3a9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 15:16:12 -0700 Subject: Fix oddly placed whitespace --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 0d1a15b..18d55d8 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -963,7 +963,7 @@ cycle_reduce(cycleobject *lz) */ return Py_BuildValue("O(O)(Oi)", Py_TYPE(lz), lz->it, lz->saved, lz->firstpass); - } +} static PyObject * cycle_setstate(cycleobject *lz, PyObject *state) -- cgit v0.12 From 2a75e8f95899431f44a9af9e23ca504d3e73a2ad Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 08:32:01 -0700 Subject: Issue #24842: Cross-reference types.SimpleNamespace from the namedtuple docs --- Doc/library/collections.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index f2befe7..aeb1c52 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -962,6 +962,9 @@ and more efficient to use a simple :class:`~enum.Enum`: constructor that is convenient for use cases where named tuples are being subclassed. + * :meth:`types.SimpleNamespace` for a mutable namespace based on an underlying + dictionary instead of a tuple. + :class:`OrderedDict` objects ---------------------------- -- cgit v0.12 From b5244a3fe5aa15c41311fe124f5ba1b209f7c316 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 14:24:20 -0700 Subject: Inline PyIter_Next() matching what was done for other itertools. --- Modules/itertoolsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 18d55d8..bba0935 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1860,7 +1860,7 @@ chain_next(chainobject *lz) return NULL; /* input not iterable */ } } - item = PyIter_Next(lz->active); + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); if (item != NULL) return item; if (PyErr_Occurred()) { @@ -3434,7 +3434,7 @@ accumulate_next(accumulateobject *lz) { PyObject *val, *oldtotal, *newtotal; - val = PyIter_Next(lz->it); + val = (*Py_TYPE(lz->it)->tp_iternext)(lz->it); if (val == NULL) return NULL; -- cgit v0.12 From a6a2d44dc7f50a75c3df742d9fc2c30217436c40 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 14:38:07 -0700 Subject: Neaten-up whitespace, vertical alignment, and line-wrapping. --- Modules/itertoolsmodule.c | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index bba0935..533e1d1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -759,9 +759,9 @@ PyDoc_STRVAR(teeobject_doc, "Iterator wrapped to make it copyable"); static PyMethodDef tee_methods[] = { - {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, - {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, + {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, + {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -1407,7 +1407,8 @@ islice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyErr_Occurred()) PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize."); + "Stop argument for islice() must be None or " + "an integer: 0 <= x <= sys.maxsize."); return NULL; } } @@ -1422,14 +1423,16 @@ islice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyErr_Occurred()) PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize."); + "Stop argument for islice() must be None or " + "an integer: 0 <= x <= sys.maxsize."); return NULL; } } } if (start<0 || stop<-1) { PyErr_SetString(PyExc_ValueError, - "Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize."); + "Indices for islice() must be None or " + "an integer: 0 <= x <= sys.maxsize."); return NULL; } @@ -1845,19 +1848,19 @@ chain_next(chainobject *lz) PyObject *item; if (lz->source == NULL) - return NULL; /* already stopped */ + return NULL; /* already stopped */ if (lz->active == NULL) { PyObject *iterable = PyIter_Next(lz->source); if (iterable == NULL) { Py_CLEAR(lz->source); - return NULL; /* no more input sources */ + return NULL; /* no more input sources */ } lz->active = PyObject_GetIter(iterable); Py_DECREF(iterable); if (lz->active == NULL) { Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + return NULL; /* input not iterable */ } } item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); @@ -1867,10 +1870,10 @@ chain_next(chainobject *lz) if (PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); else - return NULL; /* input raised an exception */ + return NULL; /* input raised an exception */ } Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + return chain_next(lz); /* recurse and use next active */ } static PyObject * @@ -1928,7 +1931,7 @@ static PyMethodDef chain_methods[] = { reduce_doc}, {"__setstate__", (PyCFunction)chain_setstate, METH_O, setstate_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; static PyTypeObject chain_type = { @@ -1980,7 +1983,7 @@ static PyTypeObject chain_type = { typedef struct { PyObject_HEAD - PyObject *pools; /* tuple of pool tuples */ + PyObject *pools; /* tuple of pool tuples */ Py_ssize_t *indices; /* one index per pool */ PyObject *result; /* most recently returned result tuple */ int stopped; /* set to 1 when the product iterator is exhausted */ @@ -3779,8 +3782,7 @@ filterfalse_next(filterfalseobject *lz) ok = PyObject_IsTrue(item); } else { PyObject *good; - good = PyObject_CallFunctionObjArgs(lz->func, - item, NULL); + good = PyObject_CallFunctionObjArgs(lz->func, item, NULL); if (good == NULL) { Py_DECREF(item); return NULL; -- cgit v0.12 From ca3788c2e8de9c60252d7a8c5fcf0e0bfa25333d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 14:49:24 -0700 Subject: Issue #24874: Speed-up itertools and make it pickles more compact. --- Misc/NEWS | 3 +++ Modules/itertoolsmodule.c | 62 +++++++++++++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index a8b93e6..a3a749d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Library - Issue #21159: Improve message in configparser.InterpolationMissingOptionError. Patch from Łukasz Langa. +- Issue #24874: Improve speed of itertools.cycle() and make its + pickle more compact. + - Fix crash in itertools.cycle.__setstate__() when the first argument wasn't a list. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 533e1d1..61ad0c8 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,4 +1,6 @@ +#define PY_SSIZE_T_CLEAN + #include "Python.h" #include "structmember.h" @@ -863,6 +865,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *saved; + Py_ssize_t index; int firstpass; } cycleobject; @@ -902,6 +905,7 @@ cycle_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } lz->it = it; lz->saved = saved; + lz->index = 0; lz->firstpass = 0; return (PyObject *)lz; @@ -911,15 +915,16 @@ static void cycle_dealloc(cycleobject *lz) { PyObject_GC_UnTrack(lz); - Py_XDECREF(lz->saved); Py_XDECREF(lz->it); + Py_XDECREF(lz->saved); Py_TYPE(lz)->tp_free(lz); } static int cycle_traverse(cycleobject *lz, visitproc visit, void *arg) { - Py_VISIT(lz->it); + if (lz->it) + Py_VISIT(lz->it); Py_VISIT(lz->saved); return 0; } @@ -928,13 +933,13 @@ static PyObject * cycle_next(cycleobject *lz) { PyObject *item; - PyObject *it; - PyObject *tmp; - while (1) { + if (lz->it != NULL) { item = PyIter_Next(lz->it); if (item != NULL) { - if (!lz->firstpass && PyList_Append(lz->saved, item)) { + if (lz->firstpass) + return item; + if (PyList_Append(lz->saved, item)) { Py_DECREF(item); return NULL; } @@ -942,27 +947,41 @@ cycle_next(cycleobject *lz) } /* Note: StopIteration is already cleared by PyIter_Next() */ if (PyErr_Occurred()) - return NULL; - if (PyList_Size(lz->saved) == 0) - return NULL; - it = PyObject_GetIter(lz->saved); - if (it == NULL) return NULL; - tmp = lz->it; - lz->it = it; - lz->firstpass = 1; - Py_DECREF(tmp); + Py_CLEAR(lz->it); } + if (Py_SIZE(lz->saved) == 0) + return NULL; + item = PyList_GET_ITEM(lz->saved, lz->index); + lz->index++; + if (lz->index >= Py_SIZE(lz->saved)) + lz->index = 0; + Py_INCREF(item); + return item; } static PyObject * cycle_reduce(cycleobject *lz) { - /* Create a new cycle with the iterator tuple, then set - * the saved state on it. - */ - return Py_BuildValue("O(O)(Oi)", Py_TYPE(lz), - lz->it, lz->saved, lz->firstpass); + /* Create a new cycle with the iterator tuple, then set the saved state */ + if (lz->it == NULL) { + PyObject *it = PyObject_GetIter(lz->saved); + if (it == NULL) + return NULL; + if (lz->index != 0) { + _Py_IDENTIFIER(__setstate__); + PyObject *res = _PyObject_CallMethodId(it, &PyId___setstate__, + "n", lz->index); + if (res == NULL) { + Py_DECREF(it); + return NULL; + } + Py_DECREF(res); + } + return Py_BuildValue("O(N)(Oi)", Py_TYPE(lz), it, lz->saved, 1); + } + return Py_BuildValue("O(O)(Oi)", Py_TYPE(lz), lz->it, lz->saved, + lz->firstpass); } static PyObject * @@ -972,10 +991,11 @@ cycle_setstate(cycleobject *lz, PyObject *state) int firstpass; if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) return NULL; + Py_INCREF(saved); Py_CLEAR(lz->saved); lz->saved = saved; - Py_XINCREF(lz->saved); lz->firstpass = firstpass != 0; + lz->index = 0; Py_RETURN_NONE; } -- cgit v0.12 From bb562a9a5659d7c97790454e1b76819614dead26 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 17:08:48 -0700 Subject: Add error information to a failing test to help diagnose a buildbot failure. --- Lib/test/test_symbol.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py index 4475bbc..c126be3 100644 --- a/Lib/test/test_symbol.py +++ b/Lib/test/test_symbol.py @@ -38,7 +38,10 @@ class TestSymbolGeneration(unittest.TestCase): self.assertFalse(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) self.assertEqual((0, b''), self._generate_symbols(GRAMMAR_FILE, TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) + self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE), + 'symbol stat: %r\ntest_py stat: %r\n' % + (os.stat(SYMBOL_FILE), + os.stat(TEST_PY_FILE))) if __name__ == "__main__": -- cgit v0.12 From 47d159f5220918f9c1216ba93583f3aa740a8944 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 19:43:34 -0700 Subject: Issue #24379: Add operator.subscript() as a convenience for building slices. --- Doc/library/operator.rst | 15 +++++++++++++++ Doc/whatsnew/3.6.rst | 7 ++++++- Lib/operator.py | 28 +++++++++++++++++++++++++++- Lib/test/test_operator.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index c01e63b..0695391 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -333,6 +333,21 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] +.. data:: subscript + + A helper to turn subscript notation into indexing objects. This can be + used to create item access patterns ahead of time to pass them into + various subscriptable objects. + + For example: + + * ``subscript[5] == 5`` + * ``subscript[3:7:2] == slice(3, 7, 2)`` + * ``subscript[5, 8] == (5, 8)`` + + .. versionadded:: 3.6 + + .. function:: methodcaller(name[, args...]) Return a callable object that calls the method *name* on its operand. If diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index e2aa1e2..48ff38a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -95,7 +95,12 @@ New Modules Improved Modules ================ -* None yet. +operator +-------- + +* New object :data:`operator.subscript` makes it easier to create complex + indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` + (Contributed by Joe Jevnik in :issue:`24379`.) Optimizations diff --git a/Lib/operator.py b/Lib/operator.py index 0e2e53e..bc2a947 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -17,7 +17,7 @@ __all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'truediv', 'truth', 'xor'] + 'setitem', 'sub', 'subscript', 'truediv', 'truth', 'xor'] from builtins import abs as _abs @@ -408,6 +408,32 @@ def ixor(a, b): return a +@object.__new__ # create a singleton instance +class subscript: + """ + A helper to turn subscript notation into indexing objects. This can be + used to create item access patterns ahead of time to pass them into + various subscriptable objects. + + For example: + subscript[5] == 5 + subscript[3:7:2] == slice(3, 7, 2) + subscript[5, 8] == (5, 8) + """ + __slots__ = () + + def __new__(cls): + raise TypeError("cannot create '{}' instances".format(cls.__name__)) + + @staticmethod + def __getitem__(key): + return key + + @staticmethod + def __reduce__(): + return 'subscript' + + try: from _operator import * except ImportError: diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 54fd1f4..27501c2 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -596,5 +596,38 @@ class CCOperatorPickleTestCase(OperatorPickleTestCase, unittest.TestCase): module2 = c_operator +class SubscriptTestCase: + def test_subscript(self): + subscript = self.module.subscript + self.assertIsNone(subscript[None]) + self.assertEqual(subscript[0], 0) + self.assertEqual(subscript[0:1:2], slice(0, 1, 2)) + self.assertEqual( + subscript[0, ..., :2, ...], + (0, Ellipsis, slice(2), Ellipsis), + ) + + def test_pickle(self): + from operator import subscript + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + self.assertIs( + pickle.loads(pickle.dumps(subscript, proto)), + subscript, + ) + + def test_singleton(self): + with self.assertRaises(TypeError): + type(self.module.subscript)() + + def test_immutable(self): + with self.assertRaises(AttributeError): + self.module.subscript.attr = None + + +class PySubscriptTestCase(SubscriptTestCase, PyOperatorTestCase): + pass + + if __name__ == "__main__": unittest.main() -- cgit v0.12 From 5b798abf5b56887d8dcde4e9fa82e405ef4b1dd3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 17 Aug 2015 22:04:45 -0700 Subject: Issue #24878: Add docstrings to selected namedtuples --- Lib/aifc.py | 8 ++++++++ Lib/dis.py | 9 +++++++++ Lib/sched.py | 11 +++++++++++ Lib/selectors.py | 12 ++++++++++-- Lib/shutil.py | 3 +++ Lib/sndhdr.py | 12 ++++++++++++ 6 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Lib/aifc.py b/Lib/aifc.py index 7ebdbeb..1556b53 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -257,6 +257,14 @@ from collections import namedtuple _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') +_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' +_aifc_params.sampwidth.__doc__ = 'Ample width in bytes' +_aifc_params.framerate.__doc__ = 'Sampling frequency' +_aifc_params.nframes.__doc__ = 'Number of audio frames' +_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' +_aifc_params.compname.__doc__ = ("""A human-readable version ofcompression type +('not compressed' for AIFF files)'""") + class Aifc_read: # Variables used in this class: diff --git a/Lib/dis.py b/Lib/dis.py index af37cdf..3540b47 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -162,6 +162,15 @@ def show_code(co, *, file=None): _Instruction = collections.namedtuple("_Instruction", "opname opcode arg argval argrepr offset starts_line is_jump_target") +_Instruction.opname.__doc__ = "Human readable name for operation" +_Instruction.opcode.__doc__ = "Numeric code for operation" +_Instruction.arg.__doc__ = "Numeric argument to operation (if any), otherwise None" +_Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" +_Instruction.argrepr.__doc__ = "Human readable description of operation argument" +_Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" +_Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None" +_Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" + class Instruction(_Instruction): """Details for a bytecode operation diff --git a/Lib/sched.py b/Lib/sched.py index b47648d..bd7c0f1 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -46,6 +46,17 @@ class Event(namedtuple('Event', 'time, priority, action, argument, kwargs')): def __gt__(s, o): return (s.time, s.priority) > (o.time, o.priority) def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority) +Event.time.__doc__ = ('''Numeric type compatible with the return value of the +timefunc function passed to the constructor.''') +Event.priority.__doc__ = ('''Events scheduled for the same time will be executed +in the order of their priority.''') +Event.action.__doc__ = ('''Executing the event means executing +action(*argument, **kwargs)''') +Event.argument.__doc__ = ('''argument is a sequence holding the positional +arguments for the action.''') +Event.kwargs.__doc__ = ('''kwargs is a dictionary holding the keyword +arguments for the action.''') + _sentinel = object() class scheduler: diff --git a/Lib/selectors.py b/Lib/selectors.py index 6d569c3..ecd8632 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -43,9 +43,17 @@ def _fileobj_to_fd(fileobj): SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) -"""Object used to associate a file object to its backing file descriptor, -selected event mask and attached data.""" +SelectorKey.__doc__ = """SelectorKey(fileobj, fd, events, data) + + Object used to associate a file object to its backing + file descriptor, selected event mask, and attached data. +""" +SelectorKey.fileobj.__doc__ = 'File object registered.' +SelectorKey.fd.__doc__ = 'Underlying file descriptor.' +SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' +SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. +For example, this could be used to store a per-client session ID.''') class _SelectorMapping(Mapping): """Mapping of file objects to selector keys.""" diff --git a/Lib/shutil.py b/Lib/shutil.py index a5da587..8edabdb 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -965,6 +965,9 @@ if hasattr(os, 'statvfs'): __all__.append('disk_usage') _ntuple_diskusage = collections.namedtuple('usage', 'total used free') + _ntuple_diskusage.total.__doc__ = 'Total space in bytes' + _ntuple_diskusage.used.__doc__ = 'Used space in bytes' + _ntuple_diskusage.free.__doc__ = 'Free space in bytes' def disk_usage(path): """Return disk usage statistics about the given path. diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py index e5901ec..7ecafb4 100644 --- a/Lib/sndhdr.py +++ b/Lib/sndhdr.py @@ -37,6 +37,18 @@ from collections import namedtuple SndHeaders = namedtuple('SndHeaders', 'filetype framerate nchannels nframes sampwidth') +SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type +and will be one of the strings 'aifc', 'aiff', 'au','hcom', +'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""") +SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual +value or 0 if unknown or difficult to decode.""") +SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be +determined or if the value is difficult to decode.""") +SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number +of frames or -1.""") +SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or +'A' for A-LAW or 'U' for u-LAW.""") + def what(filename): """Guess the type of a sound file.""" res = whathdr(filename) -- cgit v0.12 From a6ea44aed146d571a7236f2089255625c3c83693 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 17 Aug 2015 23:55:28 -0700 Subject: Minor cleanups --- Modules/itertoolsmodule.c | 156 ++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 82 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 61ad0c8..c7e1919 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,17 +1,16 @@ #define PY_SSIZE_T_CLEAN - #include "Python.h" #include "structmember.h" /* Itertools module written and maintained by Raymond D. Hettinger - Copyright (c) 2003-2013 Python Software Foundation. + Copyright (c) 2003-2015 Python Software Foundation. All rights reserved. */ -/* groupby object ***********************************************************/ +/* groupby object ************************************************************/ typedef struct { PyObject_HEAD @@ -89,8 +88,7 @@ groupby_next(groupbyobject *gbo) else { int rcmp; - rcmp = PyObject_RichCompareBool(gbo->tgtkey, - gbo->currkey, Py_EQ); + rcmp = PyObject_RichCompareBool(gbo->tgtkey, gbo->currkey, Py_EQ); if (rcmp == -1) return NULL; else if (rcmp == 0) @@ -105,8 +103,7 @@ groupby_next(groupbyobject *gbo) newkey = newvalue; Py_INCREF(newvalue); } else { - newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, - newvalue, NULL); + newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, newvalue, NULL); if (newkey == NULL) { Py_DECREF(newvalue); return NULL; @@ -303,8 +300,7 @@ _grouper_next(_grouperobject *igo) newkey = newvalue; Py_INCREF(newvalue); } else { - newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, - newvalue, NULL); + newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, newvalue, NULL); if (newkey == NULL) { Py_DECREF(newvalue); return NULL; @@ -332,8 +328,7 @@ _grouper_next(_grouperobject *igo) static PyObject * _grouper_reduce(_grouperobject *lz) { - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->parent, lz->tgtkey); + return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->parent, lz->tgtkey); } static PyMethodDef _grouper_methods[] = { @@ -387,8 +382,7 @@ static PyTypeObject _grouper_type = { }; - -/* tee object and with supporting function and objects ***************/ +/* tee object and with supporting function and objects ***********************/ /* The teedataobject pre-allocates space for LINKCELLS number of objects. To help the object fit neatly inside cache lines (space for 16 to 32 @@ -403,7 +397,7 @@ static PyTypeObject _grouper_type = { typedef struct { PyObject_HEAD PyObject *it; - int numread; /* 0 <= numread <= LINKCELLS */ + int numread; /* 0 <= numread <= LINKCELLS */ PyObject *nextlink; PyObject *(values[LINKCELLS]); } teedataobject; @@ -411,7 +405,7 @@ typedef struct { typedef struct { PyObject_HEAD teedataobject *dataobj; - int index; /* 0 <= index <= LINKCELLS */ + int index; /* 0 <= index <= LINKCELLS */ PyObject *weakreflist; } teeobject; @@ -468,6 +462,7 @@ static int teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) { int i; + Py_VISIT(tdo->it); for (i = 0; i < tdo->numread; i++) Py_VISIT(tdo->values[i]); @@ -517,6 +512,7 @@ teedataobject_reduce(teedataobject *tdo) int i; /* create a temporary list of already iterated values */ PyObject *values = PyList_New(tdo->numread); + if (!values) return NULL; for (i=0 ; inumread ; i++) { @@ -859,7 +855,7 @@ PyDoc_STRVAR(tee_doc, "tee(iterable, n=2) --> tuple of n independent iterators."); -/* cycle object **********************************************************/ +/* cycle object **************************************************************/ typedef struct { PyObject_HEAD @@ -989,6 +985,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) { PyObject *saved=NULL; int firstpass; + if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) return NULL; Py_INCREF(saved); @@ -1064,7 +1061,7 @@ typedef struct { PyObject_HEAD PyObject *func; PyObject *it; - long start; + long start; } dropwhileobject; static PyTypeObject dropwhile_type; @@ -1154,8 +1151,7 @@ dropwhile_next(dropwhileobject *lz) static PyObject * dropwhile_reduce(dropwhileobject *lz) { - return Py_BuildValue("O(OO)l", Py_TYPE(lz), - lz->func, lz->it, lz->start); + return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->start); } static PyObject * @@ -1233,7 +1229,7 @@ typedef struct { PyObject_HEAD PyObject *func; PyObject *it; - long stop; + long stop; } takewhileobject; static PyTypeObject takewhile_type; @@ -1308,7 +1304,7 @@ takewhile_next(takewhileobject *lz) } ok = PyObject_IsTrue(good); Py_DECREF(good); - if (ok == 1) + if (ok > 0) return item; Py_DECREF(item); if (ok == 0) @@ -1319,14 +1315,14 @@ takewhile_next(takewhileobject *lz) static PyObject * takewhile_reduce(takewhileobject *lz) { - return Py_BuildValue("O(OO)l", Py_TYPE(lz), - lz->func, lz->it, lz->stop); + return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->stop); } static PyObject * takewhile_reduce_setstate(takewhileobject *lz, PyObject *state) { int stop = PyObject_IsTrue(state); + if (stop < 0) return NULL; lz->stop = stop; @@ -1391,7 +1387,7 @@ static PyTypeObject takewhile_type = { }; -/* islice object ************************************************************/ +/* islice object *************************************************************/ typedef struct { PyObject_HEAD @@ -1549,6 +1545,7 @@ islice_reduce(isliceobject *lz) * then 'setstate' with the next and count */ PyObject *stop; + if (lz->it == NULL) { PyObject *empty_list; PyObject *empty_it; @@ -1578,6 +1575,7 @@ static PyObject * islice_setstate(isliceobject *lz, PyObject *state) { Py_ssize_t cnt = PyLong_AsSsize_t(state); + if (cnt == -1 && PyErr_Occurred()) return NULL; lz->cnt = cnt; @@ -1792,7 +1790,7 @@ static PyTypeObject starmap_type = { }; -/* chain object ************************************************************/ +/* chain object **************************************************************/ typedef struct { PyObject_HEAD @@ -1919,6 +1917,7 @@ static PyObject * chain_setstate(chainobject *lz, PyObject *state) { PyObject *source, *active=NULL; + if (! PyArg_ParseTuple(state, "O|O", &source, &active)) return NULL; @@ -1945,8 +1944,8 @@ Alternate chain() contructor taking a single iterable argument\n\ that evaluates lazily."); static PyMethodDef chain_methods[] = { - {"from_iterable", (PyCFunction) chain_new_from_iterable, METH_O | METH_CLASS, - chain_from_iterable_doc}, + {"from_iterable", (PyCFunction) chain_new_from_iterable, METH_O | METH_CLASS, + chain_from_iterable_doc}, {"__reduce__", (PyCFunction)chain_reduce, METH_NOARGS, reduce_doc}, {"__setstate__", (PyCFunction)chain_setstate, METH_O, @@ -2003,10 +2002,10 @@ static PyTypeObject chain_type = { typedef struct { PyObject_HEAD - PyObject *pools; /* tuple of pool tuples */ - Py_ssize_t *indices; /* one index per pool */ - PyObject *result; /* most recently returned result tuple */ - int stopped; /* set to 1 when the product iterator is exhausted */ + PyObject *pools; /* tuple of pool tuples */ + Py_ssize_t *indices; /* one index per pool */ + PyObject *result; /* most recently returned result tuple */ + int stopped; /* set to 1 when the iterator is exhausted */ } productobject; static PyTypeObject product_type; @@ -2025,7 +2024,8 @@ product_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *tmpargs = PyTuple_New(0); if (tmpargs == NULL) return NULL; - if (!PyArg_ParseTupleAndKeywords(tmpargs, kwds, "|n:product", kwlist, &repeat)) { + if (!PyArg_ParseTupleAndKeywords(tmpargs, kwds, "|n:product", + kwlist, &repeat)) { Py_DECREF(tmpargs); return NULL; } @@ -2349,15 +2349,15 @@ static PyTypeObject product_type = { }; -/* combinations object ************************************************************/ +/* combinations object *******************************************************/ typedef struct { PyObject_HEAD - PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ - Py_ssize_t r; /* size of result tuple */ - int stopped; /* set to 1 when the combinations iterator is exhausted */ + PyObject *pool; /* input converted to a tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ + Py_ssize_t r; /* size of result tuple */ + int stopped; /* set to 1 when the iterator is exhausted */ } combinationsobject; static PyTypeObject combinations_type; @@ -2567,17 +2567,16 @@ combinations_setstate(combinationsobject *lz, PyObject *state) Py_ssize_t i; Py_ssize_t n = PyTuple_GET_SIZE(lz->pool); - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) - { + if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) { PyErr_SetString(PyExc_ValueError, "invalid arguments"); return NULL; } - for (i=0; ir; i++) - { + for (i=0; ir; i++) { Py_ssize_t max; PyObject* indexObject = PyTuple_GET_ITEM(state, i); Py_ssize_t index = PyLong_AsSsize_t(indexObject); + if (index == -1 && PyErr_Occurred()) return NULL; /* not an integer */ max = i + n - lz->r; @@ -2664,7 +2663,7 @@ static PyTypeObject combinations_type = { }; -/* combinations with replacement object *******************************************/ +/* combinations with replacement object **************************************/ /* Equivalent to: @@ -2694,11 +2693,11 @@ static PyTypeObject combinations_type = { */ typedef struct { PyObject_HEAD - PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ - Py_ssize_t r; /* size of result tuple */ - int stopped; /* set to 1 when the cwr iterator is exhausted */ + PyObject *pool; /* input converted to a tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ + Py_ssize_t r; /* size of result tuple */ + int stopped; /* set to 1 when the cwr iterator is exhausted */ } cwrobject; static PyTypeObject cwr_type; @@ -2715,8 +2714,9 @@ cwr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_ssize_t i; static char *kwargs[] = {"iterable", "r", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "On:combinations_with_replacement", kwargs, - &iterable, &r)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "On:combinations_with_replacement", + kwargs, &iterable, &r)) return NULL; pool = PySequence_Tuple(iterable); @@ -2881,8 +2881,7 @@ cwr_reduce(cwrobject *lz) indices = PyTuple_New(lz->r); if (!indices) return NULL; - for (i=0; ir; i++) - { + for (i=0; ir; i++) { PyObject* index = PyLong_FromSsize_t(lz->indices[i]); if (!index) { Py_DECREF(indices); @@ -2908,10 +2907,10 @@ cwr_setstate(cwrobject *lz, PyObject *state) } n = PyTuple_GET_SIZE(lz->pool); - for (i=0; ir; i++) - { + for (i=0; ir; i++) { PyObject* indexObject = PyTuple_GET_ITEM(state, i); Py_ssize_t index = PyLong_AsSsize_t(indexObject); + if (index < 0 && PyErr_Occurred()) return NULL; /* not an integer */ /* clamp the index */ @@ -2996,7 +2995,7 @@ static PyTypeObject cwr_type = { }; -/* permutations object ************************************************************ +/* permutations object ******************************************************** def permutations(iterable, r=None): 'permutations(range(3), 2) --> (0,1) (0,2) (1,0) (1,2) (2,0) (2,1)' @@ -3023,12 +3022,12 @@ def permutations(iterable, r=None): typedef struct { PyObject_HEAD - PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per element in the pool */ - Py_ssize_t *cycles; /* one rollover counter per element in the result */ - PyObject *result; /* most recently returned result tuple */ - Py_ssize_t r; /* size of result tuple */ - int stopped; /* set to 1 when the permutations iterator is exhausted */ + PyObject *pool; /* input converted to a tuple */ + Py_ssize_t *indices; /* one index per element in the pool */ + Py_ssize_t *cycles; /* one rollover counter per element in the result */ + PyObject *result; /* most recently returned result tuple */ + Py_ssize_t r; /* size of result tuple */ + int stopped; /* set to 1 when the iterator is exhausted */ } permutationsobject; static PyTypeObject permutations_type; @@ -3243,7 +3242,7 @@ permutations_reduce(permutationsobject *po) indices = PyTuple_New(n); if (indices == NULL) goto err; - for (i=0; iindices[i]); if (!index) goto err; @@ -3253,8 +3252,7 @@ permutations_reduce(permutationsobject *po) cycles = PyTuple_New(po->r); if (cycles == NULL) goto err; - for (i=0; ir; i++) - { + for (i=0 ; ir ; i++) { PyObject* index = PyLong_FromSsize_t(po->cycles[i]); if (!index) goto err; @@ -3282,15 +3280,12 @@ permutations_setstate(permutationsobject *po, PyObject *state) return NULL; n = PyTuple_GET_SIZE(po->pool); - if (PyTuple_GET_SIZE(indices) != n || - PyTuple_GET_SIZE(cycles) != po->r) - { + if (PyTuple_GET_SIZE(indices) != n || PyTuple_GET_SIZE(cycles) != po->r) { PyErr_SetString(PyExc_ValueError, "invalid arguments"); return NULL; } - for (i=0; iindices[i] = index; } - for (i=0; ir; i++) - { + for (i=0; ir; i++) { PyObject* indexObject = PyTuple_GET_ITEM(cycles, i); Py_ssize_t index = PyLong_AsSsize_t(indexObject); if (index < 0 && PyErr_Occurred()) @@ -3388,7 +3382,7 @@ static PyTypeObject permutations_type = { PyObject_GC_Del, /* tp_free */ }; -/* accumulate object ************************************************************/ +/* accumulate object ********************************************************/ typedef struct { PyObject_HEAD @@ -3489,7 +3483,7 @@ accumulate_reduce(accumulateobject *lz) return Py_BuildValue("O(OO)O", Py_TYPE(lz), lz->it, lz->binop?lz->binop:Py_None, lz->total?lz->total:Py_None); - } +} static PyObject * accumulate_setstate(accumulateobject *lz, PyObject *state) @@ -3652,7 +3646,7 @@ compress_next(compressobject *lz) ok = PyObject_IsTrue(selector); Py_DECREF(selector); - if (ok == 1) + if (ok > 0) return datum; Py_DECREF(datum); if (ok < 0) @@ -3665,7 +3659,7 @@ compress_reduce(compressobject *lz) { return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->data, lz->selectors); - } +} static PyMethodDef compress_methods[] = { {"__reduce__", (PyCFunction)compress_reduce, METH_NOARGS, @@ -3821,9 +3815,8 @@ filterfalse_next(filterfalseobject *lz) static PyObject * filterfalse_reduce(filterfalseobject *lz) { - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->func, lz->it); - } + return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); +} static PyMethodDef filterfalse_methods[] = { {"__reduce__", (PyCFunction)filterfalse_reduce, METH_NOARGS, @@ -4274,9 +4267,7 @@ static PyTypeObject repeat_type = { PyObject_GC_Del, /* tp_free */ }; -/* ziplongest object ************************************************************/ - -#include "Python.h" +/* ziplongest object *********************************************************/ typedef struct { PyObject_HEAD @@ -4315,7 +4306,7 @@ zip_longest_new(PyTypeObject *type, PyObject *args, PyObject *kwds) ittuple = PyTuple_New(tuplesize); if (ittuple == NULL) return NULL; - for (i=0; i < tuplesize; ++i) { + for (i=0; i < tuplesize; i++) { PyObject *item = PyTuple_GET_ITEM(args, i); PyObject *it = PyObject_GetIter(item); if (it == NULL) { @@ -4456,6 +4447,7 @@ zip_longest_reduce(ziplongestobject *lz) */ int i; PyObject *args = PyTuple_New(PyTuple_GET_SIZE(lz->ittuple)); + if (args == NULL) return NULL; for (i=0; iittuple); i++) { -- cgit v0.12 From f1094140944721ca852d49fb30e107d4ffec6f12 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 00:20:20 -0700 Subject: Inline PyIter_Next() matching the other itertools code. --- Python/bltinmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2f22209..78ec3cb 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1164,7 +1164,8 @@ map_next(mapobject *lz) return NULL; for (i=0 ; iiters, i)); + PyObject *it = PyTuple_GET_ITEM(lz->iters, i); + val = Py_TYPE(it)->tp_iternext(it); if (val == NULL) { Py_DECREF(argtuple); return NULL; -- cgit v0.12 From 507343a2ef4f2992e2c318517b5833d1b1a1fddb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 00:35:52 -0700 Subject: Add missing docstring --- Lib/urllib/request.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index a7fd017..e6abf34 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -138,6 +138,71 @@ __version__ = sys.version[:3] _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *, cafile=None, capath=None, cadefault=False, context=None): + '''Open the URL url, which can be either a string or a Request object. + + *data* must be a bytes object specifying additional data to be sent to the + server, or None if no such data is needed. data may also be an iterable + object and in that case Content-Length value must be specified in the + headers. Currently HTTP requests are the only ones that use data; the HTTP + request will be a POST instead of a GET when the data parameter is + provided. + + *data* should be a buffer in the standard application/x-www-form-urlencoded + format. The urllib.parse.urlencode() function takes a mapping or sequence + of 2-tuples and returns a string in this format. It should be encoded to + bytes before being used as the data parameter. The charset parameter in + Content-Type header may be used to specify the encoding. If charset + parameter is not sent with the Content-Type header, the server following + the HTTP 1.1 recommendation may assume that the data is encoded in + ISO-8859-1 encoding. It is advisable to use charset parameter with encoding + used in Content-Type header with the Request. + + urllib.request module uses HTTP/1.1 and includes a "Connection:close" + header in its HTTP requests. + + The optional *timeout* parameter specifies a timeout in seconds for + blocking operations like the connection attempt (if not specified, the + global default timeout setting will be used). This only works for HTTP, + HTTPS and FTP connections. + + If *context* is specified, it must be a ssl.SSLContext instance describing + the various SSL options. See HTTPSConnection for more details. + + The optional *cafile* and *capath* parameters specify a set of trusted CA + certificates for HTTPS requests. cafile should point to a single file + containing a bundle of CA certificates, whereas capath should point to a + directory of hashed certificate files. More information can be found in + ssl.SSLContext.load_verify_locations(). + + The *cadefault* parameter is ignored. + + For http and https urls, this function returns a http.client.HTTPResponse + object which has the following HTTPResponse Objects methods. + + For ftp, file, and data urls and requests explicitly handled by legacy + URLopener and FancyURLopener classes, this function returns a + urllib.response.addinfourl object which can work as context manager and has + methods such as: + + * geturl() — return the URL of the resource retrieved, commonly used to + determine if a redirect was followed + + * info() — return the meta-information of the page, such as headers, in the + form of an email.message_from_string() instance (see Quick Reference to + HTTP Headers) + + * getcode() – return the HTTP status code of the response. Raises URLError + on errors. + + Note that *None& may be returned if no handler handles the request (though + the default installed global OpenerDirector uses UnknownHandler to ensure + this never happens). + + In addition, if proxy settings are detected (for example, when a *_proxy + environment variable like http_proxy is set), ProxyHandler is default + installed and makes sure the requests are handled through the proxy. + + ''' global _opener if cafile or capath or cadefault: if context is not None: -- cgit v0.12 From 501b4a73980d363fca3a69d5999d377f3ae5cc3f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 08:07:16 -0700 Subject: Use PyTuple_GET_SIZE like the adjacent code does. --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 78ec3cb..2038d2b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1158,7 +1158,7 @@ map_next(mapobject *lz) PyObject *result; Py_ssize_t numargs, i; - numargs = PyTuple_Size(lz->iters); + numargs = PyTuple_GET_SIZE(lz->iters); argtuple = PyTuple_New(numargs); if (argtuple == NULL) return NULL; -- cgit v0.12 From 631a35c71a0c1f4fb31aad04407825cee2fe8a48 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 19 Aug 2015 12:20:37 +1200 Subject: Issue #24054: decouple linecache tests from inspect tests Patch from David D. Riddle --- Lib/test/test_linecache.py | 74 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 21ef738..e74aef3 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -3,6 +3,7 @@ import linecache import unittest import os.path +import tempfile from test import support @@ -10,8 +11,6 @@ FILENAME = linecache.__file__ NONEXISTENT_FILENAME = FILENAME + '.missing' INVALID_NAME = '!@$)(!@#_1' EMPTY = '' -TESTS = 'inspect_fodder inspect_fodder2 mapping_tests' -TESTS = TESTS.split() TEST_PATH = os.path.dirname(__file__) MODULES = "linecache abc".split() MODULE_PATH = os.path.dirname(FILENAME) @@ -37,6 +36,65 @@ def f(): return 3''' # No ending newline +class TempFile: + + def setUp(self): + super().setUp() + with tempfile.NamedTemporaryFile(delete=False) as fp: + self.file_name = fp.name + fp.write(self.file_byte_string) + self.addCleanup(support.unlink, self.file_name) + + +class GetLineTestsGoodData(TempFile): + # file_list = ['list\n', 'of\n', 'good\n', 'strings\n'] + + def setUp(self): + self.file_byte_string = ''.join(self.file_list).encode('utf-8') + super().setUp() + + def test_getline(self): + with open(self.file_name) as fp: + for index, line in enumerate(fp): + if not line.endswith('\n'): + line += '\n' + + cached_line = linecache.getline(self.file_name, index + 1) + self.assertEqual(line, cached_line) + + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, self.file_list) + + +class GetLineTestsBadData(TempFile): + # file_byte_string = b'Bad data goes here' + + def test_getline(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getline, self.file_name, 1) + + def test_getlines(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getlines, self.file_name) + + +class EmptyFile(GetLineTestsGoodData, unittest.TestCase): + file_list = [] + + +class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): + file_list = ['\n'] + + +class GoodUnicode(GetLineTestsGoodData, unittest.TestCase): + file_list = ['á\n', 'b\n', 'abcdef\n', 'ááááá\n'] + + +class BadUnicode(GetLineTestsBadData, unittest.TestCase): + file_byte_string = b'\x80abc' + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -53,13 +111,6 @@ class LineCacheTests(unittest.TestCase): self.assertEqual(getline(EMPTY, 1), EMPTY) self.assertEqual(getline(INVALID_NAME, 1), EMPTY) - # Check whether lines correspond to those from file iteration - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' - with open(filename) as file: - for index, line in enumerate(file): - self.assertEqual(line, getline(filename, index + 1)) - # Check module loading for entry in MODULES: filename = os.path.join(MODULE_PATH, entry) + '.py' @@ -80,12 +131,13 @@ class LineCacheTests(unittest.TestCase): def test_clearcache(self): cached = [] - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' + for entry in MODULES: + filename = os.path.join(MODULE_PATH, entry) + '.py' cached.append(filename) linecache.getline(filename, 1) # Are all files cached? + self.assertNotEqual(cached, []) cached_empty = [fn for fn in cached if fn not in linecache.cache] self.assertEqual(cached_empty, []) -- cgit v0.12 From 15b87bfedcd40f601cb3c9caf75d2366c48e19fc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 22:03:08 -0700 Subject: Add in missing docstrings. --- Lib/http/client.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/Lib/http/client.py b/Lib/http/client.py index 80c80cf..155c2e3 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -405,6 +405,7 @@ class HTTPResponse(io.BufferedIOBase): self.fp.flush() def readable(self): + """Always returns True""" return True # End of "raw stream" methods @@ -452,6 +453,10 @@ class HTTPResponse(io.BufferedIOBase): return s def readinto(self, b): + """Read up to len(b) bytes into bytearray b and return the number + of bytes read. + """ + if self.fp is None: return 0 @@ -683,6 +688,17 @@ class HTTPResponse(io.BufferedIOBase): return self.fp.fileno() def getheader(self, name, default=None): + '''Returns the value of the header matching *name*. + + If there are multiple matching headers, the values are + combined into a single string separated by commas and spaces. + + If no matching header is found, returns *default* or None if + the *default* is not specified. + + If the headers are unknown, raises http.client.ResponseNotReady. + + ''' if self.headers is None: raise ResponseNotReady() headers = self.headers.get_all(name) or default @@ -705,12 +721,45 @@ class HTTPResponse(io.BufferedIOBase): # For compatibility with old-style urllib responses. def info(self): + '''Returns an instance of the class mimetools.Message containing + meta-information associated with the URL. + + When the method is HTTP, these headers are those returned by + the server at the head of the retrieved HTML page (including + Content-Length and Content-Type). + + When the method is FTP, a Content-Length header will be + present if (as is now usual) the server passed back a file + length in response to the FTP retrieval request. A + Content-Type header will be present if the MIME type can be + guessed. + + When the method is local-file, returned headers will include + a Date representing the file’s last-modified time, a + Content-Length giving file size, and a Content-Type + containing a guess at the file’s type. See also the + description of the mimetools module. + + ''' return self.headers def geturl(self): + '''Return the real URL of the page. + + In some cases, the HTTP server redirects a client to another + URL. The urlopen() function handles this transparently, but in + some cases the caller needs to know which URL the client was + redirected to. The geturl() method can be used to get at this + redirected URL. + + ''' return self.url def getcode(self): + '''Return the HTTP status code that was sent with the response, + or None if the URL is not an HTTP URL. + + ''' return self.status class HTTPConnection: -- cgit v0.12 From 95801bbe4e96aeb706c8fcf9fb3698c39a72b847 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 22:25:16 -0700 Subject: Issue #24879: Teach pydoc to display named tuple fields in the order they were defined. --- Lib/pydoc.py | 19 +++++++++++++++---- Lib/test/test_pydoc.py | 16 ++++++++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index ee558bf..f4f2530 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -209,6 +209,18 @@ def classify_class_attrs(object): results.append((name, kind, cls, value)) return results +def sort_attributes(attrs, object): + 'Sort the attrs list in-place by _fields and then alphabetically by name' + # This allows data descriptors to be ordered according + # to a _fields attribute if present. + fields = getattr(object, '_fields', []) + try: + field_order = {name : i-len(fields) for (i, name) in enumerate(fields)} + except TypeError: + field_order = {} + keyfunc = lambda attr: (field_order.get(attr[0], 0), attr[0]) + attrs.sort(key=keyfunc) + # ----------------------------------------------------- module manipulation def ispackage(path): @@ -867,8 +879,7 @@ class HTMLDoc(Doc): object.__module__) tag += ':
\n' - # Sort attrs by name. - attrs.sort(key=lambda t: t[0]) + sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill('Methods %s' % tag, attrs, @@ -1286,8 +1297,8 @@ location listed above. else: tag = "inherited from %s" % classname(thisclass, object.__module__) - # Sort attrs by name. - attrs.sort() + + sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill("Methods %s:\n" % tag, attrs, diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index ec5c31b..0533a03 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -811,6 +811,22 @@ class TestDescriptions(unittest.TestCase): self.assertEqual(self._get_summary_line(t.wrap), "wrap(text) method of textwrap.TextWrapper instance") + def test_field_order_for_named_tuples(self): + Person = namedtuple('Person', ['nickname', 'firstname', 'agegroup']) + s = pydoc.render_doc(Person) + self.assertLess(s.index('nickname'), s.index('firstname')) + self.assertLess(s.index('firstname'), s.index('agegroup')) + + class NonIterableFields: + _fields = None + + class NonHashableFields: + _fields = [[]] + + # Make sure these doesn't fail + pydoc.render_doc(NonIterableFields) + pydoc.render_doc(NonHashableFields) + @requires_docstrings def test_bound_builtin_method(self): s = StringIO() diff --git a/Misc/NEWS b/Misc/NEWS index b2ec4f9..0bb5697 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,10 @@ Library header in part headers. Patch written by Peter Landry and reviewed by Pierre Quentel. +- Issue #24879: help() and pydoc can now list named tuple fields in the + order they were defined rather than alphabetically. The ordering is + determined by the _fields attribute if present. + - Issue #24774: Fix docstring in http.server.test. Patch from Chiu-Hsiang Hsu. - Issue #21159: Improve message in configparser.InterpolationMissingOptionError. -- cgit v0.12 From e4d35dc8b6fb3c071d85e2d8830566876288c180 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 20 Aug 2015 10:48:46 +1200 Subject: Fix test_linecache change. In non-UTF8 locales open() will have the wrong encoding for the test data. We need to open the file via tokenize, to get a detected encoding instead. --- Lib/test/test_linecache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e74aef3..240db7f 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -4,6 +4,7 @@ import linecache import unittest import os.path import tempfile +import tokenize from test import support @@ -54,7 +55,7 @@ class GetLineTestsGoodData(TempFile): super().setUp() def test_getline(self): - with open(self.file_name) as fp: + with tokenize.open(self.file_name) as fp: for index, line in enumerate(fp): if not line.endswith('\n'): line += '\n' -- cgit v0.12 From 163b5c668d33689aad9f2490bfc5318fba64fa47 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 21 Aug 2015 09:40:38 -0400 Subject: Fix a trivial typo. --- Python/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index c1c3907..8b356fe 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -511,7 +511,7 @@ PyAST_Validate(mod_ty mod) /* Data structure used internally */ struct compiling { char *c_encoding; /* source encoding */ - PyArena *c_arena; /* arena for allocating memeory */ + PyArena *c_arena; /* Arena for allocating memory. */ PyObject *c_filename; /* filename */ PyObject *c_normalize; /* Normalization function from unicodedata. */ PyObject *c_normalize_args; /* Normalization argument tuple. */ -- cgit v0.12 From 4e70772b52b21c39488261d66b3feae2b75352dc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 23 Aug 2015 11:28:01 -0700 Subject: Issue #24878: Fix typos and line wrap --- Lib/aifc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/aifc.py b/Lib/aifc.py index 1556b53..692d0bf 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -258,12 +258,13 @@ _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') _aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' -_aifc_params.sampwidth.__doc__ = 'Ample width in bytes' +_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' _aifc_params.framerate.__doc__ = 'Sampling frequency' _aifc_params.nframes.__doc__ = 'Number of audio frames' _aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' -_aifc_params.compname.__doc__ = ("""A human-readable version ofcompression type -('not compressed' for AIFF files)'""") +_aifc_params.compname.__doc__ = ("""\ +A human-readable version of the compression type +('not compressed' for AIFF files)""") class Aifc_read: -- cgit v0.12 From 4bffa5fa3a4b70a149045a91d7dae8deb5a388ad Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 24 Aug 2015 11:46:56 +1200 Subject: Issue #24633: site-packages/README -> README.txt. --- Lib/site-packages/README | 2 -- Lib/site-packages/README.txt | 2 ++ Misc/NEWS | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 Lib/site-packages/README create mode 100644 Lib/site-packages/README.txt diff --git a/Lib/site-packages/README b/Lib/site-packages/README deleted file mode 100644 index 273f625..0000000 --- a/Lib/site-packages/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory exists so that 3rd party packages can be installed -here. Read the source for site.py for more details. diff --git a/Lib/site-packages/README.txt b/Lib/site-packages/README.txt new file mode 100644 index 0000000..273f625 --- /dev/null +++ b/Lib/site-packages/README.txt @@ -0,0 +1,2 @@ +This directory exists so that 3rd party packages can be installed +here. Read the source for site.py for more details. diff --git a/Misc/NEWS b/Misc/NEWS index 7b11b88..5207e83 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,8 @@ Core and Builtins Library ------- +- Issue #24633: site-packages/README -> README.txt. + - Issue #20362: Honour TestCase.longMessage correctly in assertRegex. Patch from Ilia Kurenkov. -- cgit v0.12 From 69de2a576cde2d1cfeac67472ad9d22aa1e87a4b Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 26 Aug 2015 12:40:28 +1200 Subject: Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. --- Lib/timeit.py | 32 +++++++++++++++++++------------- Misc/NEWS | 3 +++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/Lib/timeit.py b/Lib/timeit.py index 2de88f7..98cb3eb 100755 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -317,20 +317,26 @@ def main(args=None, *, _wrap_timer=None): print("%d loops," % number, end=' ') usec = best * 1e6 / number if time_unit is not None: - print("best of %d: %.*g %s per loop" % (repeat, precision, - usec/units[time_unit], time_unit)) + scale = units[time_unit] else: - if usec < 1000: - print("best of %d: %.*g usec per loop" % (repeat, precision, usec)) - else: - msec = usec / 1000 - if msec < 1000: - print("best of %d: %.*g msec per loop" % (repeat, - precision, msec)) - else: - sec = msec / 1000 - print("best of %d: %.*g sec per loop" % (repeat, - precision, sec)) + scales = [(scale, unit) for unit, scale in units.items()] + scales.sort(reverse=True) + for scale, time_unit in scales: + if usec >= scale: + break + print("best of %d: %.*g %s per loop" % (repeat, precision, + usec/scale, time_unit)) + best = min(r) + usec = best * 1e6 / number + worst = max(r) + if worst >= best * 4: + usec = worst * 1e6 / number + import warnings + warnings.warn_explicit( + "The test results are likely unreliable. The worst\n" + "time (%.*g %s) was more than four times slower than the best time." % + (precision, usec/scale, time_unit), + UserWarning, '', 0) return None if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index dabbeea..3404b09 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Core and Builtins Library ------- +- Issue #23552: Timeit now warns when there is substantial (4x) variance + between best and worst times. Patch from Serhiy Storchaka. + - Issue #24633: site-packages/README -> README.txt. - Issue #24879: help() and pydoc can now list named tuple fields in the -- cgit v0.12 From ab2aa6dc9197a3fc8fca5958a8578e697f88034e Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 26 Aug 2015 14:10:32 -0400 Subject: Fixed an incorrect comment. --- Objects/stringlib/unicode_format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index aec221a..d72e47d 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -67,7 +67,7 @@ SubString_new_object(SubString *str) return PyUnicode_Substring(str->str, str->start, str->end); } -/* return a new string. if str->str is NULL, return None */ +/* return a new string. if str->str is NULL, return a new empty string */ Py_LOCAL_INLINE(PyObject *) SubString_new_object_or_empty(SubString *str) { -- cgit v0.12 From 357cb986b2e330d63dbe3115b32a4acaa93e3918 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Fri, 28 Aug 2015 16:56:45 -0400 Subject: Fixed typos in TZ format description --- Doc/library/time.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 3d335c8..73436ca 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -634,11 +634,11 @@ The module defines the following functions and data items: it is possible to refer to February 29. :samp:`M{m}.{n}.{d}` - The *d*'th day (0 <= *d* <= 6) or week *n* of month *m* of the year (1 + The *d*'th day (0 <= *d* <= 6) of week *n* of month *m* of the year (1 <= *n* <= 5, 1 <= *m* <= 12, where week 5 means "the last *d* day in month *m*" which may occur in either the fourth or the fifth week). Week 1 is the first week in which the *d*'th day occurs. Day - zero is Sunday. + zero is a Sunday. ``time`` has the same format as ``offset`` except that no leading sign ('-' or '+') is allowed. The default, if time is not given, is 02:00:00. -- cgit v0.12 From 53e137c8dde65eb0d48d26d262f1f6bc5d103e68 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 00:49:16 +0200 Subject: Refactor pytime.c Move code to convert double timestamp to subfunctions. --- Python/pytime.c | 113 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 02a9374..17ef197 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -61,43 +61,51 @@ _PyLong_FromTime_t(time_t t) } static int -_PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, +_PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) { - assert(denominator <= LONG_MAX); - if (PyFloat_Check(obj)) { - double d, intpart, err; - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ - volatile double floatpart; + double intpart, err; + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double floatpart; + + floatpart = modf(d, &intpart); + if (floatpart < 0) { + floatpart = 1.0 + floatpart; + intpart -= 1.0; + } - d = PyFloat_AsDouble(obj); - floatpart = modf(d, &intpart); - if (floatpart < 0) { - floatpart = 1.0 + floatpart; - intpart -= 1.0; + floatpart *= denominator; + if (round == _PyTime_ROUND_CEILING) { + floatpart = ceil(floatpart); + if (floatpart >= denominator) { + floatpart = 0.0; + intpart += 1.0; } + } + else { + floatpart = floor(floatpart); + } - floatpart *= denominator; - if (round == _PyTime_ROUND_CEILING) { - floatpart = ceil(floatpart); - if (floatpart >= denominator) { - floatpart = 0.0; - intpart += 1.0; - } - } - else { - floatpart = floor(floatpart); - } + *sec = (time_t)intpart; + err = intpart - (double)*sec; + if (err <= -1.0 || err >= 1.0) { + error_time_t_overflow(); + return -1; + } - *sec = (time_t)intpart; - err = intpart - (double)*sec; - if (err <= -1.0 || err >= 1.0) { - error_time_t_overflow(); - return -1; - } + *numerator = (long)floatpart; + return 0; +} - *numerator = (long)floatpart; - return 0; +static int +_PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, + double denominator, _PyTime_round_t round) +{ + assert(denominator <= LONG_MAX); + if (PyFloat_Check(obj)) { + double d = PyFloat_AsDouble(obj); + return _PyTime_DoubleToDenominator(d, sec, numerator, + denominator, round); } else { *sec = _PyLong_AsTime_t(obj); @@ -221,29 +229,38 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) #endif static int +_PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, + long to_nanoseconds) +{ + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double d, err; + + /* convert to a number of nanoseconds */ + d = value; + d *= to_nanoseconds; + + if (round == _PyTime_ROUND_CEILING) + d = ceil(d); + else + d = floor(d); + + *t = (_PyTime_t)d; + err = d - (double)*t; + if (fabs(err) >= 1.0) { + _PyTime_overflow(); + return -1; + } + return 0; +} + +static int _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, long to_nanoseconds) { if (PyFloat_Check(obj)) { - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ - volatile double d, err; - - /* convert to a number of nanoseconds */ + double d; d = PyFloat_AsDouble(obj); - d *= to_nanoseconds; - - if (round == _PyTime_ROUND_CEILING) - d = ceil(d); - else - d = floor(d); - - *t = (_PyTime_t)d; - err = d - (double)*t; - if (fabs(err) >= 1.0) { - _PyTime_overflow(); - return -1; - } - return 0; + return _PyTime_FromFloatObject(t, d, round, to_nanoseconds); } else { #ifdef HAVE_LONG_LONG -- cgit v0.12 From bbdda21a7a54c30211b33ad736d7bbbf19ea08df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 00:50:43 +0200 Subject: Move assertion inside _PyTime_ObjectToTimeval() Change also _PyTime_FromSeconds() assertion to ensure that the _PyTime_t type is used. --- Modules/_datetimemodule.c | 1 - Python/pytime.c | 20 ++++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 7e4be5b..5cff3f8 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4100,7 +4100,6 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, if (_PyTime_ObjectToTimeval(timestamp, &timet, &us, _PyTime_ROUND_FLOOR) == -1) return NULL; - assert(0 <= us && us <= 999999); return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); } diff --git a/Python/pytime.c b/Python/pytime.c index 17ef197..fcac68a 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -101,7 +101,8 @@ static int _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) { - assert(denominator <= LONG_MAX); + assert(denominator <= (double)LONG_MAX); + if (PyFloat_Check(obj)) { double d = PyFloat_AsDouble(obj); return _PyTime_DoubleToDenominator(d, sec, numerator, @@ -149,14 +150,20 @@ int _PyTime_ObjectToTimespec(PyObject *obj, time_t *sec, long *nsec, _PyTime_round_t round) { - return _PyTime_ObjectToDenominator(obj, sec, nsec, 1e9, round); + int res; + res = _PyTime_ObjectToDenominator(obj, sec, nsec, 1e9, round); + assert(0 <= *nsec && *nsec < SEC_TO_NS ); + return res; } int _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, _PyTime_round_t round) { - return _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); + int res; + res = _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); + assert(0 <= *usec && *usec < SEC_TO_US ); + return res; } static void @@ -170,12 +177,13 @@ _PyTime_t _PyTime_FromSeconds(int seconds) { _PyTime_t t; + t = (_PyTime_t)seconds; /* ensure that integer overflow cannot happen, int type should have 32 bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_MS takes 30 bits). */ - assert((seconds >= 0 && seconds <= _PyTime_MAX / SEC_TO_NS) - || (seconds < 0 && seconds >= _PyTime_MIN / SEC_TO_NS)); - t = (_PyTime_t)seconds * SEC_TO_NS; + assert((t >= 0 && t <= _PyTime_MAX / SEC_TO_NS) + || (t < 0 && t >= _PyTime_MIN / SEC_TO_NS)); + t *= SEC_TO_NS; return t; } -- cgit v0.12 From 744742320f259776dab1c8ef86e22225ef569d17 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 01:43:56 +0200 Subject: Issue #23517: Add "half up" rounding mode to the _PyTime API --- Include/pytime.h | 5 +++- Lib/test/test_time.py | 64 ++++++++++++++++++++++++++++++++++++++++++++--- Modules/_testcapimodule.c | 4 ++- Python/pytime.c | 64 +++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 122 insertions(+), 15 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 027c3d8..98ae12b 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -30,7 +30,10 @@ typedef enum { _PyTime_ROUND_FLOOR=0, /* Round towards infinity (+inf). For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_CEILING + _PyTime_ROUND_CEILING=1, + /* Round to nearest with ties going away from zero. + For example, used to round from a Python float. */ + _PyTime_ROUND_HALF_UP } _PyTime_round_t; /* Convert a time_t to a PyLong. */ diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 6334e02..ed20470 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -30,8 +30,11 @@ class _PyTime(enum.IntEnum): ROUND_FLOOR = 0 # Round towards infinity (+inf) ROUND_CEILING = 1 + # Round to nearest with ties going away from zero + ROUND_HALF_UP = 2 -ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING) +ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, + _PyTime.ROUND_HALF_UP) class TimeTestCase(unittest.TestCase): @@ -753,11 +756,11 @@ class TestPyTime_t(unittest.TestCase): (123.0, 123 * SEC_TO_NS), (-7.0, -7 * SEC_TO_NS), - # nanosecond are kept for value <= 2^23 seconds + # nanosecond are kept for value <= 2^23 seconds, + # except 2**23-1e-9 with HALF_UP (2**22 - 1e-9, 4194303999999999), (2**22, 4194304000000000), (2**22 + 1e-9, 4194304000000001), - (2**23 - 1e-9, 8388607999999999), (2**23, 8388608000000000), # start loosing precision for value > 2^23 seconds @@ -790,24 +793,36 @@ class TestPyTime_t(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, ts, rnd in ( # close to zero ( 1e-10, 0, FLOOR), ( 1e-10, 1, CEILING), + ( 1e-10, 0, HALF_UP), (-1e-10, -1, FLOOR), (-1e-10, 0, CEILING), + (-1e-10, 0, HALF_UP), # test rounding of the last nanosecond ( 1.1234567899, 1123456789, FLOOR), ( 1.1234567899, 1123456790, CEILING), + ( 1.1234567899, 1123456790, HALF_UP), (-1.1234567899, -1123456790, FLOOR), (-1.1234567899, -1123456789, CEILING), + (-1.1234567899, -1123456790, HALF_UP), # close to 1 second ( 0.9999999999, 999999999, FLOOR), ( 0.9999999999, 1000000000, CEILING), + ( 0.9999999999, 1000000000, HALF_UP), (-0.9999999999, -1000000000, FLOOR), (-0.9999999999, -999999999, CEILING), + (-0.9999999999, -1000000000, HALF_UP), + + # close to 2^23 seconds + (2**23 - 1e-9, 8388607999999999, FLOOR), + (2**23 - 1e-9, 8388607999999999, CEILING), + (2**23 - 1e-9, 8388608000000000, HALF_UP), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -875,18 +890,33 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, tv, rnd in ( # nanoseconds (1, (0, 0), FLOOR), (1, (0, 1), CEILING), + (1, (0, 0), HALF_UP), (-1, (-1, 999999), FLOOR), (-1, (0, 0), CEILING), + (-1, (0, 0), HALF_UP), # seconds + nanoseconds (1234567001, (1, 234567), FLOOR), (1234567001, (1, 234568), CEILING), + (1234567001, (1, 234567), HALF_UP), (-1234567001, (-2, 765432), FLOOR), (-1234567001, (-2, 765433), CEILING), + (-1234567001, (-2, 765433), HALF_UP), + + # half up + (499, (0, 0), HALF_UP), + (500, (0, 1), HALF_UP), + (501, (0, 1), HALF_UP), + (999, (0, 1), HALF_UP), + (-499, (0, 0), HALF_UP), + (-500, (0, 0), HALF_UP), + (-501, (-1, 999999), HALF_UP), + (-999, (-1, 999999), HALF_UP), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) @@ -929,18 +959,33 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), + (1, 0, HALF_UP), (-1, 0, FLOOR), (-1, -1, CEILING), + (-1, 0, HALF_UP), # seconds + nanoseconds (1234 * MS_TO_NS + 1, 1234, FLOOR), (1234 * MS_TO_NS + 1, 1235, CEILING), + (1234 * MS_TO_NS + 1, 1234, HALF_UP), (-1234 * MS_TO_NS - 1, -1234, FLOOR), (-1234 * MS_TO_NS - 1, -1235, CEILING), + (-1234 * MS_TO_NS - 1, -1234, HALF_UP), + + # half up + (499999, 0, HALF_UP), + (499999, 0, HALF_UP), + (500000, 1, HALF_UP), + (999999, 1, HALF_UP), + (-499999, 0, HALF_UP), + (-500000, -1, HALF_UP), + (-500001, -1, HALF_UP), + (-999999, -1, HALF_UP), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) @@ -966,18 +1011,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), + (1, 0, HALF_UP), (-1, 0, FLOOR), (-1, -1, CEILING), + (-1, 0, HALF_UP), # seconds + nanoseconds (1234 * US_TO_NS + 1, 1234, FLOOR), (1234 * US_TO_NS + 1, 1235, CEILING), + (1234 * US_TO_NS + 1, 1234, HALF_UP), (-1234 * US_TO_NS - 1, -1234, FLOOR), (-1234 * US_TO_NS - 1, -1235, CEILING), + (-1234 * US_TO_NS - 1, -1234, HALF_UP), + + # half up + (1499, 1, HALF_UP), + (1500, 2, HALF_UP), + (1501, 2, HALF_UP), + (-1499, -1, HALF_UP), + (-1500, -2, HALF_UP), + (-1501, -2, HALF_UP), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ba0a24b..c38d9ae 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2646,7 +2646,9 @@ run_in_subinterp(PyObject *self, PyObject *args) static int check_time_rounding(int round) { - if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { + if (round != _PyTime_ROUND_FLOOR + && round != _PyTime_ROUND_CEILING + && round != _PyTime_ROUND_HALF_UP) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index fcac68a..3294e0f 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,6 +60,17 @@ _PyLong_FromTime_t(time_t t) #endif } +static double +_PyTime_RoundHalfUp(double x) +{ + if (x >= 0.0) + x = floor(x + 0.5); + else + x = ceil(x - 0.5); + return x; +} + + static int _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) @@ -75,7 +86,9 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, } floatpart *= denominator; - if (round == _PyTime_ROUND_CEILING) { + if (round == _PyTime_ROUND_HALF_UP) + floatpart = _PyTime_RoundHalfUp(floatpart); + else if (round == _PyTime_ROUND_CEILING) { floatpart = ceil(floatpart); if (floatpart >= denominator) { floatpart = 0.0; @@ -124,7 +137,9 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) double d, intpart, err; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_CEILING) + if (round == _PyTime_ROUND_HALF_UP) + d = _PyTime_RoundHalfUp(d); + else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -247,7 +262,9 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, d = value; d *= to_nanoseconds; - if (round == _PyTime_ROUND_CEILING) + if (round == _PyTime_ROUND_HALF_UP) + d = _PyTime_RoundHalfUp(d); + else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -333,7 +350,19 @@ static _PyTime_t _PyTime_Divide(_PyTime_t t, _PyTime_t k, _PyTime_round_t round) { assert(k > 1); - if (round == _PyTime_ROUND_CEILING) { + if (round == _PyTime_ROUND_HALF_UP) { + _PyTime_t x, r; + x = t / k; + r = t % k; + if (Py_ABS(r) >= k / 2) { + if (t >= 0) + x++; + else + x--; + } + return x; + } + else if (round == _PyTime_ROUND_CEILING) { if (t >= 0) return (t + k - 1) / k; else @@ -359,8 +388,10 @@ static int _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { + const long k = US_TO_NS; _PyTime_t secs, ns; int res = 0; + int usec; secs = t / SEC_TO_NS; ns = t % SEC_TO_NS; @@ -392,20 +423,33 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, res = -1; #endif - if (round == _PyTime_ROUND_CEILING) - tv->tv_usec = (int)((ns + US_TO_NS - 1) / US_TO_NS); + if (round == _PyTime_ROUND_HALF_UP) { + _PyTime_t r; + usec = (int)(ns / k); + r = ns % k; + if (Py_ABS(r) >= k / 2) { + if (ns >= 0) + usec++; + else + usec--; + } + } + else if (round == _PyTime_ROUND_CEILING) + usec = (int)((ns + k - 1) / k); else - tv->tv_usec = (int)(ns / US_TO_NS); + usec = (int)(ns / k); - if (tv->tv_usec >= SEC_TO_US) { - tv->tv_usec -= SEC_TO_US; + if (usec >= SEC_TO_US) { + usec -= SEC_TO_US; tv->tv_sec += 1; } if (res && raise) _PyTime_overflow(); - assert(0 <= tv->tv_usec && tv->tv_usec <= 999999); + assert(0 <= usec && usec <= 999999); + + tv->tv_usec = usec; return res; } -- cgit v0.12 From 265e1259e4ad7001ea366d52c56adde8daed7a69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 01:57:23 +0200 Subject: Issue #23517: datetime.datetime.fromtimestamp() and datetime.datetime.utcfromtimestamp() now rounds to nearest with ties going away from zero, instead of rounding towards minus infinity (-inf), as Python 2 and Python older than 3.3. --- Misc/NEWS | 5 +++++ Modules/_datetimemodule.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 38f31e4..8226f20 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,11 @@ Core and Builtins Library ------- +- Issue #23517: datetime.datetime.fromtimestamp() and + datetime.datetime.utcfromtimestamp() now rounds to nearest with ties going + away from zero, instead of rounding towards minus infinity (-inf), as Python + 2 and Python older than 3.3. + - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 5cff3f8..e1cd2b5 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4098,7 +4098,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_FLOOR) == -1) + &timet, &us, _PyTime_ROUND_HALF_UP) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); -- cgit v0.12 From a53ec7a91abb7cdbb1a13862f39d3183cab7f2a4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 10:10:26 +0200 Subject: Backed out changeset b690bf218702 Issue #23517: the change broke test_datetime. datetime.timedelta() rounding mode must also be changed, and test_datetime must be updated for the new rounding mode (half up). --- Misc/NEWS | 5 ----- Modules/_datetimemodule.c | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 8226f20..38f31e4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,11 +17,6 @@ Core and Builtins Library ------- -- Issue #23517: datetime.datetime.fromtimestamp() and - datetime.datetime.utcfromtimestamp() now rounds to nearest with ties going - away from zero, instead of rounding towards minus infinity (-inf), as Python - 2 and Python older than 3.3. - - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index e1cd2b5..5cff3f8 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4098,7 +4098,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_HALF_UP) == -1) + &timet, &us, _PyTime_ROUND_FLOOR) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); -- cgit v0.12 From 67edcc905d3574d87988537e1a5290a1c7717da0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 10:37:46 +0200 Subject: Issue #23517: Fix _PyTime_ObjectToDenominator() * initialize numerator on overflow error ensure that numerator is smaller than * denominator. --- Python/pytime.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 3294e0f..8a2579b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,6 +60,7 @@ _PyLong_FromTime_t(time_t t) #endif } +/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ static double _PyTime_RoundHalfUp(double x) { @@ -81,32 +82,31 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart = modf(d, &intpart); if (floatpart < 0) { - floatpart = 1.0 + floatpart; + floatpart += 1.0; intpart -= 1.0; } floatpart *= denominator; if (round == _PyTime_ROUND_HALF_UP) floatpart = _PyTime_RoundHalfUp(floatpart); - else if (round == _PyTime_ROUND_CEILING) { + else if (round == _PyTime_ROUND_CEILING) floatpart = ceil(floatpart); - if (floatpart >= denominator) { - floatpart = 0.0; - intpart += 1.0; - } - } - else { + else floatpart = floor(floatpart); + if (floatpart >= denominator) { + floatpart -= denominator; + intpart += 1.0; } + assert(0.0 <= floatpart && floatpart < denominator); *sec = (time_t)intpart; + *numerator = (long)floatpart; + err = intpart - (double)*sec; if (err <= -1.0 || err >= 1.0) { error_time_t_overflow(); return -1; } - - *numerator = (long)floatpart; return 0; } @@ -123,9 +123,9 @@ _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, } else { *sec = _PyLong_AsTime_t(obj); + *numerator = 0; if (*sec == (time_t)-1 && PyErr_Occurred()) return -1; - *numerator = 0; return 0; } } @@ -167,7 +167,7 @@ _PyTime_ObjectToTimespec(PyObject *obj, time_t *sec, long *nsec, { int res; res = _PyTime_ObjectToDenominator(obj, sec, nsec, 1e9, round); - assert(0 <= *nsec && *nsec < SEC_TO_NS ); + assert(0 <= *nsec && *nsec < SEC_TO_NS); return res; } @@ -177,7 +177,7 @@ _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, { int res; res = _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); - assert(0 <= *usec && *usec < SEC_TO_US ); + assert(0 <= *usec && *usec < SEC_TO_US); return res; } @@ -444,12 +444,11 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, tv->tv_sec += 1; } + assert(0 <= usec && usec < SEC_TO_US); + tv->tv_usec = usec; + if (res && raise) _PyTime_overflow(); - - assert(0 <= usec && usec <= 999999); - - tv->tv_usec = usec; return res; } @@ -484,7 +483,7 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) } ts->tv_nsec = nsec; - assert(0 <= ts->tv_nsec && ts->tv_nsec <= 999999999); + assert(0 <= ts->tv_nsec && ts->tv_nsec < SEC_TO_NS); return 0; } #endif -- cgit v0.12 From acea9f6208c74186df62f8c4cfd269919e33779f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 10:39:40 +0200 Subject: Issue #23517: Reintroduce unit tests for the old PyTime API since it's still used. --- Lib/test/test_time.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index ed20470..926e700 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -727,6 +727,9 @@ class TestPytime(unittest.TestCase): @unittest.skipUnless(_testcapi is not None, 'need the _testcapi module') class TestPyTime_t(unittest.TestCase): + """ + Test the _PyTime_t API. + """ def test_FromSeconds(self): from _testcapi import PyTime_FromSeconds for seconds in (0, 3, -456, _testcapi.INT_MAX, _testcapi.INT_MIN): @@ -1041,5 +1044,156 @@ class TestPyTime_t(unittest.TestCase): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) +@unittest.skipUnless(_testcapi is not None, + 'need the _testcapi module') +class TestOldPyTime(unittest.TestCase): + """ + Test the old _PyTime_t API: _PyTime_ObjectToXXX() functions. + """ + def setUp(self): + self.invalid_values = ( + -(2 ** 100), 2 ** 100, + -(2.0 ** 100.0), 2.0 ** 100.0, + ) + + @support.cpython_only + def test_time_t(self): + from _testcapi import pytime_object_to_time_t + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, time_t in ( + # int + (0, 0), + (-1, -1), + + # float + (1.0, 1), + (-1.0, -1), + ): + self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, time_t, rnd in ( + (-1.9, -2, FLOOR), + (-1.9, -2, HALF_UP), + (-1.9, -1, CEILING), + (1.9, 1, FLOOR), + (1.9, 2, HALF_UP), + (1.9, 2, CEILING), + ): + self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + + # Test OverflowError + rnd = _PyTime.ROUND_FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_time_t, invalid, rnd) + + def test_timeval(self): + from _testcapi import pytime_object_to_timeval + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timeval in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.0, (-1, 0)), + (-1.2, (-2, 800000)), + (-1e-6, (-1, 999999)), + (1e-6, (0, 1)), + ): + with self.subTest(obj=obj, round=rnd, timeval=timeval): + self.assertEqual(pytime_object_to_timeval(obj, rnd), + timeval) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, timeval, rnd in ( + (-1e-7, (-1, 999999), FLOOR), + (-1e-7, (0, 0), CEILING), + (-1e-7, (0, 0), HALF_UP), + + (1e-7, (0, 0), FLOOR), + (1e-7, (0, 1), CEILING), + (1e-7, (0, 0), HALF_UP), + + (0.4e-6, (0, 0), HALF_UP), + (0.5e-6, (0, 1), HALF_UP), + (0.6e-6, (0, 1), HALF_UP), + + (0.9999999, (0, 999999), FLOOR), + (0.9999999, (1, 0), CEILING), + (0.9999999, (1, 0), HALF_UP), + ): + with self.subTest(obj=obj, round=rnd, timeval=timeval): + self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) + + rnd = _PyTime.ROUND_FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_timeval, invalid, rnd) + + @support.cpython_only + def test_timespec(self): + from _testcapi import pytime_object_to_timespec + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timespec in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.0, (-1, 0)), + (-1e-9, (-1, 999999999)), + (1e-9, (0, 1)), + (-1.2, (-2, 800000000)), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), + timespec) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, timespec, rnd in ( + (-1e-10, (-1, 999999999), FLOOR), + (-1e-10, (0, 0), CEILING), + (-1e-10, (0, 0), HALF_UP), + + (1e-10, (0, 0), FLOOR), + (1e-10, (0, 1), CEILING), + (1e-10, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), + + (0.9999999999, (0, 999999999), FLOOR), + (0.9999999999, (1, 0), CEILING), + (0.9999999999, (1, 0), HALF_UP), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + + # Test OverflowError + rnd = FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_timespec, invalid, rnd) + + if __name__ == "__main__": unittest.main() -- cgit v0.12 From ead144c19b2226f82f5c0d9ae3cb26b59dcbf0c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 11:05:32 +0200 Subject: test_time: add more tests on HALF_UP rounding mode --- Lib/test/test_time.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 926e700..db96277 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1082,9 +1082,18 @@ class TestOldPyTime(unittest.TestCase): (-1.9, -2, FLOOR), (-1.9, -2, HALF_UP), (-1.9, -1, CEILING), + (1.9, 1, FLOOR), (1.9, 2, HALF_UP), (1.9, 2, CEILING), + + (-0.6, -1, HALF_UP), + (-0.5, -1, HALF_UP), + (-0.4, 0, HALF_UP), + + (0.4, 0, HALF_UP), + (0.5, 1, HALF_UP), + (0.6, 1, HALF_UP), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) @@ -1127,13 +1136,19 @@ class TestOldPyTime(unittest.TestCase): (1e-7, (0, 1), CEILING), (1e-7, (0, 0), HALF_UP), - (0.4e-6, (0, 0), HALF_UP), - (0.5e-6, (0, 1), HALF_UP), - (0.6e-6, (0, 1), HALF_UP), - (0.9999999, (0, 999999), FLOOR), (0.9999999, (1, 0), CEILING), (0.9999999, (1, 0), HALF_UP), + + (-0.6e-6, (-1, 999999), HALF_UP), + # skipped, -0.5e-6 is inexact in base 2 + #(-0.5e-6, (-1, 999999), HALF_UP), + (-0.4e-6, (0, 0), HALF_UP), + + (0.4e-6, (0, 0), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(0.5e-6, (0, 1), HALF_UP), + (0.6e-6, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timeval=timeval): self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) @@ -1177,13 +1192,18 @@ class TestOldPyTime(unittest.TestCase): (1e-10, (0, 1), CEILING), (1e-10, (0, 0), HALF_UP), - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), - (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), (0.9999999999, (1, 0), HALF_UP), + + (-0.6e-9, (-1, 999999999), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(-0.5e-9, (-1, 999999999), HALF_UP), + (-0.4e-9, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) -- cgit v0.12 From 24b822e21e19984eadd20e062a11ce7182cb81da Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 11:58:56 +0200 Subject: Issue #23517: Try to fix test_time on "x86 Ubuntu Shared 3.x" buildbot --- Python/pytime.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 8a2579b..ffb390a 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -64,11 +64,13 @@ _PyLong_FromTime_t(time_t t) static double _PyTime_RoundHalfUp(double x) { - if (x >= 0.0) - x = floor(x + 0.5); + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d = x; + if (d >= 0.0) + d = floor(d + 0.5); else - x = ceil(x - 0.5); - return x; + d = ceil(d - 0.5); + return d; } @@ -77,7 +79,7 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) { double intpart, err; - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + /* volatile avoids optimization changing how numbers are rounded */ volatile double floatpart; floatpart = modf(d, &intpart); @@ -134,7 +136,8 @@ int _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) { if (PyFloat_Check(obj)) { - double d, intpart, err; + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d, intpart, err; d = PyFloat_AsDouble(obj); if (round == _PyTime_ROUND_HALF_UP) @@ -255,7 +258,7 @@ static int _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, long to_nanoseconds) { - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + /* volatile avoids optimization changing how numbers are rounded */ volatile double d, err; /* convert to a number of nanoseconds */ -- cgit v0.12 From 8aad8d6ad32de1f8fb87ff55ecde8ea3de8a72c4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 13:54:28 +0200 Subject: Issue #23517: test_time, skip a test checking a corner case on floating point rounding --- Lib/test/test_time.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index db96277..5947f40 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -825,7 +825,9 @@ class TestPyTime_t(unittest.TestCase): # close to 2^23 seconds (2**23 - 1e-9, 8388607999999999, FLOOR), (2**23 - 1e-9, 8388607999999999, CEILING), - (2**23 - 1e-9, 8388608000000000, HALF_UP), + # Issue #23517: skip HALF_UP test because the result is different + # depending on the FPU and how the compiler optimize the code :-/ + #(2**23 - 1e-9, 8388608000000000, HALF_UP), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) -- cgit v0.12 From f08fea9ee8a4ff12445b994dc7956b58e85762f4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 14:23:40 +0200 Subject: Issue 24297: Fix test_symbol on Windows Don't rely on end of line. Open files in text mode, not in binary mode. --- Lib/test/test_symbol.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py index c126be3..2dcb9de 100644 --- a/Lib/test/test_symbol.py +++ b/Lib/test/test_symbol.py @@ -15,12 +15,11 @@ TEST_PY_FILE = 'symbol_test.py' class TestSymbolGeneration(unittest.TestCase): def _copy_file_without_generated_symbols(self, source_file, dest_file): - with open(source_file, 'rb') as fp: + with open(source_file) as fp: lines = fp.readlines() - nl = lines[0][len(lines[0].rstrip()):] - with open(dest_file, 'wb') as fp: - fp.writelines(lines[:lines.index(b"#--start constants--" + nl) + 1]) - fp.writelines(lines[lines.index(b"#--end constants--" + nl):]) + with open(dest_file, 'w') as fp: + fp.writelines(lines[:lines.index("#--start constants--\n") + 1]) + fp.writelines(lines[lines.index("#--end constants--\n"):]) def _generate_symbols(self, grammar_file, target_symbol_py_file): proc = subprocess.Popen([sys.executable, @@ -30,18 +29,26 @@ class TestSymbolGeneration(unittest.TestCase): stderr = proc.communicate()[1] return proc.returncode, stderr + def compare_files(self, file1, file2): + with open(file1) as fp: + lines1 = fp.readlines() + with open(file2) as fp: + lines2 = fp.readlines() + self.assertEqual(lines1, lines2) + @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), 'test only works from source build directory') def test_real_grammar_and_symbol_file(self): - self._copy_file_without_generated_symbols(SYMBOL_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - self.assertFalse(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) - self.assertEqual((0, b''), self._generate_symbols(GRAMMAR_FILE, - TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE), - 'symbol stat: %r\ntest_py stat: %r\n' % - (os.stat(SYMBOL_FILE), - os.stat(TEST_PY_FILE))) + output = support.TESTFN + self.addCleanup(support.unlink, output) + + self._copy_file_without_generated_symbols(SYMBOL_FILE, output) + + exitcode, stderr = self._generate_symbols(GRAMMAR_FILE, output) + self.assertEqual(b'', stderr) + self.assertEqual(0, exitcode) + + self.compare_files(SYMBOL_FILE, output) if __name__ == "__main__": -- cgit v0.12 From 6dad8f89623e588f486d069b7a336c1430c3c147 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 15:44:22 +0200 Subject: test_gdb: add debug info to investigate failure on "s390x SLES 3.x" buildbot --- Lib/test/test_gdb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 0322677..193c97a 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -28,9 +28,13 @@ except OSError: # This is what "no gdb" looks like. There may, however, be other # errors that manifest this way too. raise unittest.SkipTest("Couldn't find gdb on the path") -gdb_version_number = re.search(b"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) -gdb_major_version = int(gdb_version_number.group(1)) -gdb_minor_version = int(gdb_version_number.group(2)) +try: + gdb_version_number = re.search(b"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) + gdb_major_version = int(gdb_version_number.group(1)) + gdb_minor_version = int(gdb_version_number.group(2)) +except Exception: + raise ValueError("unable to parse GDB version: %r" % gdb_version) + if gdb_major_version < 7: raise unittest.SkipTest("gdb versions before 7.0 didn't support python embedding" " Saw:\n" + gdb_version.decode('ascii', 'replace')) -- cgit v0.12 From e801c36037bf3875de0ecbb79c9f9ba8938027da Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 15:46:00 +0200 Subject: test_gdb: fix ResourceWarning if the test is interrupted --- Lib/test/test_gdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 193c97a..a51b552 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -63,9 +63,11 @@ def run_gdb(*args, **env_vars): base_cmd = ('gdb', '--batch', '-nx') if (gdb_major_version, gdb_minor_version) >= (7, 4): base_cmd += ('-iex', 'add-auto-load-safe-path ' + checkout_hook_path) - out, err = subprocess.Popen(base_cmd + args, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, - ).communicate() + proc = subprocess.Popen(base_cmd + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + with proc: + out, err = proc.communicate() return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace') # Verify that "gdb" was built with the embedded python support enabled: -- cgit v0.12 From 177b8eb34fed9a299ad00cea8e8be45fd836ab91 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 17:19:04 +0200 Subject: test_eintr: try to debug hang on FreeBSD --- Lib/test/eintrdata/eintr_tester.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index f755880..73711a4 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import faulthandler import io import os import select @@ -36,10 +37,17 @@ class EINTRBaseTest(unittest.TestCase): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) + if hasattr(faulthandler, 'dump_traceback_later'): + # Most tests take less than 30 seconds, so 15 minutes should be + # enough. dump_traceback_later() is implemented with a thread, but + # pthread_sigmask() is used to mask all signaled on this thread. + faulthandler.dump_traceback_later(15 * 60, exit=True) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): -- cgit v0.12 From 2ec558739e6bd32365e1a883889f9d5372b35719 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 19:16:07 +0200 Subject: Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to nearest even integer (ROUND_HALF_EVEN). --- Include/pytime.h | 4 ++++ Lib/datetime.py | 12 ++++++++++-- Lib/test/datetimetester.py | 14 +++++--------- Misc/NEWS | 5 +++++ Modules/_datetimemodule.c | 22 +--------------------- Python/pytime.c | 3 +-- 6 files changed, 26 insertions(+), 34 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 98ae12b..41fb806 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -44,6 +44,10 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); +/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ +PyAPI_FUNC(double) _PyTime_RoundHalfUp( + double x); + /* Convert a number of seconds, int or float, to time_t. */ PyAPI_FUNC(int) _PyTime_ObjectToTime_t( PyObject *obj, diff --git a/Lib/datetime.py b/Lib/datetime.py index db13b12..d661460 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -316,6 +316,14 @@ def _divide_and_round(a, b): return q +def _round_half_up(x): + """Round to nearest with ties going away from zero.""" + if x >= 0.0: + return _math.floor(x + 0.5) + else: + return _math.ceil(x - 0.5) + + class timedelta: """Represent the difference between two datetime objects. @@ -399,7 +407,7 @@ class timedelta: # secondsfrac isn't referenced again if isinstance(microseconds, float): - microseconds = round(microseconds + usdouble) + microseconds = _round_half_up(microseconds + usdouble) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 24*3600) d += days @@ -410,7 +418,7 @@ class timedelta: days, seconds = divmod(seconds, 24*3600) d += days s += seconds - microseconds = round(microseconds + usdouble) + microseconds = _round_half_up(microseconds + usdouble) assert isinstance(s, int) assert isinstance(microseconds, int) assert abs(s) <= 3 * 24 * 3600 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index babeb44..62f5527 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -662,28 +662,24 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): # Single-field rounding. eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 - eq(td(milliseconds=0.5/1000), td(microseconds=0)) - eq(td(milliseconds=-0.5/1000), td(microseconds=0)) + eq(td(milliseconds=0.5/1000), td(microseconds=1)) + eq(td(milliseconds=-0.5/1000), td(microseconds=-1)) eq(td(milliseconds=0.6/1000), td(microseconds=1)) eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) - eq(td(seconds=0.5/10**6), td(microseconds=0)) - eq(td(seconds=-0.5/10**6), td(microseconds=0)) + eq(td(seconds=0.5/10**6), td(microseconds=1)) + eq(td(seconds=-0.5/10**6), td(microseconds=-1)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 us_per_day = us_per_hour * 24 eq(td(days=.4/us_per_day), td(0)) eq(td(hours=.2/us_per_hour), td(0)) - eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) + eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1), td) eq(td(days=-.4/us_per_day), td(0)) eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) - # Test for a patch in Issue 8860 - eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) - eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) - def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), diff --git a/Misc/NEWS b/Misc/NEWS index e08905c..c9b925a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,11 @@ Core and Builtins Library ------- +- Issue #23517: datetime.timedelta constructor now rounds microseconds to + nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and + Python older than 3.3, instead of rounding to nearest with ties going to + nearest even integer (ROUND_HALF_EVEN). + - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 5cff3f8..6cab1e2 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -2149,29 +2149,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (leftover_us) { /* Round to nearest whole # of us, and add into x. */ double whole_us = round(leftover_us); - int x_is_odd; PyObject *temp; - whole_us = round(leftover_us); - if (fabs(whole_us - leftover_us) == 0.5) { - /* We're exactly halfway between two integers. In order - * to do round-half-to-even, we must determine whether x - * is odd. Note that x is odd when it's last bit is 1. The - * code below uses bitwise and operation to check the last - * bit. */ - temp = PyNumber_And(x, one); /* temp <- x & 1 */ - if (temp == NULL) { - Py_DECREF(x); - goto Done; - } - x_is_odd = PyObject_IsTrue(temp); - Py_DECREF(temp); - if (x_is_odd == -1) { - Py_DECREF(x); - goto Done; - } - whole_us = 2.0 * round((leftover_us + x_is_odd) * 0.5) - x_is_odd; - } + whole_us = _PyTime_RoundHalfUp(leftover_us); temp = PyLong_FromLong((long)whole_us); diff --git a/Python/pytime.c b/Python/pytime.c index ffb390a..02a1edf 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,8 +60,7 @@ _PyLong_FromTime_t(time_t t) #endif } -/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ -static double +double _PyTime_RoundHalfUp(double x) { /* volatile avoids optimization changing how numbers are rounded */ -- cgit v0.12 From be923ac948635e1a93d55398648f8d03996d3ebd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 01:38:44 +0200 Subject: Rewrite eintr_tester.py to avoid os.fork() eintr_tester.py calls signal.setitimer() to send signals to the current process every 100 ms. The test sometimes hangs on FreeBSD. Maybe there is a race condition in the child process after fork(). It's unsafe to run arbitrary code after fork(). This change replace os.fork() with a regular call to subprocess.Popen(). This change avoids the risk of having a child process which continue to execute eintr_tester.py instead of exiting. It also ensures that the child process doesn't inherit unexpected file descriptors by mistake. Since I'm unable to reproduce the issue on FreeBSD, I will have to watch FreeBSD buildbots to check if the issue is fixed or not. Remove previous attempt to debug: remove call to faulthandler.dump_traceback_later(). --- Lib/test/eintrdata/eintr_tester.py | 259 ++++++++++++++++++++++--------------- 1 file changed, 158 insertions(+), 101 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 73711a4..0616df6 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,12 +8,13 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ -import faulthandler import io import os import select import signal import socket +import subprocess +import sys import time import unittest @@ -29,7 +30,7 @@ class EINTRBaseTest(unittest.TestCase): # signal delivery periodicity signal_period = 0.1 # default sleep time for tests - should obviously have: - # sleep_time > signal_period + # sleep_time > signal_period sleep_time = 0.2 @classmethod @@ -37,17 +38,10 @@ class EINTRBaseTest(unittest.TestCase): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) - if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 15 minutes should be - # enough. dump_traceback_later() is implemented with a thread, but - # pthread_sigmask() is used to mask all signaled on this thread. - faulthandler.dump_traceback_later(15 * 60, exit=True) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) - if hasattr(faulthandler, 'cancel_dump_traceback_later'): - faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): @@ -59,18 +53,22 @@ class EINTRBaseTest(unittest.TestCase): # default sleep time time.sleep(cls.sleep_time) + def subprocess(self, *args, **kw): + cmd_args = (sys.executable, '-c') + args + return subprocess.Popen(cmd_args, **kw) + @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") class OSEINTRTest(EINTRBaseTest): """ EINTR tests for the os module. """ + def new_sleep_process(self): + code = 'import time; time.sleep(%r)' % self.sleep_time + return self.subprocess(code) + def _test_wait_multiple(self, wait_func): num = 3 - for _ in range(num): - pid = os.fork() - if pid == 0: - self._sleep() - os._exit(0) + processes = [self.new_sleep_process() for _ in range(num)] for _ in range(num): wait_func() @@ -82,12 +80,8 @@ class OSEINTRTest(EINTRBaseTest): self._test_wait_multiple(lambda: os.wait3(0)) def _test_wait_single(self, wait_func): - pid = os.fork() - if pid == 0: - self._sleep() - os._exit(0) - else: - wait_func(pid) + proc = self.new_sleep_process() + wait_func(proc.pid) def test_waitpid(self): self._test_wait_single(lambda pid: os.waitpid(pid, 0)) @@ -105,19 +99,24 @@ class OSEINTRTest(EINTRBaseTest): # atomic datas = [b"hello", b"world", b"spam"] - pid = os.fork() - if pid == 0: - os.close(rd) - for data in datas: - # let the parent block on read() - self._sleep() - os.write(wr, data) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import os, sys, time', + '', + 'wr = int(sys.argv[1])', + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'for data in datas:', + ' # let the parent block on read()', + ' time.sleep(sleep_time)', + ' os.write(wr, data)', + )) + + with self.subprocess(code, str(wr), pass_fds=[wr]) as proc: os.close(wr) for data in datas: self.assertEqual(data, os.read(rd, len(data))) + self.assertEqual(proc.wait(), 0) def test_write(self): rd, wr = os.pipe() @@ -127,23 +126,34 @@ class OSEINTRTest(EINTRBaseTest): # we must write enough data for the write() to block data = b"xyz" * support.PIPE_MAX_SIZE - pid = os.fork() - if pid == 0: - os.close(wr) - read_data = io.BytesIO() - # let the parent block on write() - self._sleep() - while len(read_data.getvalue()) < len(data): - chunk = os.read(rd, 2 * len(data)) - read_data.write(chunk) - self.assertEqual(read_data.getvalue(), data) - os._exit(0) - else: + code = '\n'.join(( + 'import io, os, sys, time', + '', + 'rd = int(sys.argv[1])', + 'sleep_time = %r' % self.sleep_time, + 'data = b"xyz" * %s' % support.PIPE_MAX_SIZE, + 'data_len = len(data)', + '', + '# let the parent block on write()', + 'time.sleep(sleep_time)', + '', + 'read_data = io.BytesIO()', + 'while len(read_data.getvalue()) < data_len:', + ' chunk = os.read(rd, 2 * data_len)', + ' read_data.write(chunk)', + '', + 'value = read_data.getvalue()', + 'if value != data:', + ' raise Exception("read error: %s vs %s bytes"', + ' % (len(value), data_len))', + )) + + with self.subprocess(code, str(rd), pass_fds=[rd]) as proc: os.close(rd) written = 0 while written < len(data): written += os.write(wr, memoryview(data)[written:]) - self.assertEqual(0, os.waitpid(pid, 0)[1]) + self.assertEqual(proc.wait(), 0) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") @@ -159,19 +169,32 @@ class SocketEINTRTest(EINTRBaseTest): # single-byte payload guard us against partial recv datas = [b"x", b"y", b"z"] - pid = os.fork() - if pid == 0: - rd.close() - for data in datas: - # let the parent block on recv() - self._sleep() - wr.sendall(data) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(wr.family), + 'sock_type = %s' % int(wr.type), + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'wr = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with wr:', + ' for data in datas:', + ' # let the parent block on recv()', + ' time.sleep(sleep_time)', + ' wr.sendall(data)', + )) + + fd = wr.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with proc: wr.close() for data in datas: self.assertEqual(data, recv_func(rd, len(data))) + self.assertEqual(proc.wait(), 0) def test_recv(self): self._test_recv(socket.socket.recv) @@ -188,25 +211,43 @@ class SocketEINTRTest(EINTRBaseTest): # we must send enough data for the send() to block data = b"xyz" * (support.SOCK_MAX_SIZE // 3) - pid = os.fork() - if pid == 0: - wr.close() - # let the parent block on send() - self._sleep() - received_data = bytearray(len(data)) - n = 0 - while n < len(data): - n += rd.recv_into(memoryview(received_data)[n:]) - self.assertEqual(received_data, data) - os._exit(0) - else: + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(rd.family), + 'sock_type = %s' % int(rd.type), + 'sleep_time = %r' % self.sleep_time, + 'data = b"xyz" * %s' % (support.SOCK_MAX_SIZE // 3), + 'data_len = len(data)', + '', + 'rd = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with rd:', + ' # let the parent block on send()', + ' time.sleep(sleep_time)', + '', + ' received_data = bytearray(data_len)', + ' n = 0', + ' while n < data_len:', + ' n += rd.recv_into(memoryview(received_data)[n:])', + '', + 'if received_data != data:', + ' raise Exception("recv error: %s vs %s bytes"', + ' % (len(received_data), data_len))', + )) + + fd = rd.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with proc: rd.close() written = 0 while written < len(data): sent = send_func(wr, memoryview(data)[written:]) # sendall() returns None written += len(data) if sent is None else sent - self.assertEqual(0, os.waitpid(pid, 0)[1]) + self.assertEqual(proc.wait(), 0) def test_send(self): self._test_send(socket.socket.send) @@ -223,45 +264,60 @@ class SocketEINTRTest(EINTRBaseTest): self.addCleanup(sock.close) sock.bind((support.HOST, 0)) - _, port = sock.getsockname() + port = sock.getsockname()[1] sock.listen() - pid = os.fork() - if pid == 0: - # let parent block on accept() - self._sleep() - with socket.create_connection((support.HOST, port)): - self._sleep() - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import socket, time', + '', + 'host = %r' % support.HOST, + 'port = %s' % port, + 'sleep_time = %r' % self.sleep_time, + '', + '# let parent block on accept()', + 'time.sleep(sleep_time)', + 'with socket.create_connection((host, port)):', + ' time.sleep(sleep_time)', + )) + + with self.subprocess(code) as proc: client_sock, _ = sock.accept() client_sock.close() + self.assertEqual(proc.wait(), 0) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') def _test_open(self, do_open_close_reader, do_open_close_writer): + filename = support.TESTFN + # Use a fifo: until the child opens it for reading, the parent will # block when trying to open it for writing. - support.unlink(support.TESTFN) - os.mkfifo(support.TESTFN) - self.addCleanup(support.unlink, support.TESTFN) - - pid = os.fork() - if pid == 0: - # let the parent block - self._sleep() - do_open_close_reader(support.TESTFN) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) - do_open_close_writer(support.TESTFN) + support.unlink(filename) + os.mkfifo(filename) + self.addCleanup(support.unlink, filename) + + code = '\n'.join(( + 'import os, time', + '', + 'path = %a' % filename, + 'sleep_time = %r' % self.sleep_time, + '', + '# let the parent block', + 'time.sleep(sleep_time)', + '', + do_open_close_reader, + )) + + with self.subprocess(code) as proc: + do_open_close_writer(filename) + + self.assertEqual(proc.wait(), 0) def test_open(self): - self._test_open(lambda path: open(path, 'r').close(), + self._test_open("open(path, 'r').close()", lambda path: open(path, 'w').close()) def test_os_open(self): - self._test_open(lambda path: os.close(os.open(path, os.O_RDONLY)), + self._test_open("os.close(os.open(path, os.O_RDONLY))", lambda path: os.close(os.open(path, os.O_WRONLY))) @@ -298,20 +354,21 @@ class SignalEINTRTest(EINTRBaseTest): old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) + code = '\n'.join(( + 'import os, time', + 'pid = %s' % os.getpid(), + 'signum = %s' % int(signum), + 'sleep_time = %r' % self.sleep_time, + 'time.sleep(sleep_time)', + 'os.kill(pid, signum)', + )) + t0 = time.monotonic() - child_pid = os.fork() - if child_pid == 0: - # child - try: - self._sleep() - os.kill(pid, signum) - finally: - os._exit(0) - else: + with self.subprocess(code) as proc: # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 - os.waitpid(child_pid, 0) + self.assertEqual(proc.wait(), 0) self.assertGreaterEqual(dt, self.sleep_time) -- cgit v0.12 From 2ec5bd6fb22ff71ffbe5987f55a31bc4177e39a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 09:06:44 +0200 Subject: Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding towards -Infinity (ROUND_FLOOR). --- Lib/datetime.py | 4 ++-- Lib/test/datetimetester.py | 11 ++++++----- Misc/NEWS | 5 +++++ Modules/_datetimemodule.c | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index d661460..5ba2ddb 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1384,7 +1384,7 @@ class datetime(date): converter = _time.localtime if tz is None else _time.gmtime t, frac = divmod(t, 1.0) - us = int(frac * 1e6) + us = _round_half_up(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, @@ -1404,7 +1404,7 @@ class datetime(date): def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" t, frac = divmod(t, 1.0) - us = int(frac * 1e6) + us = _round_half_up(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 62f5527..f516434 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1847,6 +1847,7 @@ class TestDateTime(TestDate): zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) + one = fts(1e-6) try: minus_one = fts(-1e-6) except OSError: @@ -1857,22 +1858,22 @@ class TestDateTime(TestDate): self.assertEqual(minus_one.microsecond, 999999) t = fts(-1e-8) - self.assertEqual(t, minus_one) + self.assertEqual(t, zero) t = fts(-9e-7) self.assertEqual(t, minus_one) t = fts(-1e-7) - self.assertEqual(t, minus_one) + self.assertEqual(t, zero) t = fts(1e-7) self.assertEqual(t, zero) t = fts(9e-7) - self.assertEqual(t, zero) + self.assertEqual(t, one) t = fts(0.99999949) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 999999) t = fts(0.9999999) - self.assertEqual(t.second, 0) - self.assertEqual(t.microsecond, 999999) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Misc/NEWS b/Misc/NEWS index c9b925a..9af32ad 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,11 @@ Core and Builtins Library ------- +- Issue #23517: fromtimestamp() and utcfromtimestamp() methods of + datetime.datetime now round microseconds to nearest with ties going away from + zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of + rounding towards -Infinity (ROUND_FLOOR). + - Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 6cab1e2..ae459df 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4078,7 +4078,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_FLOOR) == -1) + &timet, &us, _PyTime_ROUND_HALF_UP) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); -- cgit v0.12 From 99bb14bf0c4b9bdef2535710cc14abbdf5aa648a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 12:16:49 +0200 Subject: type_call() now detect bugs in type new and init * Call _Py_CheckFunctionResult() to check for bugs in type constructors (tp_new) * Add assertions to ensure an exception was raised if tp_init failed or that no exception was raised if tp_init succeed Refactor also the function to have less indentation. --- Objects/typeobject.c | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1beed72..10115ef 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -906,25 +906,33 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) #endif obj = type->tp_new(type, args, kwds); - if (obj != NULL) { - /* Ugly exception: when the call was type(something), - don't call tp_init on the result. */ - if (type == &PyType_Type && - PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1 && - (kwds == NULL || - (PyDict_Check(kwds) && PyDict_Size(kwds) == 0))) - return obj; - /* If the returned object is not an instance of type, - it won't be initialized. */ - if (!PyType_IsSubtype(Py_TYPE(obj), type)) - return obj; - type = Py_TYPE(obj); - if (type->tp_init != NULL) { - int res = type->tp_init(obj, args, kwds); - if (res < 0) { - Py_DECREF(obj); - obj = NULL; - } + obj = _Py_CheckFunctionResult((PyObject*)type, obj, NULL); + if (obj == NULL) + return NULL; + + /* Ugly exception: when the call was type(something), + don't call tp_init on the result. */ + if (type == &PyType_Type && + PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1 && + (kwds == NULL || + (PyDict_Check(kwds) && PyDict_Size(kwds) == 0))) + return obj; + + /* If the returned object is not an instance of type, + it won't be initialized. */ + if (!PyType_IsSubtype(Py_TYPE(obj), type)) + return obj; + + type = Py_TYPE(obj); + if (type->tp_init != NULL) { + int res = type->tp_init(obj, args, kwds); + if (res < 0) { + assert(PyErr_Occurred()); + Py_DECREF(obj); + obj = NULL; + } + else { + assert(!PyErr_Occurred()); } } return obj; -- cgit v0.12 From 00723e03536c8494e692767f8495800188843d6f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 12:57:11 +0200 Subject: Fix ast_for_atom() Clear PyObject_Str() exception if it failed, ast_error() should not be called with an exception set. --- Python/ast.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/ast.c b/Python/ast.c index c1ce0aa..1f7ddfc 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2040,6 +2040,7 @@ ast_for_atom(struct compiling *c, const node *n) PyOS_snprintf(buf, sizeof(buf), "(%s) %s", errtype, s); Py_DECREF(errstr); } else { + PyErr_Clear(); PyOS_snprintf(buf, sizeof(buf), "(%s) unknown error", errtype); } ast_error(c, n, buf); -- cgit v0.12 From 29ee6745af8ace80cef04c18b232771e8b00acdb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 16:25:45 +0200 Subject: Enhance _PyTime_AsTimespec() Ensure that the tv_nsec field is set, even if the function fails with an overflow. --- Python/pytime.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 8166cec..cea3cf5 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -479,13 +479,13 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) secs -= 1; } ts->tv_sec = (time_t)secs; + assert(0 <= nsec && nsec < SEC_TO_NS); + ts->tv_nsec = nsec; + if ((_PyTime_t)ts->tv_sec != secs) { _PyTime_overflow(); return -1; } - ts->tv_nsec = nsec; - - assert(0 <= ts->tv_nsec && ts->tv_nsec < SEC_TO_NS); return 0; } #endif -- cgit v0.12 From 5786aef382b0f64720060d6372c215e939461957 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 16:33:16 +0200 Subject: Don't abuse volatile keyword in pytime.c Only use it on the most important number. This change fixes also a compiler warning on modf(). --- Python/pytime.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index cea3cf5..d226389 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -135,8 +135,9 @@ int _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) { if (PyFloat_Check(obj)) { + double intpart, err; /* volatile avoids optimization changing how numbers are rounded */ - volatile double d, intpart, err; + volatile double d; d = PyFloat_AsDouble(obj); if (round == _PyTime_ROUND_HALF_UP) @@ -257,8 +258,9 @@ static int _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, long to_nanoseconds) { + double err; /* volatile avoids optimization changing how numbers are rounded */ - volatile double d, err; + volatile double d; /* convert to a number of nanoseconds */ d = value; -- cgit v0.12 From 110f9e35383373022b169756e64974814733aef4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Sep 2015 10:31:16 +0200 Subject: test_time: add tests on HALF_UP rounding mode for _PyTime_ObjectToTime_t() and _PyTime_ObjectToTimespec() --- Lib/test/test_time.py | 130 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 40 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 5947f40..590425a 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -606,24 +606,49 @@ class TestPytime(unittest.TestCase): @support.cpython_only def test_time_t(self): from _testcapi import pytime_object_to_time_t + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, seconds in ( + # int + (-1, -1), + (0, 0), + (1, 1), + + # float + (-1.0, -1), + (1.0, 1), + ): + with self.subTest(obj=obj, round=rnd, seconds=seconds): + self.assertEqual(pytime_object_to_time_t(obj, rnd), + seconds) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, time_t, rnd in ( - # Round towards minus infinity (-inf) - (0, 0, _PyTime.ROUND_FLOOR), - (-1, -1, _PyTime.ROUND_FLOOR), - (-1.0, -1, _PyTime.ROUND_FLOOR), - (-1.9, -2, _PyTime.ROUND_FLOOR), - (1.0, 1, _PyTime.ROUND_FLOOR), - (1.9, 1, _PyTime.ROUND_FLOOR), - # Round towards infinity (+inf) - (0, 0, _PyTime.ROUND_CEILING), - (-1, -1, _PyTime.ROUND_CEILING), - (-1.0, -1, _PyTime.ROUND_CEILING), - (-1.9, -1, _PyTime.ROUND_CEILING), - (1.0, 1, _PyTime.ROUND_CEILING), - (1.9, 2, _PyTime.ROUND_CEILING), + (-1.9, -2, FLOOR), + (-1.9, -1, CEILING), + (-1.9, -2, HALF_UP), + + (1.9, 1, FLOOR), + (1.9, 2, CEILING), + (1.9, 2, HALF_UP), + + # half up + (-0.999, -1, HALF_UP), + (-0.510, -1, HALF_UP), + (-0.500, -1, HALF_UP), + (-0.490, 0, HALF_UP), + ( 0.490, 0, HALF_UP), + ( 0.500, 1, HALF_UP), + ( 0.510, 1, HALF_UP), + ( 0.999, 1, HALF_UP), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + # Test OverflowError rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, @@ -632,39 +657,64 @@ class TestPytime(unittest.TestCase): @support.cpython_only def test_timespec(self): from _testcapi import pytime_object_to_timespec + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timespec in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.2, (-2, 800000000)), + (-1.0, (-1, 0)), + (-1e-9, (-1, 999999999)), + (1e-9, (0, 1)), + (1.0, (1, 0)), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, timespec, rnd in ( # Round towards minus infinity (-inf) - (0, (0, 0), _PyTime.ROUND_FLOOR), - (-1, (-1, 0), _PyTime.ROUND_FLOOR), - (-1.0, (-1, 0), _PyTime.ROUND_FLOOR), - (1e-9, (0, 1), _PyTime.ROUND_FLOOR), - (1e-10, (0, 0), _PyTime.ROUND_FLOOR), - (-1e-9, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1e-10, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1.2, (-2, 800000000), _PyTime.ROUND_FLOOR), - (0.9999999999, (0, 999999999), _PyTime.ROUND_FLOOR), - (1.1234567890, (1, 123456789), _PyTime.ROUND_FLOOR), - (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), + (-1e-10, (0, 0), CEILING), + (-1e-10, (-1, 999999999), FLOOR), + (-1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), FLOOR), + (1e-10, (0, 1), CEILING), + (1e-10, (0, 0), HALF_UP), + + (0.9999999999, (0, 999999999), FLOOR), + (0.9999999999, (1, 0), CEILING), + + (1.1234567890, (1, 123456789), FLOOR), + (1.1234567899, (1, 123456789), FLOOR), + (-1.1234567890, (-2, 876543211), FLOOR), + (-1.1234567891, (-2, 876543210), FLOOR), # Round towards infinity (+inf) - (0, (0, 0), _PyTime.ROUND_CEILING), - (-1, (-1, 0), _PyTime.ROUND_CEILING), - (-1.0, (-1, 0), _PyTime.ROUND_CEILING), - (1e-9, (0, 1), _PyTime.ROUND_CEILING), - (1e-10, (0, 1), _PyTime.ROUND_CEILING), - (-1e-9, (-1, 999999999), _PyTime.ROUND_CEILING), - (-1e-10, (0, 0), _PyTime.ROUND_CEILING), - (-1.2, (-2, 800000000), _PyTime.ROUND_CEILING), - (0.9999999999, (1, 0), _PyTime.ROUND_CEILING), - (1.1234567890, (1, 123456790), _PyTime.ROUND_CEILING), - (1.1234567899, (1, 123456790), _PyTime.ROUND_CEILING), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_CEILING), - (-1.1234567891, (-2, 876543211), _PyTime.ROUND_CEILING), + (1.1234567890, (1, 123456790), CEILING), + (1.1234567899, (1, 123456790), CEILING), + (-1.1234567890, (-2, 876543211), CEILING), + (-1.1234567891, (-2, 876543211), CEILING), + + # half up + (-0.6e-9, (-1, 999999999), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(-0.5e-9, (-1, 999999999), HALF_UP), + (-0.4e-9, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + # Test OverflowError rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, -- cgit v0.12 From adfefa527a32e711c1bea9c1ac32c20e9cce0660 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Sep 2015 23:57:25 +0200 Subject: Issue #23517: Fix implementation of the ROUND_HALF_UP rounding mode in datetime.datetime.fromtimestamp() and datetime.datetime.utcfromtimestamp(). microseconds sign should be kept before rounding. --- Lib/datetime.py | 48 ++++++++++++++++++++-------------------------- Lib/test/datetimetester.py | 12 ++++++++++-- Lib/test/test_time.py | 13 +++++++------ Python/pytime.c | 8 ++++---- 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 5ba2ddb..3bf9edc 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1374,28 +1374,34 @@ class datetime(date): return self._tzinfo @classmethod - def fromtimestamp(cls, t, tz=None): + def _fromtimestamp(cls, t, utc, tz): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ - _check_tzinfo_arg(tz) - - converter = _time.localtime if tz is None else _time.gmtime - - t, frac = divmod(t, 1.0) + frac, t = _math.modf(t) us = _round_half_up(frac * 1e6) - - # If timestamp is less than one microsecond smaller than a - # full second, us can be rounded up to 1000000. In this case, - # roll over to seconds, otherwise, ValueError is raised - # by the constructor. - if us == 1000000: + if us >= 1000000: t += 1 - us = 0 + us -= 1000000 + elif us < 0: + t -= 1 + us += 1000000 + + converter = _time.gmtime if utc else _time.localtime y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) ss = min(ss, 59) # clamp out leap seconds if the platform has them - result = cls(y, m, d, hh, mm, ss, us, tz) + return cls(y, m, d, hh, mm, ss, us, tz) + + @classmethod + def fromtimestamp(cls, t, tz=None): + """Construct a datetime from a POSIX timestamp (like time.time()). + + A timezone info object may be passed in as well. + """ + _check_tzinfo_arg(tz) + + result = cls._fromtimestamp(t, tz is not None, tz) if tz is not None: result = tz.fromutc(result) return result @@ -1403,19 +1409,7 @@ class datetime(date): @classmethod def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" - t, frac = divmod(t, 1.0) - us = _round_half_up(frac * 1e6) - - # If timestamp is less than one microsecond smaller than a - # full second, us can be rounded up to 1000000. In this case, - # roll over to seconds, otherwise, ValueError is raised - # by the constructor. - if us == 1000000: - t += 1 - us = 0 - y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us) + return cls._fromtimestamp(t, True, None) @classmethod def now(cls, tz=None): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f516434..58873af 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -668,6 +668,8 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) eq(td(seconds=0.5/10**6), td(microseconds=1)) eq(td(seconds=-0.5/10**6), td(microseconds=-1)) + eq(td(seconds=1/2**7), td(microseconds=7813)) + eq(td(seconds=-1/2**7), td(microseconds=-7813)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 @@ -1842,8 +1844,8 @@ class TestDateTime(TestDate): 18000 + 3600 + 2*60 + 3 + 4*1e-6) def test_microsecond_rounding(self): - for fts in [self.theclass.fromtimestamp, - self.theclass.utcfromtimestamp]: + for fts in (datetime.fromtimestamp, + self.theclass.utcfromtimestamp): zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) @@ -1874,6 +1876,12 @@ class TestDateTime(TestDate): t = fts(0.9999999) self.assertEqual(t.second, 1) self.assertEqual(t.microsecond, 0) + t = fts(1/2**7) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 7813) + t = fts(-1/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992187) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 590425a..75ab666 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -655,7 +655,7 @@ class TestPytime(unittest.TestCase): pytime_object_to_time_t, invalid, rnd) @support.cpython_only - def test_timespec(self): + def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec # Conversion giving the same result for all rounding methods @@ -666,7 +666,7 @@ class TestPytime(unittest.TestCase): (-1, (-1, 0)), # float - (-1.2, (-2, 800000000)), + (-1/2**7, (-1, 992187500)), (-1.0, (-1, 0)), (-1e-9, (-1, 999999999)), (1e-9, (0, 1)), @@ -693,7 +693,7 @@ class TestPytime(unittest.TestCase): (1.1234567890, (1, 123456789), FLOOR), (1.1234567899, (1, 123456789), FLOOR), - (-1.1234567890, (-2, 876543211), FLOOR), + (-1.1234567890, (-2, 876543210), FLOOR), (-1.1234567891, (-2, 876543210), FLOOR), # Round towards infinity (+inf) (1.1234567890, (1, 123456790), CEILING), @@ -1155,7 +1155,7 @@ class TestOldPyTime(unittest.TestCase): self.assertRaises(OverflowError, pytime_object_to_time_t, invalid, rnd) - def test_timeval(self): + def test_object_to_timeval(self): from _testcapi import pytime_object_to_timeval # Conversion giving the same result for all rounding methods @@ -1167,7 +1167,8 @@ class TestOldPyTime(unittest.TestCase): # float (-1.0, (-1, 0)), - (-1.2, (-2, 800000)), + (1/2**6, (0, 15625)), + (-1/2**6, (-1, 984375)), (-1e-6, (-1, 999999)), (1e-6, (0, 1)), ): @@ -1225,7 +1226,7 @@ class TestOldPyTime(unittest.TestCase): (-1.0, (-1, 0)), (-1e-9, (-1, 999999999)), (1e-9, (0, 1)), - (-1.2, (-2, 800000000)), + (-1/2**9, (-1, 998046875)), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), diff --git a/Python/pytime.c b/Python/pytime.c index d226389..0364473 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -82,10 +82,6 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, volatile double floatpart; floatpart = modf(d, &intpart); - if (floatpart < 0) { - floatpart += 1.0; - intpart -= 1.0; - } floatpart *= denominator; if (round == _PyTime_ROUND_HALF_UP) @@ -98,6 +94,10 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart -= denominator; intpart += 1.0; } + else if (floatpart < 0) { + floatpart += denominator; + intpart -= 1.0; + } assert(0.0 <= floatpart && floatpart < denominator); *sec = (time_t)intpart; -- cgit v0.12 From 8820a350d77c1cf93a6c720981e562407c2dd71d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 5 Sep 2015 10:50:20 +0200 Subject: Issue #23517: Skip a datetime test on Windows The test calls gmtime(-1)/localtime(-1) which is not supported on Windows. --- Lib/test/datetimetester.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 58873af..b8c9b0d 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1865,6 +1865,9 @@ class TestDateTime(TestDate): self.assertEqual(t, minus_one) t = fts(-1e-7) self.assertEqual(t, zero) + t = fts(-1/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992187) t = fts(1e-7) self.assertEqual(t, zero) @@ -1879,9 +1882,6 @@ class TestDateTime(TestDate): t = fts(1/2**7) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 7813) - t = fts(-1/2**7) - self.assertEqual(t.second, 59) - self.assertEqual(t.microsecond, 992187) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, -- cgit v0.12 From 7827a5b7c29ae71daf0175ce3398115374ceb50e Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 6 Sep 2015 13:07:21 -0400 Subject: Closes Issue#22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. --- Doc/library/datetime.rst | 19 ++++++++++++------- Lib/datetime.py | 2 ++ Lib/test/datetimetester.py | 3 ++- Misc/NEWS | 2 ++ Modules/_datetimemodule.c | 5 +++++ 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 976cd49..9a0f936 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1734,10 +1734,7 @@ made to civil time. otherwise :exc:`ValueError` is raised. The *name* argument is optional. If specified it must be a string that - is used as the value returned by the ``tzname(dt)`` method. Otherwise, - ``tzname(dt)`` returns a string 'UTCsHH:MM', where s is the sign of - *offset*, HH and MM are two digits of ``offset.hours`` and - ``offset.minutes`` respectively. + will be used as the value returned by the :meth:`datetime.tzname` method. .. versionadded:: 3.2 @@ -1750,11 +1747,19 @@ made to civil time. .. method:: timezone.tzname(dt) - Return the fixed value specified when the :class:`timezone` instance is - constructed or a string 'UTCsHH:MM', where s is the sign of - *offset*, HH and MM are two digits of ``offset.hours`` and + Return the fixed value specified when the :class:`timezone` instance + is constructed. If *name* is not provided in the constructor, the + name returned by ``tzname(dt)`` is generated from the value of the + ``offset`` as follows. If *offset* is ``timedelta(0)``, the name + is "UTC", otherwise it is a string 'UTC±HH:MM', where ± is the sign + of ``offset``, HH and MM are two digits of ``offset.hours`` and ``offset.minutes`` respectively. + .. versionchanged:: 3.6 + Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not + 'UTC+00:00'. + + .. method:: timezone.dst(dt) Always returns ``None``. diff --git a/Lib/datetime.py b/Lib/datetime.py index 3bf9edc..6b2ac33 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1920,6 +1920,8 @@ class timezone(tzinfo): @staticmethod def _name_from_offset(delta): + if not delta: + return 'UTC' if delta < timedelta(0): sign = '-' delta = -delta diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b8c9b0d..f23add3 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -258,7 +258,8 @@ class TestTimeZone(unittest.TestCase): with self.assertRaises(TypeError): self.EST.dst(5) def test_tzname(self): - self.assertEqual('UTC+00:00', timezone(ZERO).tzname(None)) + self.assertEqual('UTC', timezone.utc.tzname(None)) + self.assertEqual('UTC', timezone(ZERO).tzname(None)) self.assertEqual('UTC-05:00', timezone(-5 * HOUR).tzname(None)) self.assertEqual('UTC+09:30', timezone(9.5 * HOUR).tzname(None)) self.assertEqual('UTC-00:01', timezone(timedelta(minutes=-1)).tzname(None)) diff --git a/Misc/NEWS b/Misc/NEWS index e88106a..096ab67 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,8 @@ Core and Builtins Library ------- +- Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. + - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ae459df..008b733 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3267,6 +3267,11 @@ timezone_str(PyDateTime_TimeZone *self) Py_INCREF(self->name); return self->name; } + if (self == PyDateTime_TimeZone_UTC || + (GET_TD_DAYS(self->offset) == 0 && + GET_TD_SECONDS(self->offset) == 0 && + GET_TD_MICROSECONDS(self->offset) == 0)) + return PyUnicode_FromString("UTC"); /* Offset is normalized, so it is negative if days < 0 */ if (GET_TD_DAYS(self->offset) < 0) { sign = '-'; -- cgit v0.12 From 56f6e76c680f47ad2b11bed9406305a000a1889a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Sep 2015 21:25:30 +0300 Subject: Issue #15989: Fixed some scarcely probable integer overflows. It is very unlikely that they can occur in real code for now. --- Modules/_datetimemodule.c | 2 +- Modules/_io/_iomodule.c | 3 ++- Modules/posixmodule.c | 7 +++++-- Modules/readline.c | 2 +- Objects/structseq.c | 24 ++++++++++++------------ Python/Python-ast.c | 2 +- Python/pythonrun.c | 10 +++++----- 7 files changed, 27 insertions(+), 23 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 008b733..fe9a948 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4692,7 +4692,7 @@ local_timezone(PyDateTime_DateTime *utc_time) if (seconds == NULL) goto error; Py_DECREF(delta); - timestamp = PyLong_AsLong(seconds); + timestamp = _PyLong_AsTime_t(seconds); Py_DECREF(seconds); if (timestamp == -1 && PyErr_Occurred()) return NULL; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 1c2d3a0..7428aed 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -238,7 +238,8 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, int text = 0, binary = 0, universal = 0; char rawmode[6], *m; - int line_buffering, isatty; + int line_buffering; + long isatty; PyObject *raw, *modeobj = NULL, *buffer, *wrapper, *result = NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d2b8dfd..270de0f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9481,7 +9481,7 @@ os__getdiskusage_impl(PyModuleDef *module, Py_UNICODE *path) */ struct constdef { char *name; - long value; + int value; }; static int @@ -9489,7 +9489,10 @@ conv_confname(PyObject *arg, int *valuep, struct constdef *table, size_t tablesize) { if (PyLong_Check(arg)) { - *valuep = PyLong_AS_LONG(arg); + int value = _PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) + return 0; + *valuep = value; return 1; } else { diff --git a/Modules/readline.c b/Modules/readline.c index f6b52a0..09877f2 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -840,7 +840,7 @@ on_hook(PyObject *func) if (r == Py_None) result = 0; else { - result = PyLong_AsLong(r); + result = _PyLong_AsInt(r); if (result == -1 && PyErr_Occurred()) goto error; } diff --git a/Objects/structseq.c b/Objects/structseq.c index 664344b..7209738 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -16,14 +16,14 @@ _Py_IDENTIFIER(n_fields); _Py_IDENTIFIER(n_unnamed_fields); #define VISIBLE_SIZE(op) Py_SIZE(op) -#define VISIBLE_SIZE_TP(tp) PyLong_AsLong( \ +#define VISIBLE_SIZE_TP(tp) PyLong_AsSsize_t( \ _PyDict_GetItemId((tp)->tp_dict, &PyId_n_sequence_fields)) -#define REAL_SIZE_TP(tp) PyLong_AsLong( \ +#define REAL_SIZE_TP(tp) PyLong_AsSsize_t( \ _PyDict_GetItemId((tp)->tp_dict, &PyId_n_fields)) #define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op)) -#define UNNAMED_FIELDS_TP(tp) PyLong_AsLong( \ +#define UNNAMED_FIELDS_TP(tp) PyLong_AsSsize_t( \ _PyDict_GetItemId((tp)->tp_dict, &PyId_n_unnamed_fields)) #define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op)) @@ -164,7 +164,8 @@ structseq_repr(PyStructSequence *obj) #define TYPE_MAXSIZE 100 PyTypeObject *typ = Py_TYPE(obj); - int i, removelast = 0; + Py_ssize_t i; + int removelast = 0; Py_ssize_t len; char buf[REPR_BUFFER_SIZE]; char *endofbuf, *pbuf = buf; @@ -236,8 +237,7 @@ structseq_reduce(PyStructSequence* self) PyObject* tup = NULL; PyObject* dict = NULL; PyObject* result; - Py_ssize_t n_fields, n_visible_fields, n_unnamed_fields; - int i; + Py_ssize_t n_fields, n_visible_fields, n_unnamed_fields, i; n_fields = REAL_SIZE(self); n_visible_fields = VISIBLE_SIZE(self); @@ -325,7 +325,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) { PyObject *dict; PyMemberDef* members; - int n_members, n_unnamed_members, i, k; + Py_ssize_t n_members, n_unnamed_members, i, k; PyObject *v; #ifdef Py_TRACE_REFS @@ -373,9 +373,9 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) Py_INCREF(type); dict = type->tp_dict; -#define SET_DICT_FROM_INT(key, value) \ +#define SET_DICT_FROM_SIZE(key, value) \ do { \ - v = PyLong_FromLong((long) value); \ + v = PyLong_FromSsize_t(value); \ if (v == NULL) \ return -1; \ if (PyDict_SetItemString(dict, key, v) < 0) { \ @@ -385,9 +385,9 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) Py_DECREF(v); \ } while (0) - SET_DICT_FROM_INT(visible_length_key, desc->n_in_sequence); - SET_DICT_FROM_INT(real_length_key, n_members); - SET_DICT_FROM_INT(unnamed_fields_key, n_unnamed_members); + SET_DICT_FROM_SIZE(visible_length_key, desc->n_in_sequence); + SET_DICT_FROM_SIZE(real_length_key, n_members); + SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members); return 0; } diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 8a2dc7c..fd7f17e 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -769,7 +769,7 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) return 1; } - i = (int)PyLong_AsLong(obj); + i = _PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) return 1; *out = i; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index ebedd12..1a5dab5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -431,7 +431,7 @@ static int parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename, int *lineno, int *offset, PyObject **text) { - long hold; + int hold; PyObject *v; _Py_IDENTIFIER(msg); _Py_IDENTIFIER(filename); @@ -464,11 +464,11 @@ parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename, v = _PyObject_GetAttrId(err, &PyId_lineno); if (!v) goto finally; - hold = PyLong_AsLong(v); + hold = _PyLong_AsInt(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; - *lineno = (int)hold; + *lineno = hold; v = _PyObject_GetAttrId(err, &PyId_offset); if (!v) @@ -477,11 +477,11 @@ parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename, *offset = -1; Py_DECREF(v); } else { - hold = PyLong_AsLong(v); + hold = _PyLong_AsInt(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; - *offset = (int)hold; + *offset = hold; } v = _PyObject_GetAttrId(err, &PyId_text); -- cgit v0.12 From 481d3af82e4fe507a995ade313c68d6b635e8757 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Sep 2015 23:29:04 +0300 Subject: Make asdl_c.py to generate Python-ast.c changed in issue #15989. --- Parser/asdl_c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index a5e35d9..d597122 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -898,7 +898,7 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) return 1; } - i = (int)PyLong_AsLong(obj); + i = _PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) return 1; *out = i; -- cgit v0.12 From 90fd895197f485583af630ef2e34cbdea298abf7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Sep 2015 00:12:49 +0200 Subject: Issue #22241: Fix a compiler waring --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index fe9a948..ea2bae6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3267,7 +3267,7 @@ timezone_str(PyDateTime_TimeZone *self) Py_INCREF(self->name); return self->name; } - if (self == PyDateTime_TimeZone_UTC || + if ((PyObject *)self == PyDateTime_TimeZone_UTC || (GET_TD_DAYS(self->offset) == 0 && GET_TD_SECONDS(self->offset) == 0 && GET_TD_MICROSECONDS(self->offset) == 0)) -- cgit v0.12 From 69cc487df42d9064a74551ae26a8c115dade3e3a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Sep 2015 23:58:54 +0200 Subject: Revert change 0eb8c182131e: """Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to nearest even integer (ROUND_HALF_EVEN).""" datetime.timedelta uses rounding mode ROUND_HALF_EVEN again. --- Lib/datetime.py | 4 ++-- Lib/test/datetimetester.py | 18 ++++++++++++------ Misc/NEWS | 7 +------ Modules/_datetimemodule.c | 22 +++++++++++++++++++++- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 6b2ac33..3c25ef8 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -407,7 +407,7 @@ class timedelta: # secondsfrac isn't referenced again if isinstance(microseconds, float): - microseconds = _round_half_up(microseconds + usdouble) + microseconds = round(microseconds + usdouble) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 24*3600) d += days @@ -418,7 +418,7 @@ class timedelta: days, seconds = divmod(seconds, 24*3600) d += days s += seconds - microseconds = _round_half_up(microseconds + usdouble) + microseconds = round(microseconds + usdouble) assert isinstance(s, int) assert isinstance(microseconds, int) assert abs(s) <= 3 * 24 * 3600 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f23add3..d87b106 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -663,14 +663,16 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): # Single-field rounding. eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 - eq(td(milliseconds=0.5/1000), td(microseconds=1)) - eq(td(milliseconds=-0.5/1000), td(microseconds=-1)) + eq(td(milliseconds=0.5/1000), td(microseconds=0)) + eq(td(milliseconds=-0.5/1000), td(microseconds=-0)) eq(td(milliseconds=0.6/1000), td(microseconds=1)) eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) - eq(td(seconds=0.5/10**6), td(microseconds=1)) - eq(td(seconds=-0.5/10**6), td(microseconds=-1)) - eq(td(seconds=1/2**7), td(microseconds=7813)) - eq(td(seconds=-1/2**7), td(microseconds=-7813)) + eq(td(milliseconds=1.5/1000), td(microseconds=2)) + eq(td(milliseconds=-1.5/1000), td(microseconds=-2)) + eq(td(seconds=0.5/10**6), td(microseconds=0)) + eq(td(seconds=-0.5/10**6), td(microseconds=-0)) + eq(td(seconds=1/2**7), td(microseconds=7812)) + eq(td(seconds=-1/2**7), td(microseconds=-7812)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 @@ -683,6 +685,10 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) + # Test for a patch in Issue 8860 + eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) + eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) + def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), diff --git a/Misc/NEWS b/Misc/NEWS index c066a73..837288e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,18 +17,13 @@ Core and Builtins Library ------- -- Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. +- Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding towards -Infinity (ROUND_FLOOR). -- Issue #23517: datetime.timedelta constructor now rounds microseconds to - nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and - Python older than 3.3, instead of rounding to nearest with ties going to - nearest even integer (ROUND_HALF_EVEN). - - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ea2bae6..24e83d3 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -2149,9 +2149,29 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (leftover_us) { /* Round to nearest whole # of us, and add into x. */ double whole_us = round(leftover_us); + int x_is_odd; PyObject *temp; - whole_us = _PyTime_RoundHalfUp(leftover_us); + whole_us = round(leftover_us); + if (fabs(whole_us - leftover_us) == 0.5) { + /* We're exactly halfway between two integers. In order + * to do round-half-to-even, we must determine whether x + * is odd. Note that x is odd when it's last bit is 1. The + * code below uses bitwise and operation to check the last + * bit. */ + temp = PyNumber_And(x, one); /* temp <- x & 1 */ + if (temp == NULL) { + Py_DECREF(x); + goto Done; + } + x_is_odd = PyObject_IsTrue(temp); + Py_DECREF(temp); + if (x_is_odd == -1) { + Py_DECREF(x); + goto Done; + } + whole_us = 2.0 * round((leftover_us + x_is_odd) * 0.5) - x_is_odd; + } temp = PyLong_FromLong((long)whole_us); -- cgit v0.12 From 7667f58151d5efbbae4f0b1d7178f99dad0d74c0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 01:02:23 +0200 Subject: Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going to nearest even integer (ROUND_HALF_EVEN), as round(float), instead of rounding towards -Infinity (ROUND_FLOOR). pytime API: replace _PyTime_ROUND_HALF_UP with _PyTime_ROUND_HALF_EVEN. Fix also _PyTime_Divide() for negative numbers. _PyTime_AsTimeval_impl() now reuses _PyTime_Divide() instead of reimplementing rounding modes. --- Include/pytime.h | 9 +- Lib/datetime.py | 2 +- Lib/test/datetimetester.py | 4 +- Lib/test/test_time.py | 245 ++++++++++++++++++++------------------------- Misc/NEWS | 6 +- Modules/_datetimemodule.c | 2 +- Modules/_testcapimodule.c | 2 +- Python/pytime.c | 71 ++++++------- 8 files changed, 152 insertions(+), 189 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 41fb806..54a0cc4 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -31,9 +31,9 @@ typedef enum { /* Round towards infinity (+inf). For example, used for timeout to wait "at least" N seconds. */ _PyTime_ROUND_CEILING=1, - /* Round to nearest with ties going away from zero. + /* Round to nearest with ties going to nearest even integer. For example, used to round from a Python float. */ - _PyTime_ROUND_HALF_UP + _PyTime_ROUND_HALF_EVEN } _PyTime_round_t; /* Convert a time_t to a PyLong. */ @@ -44,8 +44,9 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); -/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ -PyAPI_FUNC(double) _PyTime_RoundHalfUp( +/* Round to nearest with ties going to nearest even integer + (_PyTime_ROUND_HALF_EVEN) */ +PyAPI_FUNC(double) _PyTime_RoundHalfEven( double x); /* Convert a number of seconds, int or float, to time_t. */ diff --git a/Lib/datetime.py b/Lib/datetime.py index 3c25ef8..3f29bc4 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1380,7 +1380,7 @@ class datetime(date): A timezone info object may be passed in as well. """ frac, t = _math.modf(t) - us = _round_half_up(frac * 1e6) + us = round(frac * 1e6) if us >= 1000000: t += 1 us -= 1000000 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index d87b106..467fbe2 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1874,7 +1874,7 @@ class TestDateTime(TestDate): self.assertEqual(t, zero) t = fts(-1/2**7) self.assertEqual(t.second, 59) - self.assertEqual(t.microsecond, 992187) + self.assertEqual(t.microsecond, 992188) t = fts(1e-7) self.assertEqual(t, zero) @@ -1888,7 +1888,7 @@ class TestDateTime(TestDate): self.assertEqual(t.microsecond, 0) t = fts(1/2**7) self.assertEqual(t.second, 0) - self.assertEqual(t.microsecond, 7813) + self.assertEqual(t.microsecond, 7812) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d68dc4f..493b197 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -30,11 +30,11 @@ class _PyTime(enum.IntEnum): ROUND_FLOOR = 0 # Round towards infinity (+inf) ROUND_CEILING = 1 - # Round to nearest with ties going away from zero - ROUND_HALF_UP = 2 + # Round to nearest with ties going to nearest even integer + ROUND_HALF_EVEN = 2 ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, - _PyTime.ROUND_HALF_UP) + _PyTime.ROUND_HALF_EVEN) class TimeTestCase(unittest.TestCase): @@ -639,27 +639,26 @@ class TestPytime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP - for obj, time_t, rnd in ( + HALF_EVEN = _PyTime.ROUND_HALF_EVEN + for obj, seconds, rnd in ( (-1.9, -2, FLOOR), (-1.9, -1, CEILING), - (-1.9, -2, HALF_UP), + (-1.9, -2, HALF_EVEN), (1.9, 1, FLOOR), (1.9, 2, CEILING), - (1.9, 2, HALF_UP), - - # half up - (-0.999, -1, HALF_UP), - (-0.510, -1, HALF_UP), - (-0.500, -1, HALF_UP), - (-0.490, 0, HALF_UP), - ( 0.490, 0, HALF_UP), - ( 0.500, 1, HALF_UP), - ( 0.510, 1, HALF_UP), - ( 0.999, 1, HALF_UP), + (1.9, 2, HALF_EVEN), + + # half even + (-1.5, -2, HALF_EVEN), + (-0.9, -1, HALF_EVEN), + (-0.5, 0, HALF_EVEN), + ( 0.5, 0, HALF_EVEN), + ( 0.9, 1, HALF_EVEN), + ( 1.5, 2, HALF_EVEN), ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + with self.subTest(obj=obj, round=rnd, seconds=seconds): + self.assertEqual(pytime_object_to_time_t(obj, rnd), seconds) # Test OverflowError rnd = _PyTime.ROUND_FLOOR @@ -691,15 +690,15 @@ class TestPytime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timespec, rnd in ( # Round towards minus infinity (-inf) (-1e-10, (0, 0), CEILING), (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), HALF_UP), + (-1e-10, (0, 0), HALF_EVEN), (1e-10, (0, 0), FLOOR), (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), HALF_EVEN), (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), @@ -714,15 +713,13 @@ class TestPytime(unittest.TestCase): (-1.1234567890, (-2, 876543211), CEILING), (-1.1234567891, (-2, 876543211), CEILING), - # half up - (-0.6e-9, (-1, 999999999), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(-0.5e-9, (-1, 999999999), HALF_UP), - (-0.4e-9, (0, 0), HALF_UP), - - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), + # half even + (-1.5e-9, (-1, 999999998), HALF_EVEN), + (-0.9e-9, (-1, 999999999), HALF_EVEN), + (-0.5e-9, (0, 0), HALF_EVEN), + (0.5e-9, (0, 0), HALF_EVEN), + (0.9e-9, (0, 1), HALF_EVEN), + (1.5e-9, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) @@ -823,10 +820,10 @@ class TestPyTime_t(unittest.TestCase): (-7.0, -7 * SEC_TO_NS), # nanosecond are kept for value <= 2^23 seconds, - # except 2**23-1e-9 with HALF_UP (2**22 - 1e-9, 4194303999999999), (2**22, 4194304000000000), (2**22 + 1e-9, 4194304000000001), + (2**23 - 1e-9, 8388607999999999), (2**23, 8388608000000000), # start loosing precision for value > 2^23 seconds @@ -859,38 +856,31 @@ class TestPyTime_t(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, ts, rnd in ( # close to zero ( 1e-10, 0, FLOOR), ( 1e-10, 1, CEILING), - ( 1e-10, 0, HALF_UP), + ( 1e-10, 0, HALF_EVEN), (-1e-10, -1, FLOOR), (-1e-10, 0, CEILING), - (-1e-10, 0, HALF_UP), + (-1e-10, 0, HALF_EVEN), # test rounding of the last nanosecond ( 1.1234567899, 1123456789, FLOOR), ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, HALF_UP), + ( 1.1234567899, 1123456790, HALF_EVEN), (-1.1234567899, -1123456790, FLOOR), (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456790, HALF_UP), + (-1.1234567899, -1123456790, HALF_EVEN), # close to 1 second ( 0.9999999999, 999999999, FLOOR), ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, HALF_UP), + ( 0.9999999999, 1000000000, HALF_EVEN), (-0.9999999999, -1000000000, FLOOR), (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -1000000000, HALF_UP), - - # close to 2^23 seconds - (2**23 - 1e-9, 8388607999999999, FLOOR), - (2**23 - 1e-9, 8388607999999999, CEILING), - # Issue #23517: skip HALF_UP test because the result is different - # depending on the FPU and how the compiler optimize the code :-/ - #(2**23 - 1e-9, 8388608000000000, HALF_UP), + (-0.9999999999, -1000000000, HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -958,33 +948,23 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, tv, rnd in ( # nanoseconds (1, (0, 0), FLOOR), (1, (0, 1), CEILING), - (1, (0, 0), HALF_UP), + (1, (0, 0), HALF_EVEN), (-1, (-1, 999999), FLOOR), (-1, (0, 0), CEILING), - (-1, (0, 0), HALF_UP), - - # seconds + nanoseconds - (1234567001, (1, 234567), FLOOR), - (1234567001, (1, 234568), CEILING), - (1234567001, (1, 234567), HALF_UP), - (-1234567001, (-2, 765432), FLOOR), - (-1234567001, (-2, 765433), CEILING), - (-1234567001, (-2, 765433), HALF_UP), - - # half up - (499, (0, 0), HALF_UP), - (500, (0, 1), HALF_UP), - (501, (0, 1), HALF_UP), - (999, (0, 1), HALF_UP), - (-499, (0, 0), HALF_UP), - (-500, (0, 0), HALF_UP), - (-501, (-1, 999999), HALF_UP), - (-999, (-1, 999999), HALF_UP), + (-1, (0, 0), HALF_EVEN), + + # half even + (-1500, (-1, 999998), HALF_EVEN), + (-999, (-1, 999999), HALF_EVEN), + (-500, (0, 0), HALF_EVEN), + (500, (0, 0), HALF_EVEN), + (999, (0, 1), HALF_EVEN), + (1500, (0, 2), HALF_EVEN), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) @@ -1027,33 +1007,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), - (1, 0, HALF_UP), - (-1, 0, FLOOR), - (-1, -1, CEILING), - (-1, 0, HALF_UP), + (1, 0, HALF_EVEN), + (-1, -1, FLOOR), + (-1, 0, CEILING), + (-1, 0, HALF_EVEN), # seconds + nanoseconds (1234 * MS_TO_NS + 1, 1234, FLOOR), (1234 * MS_TO_NS + 1, 1235, CEILING), - (1234 * MS_TO_NS + 1, 1234, HALF_UP), - (-1234 * MS_TO_NS - 1, -1234, FLOOR), - (-1234 * MS_TO_NS - 1, -1235, CEILING), - (-1234 * MS_TO_NS - 1, -1234, HALF_UP), + (1234 * MS_TO_NS + 1, 1234, HALF_EVEN), + (-1234 * MS_TO_NS - 1, -1235, FLOOR), + (-1234 * MS_TO_NS - 1, -1234, CEILING), + (-1234 * MS_TO_NS - 1, -1234, HALF_EVEN), # half up - (499999, 0, HALF_UP), - (499999, 0, HALF_UP), - (500000, 1, HALF_UP), - (999999, 1, HALF_UP), - (-499999, 0, HALF_UP), - (-500000, -1, HALF_UP), - (-500001, -1, HALF_UP), - (-999999, -1, HALF_UP), + (-1500000, -2, HALF_EVEN), + (-999999, -1, HALF_EVEN), + (-500000, 0, HALF_EVEN), + (500000, 0, HALF_EVEN), + (999999, 1, HALF_EVEN), + (1500000, 2, HALF_EVEN), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) @@ -1079,31 +1057,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), - (1, 0, HALF_UP), - (-1, 0, FLOOR), - (-1, -1, CEILING), - (-1, 0, HALF_UP), + (1, 0, HALF_EVEN), + (-1, -1, FLOOR), + (-1, 0, CEILING), + (-1, 0, HALF_EVEN), # seconds + nanoseconds (1234 * US_TO_NS + 1, 1234, FLOOR), (1234 * US_TO_NS + 1, 1235, CEILING), - (1234 * US_TO_NS + 1, 1234, HALF_UP), - (-1234 * US_TO_NS - 1, -1234, FLOOR), - (-1234 * US_TO_NS - 1, -1235, CEILING), - (-1234 * US_TO_NS - 1, -1234, HALF_UP), + (1234 * US_TO_NS + 1, 1234, HALF_EVEN), + (-1234 * US_TO_NS - 1, -1235, FLOOR), + (-1234 * US_TO_NS - 1, -1234, CEILING), + (-1234 * US_TO_NS - 1, -1234, HALF_EVEN), # half up - (1499, 1, HALF_UP), - (1500, 2, HALF_UP), - (1501, 2, HALF_UP), - (-1499, -1, HALF_UP), - (-1500, -2, HALF_UP), - (-1501, -2, HALF_UP), + (-1500, -2, HALF_EVEN), + (-999, -1, HALF_EVEN), + (-500, 0, HALF_EVEN), + (500, 0, HALF_EVEN), + (999, 1, HALF_EVEN), + (1500, 2, HALF_EVEN), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) @@ -1142,23 +1120,23 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, time_t, rnd in ( (-1.9, -2, FLOOR), - (-1.9, -2, HALF_UP), + (-1.9, -2, HALF_EVEN), (-1.9, -1, CEILING), (1.9, 1, FLOOR), - (1.9, 2, HALF_UP), + (1.9, 2, HALF_EVEN), (1.9, 2, CEILING), - (-0.6, -1, HALF_UP), - (-0.5, -1, HALF_UP), - (-0.4, 0, HALF_UP), - - (0.4, 0, HALF_UP), - (0.5, 1, HALF_UP), - (0.6, 1, HALF_UP), + # half even + (-1.5, -2, HALF_EVEN), + (-0.9, -1, HALF_EVEN), + (-0.5, 0, HALF_EVEN), + ( 0.5, 0, HALF_EVEN), + ( 0.9, 1, HALF_EVEN), + ( 1.5, 2, HALF_EVEN), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) @@ -1192,29 +1170,27 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timeval, rnd in ( (-1e-7, (-1, 999999), FLOOR), (-1e-7, (0, 0), CEILING), - (-1e-7, (0, 0), HALF_UP), + (-1e-7, (0, 0), HALF_EVEN), (1e-7, (0, 0), FLOOR), (1e-7, (0, 1), CEILING), - (1e-7, (0, 0), HALF_UP), + (1e-7, (0, 0), HALF_EVEN), (0.9999999, (0, 999999), FLOOR), (0.9999999, (1, 0), CEILING), - (0.9999999, (1, 0), HALF_UP), - - (-0.6e-6, (-1, 999999), HALF_UP), - # skipped, -0.5e-6 is inexact in base 2 - #(-0.5e-6, (-1, 999999), HALF_UP), - (-0.4e-6, (0, 0), HALF_UP), - - (0.4e-6, (0, 0), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(0.5e-6, (0, 1), HALF_UP), - (0.6e-6, (0, 1), HALF_UP), + (0.9999999, (1, 0), HALF_EVEN), + + # half even + (-1.5e-6, (-1, 999998), HALF_EVEN), + (-0.9e-6, (-1, 999999), HALF_EVEN), + (-0.5e-6, (0, 0), HALF_EVEN), + (0.5e-6, (0, 0), HALF_EVEN), + (0.9e-6, (0, 1), HALF_EVEN), + (1.5e-6, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timeval=timeval): self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) @@ -1248,28 +1224,27 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timespec, rnd in ( (-1e-10, (-1, 999999999), FLOOR), (-1e-10, (0, 0), CEILING), - (-1e-10, (0, 0), HALF_UP), + (-1e-10, (0, 0), HALF_EVEN), (1e-10, (0, 0), FLOOR), (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), HALF_EVEN), (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), - (0.9999999999, (1, 0), HALF_UP), - - (-0.6e-9, (-1, 999999999), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(-0.5e-9, (-1, 999999999), HALF_UP), - (-0.4e-9, (0, 0), HALF_UP), - - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), + (0.9999999999, (1, 0), HALF_EVEN), + + # half even + (-1.5e-9, (-1, 999999998), HALF_EVEN), + (-0.9e-9, (-1, 999999999), HALF_EVEN), + (-0.5e-9, (0, 0), HALF_EVEN), + (0.5e-9, (0, 0), HALF_EVEN), + (0.9e-9, (0, 1), HALF_EVEN), + (1.5e-9, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) diff --git a/Misc/NEWS b/Misc/NEWS index 837288e..c2f2e87 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -20,9 +20,9 @@ Library - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of - datetime.datetime now round microseconds to nearest with ties going away from - zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of - rounding towards -Infinity (ROUND_FLOOR). + datetime.datetime now round microseconds to nearest with ties going to + nearest even integer (ROUND_HALF_EVEN), as round(float), instead of rounding + towards -Infinity (ROUND_FLOOR). - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 24e83d3..bbc51c6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4103,7 +4103,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_HALF_UP) == -1) + &timet, &us, _PyTime_ROUND_HALF_EVEN) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c38d9ae..fed3286 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2648,7 +2648,7 @@ check_time_rounding(int round) { if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING - && round != _PyTime_ROUND_HALF_UP) { + && round != _PyTime_ROUND_HALF_EVEN) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index 0364473..9f7ee2d 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -61,18 +61,15 @@ _PyLong_FromTime_t(time_t t) } double -_PyTime_RoundHalfUp(double x) +_PyTime_RoundHalfEven(double x) { - /* volatile avoids optimization changing how numbers are rounded */ - volatile double d = x; - if (d >= 0.0) - d = floor(d + 0.5); - else - d = ceil(d - 0.5); - return d; + double rounded = round(x); + if (fabs(x-rounded) == 0.5) + /* halfway case: round to even */ + rounded = 2.0*round(x/2.0); + return rounded; } - static int _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) @@ -84,8 +81,8 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart = modf(d, &intpart); floatpart *= denominator; - if (round == _PyTime_ROUND_HALF_UP) - floatpart = _PyTime_RoundHalfUp(floatpart); + if (round == _PyTime_ROUND_HALF_EVEN) + floatpart = _PyTime_RoundHalfEven(floatpart); else if (round == _PyTime_ROUND_CEILING) floatpart = ceil(floatpart); else @@ -140,8 +137,8 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) volatile double d; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_HALF_UP) - d = _PyTime_RoundHalfUp(d); + if (round == _PyTime_ROUND_HALF_EVEN) + d = _PyTime_RoundHalfEven(d); else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else @@ -266,8 +263,8 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, d = value; d *= to_nanoseconds; - if (round == _PyTime_ROUND_HALF_UP) - d = _PyTime_RoundHalfUp(d); + if (round == _PyTime_ROUND_HALF_EVEN) + d = _PyTime_RoundHalfEven(d); else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else @@ -351,14 +348,16 @@ _PyTime_AsNanosecondsObject(_PyTime_t t) } static _PyTime_t -_PyTime_Divide(_PyTime_t t, _PyTime_t k, _PyTime_round_t round) +_PyTime_Divide(const _PyTime_t t, const _PyTime_t k, + const _PyTime_round_t round) { assert(k > 1); - if (round == _PyTime_ROUND_HALF_UP) { - _PyTime_t x, r; + if (round == _PyTime_ROUND_HALF_EVEN) { + _PyTime_t x, r, abs_r; x = t / k; r = t % k; - if (Py_ABS(r) >= k / 2) { + abs_r = Py_ABS(r); + if (abs_r > k / 2 || (abs_r == k / 2 && (Py_ABS(x) & 1))) { if (t >= 0) x++; else @@ -370,10 +369,14 @@ _PyTime_Divide(_PyTime_t t, _PyTime_t k, _PyTime_round_t round) if (t >= 0) return (t + k - 1) / k; else + return t / k; + } + else { + if (t >= 0) + return t / k; + else return (t - (k - 1)) / k; } - else - return t / k; } _PyTime_t @@ -392,17 +395,12 @@ static int _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - const long k = US_TO_NS; _PyTime_t secs, ns; int res = 0; int usec; secs = t / SEC_TO_NS; ns = t % SEC_TO_NS; - if (ns < 0) { - ns += SEC_TO_NS; - secs -= 1; - } #ifdef MS_WINDOWS /* On Windows, timeval.tv_sec is a long (32 bit), @@ -427,23 +425,12 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, res = -1; #endif - if (round == _PyTime_ROUND_HALF_UP) { - _PyTime_t r; - usec = (int)(ns / k); - r = ns % k; - if (Py_ABS(r) >= k / 2) { - if (ns >= 0) - usec++; - else - usec--; - } + usec = (int)_PyTime_Divide(ns, US_TO_NS, round); + if (usec < 0) { + usec += SEC_TO_US; + tv->tv_sec -= 1; } - else if (round == _PyTime_ROUND_CEILING) - usec = (int)((ns + k - 1) / k); - else - usec = (int)(ns / k); - - if (usec >= SEC_TO_US) { + else if (usec >= SEC_TO_US) { usec -= SEC_TO_US; tv->tv_sec += 1; } -- cgit v0.12 From cd5d765b0ee861509b14a1f21b630b8fb6eb258f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 01:09:21 +0200 Subject: cleanup datetime code remove scories of round half up code and debug code. --- Lib/datetime.py | 7 ------- Lib/test/datetimetester.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 3f29bc4..b734a74 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -316,13 +316,6 @@ def _divide_and_round(a, b): return q -def _round_half_up(x): - """Round to nearest with ties going away from zero.""" - if x >= 0.0: - return _math.floor(x + 0.5) - else: - return _math.ceil(x - 0.5) - class timedelta: """Represent the difference between two datetime objects. diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 467fbe2..5d2df32 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -679,7 +679,7 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): us_per_day = us_per_hour * 24 eq(td(days=.4/us_per_day), td(0)) eq(td(hours=.2/us_per_hour), td(0)) - eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1), td) + eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) eq(td(days=-.4/us_per_day), td(0)) eq(td(hours=-.2/us_per_hour), td(0)) -- cgit v0.12 From df326eb1bb2210f42942a06fa386c1b45b8f2f0d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 9 Sep 2015 18:32:50 +0300 Subject: Fix versionchanged directive in datetime.rst --- Doc/library/datetime.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 9a0f936..5c659e7 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1756,8 +1756,8 @@ made to civil time. ``offset.minutes`` respectively. .. versionchanged:: 3.6 - Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not - 'UTC+00:00'. + Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not + 'UTC+00:00'. .. method:: timezone.dst(dt) -- cgit v0.12 From ce6aa749b40d39c8eb82463068cc58f974fbfdf5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 22:28:09 +0200 Subject: Make _PyTime_RoundHalfEven() private again --- Include/pytime.h | 5 ----- Python/pytime.c | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 54a0cc4..5de756a 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -44,11 +44,6 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); -/* Round to nearest with ties going to nearest even integer - (_PyTime_ROUND_HALF_EVEN) */ -PyAPI_FUNC(double) _PyTime_RoundHalfEven( - double x); - /* Convert a number of seconds, int or float, to time_t. */ PyAPI_FUNC(int) _PyTime_ObjectToTime_t( PyObject *obj, diff --git a/Python/pytime.c b/Python/pytime.c index 9f7ee2d..37dfabb 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,7 +60,9 @@ _PyLong_FromTime_t(time_t t) #endif } -double +/* Round to nearest with ties going to nearest even integer + (_PyTime_ROUND_HALF_EVEN) */ +static double _PyTime_RoundHalfEven(double x) { double rounded = round(x); -- cgit v0.12 From 9ae47dfbd9b2f094205faad758b32803bea8e463 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 22:28:58 +0200 Subject: pytime: add _PyTime_Round() helper to factorize code --- Python/pytime.c | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 37dfabb..9735506 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -72,6 +72,17 @@ _PyTime_RoundHalfEven(double x) return rounded; } +static double +_PyTime_Round(double x, _PyTime_round_t round) +{ + if (round == _PyTime_ROUND_HALF_EVEN) + return _PyTime_RoundHalfEven(x); + else if (round == _PyTime_ROUND_CEILING) + return ceil(x); + else + return floor(x); +} + static int _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) @@ -83,12 +94,7 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart = modf(d, &intpart); floatpart *= denominator; - if (round == _PyTime_ROUND_HALF_EVEN) - floatpart = _PyTime_RoundHalfEven(floatpart); - else if (round == _PyTime_ROUND_CEILING) - floatpart = ceil(floatpart); - else - floatpart = floor(floatpart); + floatpart = _PyTime_Round(floatpart, round); if (floatpart >= denominator) { floatpart -= denominator; intpart += 1.0; @@ -139,12 +145,7 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) volatile double d; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_HALF_EVEN) - d = _PyTime_RoundHalfEven(d); - else if (round == _PyTime_ROUND_CEILING) - d = ceil(d); - else - d = floor(d); + d = _PyTime_Round(d, round); (void)modf(d, &intpart); *sec = (time_t)intpart; @@ -255,7 +256,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) static int _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, - long to_nanoseconds) + long unit_to_ns) { double err; /* volatile avoids optimization changing how numbers are rounded */ @@ -263,14 +264,8 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, /* convert to a number of nanoseconds */ d = value; - d *= to_nanoseconds; - - if (round == _PyTime_ROUND_HALF_EVEN) - d = _PyTime_RoundHalfEven(d); - else if (round == _PyTime_ROUND_CEILING) - d = ceil(d); - else - d = floor(d); + d *= (double)unit_to_ns; + d = _PyTime_Round(d, round); *t = (_PyTime_t)d; err = d - (double)*t; @@ -283,12 +278,12 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, static int _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, - long to_nanoseconds) + long unit_to_ns) { if (PyFloat_Check(obj)) { double d; d = PyFloat_AsDouble(obj); - return _PyTime_FromFloatObject(t, d, round, to_nanoseconds); + return _PyTime_FromFloatObject(t, d, round, unit_to_ns); } else { #ifdef HAVE_LONG_LONG @@ -305,8 +300,8 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, _PyTime_overflow(); return -1; } - *t = sec * to_nanoseconds; - if (*t / to_nanoseconds != sec) { + *t = sec * unit_to_ns; + if (*t / unit_to_ns != sec) { _PyTime_overflow(); return -1; } -- cgit v0.12 From 3e2c8d84c69063b8042a75ba04b5e75b1de0704e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 22:32:48 +0200 Subject: test_time: rewrite PyTime API rounding tests Drop all hardcoded tests. Instead, reimplement each function in Python, usually using decimal.Decimal for the rounding mode. Add much more values to the dataset. Test various timestamp units from picroseconds to seconds, in integer and float. Enhance also _PyTime_AsSecondsDouble(). --- Lib/test/test_time.py | 796 +++++++++++++++----------------------------------- Python/pytime.c | 16 +- 2 files changed, 244 insertions(+), 568 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 493b197..bd697ae 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1,6 +1,8 @@ from test import support +import decimal import enum import locale +import math import platform import sys import sysconfig @@ -21,9 +23,11 @@ SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 TIME_MAXYEAR = (1 << 8 * SIZEOF_INT - 1) - 1 TIME_MINYEAR = -TIME_MAXYEAR - 1 +SEC_TO_US = 10 ** 6 US_TO_NS = 10 ** 3 MS_TO_NS = 10 ** 6 SEC_TO_NS = 10 ** 9 +NS_TO_SEC = 10 ** 9 class _PyTime(enum.IntEnum): # Round towards minus infinity (-inf) @@ -33,8 +37,13 @@ class _PyTime(enum.IntEnum): # Round to nearest with ties going to nearest even integer ROUND_HALF_EVEN = 2 -ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, - _PyTime.ROUND_HALF_EVEN) +# Rounding modes supported by PyTime +ROUNDING_MODES = ( + # (PyTime rounding method, decimal rounding method) + (_PyTime.ROUND_FLOOR, decimal.ROUND_FLOOR), + (_PyTime.ROUND_CEILING, decimal.ROUND_CEILING), + (_PyTime.ROUND_HALF_EVEN, decimal.ROUND_HALF_EVEN), +) class TimeTestCase(unittest.TestCase): @@ -610,126 +619,6 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): - def setUp(self): - self.invalid_values = ( - -(2 ** 100), 2 ** 100, - -(2.0 ** 100.0), 2.0 ** 100.0, - ) - - @support.cpython_only - def test_time_t(self): - from _testcapi import pytime_object_to_time_t - - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, seconds in ( - # int - (-1, -1), - (0, 0), - (1, 1), - - # float - (-1.0, -1), - (1.0, 1), - ): - with self.subTest(obj=obj, round=rnd, seconds=seconds): - self.assertEqual(pytime_object_to_time_t(obj, rnd), - seconds) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, seconds, rnd in ( - (-1.9, -2, FLOOR), - (-1.9, -1, CEILING), - (-1.9, -2, HALF_EVEN), - - (1.9, 1, FLOOR), - (1.9, 2, CEILING), - (1.9, 2, HALF_EVEN), - - # half even - (-1.5, -2, HALF_EVEN), - (-0.9, -1, HALF_EVEN), - (-0.5, 0, HALF_EVEN), - ( 0.5, 0, HALF_EVEN), - ( 0.9, 1, HALF_EVEN), - ( 1.5, 2, HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, seconds=seconds): - self.assertEqual(pytime_object_to_time_t(obj, rnd), seconds) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_time_t, invalid, rnd) - - @support.cpython_only - def test_object_to_timespec(self): - from _testcapi import pytime_object_to_timespec - - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timespec in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1/2**7, (-1, 992187500)), - (-1.0, (-1, 0)), - (-1e-9, (-1, 999999999)), - (1e-9, (0, 1)), - (1.0, (1, 0)), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timespec, rnd in ( - # Round towards minus infinity (-inf) - (-1e-10, (0, 0), CEILING), - (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), HALF_EVEN), - (1e-10, (0, 0), FLOOR), - (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_EVEN), - - (0.9999999999, (0, 999999999), FLOOR), - (0.9999999999, (1, 0), CEILING), - - (1.1234567890, (1, 123456789), FLOOR), - (1.1234567899, (1, 123456789), FLOOR), - (-1.1234567890, (-2, 876543210), FLOOR), - (-1.1234567891, (-2, 876543210), FLOOR), - # Round towards infinity (+inf) - (1.1234567890, (1, 123456790), CEILING), - (1.1234567899, (1, 123456790), CEILING), - (-1.1234567890, (-2, 876543211), CEILING), - (-1.1234567891, (-2, 876543211), CEILING), - - # half even - (-1.5e-9, (-1, 999999998), HALF_EVEN), - (-0.9e-9, (-1, 999999999), HALF_EVEN), - (-0.5e-9, (0, 0), HALF_EVEN), - (0.5e-9, (0, 0), HALF_EVEN), - (0.9e-9, (0, 1), HALF_EVEN), - (1.5e-9, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timespec, invalid, rnd) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): @@ -784,476 +673,259 @@ class TestPytime(unittest.TestCase): self.assertIs(lt.tm_zone, None) -@unittest.skipUnless(_testcapi is not None, - 'need the _testcapi module') -class TestPyTime_t(unittest.TestCase): +@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +class CPyTimeTestCase: """ - Test the _PyTime_t API. + Base class to test the C _PyTime_t API. """ + OVERFLOW_SECONDS = None + + def _rounding_values(self, use_float): + "Build timestamps used to test rounding." + + units = [1, US_TO_NS, MS_TO_NS, SEC_TO_NS] + if use_float: + # picoseconds are only tested to pytime_converter accepting floats + units.append(1e-3) + + values = ( + # small values + 1, 2, 5, 7, 123, 456, 1234, + # 10^k - 1 + 9, + 99, + 999, + 9999, + 99999, + 999999, + # test half even rounding near 0.5, 1.5, 2.5, 3.5, 4.5 + 499, 500, 501, + 1499, 1500, 1501, + 2500, + 3500, + 4500, + ) + + ns_timestamps = [0] + for unit in units: + for value in values: + ns = value * unit + ns_timestamps.extend((-ns, ns)) + for pow2 in (0, 5, 10, 15, 22, 23, 24, 30, 33): + ns = (2 ** pow2) * SEC_TO_NS + ns_timestamps.extend(( + -ns-1, -ns, -ns+1, + ns-1, ns, ns+1 + )) + for seconds in (_testcapi.INT_MIN, _testcapi.INT_MAX): + ns_timestamps.append(seconds * SEC_TO_NS) + if use_float: + # numbers with an extract representation in IEEE 754 (base 2) + for pow2 in (3, 7, 10, 15): + ns = 2.0 ** (-pow2) + ns_timestamps.extend((-ns, ns)) + + # seconds close to _PyTime_t type limit + ns = (2 ** 63 // SEC_TO_NS) * SEC_TO_NS + ns_timestamps.extend((-ns, ns)) + + return ns_timestamps + + def _check_rounding(self, pytime_converter, expected_func, + use_float, unit_to_sec, value_filter=None): + + def convert_values(ns_timestamps): + if use_float: + unit_to_ns = SEC_TO_NS / float(unit_to_sec) + values = [ns / unit_to_ns for ns in ns_timestamps] + else: + unit_to_ns = SEC_TO_NS // unit_to_sec + values = [ns // unit_to_ns for ns in ns_timestamps] + + if value_filter: + values = filter(value_filter, values) + + # remove duplicates and sort + return sorted(set(values)) + + # test rounding + ns_timestamps = self._rounding_values(use_float) + valid_values = convert_values(ns_timestamps) + for time_rnd, decimal_rnd in ROUNDING_MODES : + context = decimal.getcontext() + context.rounding = decimal_rnd + + for value in valid_values: + expected = expected_func(value) + self.assertEqual(pytime_converter(value, time_rnd), + expected, + {'value': value, 'rounding': decimal_rnd}) + + # test overflow + ns = self.OVERFLOW_SECONDS * SEC_TO_NS + ns_timestamps = (-ns, ns) + overflow_values = convert_values(ns_timestamps) + for time_rnd, _ in ROUNDING_MODES : + for value in overflow_values: + with self.assertRaises(OverflowError): + pytime_converter(value, time_rnd) + + def check_int_rounding(self, pytime_converter, expected_func, unit_to_sec=1, + value_filter=None): + self._check_rounding(pytime_converter, expected_func, + False, unit_to_sec, value_filter) + + def check_float_rounding(self, pytime_converter, expected_func, unit_to_sec=1): + self._check_rounding(pytime_converter, expected_func, + True, unit_to_sec) + + def decimal_round(self, x): + d = decimal.Decimal(x) + d = d.quantize(1) + return int(d) + + +class TestCPyTime(CPyTimeTestCase, unittest.TestCase): + """ + Test the C _PyTime_t API. + """ + # _PyTime_t is a 64-bit signed integer + OVERFLOW_SECONDS = math.ceil((2**63 + 1) / SEC_TO_NS) + def test_FromSeconds(self): from _testcapi import PyTime_FromSeconds - for seconds in (0, 3, -456, _testcapi.INT_MAX, _testcapi.INT_MIN): - with self.subTest(seconds=seconds): - self.assertEqual(PyTime_FromSeconds(seconds), - seconds * SEC_TO_NS) + + # PyTime_FromSeconds() expects a C int, reject values out of range + def c_int_filter(secs): + return (_testcapi.INT_MIN <= secs <= _testcapi.INT_MAX) + + self.check_int_rounding(lambda secs, rnd: PyTime_FromSeconds(secs), + lambda secs: secs * SEC_TO_NS, + value_filter=c_int_filter) def test_FromSecondsObject(self): from _testcapi import PyTime_FromSecondsObject - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, ts in ( - # integers - (0, 0), - (1, SEC_TO_NS), - (-3, -3 * SEC_TO_NS), - - # float: subseconds - (0.0, 0), - (1e-9, 1), - (1e-6, 10 ** 3), - (1e-3, 10 ** 6), - - # float: seconds - (2.0, 2 * SEC_TO_NS), - (123.0, 123 * SEC_TO_NS), - (-7.0, -7 * SEC_TO_NS), - - # nanosecond are kept for value <= 2^23 seconds, - (2**22 - 1e-9, 4194303999999999), - (2**22, 4194304000000000), - (2**22 + 1e-9, 4194304000000001), - (2**23 - 1e-9, 8388607999999999), - (2**23, 8388608000000000), - - # start loosing precision for value > 2^23 seconds - (2**23 + 1e-9, 8388608000000002), - - # nanoseconds are lost for value > 2^23 seconds - (2**24 - 1e-9, 16777215999999998), - (2**24, 16777216000000000), - (2**24 + 1e-9, 16777216000000000), - (2**25 - 1e-9, 33554432000000000), - (2**25 , 33554432000000000), - (2**25 + 1e-9, 33554432000000000), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036, 9223372036 * SEC_TO_NS), - (9223372036.0, 9223372036 * SEC_TO_NS), - (-9223372036, -9223372036 * SEC_TO_NS), - (-9223372036.0, -9223372036 * SEC_TO_NS), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) - - with self.subTest(round=rnd): - with self.assertRaises(OverflowError): - PyTime_FromSecondsObject(9223372037, rnd) - PyTime_FromSecondsObject(9223372037.0, rnd) - PyTime_FromSecondsObject(-9223372037, rnd) - PyTime_FromSecondsObject(-9223372037.0, rnd) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, ts, rnd in ( - # close to zero - ( 1e-10, 0, FLOOR), - ( 1e-10, 1, CEILING), - ( 1e-10, 0, HALF_EVEN), - (-1e-10, -1, FLOOR), - (-1e-10, 0, CEILING), - (-1e-10, 0, HALF_EVEN), - - # test rounding of the last nanosecond - ( 1.1234567899, 1123456789, FLOOR), - ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, HALF_EVEN), - (-1.1234567899, -1123456790, FLOOR), - (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456790, HALF_EVEN), - - # close to 1 second - ( 0.9999999999, 999999999, FLOOR), - ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, HALF_EVEN), - (-0.9999999999, -1000000000, FLOOR), - (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -1000000000, HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) + self.check_int_rounding( + PyTime_FromSecondsObject, + lambda secs: secs * SEC_TO_NS) + + self.check_float_rounding( + PyTime_FromSecondsObject, + lambda ns: self.decimal_round(ns * SEC_TO_NS)) def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble - for nanoseconds, seconds in ( - # near 1 nanosecond - ( 0, 0.0), - ( 1, 1e-9), - (-1, -1e-9), - - # near 1 second - (SEC_TO_NS + 1, 1.0 + 1e-9), - (SEC_TO_NS, 1.0), - (SEC_TO_NS - 1, 1.0 - 1e-9), - - # a few seconds - (123 * SEC_TO_NS, 123.0), - (-567 * SEC_TO_NS, -567.0), - - # nanosecond are kept for value <= 2^23 seconds - (4194303999999999, 2**22 - 1e-9), - (4194304000000000, 2**22), - (4194304000000001, 2**22 + 1e-9), - - # start loosing precision for value > 2^23 seconds - (8388608000000002, 2**23 + 1e-9), - - # nanoseconds are lost for value > 2^23 seconds - (16777215999999998, 2**24 - 1e-9), - (16777215999999999, 2**24 - 1e-9), - (16777216000000000, 2**24 ), - (16777216000000001, 2**24 ), - (16777216000000002, 2**24 + 2e-9), - - (33554432000000000, 2**25 ), - (33554432000000002, 2**25 ), - (33554432000000004, 2**25 + 4e-9), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036 * SEC_TO_NS, 9223372036.0), - (-9223372036 * SEC_TO_NS, -9223372036.0), - ): - with self.subTest(nanoseconds=nanoseconds, seconds=seconds): - self.assertEqual(PyTime_AsSecondsDouble(nanoseconds), - seconds) - - def test_timeval(self): + def float_converter(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), + float_converter, + NS_TO_SEC) + + def create_decimal_converter(self, denominator): + denom = decimal.Decimal(denominator) + + def converter(value): + d = decimal.Decimal(value) / denom + return self.decimal_round(d) + + return converter + + def test_AsTimeval(self): from _testcapi import PyTime_AsTimeval - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (0, (0, 0)), - (1000, (0, 1)), - (-1000, (-1, 999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, tv, rnd in ( - # nanoseconds - (1, (0, 0), FLOOR), - (1, (0, 1), CEILING), - (1, (0, 0), HALF_EVEN), - (-1, (-1, 999999), FLOOR), - (-1, (0, 0), CEILING), - (-1, (0, 0), HALF_EVEN), - - # half even - (-1500, (-1, 999998), HALF_EVEN), - (-999, (-1, 999999), HALF_EVEN), - (-500, (0, 0), HALF_EVEN), - (500, (0, 0), HALF_EVEN), - (999, (0, 1), HALF_EVEN), - (1500, (0, 2), HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) + + us_converter = self.create_decimal_converter(US_TO_NS) + + def timeval_converter(ns): + us = us_converter(ns) + return divmod(us, SEC_TO_US) + + self.check_int_rounding(PyTime_AsTimeval, + timeval_converter, + NS_TO_SEC) @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), 'need _testcapi.PyTime_AsTimespec') - def test_timespec(self): + def test_AsTimespec(self): from _testcapi import PyTime_AsTimespec - for ns, ts in ( - # nanoseconds - (0, (0, 0)), - (1, (0, 1)), - (-1, (-1, 999999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - - # seconds + nanoseconds - (1234567890, (1, 234567890)), - (-1234567890, (-2, 765432110)), - ): - with self.subTest(nanoseconds=ns, timespec=ts): - self.assertEqual(PyTime_AsTimespec(ns), ts) - - def test_milliseconds(self): + + def timespec_converter(ns): + return divmod(ns, SEC_TO_NS) + + self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), + timespec_converter, + NS_TO_SEC) + + def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # milliseconds - (1 * MS_TO_NS, 1), - (-2 * MS_TO_NS, -2), - - # seconds - (2 * SEC_TO_NS, 2000), - (-3 * SEC_TO_NS, -3000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (1, 0, HALF_EVEN), - (-1, -1, FLOOR), - (-1, 0, CEILING), - (-1, 0, HALF_EVEN), - - # seconds + nanoseconds - (1234 * MS_TO_NS + 1, 1234, FLOOR), - (1234 * MS_TO_NS + 1, 1235, CEILING), - (1234 * MS_TO_NS + 1, 1234, HALF_EVEN), - (-1234 * MS_TO_NS - 1, -1235, FLOOR), - (-1234 * MS_TO_NS - 1, -1234, CEILING), - (-1234 * MS_TO_NS - 1, -1234, HALF_EVEN), - - # half up - (-1500000, -2, HALF_EVEN), - (-999999, -1, HALF_EVEN), - (-500000, 0, HALF_EVEN), - (500000, 0, HALF_EVEN), - (999999, 1, HALF_EVEN), - (1500000, 2, HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) - - def test_microseconds(self): + + self.check_int_rounding(PyTime_AsMilliseconds, + self.create_decimal_converter(MS_TO_NS), + NS_TO_SEC) + + def test_AsMicroseconds(self): from _testcapi import PyTime_AsMicroseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (1 * US_TO_NS, 1), - (-2 * US_TO_NS, -2), - - # milliseconds - (1 * MS_TO_NS, 1000), - (-2 * MS_TO_NS, -2000), - - # seconds - (2 * SEC_TO_NS, 2000000), - (-3 * SEC_TO_NS, -3000000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (1, 0, HALF_EVEN), - (-1, -1, FLOOR), - (-1, 0, CEILING), - (-1, 0, HALF_EVEN), - - # seconds + nanoseconds - (1234 * US_TO_NS + 1, 1234, FLOOR), - (1234 * US_TO_NS + 1, 1235, CEILING), - (1234 * US_TO_NS + 1, 1234, HALF_EVEN), - (-1234 * US_TO_NS - 1, -1235, FLOOR), - (-1234 * US_TO_NS - 1, -1234, CEILING), - (-1234 * US_TO_NS - 1, -1234, HALF_EVEN), - - # half up - (-1500, -2, HALF_EVEN), - (-999, -1, HALF_EVEN), - (-500, 0, HALF_EVEN), - (500, 0, HALF_EVEN), - (999, 1, HALF_EVEN), - (1500, 2, HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) - - -@unittest.skipUnless(_testcapi is not None, - 'need the _testcapi module') -class TestOldPyTime(unittest.TestCase): + + self.check_int_rounding(PyTime_AsMicroseconds, + self.create_decimal_converter(US_TO_NS), + NS_TO_SEC) + + +class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): """ - Test the old _PyTime_t API: _PyTime_ObjectToXXX() functions. + Test the old C _PyTime_t API: _PyTime_ObjectToXXX() functions. """ - def setUp(self): - self.invalid_values = ( - -(2 ** 100), 2 ** 100, - -(2.0 ** 100.0), 2.0 ** 100.0, - ) - @support.cpython_only - def test_time_t(self): + # time_t is a 32-bit or 64-bit signed integer + OVERFLOW_SECONDS = 2 ** 64 + + def test_object_to_time_t(self): from _testcapi import pytime_object_to_time_t - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, time_t in ( - # int - (0, 0), - (-1, -1), - - # float - (1.0, 1), - (-1.0, -1), - ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, time_t, rnd in ( - (-1.9, -2, FLOOR), - (-1.9, -2, HALF_EVEN), - (-1.9, -1, CEILING), - - (1.9, 1, FLOOR), - (1.9, 2, HALF_EVEN), - (1.9, 2, CEILING), - - # half even - (-1.5, -2, HALF_EVEN), - (-0.9, -1, HALF_EVEN), - (-0.5, 0, HALF_EVEN), - ( 0.5, 0, HALF_EVEN), - ( 0.9, 1, HALF_EVEN), - ( 1.5, 2, HALF_EVEN), - ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_time_t, invalid, rnd) + self.check_int_rounding(pytime_object_to_time_t, + lambda secs: secs) + + self.check_float_rounding(pytime_object_to_time_t, + self.decimal_round) + + def create_converter(self, sec_to_unit): + def converter(secs): + floatpart, intpart = math.modf(secs) + intpart = int(intpart) + floatpart *= sec_to_unit + floatpart = self.decimal_round(floatpart) + if floatpart < 0: + floatpart += sec_to_unit + intpart -= 1 + elif floatpart >= sec_to_unit: + floatpart -= sec_to_unit + intpart += 1 + return (intpart, floatpart) + return converter def test_object_to_timeval(self): from _testcapi import pytime_object_to_timeval - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timeval in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1.0, (-1, 0)), - (1/2**6, (0, 15625)), - (-1/2**6, (-1, 984375)), - (-1e-6, (-1, 999999)), - (1e-6, (0, 1)), - ): - with self.subTest(obj=obj, round=rnd, timeval=timeval): - self.assertEqual(pytime_object_to_timeval(obj, rnd), - timeval) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timeval, rnd in ( - (-1e-7, (-1, 999999), FLOOR), - (-1e-7, (0, 0), CEILING), - (-1e-7, (0, 0), HALF_EVEN), - - (1e-7, (0, 0), FLOOR), - (1e-7, (0, 1), CEILING), - (1e-7, (0, 0), HALF_EVEN), - - (0.9999999, (0, 999999), FLOOR), - (0.9999999, (1, 0), CEILING), - (0.9999999, (1, 0), HALF_EVEN), - - # half even - (-1.5e-6, (-1, 999998), HALF_EVEN), - (-0.9e-6, (-1, 999999), HALF_EVEN), - (-0.5e-6, (0, 0), HALF_EVEN), - (0.5e-6, (0, 0), HALF_EVEN), - (0.9e-6, (0, 1), HALF_EVEN), - (1.5e-6, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timeval=timeval): - self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) - - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timeval, invalid, rnd) - - @support.cpython_only - def test_timespec(self): + self.check_int_rounding(pytime_object_to_timeval, + lambda secs: (secs, 0)) + + self.check_float_rounding(pytime_object_to_timeval, + self.create_converter(SEC_TO_US)) + + def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timespec in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1.0, (-1, 0)), - (-1e-9, (-1, 999999999)), - (1e-9, (0, 1)), - (-1/2**9, (-1, 998046875)), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), - timespec) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timespec, rnd in ( - (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), CEILING), - (-1e-10, (0, 0), HALF_EVEN), - - (1e-10, (0, 0), FLOOR), - (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_EVEN), - - (0.9999999999, (0, 999999999), FLOOR), - (0.9999999999, (1, 0), CEILING), - (0.9999999999, (1, 0), HALF_EVEN), - - # half even - (-1.5e-9, (-1, 999999998), HALF_EVEN), - (-0.9e-9, (-1, 999999999), HALF_EVEN), - (-0.5e-9, (0, 0), HALF_EVEN), - (0.5e-9, (0, 0), HALF_EVEN), - (0.9e-9, (0, 1), HALF_EVEN), - (1.5e-9, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Test OverflowError - rnd = FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timespec, invalid, rnd) + self.check_int_rounding(pytime_object_to_timespec, + lambda secs: (secs, 0)) + + self.check_float_rounding(pytime_object_to_timespec, + self.create_converter(SEC_TO_NS)) + if __name__ == "__main__": diff --git a/Python/pytime.c b/Python/pytime.c index 9735506..4c940c9 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -324,12 +324,16 @@ _PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t roun double _PyTime_AsSecondsDouble(_PyTime_t t) { - _PyTime_t sec, ns; - /* Divide using integers to avoid rounding issues on the integer part. - 1e-9 cannot be stored exactly in IEEE 64-bit. */ - sec = t / SEC_TO_NS; - ns = t % SEC_TO_NS; - return (double)sec + (double)ns * 1e-9; + if (t % SEC_TO_NS == 0) { + _PyTime_t secs; + /* Divide using integers to avoid rounding issues on the integer part. + 1e-9 cannot be stored exactly in IEEE 64-bit. */ + secs = t / SEC_TO_NS; + return (double)secs; + } + else { + return (double)t / 1e9; + } } PyObject * -- cgit v0.12 From f5d72f35e854575772d971e35d64d643465019c6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 9 Sep 2015 22:39:44 -0400 Subject: Simply deque repeat by reusing code in in-line repeat. Avoid unnecessary division. --- Modules/_collectionsmodule.c | 48 +++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ea3e88f..db27e92 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -539,32 +539,6 @@ deque_concat(dequeobject *deque, PyObject *other) static void deque_clear(dequeobject *deque); static PyObject * -deque_repeat(dequeobject *deque, Py_ssize_t n) -{ - dequeobject *new_deque; - PyObject *result; - - /* XXX add a special case for when maxlen is defined */ - if (n < 0) - n = 0; - else if (n > 0 && Py_SIZE(deque) > MAX_DEQUE_LEN / n) - return PyErr_NoMemory(); - - new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); - new_deque->maxlen = deque->maxlen; - - for ( ; n ; n--) { - result = deque_extend(new_deque, (PyObject *)deque); - if (result == NULL) { - Py_DECREF(new_deque); - return NULL; - } - Py_DECREF(result); - } - return (PyObject *)new_deque; -} - -static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) { Py_ssize_t i, size; @@ -583,10 +557,6 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } - if (size > MAX_DEQUE_LEN / n) { - return PyErr_NoMemory(); - } - if (size == 1) { /* common case, repeating a single element */ PyObject *item = deque->leftblock->data[deque->leftindex]; @@ -594,6 +564,9 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) if (deque->maxlen != -1 && n > deque->maxlen) n = deque->maxlen; + if (n > MAX_DEQUE_LEN) + return PyErr_NoMemory(); + for (i = 0 ; i < n-1 ; i++) { rv = deque_append(deque, item); if (rv == NULL) @@ -604,6 +577,10 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } + if ((size_t)size > MAX_DEQUE_LEN / (size_t)n) { + return PyErr_NoMemory(); + } + seq = PySequence_List((PyObject *)deque); if (seq == NULL) return seq; @@ -621,6 +598,17 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } +static PyObject * +deque_repeat(dequeobject *deque, Py_ssize_t n) +{ + dequeobject *new_deque; + + new_deque = (dequeobject *)deque_copy((PyObject *) deque); + if (new_deque == NULL) + return NULL; + return deque_inplace_repeat(new_deque, n); +} + /* The rotate() method is part of the public API and is used internally as a primitive for other methods. -- cgit v0.12 From 9c72f9b30a60d09c10b6cde23bd4b0546c6947af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 09:10:14 +0200 Subject: Fix test_time on Windows * Filter values which would overflow on conversion to the C long type (for timeval.tv_sec). * Adjust also the message of OverflowError on PyTime conversions * test_time: add debug information if a timestamp conversion fails --- Lib/test/test_time.py | 33 ++++++++++++++++++++++++--------- Python/pytime.c | 28 ++++++++-------------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index bd697ae..7f1613b 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -756,10 +756,15 @@ class CPyTimeTestCase: context.rounding = decimal_rnd for value in valid_values: - expected = expected_func(value) - self.assertEqual(pytime_converter(value, time_rnd), + debug_info = {'value': value, 'rounding': decimal_rnd} + try: + result = pytime_converter(value, time_rnd) + expected = expected_func(value) + except Exception as exc: + self.fail("Error on timestamp conversion: %s" % debug_info) + self.assertEqual(result, expected, - {'value': value, 'rounding': decimal_rnd}) + debug_info) # test overflow ns = self.OVERFLOW_SECONDS * SEC_TO_NS @@ -770,14 +775,15 @@ class CPyTimeTestCase: with self.assertRaises(OverflowError): pytime_converter(value, time_rnd) - def check_int_rounding(self, pytime_converter, expected_func, unit_to_sec=1, - value_filter=None): + def check_int_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): self._check_rounding(pytime_converter, expected_func, False, unit_to_sec, value_filter) - def check_float_rounding(self, pytime_converter, expected_func, unit_to_sec=1): + def check_float_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): self._check_rounding(pytime_converter, expected_func, - True, unit_to_sec) + True, unit_to_sec, value_filter) def decimal_round(self, x): d = decimal.Decimal(x) @@ -845,9 +851,19 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): us = us_converter(ns) return divmod(us, SEC_TO_US) + if sys.platform == 'win32': + from _testcapi import LONG_MIN, LONG_MAX + + # On Windows, timeval.tv_sec type is a C long + def seconds_filter(secs): + return LONG_MIN <= secs <= LONG_MAX + else: + seconds_filter = None + self.check_int_rounding(PyTime_AsTimeval, timeval_converter, - NS_TO_SEC) + NS_TO_SEC, + value_filter=seconds_filter) @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), 'need _testcapi.PyTime_AsTimespec') @@ -927,6 +943,5 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): self.create_converter(SEC_TO_NS)) - if __name__ == "__main__": unittest.main() diff --git a/Python/pytime.c b/Python/pytime.c index 4c940c9..243f756 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -288,18 +288,21 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, else { #ifdef HAVE_LONG_LONG PY_LONG_LONG sec; - sec = PyLong_AsLongLong(obj); assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + + sec = PyLong_AsLongLong(obj); #else long sec; - sec = PyLong_AsLong(obj); assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + + sec = PyLong_AsLong(obj); #endif if (sec == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) _PyTime_overflow(); return -1; } + *t = sec * unit_to_ns; if (*t / unit_to_ns != sec) { _PyTime_overflow(); @@ -404,27 +407,12 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, ns = t % SEC_TO_NS; #ifdef MS_WINDOWS - /* On Windows, timeval.tv_sec is a long (32 bit), - whereas time_t can be 64-bit. */ - assert(sizeof(tv->tv_sec) == sizeof(long)); -#if SIZEOF_TIME_T > SIZEOF_LONG - if (secs > LONG_MAX) { - secs = LONG_MAX; - res = -1; - } - else if (secs < LONG_MIN) { - secs = LONG_MIN; - res = -1; - } -#endif tv->tv_sec = (long)secs; #else - /* On OpenBSD 5.4, timeval.tv_sec is a long. - Example: long is 64-bit, whereas time_t is 32-bit. */ tv->tv_sec = secs; +#endif if ((_PyTime_t)tv->tv_sec != secs) res = -1; -#endif usec = (int)_PyTime_Divide(ns, US_TO_NS, round); if (usec < 0) { @@ -440,7 +428,7 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, tv->tv_usec = usec; if (res && raise) - _PyTime_overflow(); + error_time_t_overflow(); return res; } @@ -473,7 +461,7 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) ts->tv_nsec = nsec; if ((_PyTime_t)ts->tv_sec != secs) { - _PyTime_overflow(); + error_time_t_overflow(); return -1; } return 0; -- cgit v0.12 From 4237d3474c4792472009c6c6c5d46a7f7ab1410d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 10:10:39 +0200 Subject: Fix test_time on platform with 32-bit time_t type Filter values which would overflow when converted to a C time_t type. --- Lib/test/test_time.py | 23 ++++++++++++++++++----- Modules/_testcapimodule.c | 1 + 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 7f1613b..891c99d 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -680,6 +680,15 @@ class CPyTimeTestCase: """ OVERFLOW_SECONDS = None + def setUp(self): + from _testcapi import SIZEOF_TIME_T + bits = SIZEOF_TIME_T * 8 - 1 + self.time_t_min = -2 ** bits + self.time_t_max = 2 ** bits - 1 + + def time_t_filter(self, seconds): + return (self.time_t_min <= seconds <= self.time_t_max) + def _rounding_values(self, use_float): "Build timestamps used to test rounding." @@ -858,7 +867,7 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): def seconds_filter(secs): return LONG_MIN <= secs <= LONG_MAX else: - seconds_filter = None + seconds_filter = self.time_t_filter self.check_int_rounding(PyTime_AsTimeval, timeval_converter, @@ -875,7 +884,8 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), timespec_converter, - NS_TO_SEC) + NS_TO_SEC, + value_filter=self.time_t_filter) def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds @@ -904,7 +914,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_time_t self.check_int_rounding(pytime_object_to_time_t, - lambda secs: secs) + lambda secs: secs, + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_time_t, self.decimal_round) @@ -928,7 +939,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_timeval self.check_int_rounding(pytime_object_to_timeval, - lambda secs: (secs, 0)) + lambda secs: (secs, 0), + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timeval, self.create_converter(SEC_TO_US)) @@ -937,7 +949,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_timespec self.check_int_rounding(pytime_object_to_timespec, - lambda secs: (secs, 0)) + lambda secs: (secs, 0), + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timespec, self.create_converter(SEC_TO_NS)) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index fed3286..1f013c2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4114,6 +4114,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "PY_SSIZE_T_MAX", PyLong_FromSsize_t(PY_SSIZE_T_MAX)); PyModule_AddObject(m, "PY_SSIZE_T_MIN", PyLong_FromSsize_t(PY_SSIZE_T_MIN)); PyModule_AddObject(m, "SIZEOF_PYGC_HEAD", PyLong_FromSsize_t(sizeof(PyGC_Head))); + PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); -- cgit v0.12 From 350b51839aea3f5bd541b960ef1ec1b5a170854a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 11:45:06 +0200 Subject: Fix test_time on platform with 32-bit time_t type Filter also values for check_float_rounding(). --- Lib/test/test_time.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 891c99d..30b01d5 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -918,7 +918,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_time_t, - self.decimal_round) + self.decimal_round, + value_filter=self.time_t_filter) def create_converter(self, sec_to_unit): def converter(secs): @@ -943,7 +944,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timeval, - self.create_converter(SEC_TO_US)) + self.create_converter(SEC_TO_US), + value_filter=self.time_t_filter) def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec @@ -953,7 +955,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timespec, - self.create_converter(SEC_TO_NS)) + self.create_converter(SEC_TO_NS), + value_filter=self.time_t_filter) if __name__ == "__main__": -- cgit v0.12 From 1efbebaac2a29ae42f3de8d74071ea2eda660711 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 11:48:00 +0200 Subject: Try to fix test_time.test_AsSecondsDouble() on "x86 Gentoo Non-Debug with X 3.x" buildbot Use volatile keyword in _PyTime_Round() --- Python/pytime.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 243f756..9470636 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -75,12 +75,17 @@ _PyTime_RoundHalfEven(double x) static double _PyTime_Round(double x, _PyTime_round_t round) { + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d; + + d = x; if (round == _PyTime_ROUND_HALF_EVEN) - return _PyTime_RoundHalfEven(x); + d = _PyTime_RoundHalfEven(d); else if (round == _PyTime_ROUND_CEILING) - return ceil(x); + d = ceil(d); else - return floor(x); + d = floor(d); + return d; } static int -- cgit v0.12 From ff0ed3e71cb828103cf21442231686f1b348479b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 13:25:17 +0200 Subject: New try to fix test_time.test_AsSecondsDouble() on x86 buildbots. Use volatile keyword in _PyTime_AsSecondsDouble() --- Python/pytime.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 9470636..c82d598 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -332,16 +332,21 @@ _PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t roun double _PyTime_AsSecondsDouble(_PyTime_t t) { + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d; + if (t % SEC_TO_NS == 0) { _PyTime_t secs; /* Divide using integers to avoid rounding issues on the integer part. 1e-9 cannot be stored exactly in IEEE 64-bit. */ secs = t / SEC_TO_NS; - return (double)secs; + d = (double)secs; } else { - return (double)t / 1e9; + d = (double)t; + d /= 1e9; } + return d; } PyObject * -- cgit v0.12 From c60542b12bdf11487b959bbb304f4ea194be6a19 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 15:55:07 +0200 Subject: pytime: add _PyTime_check_mul_overflow() macro to avoid undefined behaviour Overflow test in test_FromSecondsObject() fails on FreeBSD 10.0 buildbot which uses clang. clang implements more aggressive optimization which gives different result than GCC on undefined behaviours. Check if a multiplication will overflow, instead of checking if a multiplicatin had overflowed, to avoid undefined behaviour. Add also debug information if the test on overflow fails. --- Lib/test/test_time.py | 3 ++- Python/pytime.c | 35 ++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 30b01d5..f883c45 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -781,7 +781,8 @@ class CPyTimeTestCase: overflow_values = convert_values(ns_timestamps) for time_rnd, _ in ROUNDING_MODES : for value in overflow_values: - with self.assertRaises(OverflowError): + debug_info = {'value': value, 'rounding': time_rnd} + with self.assertRaises(OverflowError, msg=debug_info): pytime_converter(value, time_rnd) def check_int_rounding(self, pytime_converter, expected_func, diff --git a/Python/pytime.c b/Python/pytime.c index c82d598..1b20dc7 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -7,6 +7,11 @@ #include /* mach_absolute_time(), mach_timebase_info() */ #endif +#define _PyTime_check_mul_overflow(a, b) \ + (assert(b > 0), \ + (_PyTime_t)(a) < _PyTime_MIN / (_PyTime_t)(b) \ + || _PyTime_MAX / (_PyTime_t)(b) < (_PyTime_t)(a)) + /* To millisecond (10^-3) */ #define SEC_TO_MS 1000 @@ -226,12 +231,15 @@ _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts, int raise) _PyTime_t t; int res = 0; - t = (_PyTime_t)ts->tv_sec * SEC_TO_NS; - if (t / SEC_TO_NS != ts->tv_sec) { + assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + t = (_PyTime_t)ts->tv_sec; + + if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { if (raise) _PyTime_overflow(); res = -1; } + t = t * SEC_TO_NS; t += ts->tv_nsec; @@ -245,12 +253,15 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) _PyTime_t t; int res = 0; - t = (_PyTime_t)tv->tv_sec * SEC_TO_NS; - if (t / SEC_TO_NS != tv->tv_sec) { + assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + t = (_PyTime_t)tv->tv_sec; + + if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { if (raise) _PyTime_overflow(); res = -1; } + t = t * SEC_TO_NS; t += (_PyTime_t)tv->tv_usec * US_TO_NS; @@ -308,11 +319,11 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, return -1; } - *t = sec * unit_to_ns; - if (*t / unit_to_ns != sec) { + if (_PyTime_check_mul_overflow(sec, unit_to_ns)) { _PyTime_overflow(); return -1; } + *t = sec * unit_to_ns; return 0; } } @@ -587,19 +598,20 @@ _PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) return pygettimeofday_new(t, info, 1); } - static int pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) { #if defined(MS_WINDOWS) - ULONGLONG result; + ULONGLONG ticks; + _PyTime_t t; assert(info == NULL || raise); - result = GetTickCount64(); + ticks = GetTickCount64(); + assert(sizeof(result) <= sizeof(_PyTime_t)); + t = (_PyTime_t)ticks; - *tp = result * MS_TO_NS; - if (*tp / MS_TO_NS != result) { + if (_PyTime_check_mul_overflow(t, MS_TO_NS)) { if (raise) { _PyTime_overflow(); return -1; @@ -607,6 +619,7 @@ pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) /* Hello, time traveler! */ assert(0); } + *tp = t * MS_TO_NS; if (info) { DWORD timeAdjustment, timeIncrement; -- cgit v0.12 From 51b93984449ec79908036cc7a1a5922458f87690 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 16:00:06 +0200 Subject: pytime: oops, fix typos on Windows --- Python/pytime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 1b20dc7..9d0b0fa 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -253,7 +253,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) _PyTime_t t; int res = 0; - assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + assert(sizeof(tv->tv_sec) <= sizeof(_PyTime_t)); t = (_PyTime_t)tv->tv_sec; if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { @@ -608,7 +608,7 @@ pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) assert(info == NULL || raise); ticks = GetTickCount64(); - assert(sizeof(result) <= sizeof(_PyTime_t)); + assert(sizeof(ticks) <= sizeof(_PyTime_t)); t = (_PyTime_t)ticks; if (_PyTime_check_mul_overflow(t, MS_TO_NS)) { -- cgit v0.12 From 67c78b5421ce9ebba1399e5935879ca01d5ac091 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 12 Sep 2015 11:00:20 -0400 Subject: In-line the append operations inside deque_inplace_repeat(). --- Modules/_collectionsmodule.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index db27e92..7f81460 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -567,12 +567,26 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) if (n > MAX_DEQUE_LEN) return PyErr_NoMemory(); + deque->state++; for (i = 0 ; i < n-1 ; i++) { - rv = deque_append(deque, item); - if (rv == NULL) - return NULL; - Py_DECREF(rv); + if (deque->rightindex == BLOCKLEN - 1) { + block *b = newblock(Py_SIZE(deque) + i); + if (b == NULL) { + Py_SIZE(deque) += i; + return NULL; + } + b->leftlink = deque->rightblock; + CHECK_END(deque->rightblock->rightlink); + deque->rightblock->rightlink = b; + deque->rightblock = b; + MARK_END(b->rightlink); + deque->rightindex = -1; + } + deque->rightindex++; + Py_INCREF(item); + deque->rightblock->data[deque->rightindex] = item; } + Py_SIZE(deque) += i; Py_INCREF(deque); return (PyObject *)deque; } -- cgit v0.12 From 6408dc82fabf5bbbc8b7ed7287e34b4e0080191d Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 12 Sep 2015 18:53:36 -0400 Subject: Fixed indentation. --- Parser/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 04baeaf..5046fa5 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1741,7 +1741,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) else { end_quote_size = 0; if (c == '\\') - c = tok_nextc(tok); /* skip escaped char */ + c = tok_nextc(tok); /* skip escaped char */ } } -- cgit v0.12 From 95e2cc5d122c8469c3fd3040f43311547a561663 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Sep 2015 02:41:18 -0400 Subject: Fix refcount. --- Modules/_collectionsmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 7f81460..17233e4 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -616,11 +616,14 @@ static PyObject * deque_repeat(dequeobject *deque, Py_ssize_t n) { dequeobject *new_deque; + PyObject *rv; new_deque = (dequeobject *)deque_copy((PyObject *) deque); if (new_deque == NULL) return NULL; - return deque_inplace_repeat(new_deque, n); + rv = deque_inplace_repeat(new_deque, n); + Py_DECREF(new_deque); + return rv; } /* The rotate() method is part of the public API and is used internally -- cgit v0.12 From e4f3467df17873db60dd1e8daf61e0c629abb130 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Sep 2015 19:27:01 -0400 Subject: Add an exact type match fast path for deque_copy(). --- Modules/_collectionsmodule.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 17233e4..49a46a1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1205,6 +1205,22 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) static PyObject * deque_copy(PyObject *deque) { + if (Py_TYPE(deque) == &deque_type) { + dequeobject *new_deque; + PyObject *rv; + + new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); + if (new_deque == NULL) + return NULL; + new_deque->maxlen = ((dequeobject *)deque)->maxlen; + rv = deque_extend(new_deque, deque); + if (rv != NULL) { + Py_DECREF(rv); + return (PyObject *)new_deque; + } + Py_DECREF(new_deque); + return NULL; + } if (((dequeobject *)deque)->maxlen == -1) return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); else -- cgit v0.12 From ad26225e1a62f32848669b09326835dfd0369a79 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 14 Sep 2015 01:03:04 -0400 Subject: Tighten inner-loop for deque_inplace_repeat(). --- Modules/_collectionsmodule.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 49a46a1..087f8e5 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -568,7 +568,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return PyErr_NoMemory(); deque->state++; - for (i = 0 ; i < n-1 ; i++) { + for (i = 0 ; i < n-1 ; ) { if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque) + i); if (b == NULL) { @@ -582,9 +582,11 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) MARK_END(b->rightlink); deque->rightindex = -1; } - deque->rightindex++; - Py_INCREF(item); - deque->rightblock->data[deque->rightindex] = item; + for ( ; i < n-1 && deque->rightindex != BLOCKLEN - 1 ; i++) { + deque->rightindex++; + Py_INCREF(item); + deque->rightblock->data[deque->rightindex] = item; + } } Py_SIZE(deque) += i; Py_INCREF(deque); -- cgit v0.12 From f11d0d2c0d44c812534067c76e61c366b5b79a06 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 12:15:59 +0200 Subject: Issue #25122: try to debug test_eintr hang on FreeBSD * Add verbose mode to test_eintr * Always enable verbose mode in test_eintr * Use faulthandler.dump_traceback_later() with a timeout of 15 minutes in eintr_tester.py --- Lib/test/eintrdata/eintr_tester.py | 8 ++++++++ Lib/test/test_eintr.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 0616df6..b96f09b 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import faulthandler import io import os import select @@ -36,12 +37,19 @@ class EINTRBaseTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) + if hasattr(faulthandler, 'dump_traceback_later'): + # Most tests take less than 30 seconds, so 15 minutes should be + # enough. dump_traceback_later() is implemented with a thread, but + # pthread_sigmask() is used to mask all signaled on this thread. + faulthandler.dump_traceback_later(5 * 60, exit=True) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 111ead3..1d1d16e 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -1,5 +1,7 @@ import os import signal +import subprocess +import sys import unittest from test import support @@ -14,7 +16,15 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - script_helper.assert_python_ok(tester) + + # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD + if True: #support.verbose: + args = [sys.executable, tester] + with subprocess.Popen(args, stdout=sys.stderr) as proc: + exitcode = proc.wait() + self.assertEqual(exitcode, 0) + else: + script_helper.assert_python_ok(tester) if __name__ == "__main__": -- cgit v0.12 From 44879a0b18048e2d722a866e3ec0485012bce732 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 22:38:09 +0200 Subject: Issue #25122: Fix test_eintr, kill child process on error Some test_eintr hangs on waiting for the child process completion if an error occurred on the parent. Kill the child process on error (in the parent) to avoid the hang. --- Lib/test/eintrdata/eintr_tester.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index b96f09b..e1330a7 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import contextlib import faulthandler import io import os @@ -21,6 +22,16 @@ import unittest from test import support +@contextlib.contextmanager +def kill_on_error(proc): + """Context manager killing the subprocess if a Python exception is raised.""" + with proc: + try: + yield proc + except: + proc.kill() + raise + @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") class EINTRBaseTest(unittest.TestCase): @@ -38,7 +49,7 @@ class EINTRBaseTest(unittest.TestCase): def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 15 minutes should be + # Most tests take less than 30 seconds, so 5 minutes should be # enough. dump_traceback_later() is implemented with a thread, but # pthread_sigmask() is used to mask all signaled on this thread. faulthandler.dump_traceback_later(5 * 60, exit=True) @@ -120,7 +131,8 @@ class OSEINTRTest(EINTRBaseTest): ' os.write(wr, data)', )) - with self.subprocess(code, str(wr), pass_fds=[wr]) as proc: + proc = self.subprocess(code, str(wr), pass_fds=[wr]) + with kill_on_error(proc): os.close(wr) for data in datas: self.assertEqual(data, os.read(rd, len(data))) @@ -156,7 +168,8 @@ class OSEINTRTest(EINTRBaseTest): ' % (len(value), data_len))', )) - with self.subprocess(code, str(rd), pass_fds=[rd]) as proc: + proc = self.subprocess(code, str(rd), pass_fds=[rd]) + with kill_on_error(proc): os.close(rd) written = 0 while written < len(data): @@ -198,7 +211,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = wr.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) - with proc: + with kill_on_error(proc): wr.close() for data in datas: self.assertEqual(data, recv_func(rd, len(data))) @@ -248,7 +261,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = rd.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) - with proc: + with kill_on_error(proc): rd.close() written = 0 while written < len(data): @@ -288,7 +301,8 @@ class SocketEINTRTest(EINTRBaseTest): ' time.sleep(sleep_time)', )) - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): client_sock, _ = sock.accept() client_sock.close() self.assertEqual(proc.wait(), 0) @@ -315,7 +329,8 @@ class SocketEINTRTest(EINTRBaseTest): do_open_close_reader, )) - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): do_open_close_writer(filename) self.assertEqual(proc.wait(), 0) @@ -372,7 +387,8 @@ class SignalEINTRTest(EINTRBaseTest): )) t0 = time.monotonic() - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 -- cgit v0.12 From 3731bbe8b10f7f31240fbffe7d9617365f7814dc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 22:55:52 +0200 Subject: Issue #25122: test_eintr: don't redirect stdout to stderr sys.stderr is sometimes a StringIO. The redirection was just a hack to see eintr_tester.py output in red in the buildbot output. --- Lib/test/test_eintr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 1d1d16e..a1907e8 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -20,7 +20,7 @@ class EINTRTests(unittest.TestCase): # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD if True: #support.verbose: args = [sys.executable, tester] - with subprocess.Popen(args, stdout=sys.stderr) as proc: + with subprocess.Popen(args) as proc: exitcode = proc.wait() self.assertEqual(exitcode, 0) else: -- cgit v0.12 From 1e0f8ecdd8a4fb1fa21f4da1a8849ffd42b966f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 23:59:00 +0200 Subject: Issue #25122: optimize test_eintr Fix test_write(): copy support.PIPE_MAX_SIZE bytes, not support.PIPE_MAX_SIZE*3 bytes. --- Lib/test/eintrdata/eintr_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1330a7..24e809a 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -144,14 +144,14 @@ class OSEINTRTest(EINTRBaseTest): # rd closed explicitly by parent # we must write enough data for the write() to block - data = b"xyz" * support.PIPE_MAX_SIZE + data = b"x" * support.PIPE_MAX_SIZE code = '\n'.join(( 'import io, os, sys, time', '', 'rd = int(sys.argv[1])', 'sleep_time = %r' % self.sleep_time, - 'data = b"xyz" * %s' % support.PIPE_MAX_SIZE, + 'data = b"x" * %s' % support.PIPE_MAX_SIZE, 'data_len = len(data)', '', '# let the parent block on write()', -- cgit v0.12 From e41b4712fe884551aef7dd3b8852907dcc310a29 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Sep 2015 09:23:28 +0200 Subject: Issue #25122: add debug traces to test_eintr.test_open() --- Lib/test/eintrdata/eintr_tester.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 24e809a..c3c08fe 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -326,12 +326,26 @@ class SocketEINTRTest(EINTRBaseTest): '# let the parent block', 'time.sleep(sleep_time)', '', + 'print("try to open %a fifo for reading, pid %s, ppid %s"' + ' % (path, os.getpid(), os.getppid()),' + ' flush=True)', + '', do_open_close_reader, + '', + 'print("%a fifo opened for reading and closed, pid %s, ppid %s"' + ' % (path, os.getpid(), os.getppid()),' + ' flush=True)', )) proc = self.subprocess(code) with kill_on_error(proc): + print("try to open %a fifo for writing, pid %s" + % (filename, os.getpid()), + flush=True) do_open_close_writer(filename) + print("%a fifo opened for writing and closed, pid %s" + % (filename, os.getpid()), + flush=True) self.assertEqual(proc.wait(), 0) -- cgit v0.12 From 6db1fd5fb8b2287cd4713c95f0df451e375c8853 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 21:49:12 -0700 Subject: Close issue24840: Enum._value_ is queried for bool(); original patch by Mike Lundy --- Lib/enum.py | 3 +++ Lib/test/test_enum.py | 7 +++++++ Misc/ACKS | 1 + 3 files changed, 11 insertions(+) diff --git a/Lib/enum.py b/Lib/enum.py index c28f345..616b2ea 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -476,6 +476,9 @@ class Enum(metaclass=EnumMeta): def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) + def __bool__(self): + return bool(self._value_) + def __dir__(self): added_behavior = [ m diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 4b5d0d0..0f7b769 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -270,6 +270,13 @@ class TestEnum(unittest.TestCase): class Wrong(Enum): _any_name_ = 9 + def test_bool(self): + class Logic(Enum): + true = True + false = False + self.assertTrue(Logic.true) + self.assertFalse(Logic.false) + def test_contains(self): Season = self.Season self.assertIn(Season.AUTUMN, Season) diff --git a/Misc/ACKS b/Misc/ACKS index e26be93..43b4c99 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -877,6 +877,7 @@ Kang-Hao (Kenny) Lu Lukas Lueg Loren Luke Fredrik Lundh +Mike Lundy Zhongyue Luo Mark Lutz Taras Lyapun -- cgit v0.12 From e5754ab0c4ab44cd1e7e5484ce07e640964387dd Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 22:03:52 -0700 Subject: Close issue25147: use C implementation of OrderedDict --- Lib/enum.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/enum.py b/Lib/enum.py index 616b2ea..6284b9b 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,7 +1,12 @@ import sys -from collections import OrderedDict from types import MappingProxyType, DynamicClassAttribute +try: + from _collections import OrderedDict +except ImportError: + from collections import OrderedDict + + __all__ = ['Enum', 'IntEnum', 'unique'] -- cgit v0.12 From c791507e1fd1ce218f40c7bd4b85076c278ae8d9 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 22:55:40 -0700 Subject: Issue 25147: add reason for using _collections --- Lib/enum.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/enum.py b/Lib/enum.py index 6284b9b..8d04e2d 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,6 +1,7 @@ import sys from types import MappingProxyType, DynamicClassAttribute +# try _collections first to reduce startup cost try: from _collections import OrderedDict except ImportError: -- cgit v0.12 From baab5f73417dabd5e5dc6fe9e0dc3b917f812fb2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 11:23:42 +0200 Subject: Issue #25122: Fix test_eintr.test_open() on FreeBSD Skip test_open() and test_os_open(): both tests uses a FIFO and signals, but there is a bug in the FreeBSD kernel which blocks the test. Skip the tests until the bug is fixed in FreeBSD kernel. Remove also debug traces from test_eintr: * stop using faulthandler to have a timeout * remove print() Write also open and close on two lines in test_open() and test_os_open() tests. If these tests block again, we can know if the test is stuck at open or close. test_eintr: don't always run the test in debug mode. --- Lib/test/eintrdata/eintr_tester.py | 44 +++++++++++++++----------------------- Lib/test/test_eintr.py | 3 +-- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index c3c08fe..c2407ca 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -9,7 +9,6 @@ sub-second periodicity (contrarily to signal()). """ import contextlib -import faulthandler import io import os import select @@ -48,19 +47,12 @@ class EINTRBaseTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) - if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 5 minutes should be - # enough. dump_traceback_later() is implemented with a thread, but - # pthread_sigmask() is used to mask all signaled on this thread. - faulthandler.dump_traceback_later(5 * 60, exit=True) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) - if hasattr(faulthandler, 'cancel_dump_traceback_later'): - faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): @@ -307,6 +299,11 @@ class SocketEINTRTest(EINTRBaseTest): client_sock.close() self.assertEqual(proc.wait(), 0) + # Issue #25122: There is a race condition in the FreeBSD kernel on + # handling signals in the FIFO device. Skip the test until the bug is + # fixed in the kernel. Bet that the bug will be fixed in FreeBSD 11. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 + @support.requires_freebsd_version(11) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') def _test_open(self, do_open_close_reader, do_open_close_writer): filename = support.TESTFN @@ -326,36 +323,29 @@ class SocketEINTRTest(EINTRBaseTest): '# let the parent block', 'time.sleep(sleep_time)', '', - 'print("try to open %a fifo for reading, pid %s, ppid %s"' - ' % (path, os.getpid(), os.getppid()),' - ' flush=True)', - '', do_open_close_reader, - '', - 'print("%a fifo opened for reading and closed, pid %s, ppid %s"' - ' % (path, os.getpid(), os.getppid()),' - ' flush=True)', )) proc = self.subprocess(code) with kill_on_error(proc): - print("try to open %a fifo for writing, pid %s" - % (filename, os.getpid()), - flush=True) do_open_close_writer(filename) - print("%a fifo opened for writing and closed, pid %s" - % (filename, os.getpid()), - flush=True) - self.assertEqual(proc.wait(), 0) + def python_open(self, path): + fp = open(path, 'w') + fp.close() + def test_open(self): - self._test_open("open(path, 'r').close()", - lambda path: open(path, 'w').close()) + self._test_open("fp = open(path, 'r')\nfp.close()", + self.python_open) + + def os_open(self, path): + fd = os.open(path, os.O_WRONLY) + os.close(fd) def test_os_open(self): - self._test_open("os.close(os.open(path, os.O_RDONLY))", - lambda path: os.close(os.open(path, os.O_WRONLY))) + self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", + self.os_open) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index a1907e8..aabad83 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -17,8 +17,7 @@ class EINTRTests(unittest.TestCase): # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD - if True: #support.verbose: + if support.verbose: args = [sys.executable, tester] with subprocess.Popen(args) as proc: exitcode = proc.wait() -- cgit v0.12 From 1e2b6882fc28a3cded227f55e4dc3f937afd678e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 13:23:02 +0200 Subject: Issue #25155: Add _PyTime_AsTimevalTime_t() function On Windows, the tv_sec field of the timeval structure has the type C long, whereas it has the type C time_t on all other platforms. A C long has a size of 32 bits (signed inter, 1 bit for the sign, 31 bits for the value) which is not enough to store an Epoch timestamp after the year 2038. Add the _PyTime_AsTimevalTime_t() function written for datetime.datetime.now(): convert a _PyTime_t timestamp to a (secs, us) tuple where secs type is time_t. It allows to support dates after the year 2038 on Windows. Enhance also _PyTime_AsTimeval_impl() to detect overflow on the number of seconds when rounding the number of microseconds. --- Include/pytime.h | 12 +++++++ Modules/_datetimemodule.c | 9 +++--- Python/pytime.c | 79 +++++++++++++++++++++++++++++++++++------------ 3 files changed, 77 insertions(+), 23 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 5de756a..98612e1 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -120,6 +120,18 @@ PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); +/* Convert a timestamp to a number of seconds (secs) and microseconds (us). + us is always positive. This function is similar to _PyTime_AsTimeval() + except that secs is always a time_t type, whereas the timeval structure + uses a C long for tv_sec on Windows. + Raise an exception and return -1 if the conversion overflowed, + return 0 on success. */ +PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( + _PyTime_t t, + time_t *secs, + int *us, + _PyTime_round_t round); + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) /* Convert a timestamp to a timespec structure (nanosecond resolution). tv_nsec is always positive. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index bbc51c6..5289222 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4117,13 +4117,14 @@ static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { _PyTime_t ts = _PyTime_GetSystemClock(); - struct timeval tv; + time_t secs; + int us; - if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_FLOOR) < 0) + if (_PyTime_AsTimevalTime_t(ts, &secs, &us, _PyTime_ROUND_FLOOR) < 0) return NULL; - assert(0 <= tv.tv_usec && tv.tv_usec <= 999999); + assert(0 <= us && us <= 999999); - return datetime_from_timet_and_us(cls, f, tv.tv_sec, tv.tv_usec, tzinfo); + return datetime_from_timet_and_us(cls, f, secs, us, tzinfo); } /*[clinic input] diff --git a/Python/pytime.c b/Python/pytime.c index 9d0b0fa..9889a3b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -417,54 +417,95 @@ _PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) } static int -_PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, - int raise) +_PyTime_AsTimeval_impl(_PyTime_t t, _PyTime_t *p_secs, int *p_us, + _PyTime_round_t round) { _PyTime_t secs, ns; - int res = 0; int usec; + int res = 0; secs = t / SEC_TO_NS; ns = t % SEC_TO_NS; -#ifdef MS_WINDOWS - tv->tv_sec = (long)secs; -#else - tv->tv_sec = secs; -#endif - if ((_PyTime_t)tv->tv_sec != secs) - res = -1; - usec = (int)_PyTime_Divide(ns, US_TO_NS, round); if (usec < 0) { usec += SEC_TO_US; - tv->tv_sec -= 1; + if (secs != _PyTime_MIN) + secs -= 1; + else + res = -1; } else if (usec >= SEC_TO_US) { usec -= SEC_TO_US; - tv->tv_sec += 1; + if (secs != _PyTime_MAX) + secs += 1; + else + res = -1; } - assert(0 <= usec && usec < SEC_TO_US); - tv->tv_usec = usec; - if (res && raise) - error_time_t_overflow(); + *p_secs = secs; + *p_us = usec; + return res; } +static int +_PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, + _PyTime_round_t round, int raise) +{ + _PyTime_t secs; + int us; + int res; + + res = _PyTime_AsTimeval_impl(t, &secs, &us, round); + +#ifdef MS_WINDOWS + tv->tv_sec = (long)secs; +#else + tv->tv_sec = secs; +#endif + tv->tv_usec = us; + + if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { + if (raise) + error_time_t_overflow(); + return -1; + } + return 0; +} + int _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) { - return _PyTime_AsTimeval_impl(t, tv, round, 1); + return _PyTime_AsTimevalStruct_impl(t, tv, round, 1); } int _PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) { - return _PyTime_AsTimeval_impl(t, tv, round, 0); + return _PyTime_AsTimevalStruct_impl(t, tv, round, 0); } +int +_PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, + _PyTime_round_t round) +{ + _PyTime_t secs; + int res; + + res = _PyTime_AsTimeval_impl(t, &secs, us, round); + + *p_secs = secs; + + if (res < 0 || (_PyTime_t)*p_secs != secs) { + error_time_t_overflow(); + return -1; + } + return 0; +} + + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) int _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) -- cgit v0.12 From 5ebfe42cdf7ba4ba0a573e2990960fea460540b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 14:52:15 +0200 Subject: Oops, fix test_microsecond_rounding() Test self.theclass, not datetime. Regression introduced by manual tests. --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 5d2df32..a6c7421 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1851,8 +1851,8 @@ class TestDateTime(TestDate): 18000 + 3600 + 2*60 + 3 + 4*1e-6) def test_microsecond_rounding(self): - for fts in (datetime.fromtimestamp, - self.theclass.utcfromtimestamp): + for fts in [self.theclass.fromtimestamp, + self.theclass.utcfromtimestamp]: zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) -- cgit v0.12 From 3abf44e48f973d5ffba779471a6c4dab00be2f77 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 15:38:37 +0200 Subject: Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the getrandom() function instead of the getentropy() function. The getentropy() function is blocking to generate very good quality entropy, os.urandom() doesn't need such high-quality entropy. --- Misc/NEWS | 5 +++++ Python/random.c | 49 ++++++++++++++++++++++++++++++++++--------------- configure | 43 ++++++++++++++++++++++++++++++++++++++++--- configure.ac | 31 ++++++++++++++++++++++++++++--- pyconfig.h.in | 3 +++ 5 files changed, 110 insertions(+), 21 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 827d788..77efd05 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the + getrandom() function instead of the getentropy() function. The getentropy() + function is blocking to generate very good quality entropy, os.urandom() + doesn't need such high-quality entropy. + - Issue #9232: Modify Python's grammar to allow trailing commas in the argument list of a function declaration. For example, "def f(*, a = 3,): pass" is now legal. Patch from Mark Dickinson. diff --git a/Python/random.c b/Python/random.c index ea09e84..8f3e6d6 100644 --- a/Python/random.c +++ b/Python/random.c @@ -6,7 +6,9 @@ # ifdef HAVE_SYS_STAT_H # include # endif -# ifdef HAVE_GETRANDOM_SYSCALL +# ifdef HAVE_GETRANDOM +# include +# elif defined(HAVE_GETRANDOM_SYSCALL) # include # endif #endif @@ -70,7 +72,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) return 0; } -#elif HAVE_GETENTROPY +#elif defined(HAVE_GETENTROPY) && !defined(sun) +#define PY_GETENTROPY + /* Fill buffer with size pseudo-random bytes generated by getentropy(). Return 0 on success, or raise an exception and return -1 on error. @@ -105,16 +109,19 @@ py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal) return 0; } -#else /* !HAVE_GETENTROPY */ +#else + +#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) +#define PY_GETRANDOM -#ifdef HAVE_GETRANDOM_SYSCALL static int py_getrandom(void *buffer, Py_ssize_t size, int raise) { - /* is getrandom() supported by the running kernel? - * need Linux kernel 3.17 or later */ + /* Is getrandom() supported by the running kernel? + * Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */ static int getrandom_works = 1; - /* Use /dev/urandom, block if the kernel has no entropy */ + /* Use non-blocking /dev/urandom device. On Linux at boot, the getrandom() + * syscall blocks until /dev/urandom is initialized with enough entropy. */ const int flags = 0; int n; @@ -124,7 +131,18 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise) while (0 < size) { errno = 0; - /* Use syscall() because the libc doesn't expose getrandom() yet, see: +#ifdef HAVE_GETRANDOM + if (raise) { + Py_BEGIN_ALLOW_THREADS + n = getrandom(buffer, size, flags); + Py_END_ALLOW_THREADS + } + else { + n = getrandom(buffer, size, flags); + } +#else + /* On Linux, use the syscall() function because the GNU libc doesn't + * expose the Linux getrandom() syscall yet. See: * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ if (raise) { Py_BEGIN_ALLOW_THREADS @@ -134,6 +152,7 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise) else { n = syscall(SYS_getrandom, buffer, size, flags); } +#endif if (n < 0) { if (errno == ENOSYS) { @@ -182,7 +201,7 @@ dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) assert (0 < size); -#ifdef HAVE_GETRANDOM_SYSCALL +#ifdef PY_GETRANDOM if (py_getrandom(buffer, size, 0) == 1) return; /* getrandom() is not supported by the running kernel, fall back @@ -218,14 +237,14 @@ dev_urandom_python(char *buffer, Py_ssize_t size) int fd; Py_ssize_t n; struct _Py_stat_struct st; -#ifdef HAVE_GETRANDOM_SYSCALL +#ifdef PY_GETRANDOM int res; #endif if (size <= 0) return 0; -#ifdef HAVE_GETRANDOM_SYSCALL +#ifdef PY_GETRANDOM res = py_getrandom(buffer, size, 1); if (res < 0) return -1; @@ -304,7 +323,7 @@ dev_urandom_close(void) } } -#endif /* HAVE_GETENTROPY */ +#endif /* Fill buffer with pseudo-random bytes generated by a linear congruent generator (LCG): @@ -345,7 +364,7 @@ _PyOS_URandom(void *buffer, Py_ssize_t size) #ifdef MS_WINDOWS return win32_urandom((unsigned char *)buffer, size, 1); -#elif HAVE_GETENTROPY +#elif PY_GETENTROPY return py_getentropy(buffer, size, 0); #else return dev_urandom_python((char*)buffer, size); @@ -392,7 +411,7 @@ _PyRandom_Init(void) else { #ifdef MS_WINDOWS (void)win32_urandom(secret, secret_size, 0); -#elif HAVE_GETENTROPY +#elif PY_GETENTROPY (void)py_getentropy(secret, secret_size, 1); #else dev_urandom_noraise(secret, secret_size); @@ -408,7 +427,7 @@ _PyRandom_Fini(void) CryptReleaseContext(hCryptProv, 0); hCryptProv = 0; } -#elif HAVE_GETENTROPY +#elif PY_GETENTROPY /* nothing to clean */ #else dev_urandom_close(); diff --git a/configure b/configure index 95d953a..f7b1839 100755 --- a/configure +++ b/configure @@ -16005,11 +16005,11 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include int main() { - const int flags = 0; char buffer[1]; - int n; + const size_t buflen = sizeof(buffer); + const int flags = 0; /* ignore the result, Python checks for ENOSYS at runtime */ - (void)syscall(SYS_getrandom, buffer, sizeof(buffer), flags); + (void)syscall(SYS_getrandom, buffer, buflen, flags); return 0; } @@ -16031,6 +16031,43 @@ $as_echo "#define HAVE_GETRANDOM_SYSCALL 1" >>confdefs.h fi +# check if the getrandom() function is available +# the test was written for the Solaris function of +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for the getrandom() function" >&5 +$as_echo_n "checking for the getrandom() function... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + + int main() { + char buffer[1]; + const size_t buflen = sizeof(buffer); + const int flags = 0; + /* ignore the result, Python checks for ENOSYS at runtime */ + (void)getrandom(buffer, buflen, flags); + return 0; + } + + +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + have_getrandom=yes +else + have_getrandom=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $have_getrandom" >&5 +$as_echo "$have_getrandom" >&6; } + +if test "$have_getrandom" = yes; then + +$as_echo "#define HAVE_GETRANDOM 1" >>confdefs.h + +fi + # generate output files ac_config_files="$ac_config_files Makefile.pre Modules/Setup.config Misc/python.pc Misc/python-config.sh" diff --git a/configure.ac b/configure.ac index 668234d..180a421 100644 --- a/configure.ac +++ b/configure.ac @@ -5111,11 +5111,11 @@ AC_LINK_IFELSE( #include int main() { - const int flags = 0; char buffer[1]; - int n; + const size_t buflen = sizeof(buffer); + const int flags = 0; /* ignore the result, Python checks for ENOSYS at runtime */ - (void)syscall(SYS_getrandom, buffer, sizeof(buffer), flags); + (void)syscall(SYS_getrandom, buffer, buflen, flags); return 0; } ]]) @@ -5127,6 +5127,31 @@ if test "$have_getrandom_syscall" = yes; then [Define to 1 if the Linux getrandom() syscall is available]) fi +# check if the getrandom() function is available +# the test was written for the Solaris function of +AC_MSG_CHECKING(for the getrandom() function) +AC_LINK_IFELSE( +[ + AC_LANG_SOURCE([[ + #include + + int main() { + char buffer[1]; + const size_t buflen = sizeof(buffer); + const int flags = 0; + /* ignore the result, Python checks for ENOSYS at runtime */ + (void)getrandom(buffer, buflen, flags); + return 0; + } + ]]) +],[have_getrandom=yes],[have_getrandom=no]) +AC_MSG_RESULT($have_getrandom) + +if test "$have_getrandom" = yes; then + AC_DEFINE(HAVE_GETRANDOM, 1, + [Define to 1 if the getrandom() function is available]) +fi + # generate output files AC_CONFIG_FILES(Makefile.pre Modules/Setup.config Misc/python.pc Misc/python-config.sh) AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 0d40c94..b0cafb3 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -395,6 +395,9 @@ /* Define to 1 if you have the `getpwent' function. */ #undef HAVE_GETPWENT +/* Define to 1 if the getrandom() function is available */ +#undef HAVE_GETRANDOM + /* Define to 1 if the Linux getrandom() syscall is available */ #undef HAVE_GETRANDOM_SYSCALL -- cgit v0.12 From d8f432a98c6bbe143002f45e181b7e5243d1166e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 16:24:31 +0200 Subject: Issue #25003: Skip test_os.URandomFDTests on Solaris 11.3 and newer When os.urandom() is implemented with the getrandom() function, it doesn't use a file descriptor. --- Lib/test/test_os.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b73b64b..d4b9e9c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1225,13 +1225,15 @@ class URandomTests(unittest.TestCase): self.assertNotEqual(data1, data2) -HAVE_GETENTROPY = (sysconfig.get_config_var('HAVE_GETENTROPY') == 1) -HAVE_GETRANDOM = (sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1) - -@unittest.skipIf(HAVE_GETENTROPY, - "getentropy() does not use a file descriptor") -@unittest.skipIf(HAVE_GETRANDOM, - "getrandom() does not use a file descriptor") +# os.urandom() doesn't use a file descriptor when it is implemented with the +# getentropy() function, the getrandom() function or the getrandom() syscall +OS_URANDOM_DONT_USE_FD = ( + sysconfig.get_config_var('HAVE_GETENTROPY') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1) + +@unittest.skipIf(OS_URANDOM_DONT_USE_FD , + "os.random() does not use a file descriptor") class URandomFDTests(unittest.TestCase): @unittest.skipUnless(resource, "test requires the resource module") def test_urandom_failure(self): -- cgit v0.12 From 0e14e6610b3cf692377996fad569496191a3f731 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 19 Sep 2015 00:21:33 -0600 Subject: Hoist constant expression out of an inner loop --- Modules/_collectionsmodule.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 087f8e5..dbe44e1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -371,6 +371,7 @@ static PyObject * deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -417,7 +418,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - deque_trim_left(deque); + if (trim) + deque_trim_left(deque); } if (PyErr_Occurred()) { Py_DECREF(it); @@ -434,6 +436,7 @@ static PyObject * deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -480,7 +483,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - deque_trim_right(deque); + if (trim) + deque_trim_right(deque); } if (PyErr_Occurred()) { Py_DECREF(it); -- cgit v0.12 From aed8830af3bb5a79878cf0f603ebbd8a37f5b36e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 19 Sep 2015 09:05:42 -0700 Subject: Add a fast path (no iterator creation) for a common case for repeating deques of size 1 --- Lib/test/test_deque.py | 9 +++++++++ Modules/_collectionsmodule.c | 15 +++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 8718716..c61e80b 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -654,6 +654,15 @@ class TestBasic(unittest.TestCase): self.assertNotEqual(id(d), id(e)) self.assertEqual(list(d), list(e)) + for i in range(5): + for maxlen in range(-1, 6): + s = [random.random() for j in range(i)] + d = deque(s) if maxlen == -1 else deque(s, maxlen) + e = d.copy() + self.assertEqual(d, e) + self.assertEqual(d.maxlen, e.maxlen) + self.assertTrue(all(x is y for x, y in zip(d, e))) + def test_copy_method(self): mut = [10] d = deque([mut]) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index dbe44e1..3e2c69f 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1211,6 +1211,7 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) static PyObject * deque_copy(PyObject *deque) { + dequeobject *old_deque = (dequeobject *)deque; if (Py_TYPE(deque) == &deque_type) { dequeobject *new_deque; PyObject *rv; @@ -1218,8 +1219,14 @@ deque_copy(PyObject *deque) new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); if (new_deque == NULL) return NULL; - new_deque->maxlen = ((dequeobject *)deque)->maxlen; - rv = deque_extend(new_deque, deque); + new_deque->maxlen = old_deque->maxlen; + /* Fast path for the deque_repeat() common case where len(deque) == 1 */ + if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { + PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; + rv = deque_append(new_deque, item); + } else { + rv = deque_extend(new_deque, deque); + } if (rv != NULL) { Py_DECREF(rv); return (PyObject *)new_deque; @@ -1227,11 +1234,11 @@ deque_copy(PyObject *deque) Py_DECREF(new_deque); return NULL; } - if (((dequeobject *)deque)->maxlen == -1) + if (old_deque->maxlen == -1) return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); else return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", - deque, ((dequeobject *)deque)->maxlen, NULL); + deque, old_deque->maxlen, NULL); } PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); -- cgit v0.12 From 235a6f09847ad554d8bf073d4e1d58d1e398ae8c Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 19 Sep 2015 14:51:32 -0400 Subject: Issue #24965: Implement PEP 498 "Literal String Interpolation". Documentation is still needed, I'll open an issue for that. --- Include/Python-ast.h | 23 +- Lib/test/test_fstring.py | 715 ++++++++++++++++++++++++++++++++++ Misc/NEWS | 5 + Parser/Python.asdl | 2 + Parser/tokenizer.c | 8 +- Python/Python-ast.c | 165 ++++++++ Python/ast.c | 985 ++++++++++++++++++++++++++++++++++++++++++++--- Python/compile.c | 117 +++++- Python/symtable.c | 8 + 9 files changed, 1965 insertions(+), 63 deletions(-) create mode 100644 Lib/test/test_fstring.py diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 3bc015f..ea6679c 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -201,9 +201,10 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, Await_kind=12, Yield_kind=13, YieldFrom_kind=14, Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18, - Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21, - Attribute_kind=22, Subscript_kind=23, Starred_kind=24, - Name_kind=25, List_kind=26, Tuple_kind=27}; + FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21, + NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24, + Subscript_kind=25, Starred_kind=26, Name_kind=27, + List_kind=28, Tuple_kind=29}; struct _expr { enum _expr_kind kind; union { @@ -297,6 +298,16 @@ struct _expr { } Str; struct { + expr_ty value; + int conversion; + expr_ty format_spec; + } FormattedValue; + + struct { + asdl_seq *values; + } JoinedStr; + + struct { bytes s; } Bytes; @@ -543,6 +554,12 @@ expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena); #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3) expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena); +#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5) +expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec, + int lineno, int col_offset, PyArena *arena); +#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3) +expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena + *arena); #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3) expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena); #define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py new file mode 100644 index 0000000..a6ff9cf --- /dev/null +++ b/Lib/test/test_fstring.py @@ -0,0 +1,715 @@ +import ast +import types +import decimal +import unittest + +a_global = 'global variable' + +# You could argue that I'm too strict in looking for specific error +# values with assertRaisesRegex, but without it it's way too easy to +# make a syntax error in the test strings. Especially with all of the +# triple quotes, raw strings, backslashes, etc. I think it's a +# worthwhile tradeoff. When I switched to this method, I found many +# examples where I wasn't testing what I thought I was. + +class TestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test__format__lookup(self): + # Make sure __format__ is looked up on the type, not the instance. + class X: + def __format__(self, spec): + return 'class' + + x = X() + + # Add a bound __format__ method to the 'y' instance, but not + # the 'x' instance. + y = X() + y.__format__ = types.MethodType(lambda self, spec: 'instance', y) + + self.assertEqual(f'{y}', format(y)) + self.assertEqual(f'{y}', 'class') + self.assertEqual(format(x), format(y)) + + # __format__ is not called this way, but still make sure it + # returns what we expect (so we can make sure we're bypassing + # it). + self.assertEqual(x.__format__(''), 'class') + self.assertEqual(y.__format__(''), 'instance') + + # This is how __format__ is actually called. + self.assertEqual(type(x).__format__(x, ''), 'class') + self.assertEqual(type(y).__format__(y, ''), 'class') + + def test_ast(self): + # Inspired by http://bugs.python.org/issue24975 + class X: + def __init__(self): + self.called = False + def __call__(self): + self.called = True + return 4 + x = X() + expr = """ +a = 10 +f'{a * x()}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + + # Make sure x was not called. + self.assertFalse(x.called) + + # Actually run the code. + exec(c) + + # Make sure x was called. + self.assertTrue(x.called) + + def test_literal_eval(self): + # With no expressions, an f-string is okay. + self.assertEqual(ast.literal_eval("f'x'"), 'x') + self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy') + + # But this should raise an error. + with self.assertRaisesRegex(ValueError, 'malformed node or string'): + ast.literal_eval("f'x{3}'") + + # As should this, which uses a different ast node + with self.assertRaisesRegex(ValueError, 'malformed node or string'): + ast.literal_eval("f'{3}'") + + def test_ast_compile_time_concat(self): + x = [''] + + expr = """x[0] = 'foo' f'{3}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + exec(c) + self.assertEqual(x[0], 'foo3') + + def test_literal(self): + self.assertEqual(f'', '') + self.assertEqual(f'a', 'a') + self.assertEqual(f' ', ' ') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', + '\N{GREEK CAPITAL LETTER DELTA}') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', + '\u0394') + self.assertEqual(f'\N{True}', '\u22a8') + self.assertEqual(rf'\N{True}', r'\NTrue') + + def test_escape_order(self): + # note that hex(ord('{')) == 0x7b, so this + # string becomes f'a{4*10}b' + self.assertEqual(f'a\u007b4*10}b', 'a40b') + self.assertEqual(f'a\x7b4*10}b', 'a40b') + self.assertEqual(f'a\x7b4*10\N{RIGHT CURLY BRACKET}b', 'a40b') + self.assertEqual(f'{"a"!\N{LATIN SMALL LETTER R}}', "'a'") + self.assertEqual(f'{10\x3a02X}', '0A') + self.assertEqual(f'{10:02\N{LATIN CAPITAL LETTER X}}', '0A') + + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + [r"""f'a{\u007b4*10}b'""", # mis-matched brackets + ]) + self.assertAllRaise(SyntaxError, 'unexpected character after line continuation character', + [r"""f'{"a"\!r}'""", + r"""f'{a\!r}'""", + ]) + + def test_unterminated_string(self): + self.assertAllRaise(SyntaxError, 'f-string: unterminated string', + [r"""f'{"x'""", + r"""f'{"x}'""", + r"""f'{("x'""", + r"""f'{("x}'""", + ]) + + def test_mismatched_parens(self): + self.assertAllRaise(SyntaxError, 'f-string: mismatched', + ["f'{((}'", + ]) + + def test_double_braces(self): + self.assertEqual(f'{{', '{') + self.assertEqual(f'a{{', 'a{') + self.assertEqual(f'{{b', '{b') + self.assertEqual(f'a{{b', 'a{b') + self.assertEqual(f'}}', '}') + self.assertEqual(f'a}}', 'a}') + self.assertEqual(f'}}b', '}b') + self.assertEqual(f'a}}b', 'a}b') + + self.assertEqual(f'{{{10}', '{10') + self.assertEqual(f'}}{10}', '}10') + self.assertEqual(f'}}{{{10}', '}{10') + self.assertEqual(f'}}a{{{10}', '}a{10') + + self.assertEqual(f'{10}{{', '10{') + self.assertEqual(f'{10}}}', '10}') + self.assertEqual(f'{10}}}{{', '10}{') + self.assertEqual(f'{10}}}a{{' '}', '10}a{}') + + # Inside of strings, don't interpret doubled brackets. + self.assertEqual(f'{"{{}}"}', '{{}}') + + self.assertAllRaise(TypeError, 'unhashable type', + ["f'{ {{}} }'", # dict in a set + ]) + + def test_compile_time_concat(self): + x = 'def' + self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') + self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi') + self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{x' f'{x}', '{xdef') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{{x}}' f'{x}', '{{x}}def') + self.assertEqual('{{x' f'{x}', '{{xdef') + self.assertEqual('x}}' f'{x}', 'x}}def') + self.assertEqual(f'{x}' 'x}}', 'defx}}') + self.assertEqual(f'{x}' '', 'def') + self.assertEqual('' f'{x}' '', 'def') + self.assertEqual('' f'{x}', 'def') + self.assertEqual(f'{x}' '2', 'def2') + self.assertEqual('1' f'{x}' '2', '1def2') + self.assertEqual('1' f'{x}', '1def') + self.assertEqual(f'{x}' f'-{x}', 'def-def') + self.assertEqual('' f'', '') + self.assertEqual('' f'' '', '') + self.assertEqual('' f'' '' f'', '') + self.assertEqual(f'', '') + self.assertEqual(f'' '', '') + self.assertEqual(f'' '' f'', '') + self.assertEqual(f'' '' f'' '', '') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3' f'}'", # can't concat to get a valid f-string + ]) + + def test_comments(self): + # These aren't comments, since they're in strings. + d = {'#': 'hash'} + self.assertEqual(f'{"#"}', '#') + self.assertEqual(f'{d["#"]}', 'hash') + + self.assertAllRaise(SyntaxError, "f-string cannot include '#'", + ["f'{1#}'", # error because the expression becomes "(1#)" + "f'{3(#)}'", + ]) + + def test_many_expressions(self): + # Create a string with many expressions in it. Note that + # because we have a space in here as a literal, we're actually + # going to use twice as many ast nodes: one for each literal + # plus one for each expression. + def build_fstr(n, extra=''): + return "f'" + ('{x} ' * n) + extra + "'" + + x = 'X' + width = 1 + + # Test around 256. + for i in range(250, 260): + self.assertEqual(eval(build_fstr(i)), (x+' ')*i) + + # Test concatenating 2 largs fstrings. + self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) + + s = build_fstr(253, '{x:{width}} ') + self.assertEqual(eval(s), (x+' ')*254) + + # Test lots of expressions and constants, concatenated. + s = "f'{1}' 'x' 'y'" * 1024 + self.assertEqual(eval(s), '1xy' * 1024) + + def test_format_specifier_expressions(self): + width = 10 + precision = 4 + value = decimal.Decimal('12.34567') + self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') + self.assertEqual(f'{10:#{1}0x}', ' 0xa') + self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') + self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') + self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa') + self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["""f'{"s"!r{":10"}}'""", + + # This looks like a nested format spec. + ]) + + self.assertAllRaise(SyntaxError, "invalid syntax", + [# Invalid sytax inside a nested spec. + "f'{4:{/5}}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", + [# Can't nest format specifiers. + "f'result: {value:{width:{0}}.{precision:1}}'", + ]) + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + [# No expansion inside conversion or for + # the : or ! itself. + """f'{"s"!{"r"}}'""", + ]) + + def test_side_effect_order(self): + class X: + def __init__(self): + self.i = 0 + def __format__(self, spec): + self.i += 1 + return str(self.i) + + x = X() + self.assertEqual(f'{x} {x}', '1 2') + + def test_missing_expression(self): + self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', + ["f'{}'", + "f'{ }'" + "f' {} '", + "f'{!r}'", + "f'{ !r}'", + "f'{10:{ }}'", + "f' { } '", + r"f'{\n}'", + r"f'{\n \n}'", + ]) + + def test_parens_in_expressions(self): + self.assertEqual(f'{3,}', '(3,)') + + # Add these because when an expression is evaluated, parens + # are added around it. But we shouldn't go from an invalid + # expression to a valid one. The added parens are just + # supposed to allow whitespace (including newlines). + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'{,}'", + "f'{,}'", # this is (,), which is an error + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3)+(4}'", + ]) + + self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + ["f'{\n}'", + ]) + + def test_newlines_in_expressions(self): + self.assertEqual(f'{0}', '0') + self.assertEqual(f'{0\n}', '0') + self.assertEqual(f'{0\r}', '0') + self.assertEqual(f'{\n0\n}', '0') + self.assertEqual(f'{\r0\r}', '0') + self.assertEqual(f'{\n0\r}', '0') + self.assertEqual(f'{\n0}', '0') + self.assertEqual(f'{3+\n4}', '7') + self.assertEqual(f'{3+\\\n4}', '7') + self.assertEqual(rf'''{3+ +4}''', '7') + self.assertEqual(f'''{3+\ +4}''', '7') + + self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', + [r"f'{\n}'", + ]) + + def test_lambda(self): + x = 5 + self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'") + self.assertEqual(f'{(lambda y:x*y)("8")!r:10}', "'88888' ") + self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888 ") + + # lambda doesn't work without parens, because the colon + # makes the parser think it's a format_spec + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["f'{lambda x:x}'", + ]) + + def test_yield(self): + # Not terribly useful, but make sure the yield turns + # a function into a generator + def fn(y): + f'y:{yield y*2}' + + g = fn(4) + self.assertEqual(next(g), 8) + + def test_yield_send(self): + def fn(x): + yield f'x:{yield (lambda i: x * i)}' + + g = fn(10) + the_lambda = next(g) + self.assertEqual(the_lambda(4), 40) + self.assertEqual(g.send('string'), 'x:string') + + def test_expressions_with_triple_quoted_strings(self): + self.assertEqual(f"{'''x'''}", 'x') + self.assertEqual(f"{'''eric's'''}", "eric's") + self.assertEqual(f'{"""eric\'s"""}', "eric's") + self.assertEqual(f"{'''eric\"s'''}", 'eric"s') + self.assertEqual(f'{"""eric"s"""}', 'eric"s') + + # Test concatenation within an expression + self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') + self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') + self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') + self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') + + def test_multiple_vars(self): + x = 98 + y = 'abc' + self.assertEqual(f'{x}{y}', '98abc') + + self.assertEqual(f'X{x}{y}', 'X98abc') + self.assertEqual(f'{x}X{y}', '98Xabc') + self.assertEqual(f'{x}{y}X', '98abcX') + + self.assertEqual(f'X{x}Y{y}', 'X98Yabc') + self.assertEqual(f'X{x}{y}Y', 'X98abcY') + self.assertEqual(f'{x}X{y}Y', '98XabcY') + + self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ') + + def test_closure(self): + def outer(x): + def inner(): + return f'x:{x}' + return inner + + self.assertEqual(outer('987')(), 'x:987') + self.assertEqual(outer(7)(), 'x:7') + + def test_arguments(self): + y = 2 + def f(x, width): + return f'x={x*y:{width}}' + + self.assertEqual(f('foo', 10), 'x=foofoo ') + x = 'bar' + self.assertEqual(f(10, 10), 'x= 20') + + def test_locals(self): + value = 123 + self.assertEqual(f'v:{value}', 'v:123') + + def test_missing_variable(self): + with self.assertRaises(NameError): + f'v:{value}' + + def test_missing_format_spec(self): + class O: + def __format__(self, spec): + if not spec: + return '*' + return spec + + self.assertEqual(f'{O():x}', 'x') + self.assertEqual(f'{O()}', '*') + self.assertEqual(f'{O():}', '*') + + self.assertEqual(f'{3:}', '3') + self.assertEqual(f'{3!s:}', '3') + + def test_global(self): + self.assertEqual(f'g:{a_global}', 'g:global variable') + self.assertEqual(f'g:{a_global!r}', "g:'global variable'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:global variable l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'global variable'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:global variable l:'local variable'") + + self.assertIn("module 'unittest' from", f'{unittest}') + + def test_shadowed_global(self): + a_global = 'really a local' + self.assertEqual(f'g:{a_global}', 'g:really a local') + self.assertEqual(f'g:{a_global!r}', "g:'really a local'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:really a local l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'really a local'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:really a local l:'local variable'") + + def test_call(self): + def foo(x): + return 'x=' + str(x) + + self.assertEqual(f'{foo(10)}', 'x=10') + + def test_nested_fstrings(self): + y = 5 + self.assertEqual(f'{f"{0}"*3}', '000') + self.assertEqual(f'{f"{y}"*3}', '555') + self.assertEqual(f'{f"{\'x\'}"*3}', 'xxx') + + self.assertEqual(f"{r'x' f'{\"s\"}'}", 'xs') + self.assertEqual(f"{r'x'rf'{\"s\"}'}", 'xs') + + def test_invalid_string_prefixes(self): + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["fu''", + "uf''", + "Fu''", + "fU''", + "Uf''", + "uF''", + "ufr''", + "urf''", + "fur''", + "fru''", + "rfu''", + "ruf''", + "FUR''", + "Fur''", + ]) + + def test_leading_trailing_spaces(self): + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{\t3}', '3') + self.assertEqual(f'{\t\t3}', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3\t}', '3') + self.assertEqual(f'{3\t\t}', '3') + + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}', + 'expr={1: 2}') + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }', + 'expr={1: 2}') + + def test_character_name(self): + self.assertEqual(f'{4}\N{GREEK CAPITAL LETTER DELTA}{3}', + '4\N{GREEK CAPITAL LETTER DELTA}3') + self.assertEqual(f'{{}}\N{GREEK CAPITAL LETTER DELTA}{3}', + '{}\N{GREEK CAPITAL LETTER DELTA}3') + + def test_not_equal(self): + # There's a special test for this because there's a special + # case in the f-string parser to look for != as not ending an + # expression. Normally it would, while looking for !s or !r. + + self.assertEqual(f'{3!=4}', 'True') + self.assertEqual(f'{3!=4:}', 'True') + self.assertEqual(f'{3!=4!s}', 'True') + self.assertEqual(f'{3!=4!s:.3}', 'Tru') + + def test_conversions(self): + self.assertEqual(f'{3.14:10.10}', ' 3.14') + self.assertEqual(f'{3.14!s:10.10}', '3.14 ') + self.assertEqual(f'{3.14!r:10.10}', '3.14 ') + self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + + self.assertEqual(f'{"a"}', 'a') + self.assertEqual(f'{"a"!r}', "'a'") + self.assertEqual(f'{"a"!a}', "'a'") + + # Not a conversion. + self.assertEqual(f'{"a!r"}', "a!r") + + # Not a conversion, but show that ! is allowed in a format spec. + self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!') + + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"}', '\u0394') + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!r}', "'\u0394'") + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!a}', "'\\u0394'") + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + ["f'{3!g}'", + "f'{3!A}'", + "f'{3!A}'", + "f'{3!A}'", + "f'{3!!}'", + "f'{3!:}'", + "f'{3!\N{GREEK CAPITAL LETTER DELTA}}'", + "f'{3! s}'", # no space before conversion char + "f'{x!\\x00:.<10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{x!s{y}}'", + "f'{3!ss}'", + "f'{3!ss:}'", + "f'{3!ss:s}'", + ]) + + def test_assignment(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'' = 3", + "f'{0}' = x", + "f'{x}' = x", + ]) + + def test_del(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["del f''", + "del '' f''", + ]) + + def test_mismatched_braces(self): + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + ["f'{{}'", + "f'{{}}}'", + "f'}'", + "f'x}'", + "f'x}x'", + + # Can't have { or } in a format spec. + "f'{3:}>10}'", + r"f'{3:\\}>10}'", + "f'{3:}}>10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3:{{>10}'", + "f'{3'", + "f'{3!'", + "f'{3:'", + "f'{3!s'", + "f'{3!s:'", + "f'{3!s:3'", + "f'x{'", + "f'x{x'", + "f'{3:s'", + "f'{{{'", + "f'{{}}{'", + "f'{'", + ]) + + self.assertAllRaise(SyntaxError, 'invalid syntax', + [r"f'{3:\\{>10}'", + ]) + + # But these are just normal strings. + self.assertEqual(f'{"{"}', '{') + self.assertEqual(f'{"}"}', '}') + self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3') + self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2') + + def test_if_conditional(self): + # There's special logic in compile.c to test if the + # conditional for an if (and while) are constants. Exercise + # that code. + + def test_fstring(x, expected): + flag = 0 + if f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_empty(x, expected): + flag = 0 + if '' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_non_empty(x, expected): + flag = 0 + if ' ' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + test_fstring('', 2) + test_fstring(' ', 1) + + test_concat_empty('', 2) + test_concat_empty(' ', 1) + + test_concat_non_empty('', 1) + test_concat_non_empty(' ', 1) + + def test_empty_format_specifier(self): + x = 'test' + self.assertEqual(f'{x}', 'test') + self.assertEqual(f'{x:}', 'test') + self.assertEqual(f'{x!s:}', 'test') + self.assertEqual(f'{x!r:}', "'test'") + + def test_str_format_differences(self): + d = {'a': 'string', + 0: 'integer', + } + a = 0 + self.assertEqual(f'{d[0]}', 'integer') + self.assertEqual(f'{d["a"]}', 'string') + self.assertEqual(f'{d[a]}', 'integer') + self.assertEqual('{d[a]}'.format(d=d), 'string') + self.assertEqual('{d[0]}'.format(d=d), 'integer') + + def test_invalid_expressions(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + [r"f'{a[4)}'", + r"f'{a(4]}'", + ]) + + def test_loop(self): + for i in range(1000): + self.assertEqual(f'i:{i}', 'i:' + str(i)) + + def test_dict(self): + d = {'"': 'dquote', + "'": 'squote', + 'foo': 'bar', + } + self.assertEqual(f'{d["\'"]}', 'squote') + self.assertEqual(f"{d['\"']}", 'dquote') + + self.assertEqual(f'''{d["'"]}''', 'squote') + self.assertEqual(f"""{d['"']}""", 'dquote') + + self.assertEqual(f'{d["foo"]}', 'bar') + self.assertEqual(f"{d['foo']}", 'bar') + self.assertEqual(f'{d[\'foo\']}', 'bar') + self.assertEqual(f"{d[\"foo\"]}", 'bar') + + def test_escaped_quotes(self): + d = {'"': 'a', + "'": 'b'} + + self.assertEqual(fr"{d['\"']}", 'a') + self.assertEqual(fr'{d["\'"]}', 'b') + self.assertEqual(fr"{'\"'}", '"') + self.assertEqual(fr'{"\'"}', "'") + self.assertEqual(f'{"\\"3"}', '"3') + + self.assertAllRaise(SyntaxError, 'f-string: unterminated string', + [r'''f'{"""\\}' ''', # Backslash at end of expression + ]) + self.assertAllRaise(SyntaxError, 'unexpected character after line continuation', + [r"rf'{3\}'", + ]) + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index a2c76f5..46f7a3e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,11 @@ Core and Builtins argument list of a function declaration. For example, "def f(*, a = 3,): pass" is now legal. Patch from Mark Dickinson. +- Issue #24965: Implement PEP 498 "Literal String Interpolation". This + allows you to embed expressions inside f-strings, which are + converted to normal strings at run time. Given x=3, then + f'value={x}' == 'value=3'. Patch by Eric V. Smith. + Library ------- diff --git a/Parser/Python.asdl b/Parser/Python.asdl index cd0832d..22775c6 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -71,6 +71,8 @@ module Python | Call(expr func, expr* args, keyword* keywords) | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? + | FormattedValue(expr value, int? conversion, expr? format_spec) + | JoinedStr(expr* values) | Bytes(bytes s) | NameConstant(singleton value) | Ellipsis diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5046fa5..2369be4 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1477,17 +1477,19 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) nonascii = 0; if (is_potential_identifier_start(c)) { /* Process b"", r"", u"", br"" and rb"" */ - int saw_b = 0, saw_r = 0, saw_u = 0; + int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0; while (1) { - if (!(saw_b || saw_u) && (c == 'b' || c == 'B')) + if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B')) saw_b = 1; /* Since this is a backwards compatibility support literal we don't want to support it in arbitrary order like byte literals. */ - else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U')) + else if (!(saw_b || saw_u || saw_r || saw_f) && (c == 'u' || c == 'U')) saw_u = 1; /* ur"" and ru"" are not supported */ else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) saw_r = 1; + else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) + saw_f = 1; else break; c = tok_nextc(tok); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index fd7f17e..a2e9816 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -285,6 +285,18 @@ _Py_IDENTIFIER(s); static char *Str_fields[]={ "s", }; +static PyTypeObject *FormattedValue_type; +_Py_IDENTIFIER(conversion); +_Py_IDENTIFIER(format_spec); +static char *FormattedValue_fields[]={ + "value", + "conversion", + "format_spec", +}; +static PyTypeObject *JoinedStr_type; +static char *JoinedStr_fields[]={ + "values", +}; static PyTypeObject *Bytes_type; static char *Bytes_fields[]={ "s", @@ -917,6 +929,11 @@ static int init_types(void) if (!Num_type) return 0; Str_type = make_type("Str", expr_type, Str_fields, 1); if (!Str_type) return 0; + FormattedValue_type = make_type("FormattedValue", expr_type, + FormattedValue_fields, 3); + if (!FormattedValue_type) return 0; + JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1); + if (!JoinedStr_type) return 0; Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1); if (!Bytes_type) return 0; NameConstant_type = make_type("NameConstant", expr_type, @@ -2063,6 +2080,42 @@ Str(string s, int lineno, int col_offset, PyArena *arena) } expr_ty +FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno, + int col_offset, PyArena *arena) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for FormattedValue"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = FormattedValue_kind; + p->v.FormattedValue.value = value; + p->v.FormattedValue.conversion = conversion; + p->v.FormattedValue.format_spec = format_spec; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty +JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = JoinedStr_kind; + p->v.JoinedStr.values = values; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Bytes(bytes s, int lineno, int col_offset, PyArena *arena) { expr_ty p; @@ -3161,6 +3214,34 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case FormattedValue_kind: + result = PyType_GenericNew(FormattedValue_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.FormattedValue.value); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->v.FormattedValue.conversion); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_conversion, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.FormattedValue.format_spec); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1) + goto failed; + Py_DECREF(value); + break; + case JoinedStr_kind: + result = PyType_GenericNew(JoinedStr_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_list(o->v.JoinedStr.values, ast2obj_expr); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_values, value) == -1) + goto failed; + Py_DECREF(value); + break; case Bytes_kind: result = PyType_GenericNew(Bytes_type, NULL, NULL); if (!result) goto failed; @@ -6022,6 +6103,86 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)FormattedValue_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty value; + int conversion; + expr_ty format_spec; + + if (_PyObject_HasAttrId(obj, &PyId_value)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue"); + return 1; + } + if (exists_not_none(obj, &PyId_conversion)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_conversion); + if (tmp == NULL) goto failed; + res = obj2ast_int(tmp, &conversion, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + conversion = 0; + } + if (exists_not_none(obj, &PyId_format_spec)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_format_spec); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &format_spec, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + format_spec = NULL; + } + *out = FormattedValue(value, conversion, format_spec, lineno, + col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + isinstance = PyObject_IsInstance(obj, (PyObject*)JoinedStr_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + asdl_seq* values; + + if (_PyObject_HasAttrId(obj, &PyId_values)) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = _PyObject_GetAttrId(obj, &PyId_values); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + values = _Py_asdl_seq_new(len, arena); + if (values == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(values, i, value); + } + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from JoinedStr"); + return 1; + } + *out = JoinedStr(values, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Bytes_type); if (isinstance == -1) { return 1; @@ -7319,6 +7480,10 @@ PyInit__ast(void) if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL; if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return NULL; if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return NULL; + if (PyDict_SetItemString(d, "FormattedValue", + (PyObject*)FormattedValue_type) < 0) return NULL; + if (PyDict_SetItemString(d, "JoinedStr", (PyObject*)JoinedStr_type) < 0) + return NULL; if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return NULL; if (PyDict_SetItemString(d, "NameConstant", (PyObject*)NameConstant_type) < diff --git a/Python/ast.c b/Python/ast.c index 1f7ddfc..735424b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -257,6 +257,14 @@ validate_expr(expr_ty exp, expr_context_ty ctx) } return 1; } + case JoinedStr_kind: + return validate_exprs(exp->v.JoinedStr.values, Load, 0); + case FormattedValue_kind: + if (validate_expr(exp->v.FormattedValue.value, Load) == 0) + return 0; + if (exp->v.FormattedValue.format_spec) + return validate_expr(exp->v.FormattedValue.format_spec, Load); + return 1; case Bytes_kind: { PyObject *b = exp->v.Bytes.s; if (!PyBytes_CheckExact(b)) { @@ -535,9 +543,7 @@ static stmt_ty ast_for_for_stmt(struct compiling *, const node *, int); static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); static PyObject *parsenumber(struct compiling *, const char *); -static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode); -static PyObject *parsestrplus(struct compiling *, const node *n, - int *bytesmode); +static expr_ty parsestrplus(struct compiling *, const node *n); #define COMP_GENEXP 0 #define COMP_LISTCOMP 1 @@ -986,6 +992,8 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) case Num_kind: case Str_kind: case Bytes_kind: + case JoinedStr_kind: + case FormattedValue_kind: expr_name = "literal"; break; case NameConstant_kind: @@ -2001,7 +2009,6 @@ ast_for_atom(struct compiling *c, const node *n) | '...' | 'None' | 'True' | 'False' */ node *ch = CHILD(n, 0); - int bytesmode = 0; switch (TYPE(ch)) { case NAME: { @@ -2023,7 +2030,7 @@ ast_for_atom(struct compiling *c, const node *n) return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); } case STRING: { - PyObject *str = parsestrplus(c, n, &bytesmode); + expr_ty str = parsestrplus(c, n); if (!str) { const char *errtype = NULL; if (PyErr_ExceptionMatches(PyExc_UnicodeError)) @@ -2050,14 +2057,7 @@ ast_for_atom(struct compiling *c, const node *n) } return NULL; } - if (PyArena_AddPyObject(c->c_arena, str) < 0) { - Py_DECREF(str); - return NULL; - } - if (bytesmode) - return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena); - else - return Str(str, LINENO(n), n->n_col_offset, c->c_arena); + return str; } case NUMBER: { PyObject *pynum = parsenumber(c, STR(ch)); @@ -4002,12 +4002,838 @@ decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons return v; } -/* s is a Python string literal, including the bracketing quote characters, - * and r &/or b prefixes (if any), and embedded escape sequences (if any). - * parsestr parses it, and returns the decoded Python string object. - */ +/* Compile this expression in to an expr_ty. We know that we can + temporarily modify the character before the start of this string + (it's '{'), and we know we can temporarily modify the character + after this string (it is a '}'). Leverage this to create a + sub-string with enough room for us to add parens around the + expression. This is to allow strings with embedded newlines, for + example. */ +static expr_ty +fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, + Py_ssize_t expr_end, PyArena *arena) +{ + PyCompilerFlags cf; + mod_ty mod; + char *utf_expr; + Py_ssize_t i; + int all_whitespace; + PyObject *sub = NULL; + + /* We only decref sub if we allocated it with a PyUnicode_Substring. + decref_sub records that. */ + int decref_sub = 0; + + assert(str); + + /* If the substring is all whitespace, it's an error. We need to + catch this here, and not when we call PyParser_ASTFromString, + because turning the expression '' in to '()' would go from + being invalid to valid. */ + /* Note that this code says an empty string is all + whitespace. That's important. There's a test for it: f'{}'. */ + all_whitespace = 1; + for (i = expr_start; i < expr_end; i++) { + if (!Py_UNICODE_ISSPACE(PyUnicode_READ_CHAR(str, i))) { + all_whitespace = 0; + break; + } + } + if (all_whitespace) { + PyErr_SetString(PyExc_SyntaxError, "f-string: empty expression " + "not allowed"); + goto error; + } + + /* If the substring will be the entire source string, we can't use + PyUnicode_Substring, since it will return another reference to + our original string. Because we're modifying the string in + place, that's a no-no. So, detect that case and just use our + string directly. */ + + if (expr_start-1 == 0 && expr_end+1 == PyUnicode_GET_LENGTH(str)) { + /* No need to actually remember these characters, because we + know they must be braces. */ + assert(PyUnicode_ReadChar(str, 0) == '{'); + assert(PyUnicode_ReadChar(str, expr_end-expr_start+1) == '}'); + sub = str; + } else { + /* Create a substring object. It must be a new object, with + refcount==1, so that we can modify it. */ + sub = PyUnicode_Substring(str, expr_start-1, expr_end+1); + if (!sub) + goto error; + assert(sub != str); /* Make sure it's a new string. */ + decref_sub = 1; /* Remember to deallocate it on error. */ + } + + if (PyUnicode_WriteChar(sub, 0, '(') < 0 || + PyUnicode_WriteChar(sub, expr_end-expr_start+1, ')') < 0) + goto error; + + cf.cf_flags = PyCF_ONLY_AST; + + /* No need to free the memory returned here: it's managed by the + string. */ + utf_expr = PyUnicode_AsUTF8(sub); + if (!utf_expr) + goto error; + mod = PyParser_ASTFromString(utf_expr, "", + Py_eval_input, &cf, arena); + if (!mod) + goto error; + if (sub != str) + /* Clear instead of decref in case we ever modify this code to change + the error handling: this is safest because the XDECREF won't try + and decref it when it's NULL. */ + /* No need to restore the chars in sub, since we know it's getting + ready to get deleted (refcount must be 1, since we got a new string + in PyUnicode_Substring). */ + Py_CLEAR(sub); + else { + assert(!decref_sub); + /* Restore str, which we earlier modified directly. */ + if (PyUnicode_WriteChar(str, 0, '{') < 0 || + PyUnicode_WriteChar(str, expr_end-expr_start+1, '}') < 0) + goto error; + } + return mod->v.Expression.body; + +error: + /* Only decref sub if it was the result of a call to SubString. */ + if (decref_sub) + Py_XDECREF(sub); + return NULL; +} + +/* Return -1 on error. + + Return 0 if we reached the end of the literal. + + Return 1 if we haven't reached the end of the literal, but we want + the caller to process the literal up to this point. Used for + doubled braces. +*/ +static int +fstring_find_literal(PyObject *str, Py_ssize_t *ofs, PyObject **literal, + int recurse_lvl, struct compiling *c, const node *n) +{ + /* Get any literal string. It ends when we hit an un-doubled brace, or the + end of the string. */ + + Py_ssize_t literal_start, literal_end; + int result = 0; + + enum PyUnicode_Kind kind = PyUnicode_KIND(str); + void *data = PyUnicode_DATA(str); + + assert(*literal == NULL); + + literal_start = *ofs; + for (; *ofs < PyUnicode_GET_LENGTH(str); *ofs += 1) { + Py_UCS4 ch = PyUnicode_READ(kind, data, *ofs); + if (ch == '{' || ch == '}') { + /* Check for doubled braces, but only at the top level. If + we checked at every level, then f'{0:{3}}' would fail + with the two closing braces. */ + if (recurse_lvl == 0) { + if (*ofs + 1 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs + 1) == ch) { + /* We're going to tell the caller that the literal ends + here, but that they should continue scanning. But also + skip over the second brace when we resume scanning. */ + literal_end = *ofs + 1; + *ofs += 2; + result = 1; + goto done; + } + + /* Where a single '{' is the start of a new expression, a + single '}' is not allowed. */ + if (ch == '}') { + ast_error(c, n, "f-string: single '}' is not allowed"); + return -1; + } + } + + /* We're either at a '{', which means we're starting another + expression; or a '}', which means we're at the end of this + f-string (for a nested format_spec). */ + break; + } + } + literal_end = *ofs; + + assert(*ofs == PyUnicode_GET_LENGTH(str) || + PyUnicode_READ(kind, data, *ofs) == '{' || + PyUnicode_READ(kind, data, *ofs) == '}'); +done: + if (literal_start != literal_end) { + *literal = PyUnicode_Substring(str, literal_start, literal_end); + if (!*literal) + return -1; + } + + return result; +} + +/* Forward declaration because parsing is recursive. */ +static expr_ty +fstring_parse(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + struct compiling *c, const node *n); + +/* Parse the f-string str, starting at ofs. We know *ofs starts an + expression (so it must be a '{'). Returns the FormattedValue node, + which includes the expression, conversion character, and + format_spec expression. + + Note that I don't do a perfect job here: I don't make sure that a + closing brace doesn't match an opening paren, for example. It + doesn't need to error on all invalid expressions, just correctly + find the end of all valid ones. Any errors inside the expression + will be caught when we parse it later. */ +static int +fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + expr_ty *expression, struct compiling *c, const node *n) +{ + /* Return -1 on error, else 0. */ + + Py_ssize_t expr_start; + Py_ssize_t expr_end; + expr_ty simple_expression; + expr_ty format_spec = NULL; /* Optional format specifier. */ + Py_UCS4 conversion = -1; /* The conversion char. -1 if not specified. */ + + enum PyUnicode_Kind kind = PyUnicode_KIND(str); + void *data = PyUnicode_DATA(str); + + /* 0 if we're not in a string, else the quote char we're trying to + match (single or double quote). */ + Py_UCS4 quote_char = 0; + + /* If we're inside a string, 1=normal, 3=triple-quoted. */ + int string_type = 0; + + /* Keep track of nesting level for braces/parens/brackets in + expressions. */ + Py_ssize_t nested_depth = 0; + + /* Can only nest one level deep. */ + if (recurse_lvl >= 2) { + ast_error(c, n, "f-string: expressions nested too deeply"); + return -1; + } + + /* The first char must be a left brace, or we wouldn't have gotten + here. Skip over it. */ + assert(PyUnicode_READ(kind, data, *ofs) == '{'); + *ofs += 1; + + expr_start = *ofs; + for (; *ofs < PyUnicode_GET_LENGTH(str); *ofs += 1) { + Py_UCS4 ch; + + /* Loop invariants. */ + assert(nested_depth >= 0); + assert(*ofs >= expr_start); + if (quote_char) + assert(string_type == 1 || string_type == 3); + else + assert(string_type == 0); + + ch = PyUnicode_READ(kind, data, *ofs); + if (quote_char) { + /* We're inside a string. See if we're at the end. */ + /* This code needs to implement the same non-error logic + as tok_get from tokenizer.c, at the letter_quote + label. To actually share that code would be a + nightmare. But, it's unlikely to change and is small, + so duplicate it here. Note we don't need to catch all + of the errors, since they'll be caught when parsing the + expression. We just need to match the non-error + cases. Thus we can ignore \n in single-quoted strings, + for example. Or non-terminated strings. */ + if (ch == quote_char) { + /* Does this match the string_type (single or triple + quoted)? */ + if (string_type == 3) { + if (*ofs+2 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs+1) == ch && + PyUnicode_READ(kind, data, *ofs+2) == ch) { + /* We're at the end of a triple quoted string. */ + *ofs += 2; + string_type = 0; + quote_char = 0; + continue; + } + } else { + /* We're at the end of a normal string. */ + quote_char = 0; + string_type = 0; + continue; + } + } + /* We're inside a string, and not finished with the + string. If this is a backslash, skip the next char (it + might be an end quote that needs skipping). Otherwise, + just consume this character normally. */ + if (ch == '\\' && *ofs+1 < PyUnicode_GET_LENGTH(str)) { + /* Just skip the next char, whatever it is. */ + *ofs += 1; + } + } else if (ch == '\'' || ch == '"') { + /* Is this a triple quoted string? */ + if (*ofs+2 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs+1) == ch && + PyUnicode_READ(kind, data, *ofs+2) == ch) { + string_type = 3; + *ofs += 2; + } else { + /* Start of a normal string. */ + string_type = 1; + } + /* Start looking for the end of the string. */ + quote_char = ch; + } else if (ch == '[' || ch == '{' || ch == '(') { + nested_depth++; + } else if (nested_depth != 0 && + (ch == ']' || ch == '}' || ch == ')')) { + nested_depth--; + } else if (ch == '#') { + /* Error: can't include a comment character, inside parens + or not. */ + ast_error(c, n, "f-string cannot include '#'"); + return -1; + } else if (nested_depth == 0 && + (ch == '!' || ch == ':' || ch == '}')) { + /* First, test for the special case of "!=". Since '=' is + not an allowed conversion character, nothing is lost in + this test. */ + if (ch == '!' && *ofs+1 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs+1) == '=') + /* This isn't a conversion character, just continue. */ + continue; + + /* Normal way out of this loop. */ + break; + } else { + /* Just consume this char and loop around. */ + } + } + expr_end = *ofs; + /* If we leave this loop in a string or with mismatched parens, we + don't care. We'll get a syntax error when compiling the + expression. But, we can produce a better error message, so + let's just do that.*/ + if (quote_char) { + ast_error(c, n, "f-string: unterminated string"); + return -1; + } + if (nested_depth) { + ast_error(c, n, "f-string: mismatched '(', '{', or '['"); + return -1; + } + + /* Check for a conversion char, if present. */ + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + if (PyUnicode_READ(kind, data, *ofs) == '!') { + *ofs += 1; + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + + conversion = PyUnicode_READ(kind, data, *ofs); + *ofs += 1; + + /* Validate the conversion. */ + if (!(conversion == 's' || conversion == 'r' + || conversion == 'a')) { + ast_error(c, n, "f-string: invalid conversion character: " + "expected 's', 'r', or 'a'"); + return -1; + } + } + + /* Check for the format spec, if present. */ + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + if (PyUnicode_READ(kind, data, *ofs) == ':') { + *ofs += 1; + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + + /* Parse the format spec. */ + format_spec = fstring_parse(str, ofs, recurse_lvl+1, c, n); + if (!format_spec) + return -1; + } + + if (*ofs >= PyUnicode_GET_LENGTH(str) || + PyUnicode_READ(kind, data, *ofs) != '}') + goto unexpected_end_of_string; + + /* We're at a right brace. Consume it. */ + assert(*ofs < PyUnicode_GET_LENGTH(str)); + assert(PyUnicode_READ(kind, data, *ofs) == '}'); + *ofs += 1; + + /* Compile the expression. */ + simple_expression = fstring_expression_compile(str, expr_start, expr_end, + c->c_arena); + if (!simple_expression) + return -1; + + /* And now create the FormattedValue node that represents this entire + expression with the conversion and format spec. */ + *expression = FormattedValue(simple_expression, (int)conversion, + format_spec, LINENO(n), n->n_col_offset, + c->c_arena); + if (!*expression) + return -1; + + return 0; + +unexpected_end_of_string: + ast_error(c, n, "f-string: expecting '}'"); + return -1; +} + +/* Return -1 on error. + + Return 0 if we have a literal (possible zero length) and an + expression (zero length if at the end of the string. + + Return 1 if we have a literal, but no expression, and we want the + caller to call us again. This is used to deal with doubled + braces. + + When called multiple times on the string 'a{{b{0}c', this function + will return: + + 1. the literal 'a{' with no expression, and a return value + of 1. Despite the fact that there's no expression, the return + value of 1 means we're not finished yet. + + 2. the literal 'b' and the expression '0', with a return value of + 0. The fact that there's an expression means we're not finished. + + 3. literal 'c' with no expression and a return value of 0. The + combination of the return value of 0 with no expression means + we're finished. +*/ +static int +fstring_find_literal_and_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + PyObject **literal, expr_ty *expression, + struct compiling *c, const node *n) +{ + int result; + + assert(*literal == NULL && *expression == NULL); + + /* Get any literal string. */ + result = fstring_find_literal(str, ofs, literal, recurse_lvl, c, n); + if (result < 0) + goto error; + + assert(result == 0 || result == 1); + + if (result == 1) + /* We have a literal, but don't look at the expression. */ + return 1; + + assert(*ofs <= PyUnicode_GET_LENGTH(str)); + + if (*ofs >= PyUnicode_GET_LENGTH(str) || + PyUnicode_READ_CHAR(str, *ofs) == '}') + /* We're at the end of the string or the end of a nested + f-string: no expression. The top-level error case where we + expect to be at the end of the string but we're at a '}' is + handled later. */ + return 0; + + /* We must now be the start of an expression, on a '{'. */ + assert(*ofs < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ_CHAR(str, *ofs) == '{'); + + if (fstring_find_expr(str, ofs, recurse_lvl, expression, c, n) < 0) + goto error; + + return 0; + +error: + Py_XDECREF(*literal); + *literal = NULL; + return -1; +} + +#define EXPRLIST_N_CACHED 64 + +typedef struct { + /* Incrementally build an array of expr_ty, so be used in an + asdl_seq. Cache some small but reasonably sized number of + expr_ty's, and then after that start dynamically allocating, + doubling the number allocated each time. Note that the f-string + f'{0}a{1}' contains 3 expr_ty's: 2 FormattedValue's, and one + Str for the literal 'a'. So you add expr_ty's about twice as + fast as you add exressions in an f-string. */ + + Py_ssize_t allocated; /* Number we've allocated. */ + Py_ssize_t size; /* Number we've used. */ + expr_ty *p; /* Pointer to the memory we're actually + using. Will point to 'data' until we + start dynamically allocating. */ + expr_ty data[EXPRLIST_N_CACHED]; +} ExprList; + +#ifdef NDEBUG +#define ExprList_check_invariants(l) +#else +static void +ExprList_check_invariants(ExprList *l) +{ + /* Check our invariants. Make sure this object is "live", and + hasn't been deallocated. */ + assert(l->size >= 0); + assert(l->p != NULL); + if (l->size <= EXPRLIST_N_CACHED) + assert(l->data == l->p); +} +#endif + +static void +ExprList_Init(ExprList *l) +{ + l->allocated = EXPRLIST_N_CACHED; + l->size = 0; + + /* Until we start allocating dynamically, p points to data. */ + l->p = l->data; + + ExprList_check_invariants(l); +} + +static int +ExprList_Append(ExprList *l, expr_ty exp) +{ + ExprList_check_invariants(l); + if (l->size >= l->allocated) { + /* We need to alloc (or realloc) the memory. */ + Py_ssize_t new_size = l->allocated * 2; + + /* See if we've ever allocated anything dynamically. */ + if (l->p == l->data) { + Py_ssize_t i; + /* We're still using the cached data. Switch to + alloc-ing. */ + l->p = PyMem_RawMalloc(sizeof(expr_ty) * new_size); + if (!l->p) + return -1; + /* Copy the cached data into the new buffer. */ + for (i = 0; i < l->size; i++) + l->p[i] = l->data[i]; + } else { + /* Just realloc. */ + expr_ty *tmp = PyMem_RawRealloc(l->p, sizeof(expr_ty) * new_size); + if (!tmp) { + PyMem_RawFree(l->p); + l->p = NULL; + return -1; + } + l->p = tmp; + } + + l->allocated = new_size; + assert(l->allocated == 2 * l->size); + } + + l->p[l->size++] = exp; + + ExprList_check_invariants(l); + return 0; +} + +static void +ExprList_Dealloc(ExprList *l) +{ + ExprList_check_invariants(l); + + /* If there's been an error, or we've never dynamically allocated, + do nothing. */ + if (!l->p || l->p == l->data) { + /* Do nothing. */ + } else { + /* We have dynamically allocated. Free the memory. */ + PyMem_RawFree(l->p); + } + l->p = NULL; + l->size = -1; +} + +static asdl_seq * +ExprList_Finish(ExprList *l, PyArena *arena) +{ + asdl_seq *seq; + + ExprList_check_invariants(l); + + /* Allocate the asdl_seq and copy the expressions in to it. */ + seq = _Py_asdl_seq_new(l->size, arena); + if (seq) { + Py_ssize_t i; + for (i = 0; i < l->size; i++) + asdl_seq_SET(seq, i, l->p[i]); + } + ExprList_Dealloc(l); + return seq; +} + +/* The FstringParser is designed to add a mix of strings and + f-strings, and concat them together as needed. Ultimately, it + generates an expr_ty. */ +typedef struct { + PyObject *last_str; + ExprList expr_list; +} FstringParser; + +#ifdef NDEBUG +#define FstringParser_check_invariants(state) +#else +static void +FstringParser_check_invariants(FstringParser *state) +{ + if (state->last_str) + assert(PyUnicode_CheckExact(state->last_str)); + ExprList_check_invariants(&state->expr_list); +} +#endif + +static void +FstringParser_Init(FstringParser *state) +{ + state->last_str = NULL; + ExprList_Init(&state->expr_list); + FstringParser_check_invariants(state); +} + +static void +FstringParser_Dealloc(FstringParser *state) +{ + FstringParser_check_invariants(state); + + Py_XDECREF(state->last_str); + ExprList_Dealloc(&state->expr_list); +} + +/* Make a Str node, but decref the PyUnicode object being added. */ +static expr_ty +make_str_node_and_del(PyObject **str, struct compiling *c, const node* n) +{ + PyObject *s = *str; + *str = NULL; + assert(PyUnicode_CheckExact(s)); + if (PyArena_AddPyObject(c->c_arena, s) < 0) { + Py_DECREF(s); + return NULL; + } + return Str(s, LINENO(n), n->n_col_offset, c->c_arena); +} + +/* Add a non-f-string (that is, a regular literal string). str is + decref'd. */ +static int +FstringParser_ConcatAndDel(FstringParser *state, PyObject *str) +{ + FstringParser_check_invariants(state); + + assert(PyUnicode_CheckExact(str)); + + if (PyUnicode_GET_LENGTH(str) == 0) { + Py_DECREF(str); + return 0; + } + + if (!state->last_str) { + /* We didn't have a string before, so just remember this one. */ + state->last_str = str; + } else { + /* Concatenate this with the previous string. */ + PyObject *temp = PyUnicode_Concat(state->last_str, str); + Py_DECREF(state->last_str); + Py_DECREF(str); + state->last_str = temp; + if (!temp) + return -1; + } + FstringParser_check_invariants(state); + return 0; +} + +/* Parse an f-string. The f-string is in str, starting at ofs, with no 'f' + or quotes. str is not decref'd, since we don't know if it's used elsewhere. + And if we're only looking at a part of a string, then decref'ing is + definitely not the right thing to do! */ +static int +FstringParser_ConcatFstring(FstringParser *state, PyObject *str, + Py_ssize_t *ofs, int recurse_lvl, + struct compiling *c, const node *n) +{ + FstringParser_check_invariants(state); + + /* Parse the f-string. */ + while (1) { + PyObject *literal = NULL; + expr_ty expression = NULL; + + /* If there's a zero length literal in front of the + expression, literal will be NULL. If we're at the end of + the f-string, expression will be NULL (unless result == 1, + see below). */ + int result = fstring_find_literal_and_expr(str, ofs, recurse_lvl, + &literal, &expression, + c, n); + if (result < 0) + return -1; + + /* Add the literal, if any. */ + if (!literal) { + /* Do nothing. Just leave last_str alone (and possibly + NULL). */ + } else if (!state->last_str) { + state->last_str = literal; + literal = NULL; + } else { + /* We have a literal, concatenate it. */ + assert(PyUnicode_GET_LENGTH(literal) != 0); + if (FstringParser_ConcatAndDel(state, literal) < 0) + return -1; + literal = NULL; + } + assert(!state->last_str || + PyUnicode_GET_LENGTH(state->last_str) != 0); + + /* We've dealt with the literal now. It can't be leaked on further + errors. */ + assert(literal == NULL); + + /* See if we should just loop around to get the next literal + and expression, while ignoring the expression this + time. This is used for un-doubling braces, as an + optimization. */ + if (result == 1) + continue; + + if (!expression) + /* We're done with this f-string. */ + break; + + /* We know we have an expression. Convert any existing string + to a Str node. */ + if (!state->last_str) { + /* Do nothing. No previous literal. */ + } else { + /* Convert the existing last_str literal to a Str node. */ + expr_ty str = make_str_node_and_del(&state->last_str, c, n); + if (!str || ExprList_Append(&state->expr_list, str) < 0) + return -1; + } + + if (ExprList_Append(&state->expr_list, expression) < 0) + return -1; + } + + assert(*ofs <= PyUnicode_GET_LENGTH(str)); + + /* If recurse_lvl is zero, then we must be at the end of the + string. Otherwise, we must be at a right brace. */ + + if (recurse_lvl == 0 && *ofs < PyUnicode_GET_LENGTH(str)) { + ast_error(c, n, "f-string: unexpected end of string"); + return -1; + } + if (recurse_lvl != 0 && PyUnicode_READ_CHAR(str, *ofs) != '}') { + ast_error(c, n, "f-string: expecting '}'"); + return -1; + } + + FstringParser_check_invariants(state); + return 0; +} + +/* Convert the partial state reflected in last_str and expr_list to an + expr_ty. The expr_ty can be a Str, or a JoinedStr. */ +static expr_ty +FstringParser_Finish(FstringParser *state, struct compiling *c, + const node *n) +{ + asdl_seq *seq; + + FstringParser_check_invariants(state); + + /* If we're just a constant string with no expressions, return + that. */ + if(state->expr_list.size == 0) { + if (!state->last_str) { + /* Create a zero length string. */ + state->last_str = PyUnicode_FromStringAndSize(NULL, 0); + if (!state->last_str) + goto error; + } + return make_str_node_and_del(&state->last_str, c, n); + } + + /* Create a Str node out of last_str, if needed. It will be the + last node in our expression list. */ + if (state->last_str) { + expr_ty str = make_str_node_and_del(&state->last_str, c, n); + if (!str || ExprList_Append(&state->expr_list, str) < 0) + goto error; + } + /* This has already been freed. */ + assert(state->last_str == NULL); + + seq = ExprList_Finish(&state->expr_list, c->c_arena); + if (!seq) + goto error; + + /* If there's only one expression, return it. Otherwise, we need + to join them together. */ + if (seq->size == 1) + return seq->elements[0]; + + return JoinedStr(seq, LINENO(n), n->n_col_offset, c->c_arena); + +error: + FstringParser_Dealloc(state); + return NULL; +} + +/* Given an f-string (with no 'f' or quotes) that's in str starting at + ofs, parse it into an expr_ty. Return NULL on error. Does not + decref str. */ +static expr_ty +fstring_parse(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + struct compiling *c, const node *n) +{ + FstringParser state; + + FstringParser_Init(&state); + if (FstringParser_ConcatFstring(&state, str, ofs, recurse_lvl, + c, n) < 0) { + FstringParser_Dealloc(&state); + return NULL; + } + + return FstringParser_Finish(&state, c, n); +} + +/* n is a Python string literal, including the bracketing quote + characters, and r, b, u, &/or f prefixes (if any), and embedded + escape sequences (if any). parsestr parses it, and returns the + decoded Python string object. If the string is an f-string, set + *fmode and return the unparsed string object. +*/ static PyObject * -parsestr(struct compiling *c, const node *n, int *bytesmode) +parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) { size_t len; const char *s = STR(n); @@ -4027,15 +4853,24 @@ parsestr(struct compiling *c, const node *n, int *bytesmode) quote = *++s; rawmode = 1; } + else if (quote == 'f' || quote == 'F') { + quote = *++s; + *fmode = 1; + } else { break; } } } + if (*fmode && *bytesmode) { + PyErr_BadInternalCall(); + return NULL; + } if (quote != '\'' && quote != '\"') { PyErr_BadInternalCall(); return NULL; } + /* Skip the leading quote char. */ s++; len = strlen(s); if (len > INT_MAX) { @@ -4044,12 +4879,17 @@ parsestr(struct compiling *c, const node *n, int *bytesmode) return NULL; } if (s[--len] != quote) { + /* Last quote char must match the first. */ PyErr_BadInternalCall(); return NULL; } if (len >= 4 && s[0] == quote && s[1] == quote) { + /* A triple quoted string. We've already skipped one quote at + the start and one at the end of the string. Now skip the + two at the start. */ s += 2; len -= 2; + /* And check that the last two match. */ if (s[--len] != quote || s[--len] != quote) { PyErr_BadInternalCall(); return NULL; @@ -4088,51 +4928,84 @@ parsestr(struct compiling *c, const node *n, int *bytesmode) } } return PyBytes_DecodeEscape(s, len, NULL, 1, - need_encoding ? c->c_encoding : NULL); + need_encoding ? c->c_encoding : NULL); } -/* Build a Python string object out of a STRING+ atom. This takes care of - * compile-time literal catenation, calling parsestr() on each piece, and - * pasting the intermediate results together. - */ -static PyObject * -parsestrplus(struct compiling *c, const node *n, int *bytesmode) +/* Accepts a STRING+ atom, and produces an expr_ty node. Run through + each STRING atom, and process it as needed. For bytes, just + concatenate them together, and the result will be a Bytes node. For + normal strings and f-strings, concatenate them together. The result + will be a Str node if there were no f-strings; a FormattedValue + node if there's just an f-string (with no leading or trailing + literals), or a JoinedStr node if there are multiple f-strings or + any literals involved. */ +static expr_ty +parsestrplus(struct compiling *c, const node *n) { - PyObject *v; + int bytesmode = 0; + PyObject *bytes_str = NULL; int i; - REQ(CHILD(n, 0), STRING); - v = parsestr(c, CHILD(n, 0), bytesmode); - if (v != NULL) { - /* String literal concatenation */ - for (i = 1; i < NCH(n); i++) { - PyObject *s; - int subbm = 0; - s = parsestr(c, CHILD(n, i), &subbm); - if (s == NULL) - goto onError; - if (*bytesmode != subbm) { - ast_error(c, n, "cannot mix bytes and nonbytes literals"); - Py_DECREF(s); - goto onError; - } - if (PyBytes_Check(v) && PyBytes_Check(s)) { - PyBytes_ConcatAndDel(&v, s); - if (v == NULL) - goto onError; - } - else { - PyObject *temp = PyUnicode_Concat(v, s); - Py_DECREF(s); - Py_DECREF(v); - v = temp; - if (v == NULL) - goto onError; + + FstringParser state; + FstringParser_Init(&state); + + for (i = 0; i < NCH(n); i++) { + int this_bytesmode = 0; + int this_fmode = 0; + PyObject *s; + + REQ(CHILD(n, i), STRING); + s = parsestr(c, CHILD(n, i), &this_bytesmode, &this_fmode); + if (!s) + goto error; + + /* Check that we're not mixing bytes with unicode. */ + if (i != 0 && bytesmode != this_bytesmode) { + ast_error(c, n, "cannot mix bytes and nonbytes literals"); + Py_DECREF(s); + goto error; + } + bytesmode = this_bytesmode; + + assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s)); + + if (bytesmode) { + /* For bytes, concat as we go. */ + if (i == 0) { + /* First time, just remember this value. */ + bytes_str = s; + } else { + PyBytes_ConcatAndDel(&bytes_str, s); + if (!bytes_str) + goto error; } + } else if (this_fmode) { + /* This is an f-string. Concatenate and decref it. */ + Py_ssize_t ofs = 0; + int result = FstringParser_ConcatFstring(&state, s, &ofs, 0, c, n); + Py_DECREF(s); + if (result < 0) + goto error; + } else { + /* This is a regular string. Concatenate it. */ + if (FstringParser_ConcatAndDel(&state, s) < 0) + goto error; } } - return v; + if (bytesmode) { + /* Just return the bytes object and we're done. */ + if (PyArena_AddPyObject(c->c_arena, bytes_str) < 0) + goto error; + return Bytes(bytes_str, LINENO(n), n->n_col_offset, c->c_arena); + } + + /* We're not a bytes string, bytes_str should never have been set. */ + assert(bytes_str == NULL); + + return FstringParser_Finish(&state, c, n); - onError: - Py_XDECREF(v); +error: + Py_XDECREF(bytes_str); + FstringParser_Dealloc(&state); return NULL; } diff --git a/Python/compile.c b/Python/compile.c index a6884ec..3a49ece 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -731,6 +731,7 @@ compiler_set_qualname(struct compiler *c) return 1; } + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -3209,6 +3210,117 @@ compiler_call(struct compiler *c, expr_ty e) e->v.Call.keywords); } +static int +compiler_joined_str(struct compiler *c, expr_ty e) +{ + /* Concatenate parts of a string using ''.join(parts). There are + probably better ways of doing this. + + This is used for constructs like "'x=' f'{42}'", which have to + be evaluated at compile time. */ + + static PyObject *empty_string; + static PyObject *join_string; + + if (!empty_string) { + empty_string = PyUnicode_FromString(""); + if (!empty_string) + return 0; + } + if (!join_string) { + join_string = PyUnicode_FromString("join"); + if (!join_string) + return 0; + } + + ADDOP_O(c, LOAD_CONST, empty_string, consts); + ADDOP_NAME(c, LOAD_ATTR, join_string, names); + VISIT_SEQ(c, expr, e->v.JoinedStr.values); + ADDOP_I(c, BUILD_LIST, asdl_seq_LEN(e->v.JoinedStr.values)); + ADDOP_I(c, CALL_FUNCTION, 1); + return 1; +} + +/* Note that this code uses the builtin functions format(), str(), + repr(), and ascii(). You can break this code, or make it do odd + things, by redefining those functions. */ +static int +compiler_formatted_value(struct compiler *c, expr_ty e) +{ + PyObject *conversion_name = NULL; + + static PyObject *format_string; + static PyObject *str_string; + static PyObject *repr_string; + static PyObject *ascii_string; + + if (!format_string) { + format_string = PyUnicode_InternFromString("format"); + if (!format_string) + return 0; + } + + if (!str_string) { + str_string = PyUnicode_InternFromString("str"); + if (!str_string) + return 0; + } + + if (!repr_string) { + repr_string = PyUnicode_InternFromString("repr"); + if (!repr_string) + return 0; + } + if (!ascii_string) { + ascii_string = PyUnicode_InternFromString("ascii"); + if (!ascii_string) + return 0; + } + + ADDOP_NAME(c, LOAD_GLOBAL, format_string, names); + + /* If needed, convert via str, repr, or ascii. */ + if (e->v.FormattedValue.conversion != -1) { + switch (e->v.FormattedValue.conversion) { + case 's': + conversion_name = str_string; + break; + case 'r': + conversion_name = repr_string; + break; + case 'a': + conversion_name = ascii_string; + break; + default: + PyErr_SetString(PyExc_SystemError, + "Unrecognized conversion character"); + return 0; + } + ADDOP_NAME(c, LOAD_GLOBAL, conversion_name, names); + } + + /* Evaluate the value. */ + VISIT(c, expr, e->v.FormattedValue.value); + + /* If needed, convert via str, repr, or ascii. */ + if (conversion_name) { + /* Call the function we previously pushed. */ + ADDOP_I(c, CALL_FUNCTION, 1); + } + + /* If we have a format spec, use format(value, format_spec). Otherwise, + use the single argument form. */ + if (e->v.FormattedValue.format_spec) { + VISIT(c, expr, e->v.FormattedValue.format_spec); + ADDOP_I(c, CALL_FUNCTION, 2); + } else { + /* No format spec specified, call format(value). */ + ADDOP_I(c, CALL_FUNCTION, 1); + } + + return 1; +} + /* shared code between compiler_call and compiler_class */ static int compiler_call_helper(struct compiler *c, @@ -3878,6 +3990,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e) case Str_kind: ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts); break; + case JoinedStr_kind: + return compiler_joined_str(c, e); + case FormattedValue_kind: + return compiler_formatted_value(c, e); case Bytes_kind: ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts); break; @@ -4784,4 +4900,3 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, { return PyAST_CompileEx(mod, filename, flags, -1, arena); } - diff --git a/Python/symtable.c b/Python/symtable.c index 64910d8..8431d51 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1439,6 +1439,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Call.args); VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords); break; + case FormattedValue_kind: + VISIT(st, expr, e->v.FormattedValue.value); + if (e->v.FormattedValue.format_spec) + VISIT(st, expr, e->v.FormattedValue.format_spec); + break; + case JoinedStr_kind: + VISIT_SEQ(st, expr, e->v.JoinedStr.values); + break; case Num_kind: case Str_kind: case Bytes_kind: -- cgit v0.12 From edef3ebafa3238b97d3bfe9913a904f19a1c195e Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 19 Sep 2015 15:49:57 -0400 Subject: Temporary hack for issue #25180: exclude test_fstring.py from the unparse round-tripping, while I figure out how to properly fix it. --- Lib/test/test_tools/test_unparse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_tools/test_unparse.py b/Lib/test/test_tools/test_unparse.py index 976a6c5..920fcbd 100644 --- a/Lib/test/test_tools/test_unparse.py +++ b/Lib/test/test_tools/test_unparse.py @@ -264,6 +264,8 @@ class DirectoryTestCase(ASTTestCase): for d in self.test_directories: test_dir = os.path.join(basepath, d) for n in os.listdir(test_dir): + if n == 'test_fstring.py': + continue if n.endswith('.py') and not n.startswith('bad'): names.append(os.path.join(test_dir, n)) -- cgit v0.12 From 608adf9c827a14795c21a14aef98b90a94eac287 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sun, 20 Sep 2015 15:09:15 -0400 Subject: Issue 25180: Fix Tools/parser/unparse.py for f-strings. Patch by Martin Panter. --- Lib/test/test_tools/test_unparse.py | 11 +++++++++-- Tools/parser/unparse.py | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tools/test_unparse.py b/Lib/test/test_tools/test_unparse.py index 920fcbd..4b47916 100644 --- a/Lib/test/test_tools/test_unparse.py +++ b/Lib/test/test_tools/test_unparse.py @@ -134,6 +134,15 @@ class ASTTestCase(unittest.TestCase): class UnparseTestCase(ASTTestCase): # Tests for specific bugs found in earlier versions of unparse + def test_fstrings(self): + # See issue 25180 + self.check_roundtrip(r"""f'{f"{0}"*3}'""") + self.check_roundtrip(r"""f'{f"{y}"*3}'""") + self.check_roundtrip(r"""f'{f"{\'x\'}"*3}'""") + + self.check_roundtrip(r'''f"{r'x' f'{\"s\"}'}"''') + self.check_roundtrip(r'''f"{r'x'rf'{\"s\"}'}"''') + def test_del_statement(self): self.check_roundtrip("del x, y, z") @@ -264,8 +273,6 @@ class DirectoryTestCase(ASTTestCase): for d in self.test_directories: test_dir = os.path.join(basepath, d) for n in os.listdir(test_dir): - if n == 'test_fstring.py': - continue if n.endswith('.py') and not n.startswith('bad'): names.append(os.path.join(test_dir, n)) diff --git a/Tools/parser/unparse.py b/Tools/parser/unparse.py index c828577..9972797 100644 --- a/Tools/parser/unparse.py +++ b/Tools/parser/unparse.py @@ -322,6 +322,45 @@ class Unparser: def _Str(self, tree): self.write(repr(tree.s)) + def _JoinedStr(self, t): + self.write("f") + string = io.StringIO() + self._fstring_JoinedStr(t, string.write) + self.write(repr(string.getvalue())) + + def _FormattedValue(self, t): + self.write("f") + string = io.StringIO() + self._fstring_FormattedValue(t, string.write) + self.write(repr(string.getvalue())) + + def _fstring_JoinedStr(self, t, write): + for value in t.values: + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, write) + + def _fstring_Str(self, t, write): + value = t.s.replace("{", "{{").replace("}", "}}") + write(value) + + def _fstring_FormattedValue(self, t, write): + write("{") + expr = io.StringIO() + Unparser(t.value, expr) + expr = expr.getvalue().rstrip("\n") + if expr.startswith("{"): + write(" ") # Separate pair of opening brackets as "{ {" + write(expr) + if t.conversion != -1: + conversion = chr(t.conversion) + assert conversion in "sra" + write(f"!{conversion}") + if t.format_spec: + write(":") + meth = getattr(self, "_fstring_" + type(t.format_spec).__name__) + meth(t.format_spec, write) + write("}") + def _Name(self, t): self.write(t.id) -- cgit v0.12 From 8e5bc9899522c9550e343039c207044f2c2f7c5f Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 20 Sep 2015 23:32:08 -0400 Subject: Add NEWS items for Idle to 3.6.0a1 section. --- Misc/NEWS | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index a5c5884..d16e18d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -78,6 +78,24 @@ Library - Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo functions. +IDLE +---- + +- Issue #25199: Idle: add synchronization comments for future maintainers. + +- Issue #16893: Replace help.txt with idle.html for Idle doc display. + The new idlelib/idle.html is copied from Doc/build/html/idle.html. + It looks better than help.txt and will better document Idle as released. + The tkinter html viewer that works for this file was written by Rose Roseman. + The now unused EditorWindow.HelpDialog class and helt.txt file are deprecated. + +- Issue #24199: Deprecate unused idlelib.idlever with possible removal in 3.6. + +- Issue #24782: In Idle extension config dialog, replace tabs with sorted list. + Patch by Mark Roseman. + +- Issue #24790: Remove extraneous code (which also create 2 & 3 conflicts). + Documentation ------------- -- cgit v0.12 From 5567f89c6f51a30034e07d6a4a678caf5b4bbb37 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Mon, 21 Sep 2015 13:36:09 -0400 Subject: Issue #24779: Remove unused rawmode parameter to unicode_decode. --- Python/ast.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 735424b..4b2dca4 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3938,7 +3938,7 @@ decode_utf8(struct compiling *c, const char **sPtr, const char *end) } static PyObject * -decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, const char *encoding) +decode_unicode(struct compiling *c, const char *s, size_t len, const char *encoding) { PyObject *v, *u; char *buf; @@ -3994,10 +3994,7 @@ decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons len = p - buf; s = buf; } - if (rawmode) - v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL); - else - v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); + v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); Py_XDECREF(u); return v; } @@ -4896,7 +4893,7 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) } } if (!*bytesmode && !rawmode) { - return decode_unicode(c, s, len, rawmode, c->c_encoding); + return decode_unicode(c, s, len, c->c_encoding); } if (*bytesmode) { /* Disallow non-ascii characters (but not escapes) */ -- cgit v0.12 From 1d59fee294d0bca792dacca4d5cc7dd21fedec95 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 22:29:43 +0200 Subject: Merge 3.5 (Issue #23630, fix test_asyncio) --- Lib/test/test_asyncio/test_events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 3488101..9801d22 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -764,6 +764,7 @@ class EventLoopTestsMixin: for host in hosts] self.loop.getaddrinfo = getaddrinfo_task self.loop._start_serving = mock.Mock() + self.loop._stop_serving = mock.Mock() f = self.loop.create_server(lambda: MyProto(self.loop), hosts, 80) server = self.loop.run_until_complete(f) self.addCleanup(server.close) -- cgit v0.12 From 4552ced91673b5138460e3d697b4bea8f4aab93b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 22:37:15 +0200 Subject: Issue #25207, #14626: Fix ICC compiler warnings in posixmodule.c Replace "#if XXX" with #ifdef XXX". --- Modules/posixmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6654fbb..e5b4792 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4609,7 +4609,7 @@ utime_fd(utime_t *ut, int fd) #define UTIME_HAVE_NOFOLLOW_SYMLINKS \ (defined(HAVE_UTIMENSAT) || defined(HAVE_LUTIMES)) -#if UTIME_HAVE_NOFOLLOW_SYMLINKS +#ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS static int utime_nofollow_symlinks(utime_t *ut, char *path) @@ -4771,7 +4771,7 @@ os_utime_impl(PyModuleDef *module, path_t *path, PyObject *times, utime.now = 1; } -#if !UTIME_HAVE_NOFOLLOW_SYMLINKS +#if !defined(UTIME_HAVE_NOFOLLOW_SYMLINKS) if (follow_symlinks_specified("utime", follow_symlinks)) goto exit; #endif @@ -4825,7 +4825,7 @@ os_utime_impl(PyModuleDef *module, path_t *path, PyObject *times, #else /* MS_WINDOWS */ Py_BEGIN_ALLOW_THREADS -#if UTIME_HAVE_NOFOLLOW_SYMLINKS +#ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS if ((!follow_symlinks) && (dir_fd == DEFAULT_DIR_FD)) result = utime_nofollow_symlinks(&utime, path->narrow); else -- cgit v0.12 From ba4529593877f2016215149912496e030782a57e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 22:40:28 +0200 Subject: ssue #25207: fix ICC compiler warning in msvcrtmodule.c --- PC/msvcrtmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 52d4100..d73ce79 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -541,7 +541,6 @@ PyInit_msvcrt(void) #endif /* constants for the crt versions */ - (void)st; #ifdef _VC_ASSEMBLY_PUBLICKEYTOKEN st = PyModule_AddStringConstant(m, "VC_ASSEMBLY_PUBLICKEYTOKEN", _VC_ASSEMBLY_PUBLICKEYTOKEN); @@ -567,6 +566,8 @@ PyInit_msvcrt(void) st = PyModule_AddObject(m, "CRT_ASSEMBLY_VERSION", version); if (st < 0) return NULL; #endif + /* make compiler warning quiet if st is unused */ + (void)st; return m; } -- cgit v0.12 From f96418de05fc8710f9dc1e3a19878f8e744956f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 23:06:27 +0200 Subject: Issue #24870: Optimize the ASCII decoder for error handlers: surrogateescape, ignore and replace. Initial patch written by Naoki Inada. The decoder is now up to 60 times as fast for these error handlers. Add also unit tests for the ASCII decoder. --- Doc/whatsnew/3.6.rst | 3 ++- Lib/test/test_codecs.py | 32 ++++++++++++++++++++++++ Objects/unicodeobject.c | 66 +++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 48ff38a..8a2b5d3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -106,7 +106,8 @@ operator Optimizations ============= -* None yet. +* The ASCII decoder is now up to 60 times as fast for error handlers: + ``surrogateescape``, ``ignore`` and ``replace``. Build and C API Changes diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index a4a6f95..e0e3119 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -27,6 +27,7 @@ def coding_checker(self, coder): self.assertEqual(coder(input), (expect, len(input))) return check + class Queue(object): """ queue: write bytes at one end, read bytes from the other end @@ -47,6 +48,7 @@ class Queue(object): self._buffer = self._buffer[size:] return s + class MixInCheckStateHandling: def check_state_handling_decode(self, encoding, u, s): for i in range(len(s)+1): @@ -80,6 +82,7 @@ class MixInCheckStateHandling: part2 = d.encode(u[i:], True) self.assertEqual(s, part1+part2) + class ReadTest(MixInCheckStateHandling): def check_partial(self, input, partialresults): # get a StreamReader for the encoding and feed the bytestring version @@ -383,6 +386,7 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"), before + backslashreplace + after) + class UTF32Test(ReadTest, unittest.TestCase): encoding = "utf-32" if sys.byteorder == 'little': @@ -478,6 +482,7 @@ class UTF32Test(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) + class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" @@ -523,6 +528,7 @@ class UTF32LETest(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) + class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" @@ -797,6 +803,7 @@ class UTF8Test(ReadTest, unittest.TestCase): with self.assertRaises(UnicodeDecodeError): b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + @unittest.skipUnless(sys.platform == 'win32', 'cp65001 is a Windows-only codec') class CP65001Test(ReadTest, unittest.TestCase): @@ -1136,6 +1143,7 @@ class EscapeDecodeTest(unittest.TestCase): self.assertEqual(decode(br"[\x0]\x0", "ignore"), (b"[]", 8)) self.assertEqual(decode(br"[\x0]\x0", "replace"), (b"[?]?", 8)) + class RecodingTest(unittest.TestCase): def test_recoding(self): f = io.BytesIO() @@ -1255,6 +1263,7 @@ for i in punycode_testcases: if len(i)!=2: print(repr(i)) + class PunycodeTest(unittest.TestCase): def test_encode(self): for uni, puny in punycode_testcases: @@ -1274,6 +1283,7 @@ class PunycodeTest(unittest.TestCase): puny = puny.decode("ascii").encode("ascii") self.assertEqual(uni, puny.decode("punycode")) + class UnicodeInternalTest(unittest.TestCase): @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_bug1251300(self): @@ -1528,6 +1538,7 @@ class NameprepTest(unittest.TestCase): except Exception as e: raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e))) + class IDNACodecTest(unittest.TestCase): def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") @@ -1614,6 +1625,7 @@ class IDNACodecTest(unittest.TestCase): self.assertRaises(Exception, b"python.org".decode, "idna", errors) + class CodecsModuleTest(unittest.TestCase): def test_decode(self): @@ -1722,6 +1734,7 @@ class CodecsModuleTest(unittest.TestCase): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) + class StreamReaderTest(unittest.TestCase): def setUp(self): @@ -1732,6 +1745,7 @@ class StreamReaderTest(unittest.TestCase): f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + class EncodedFileTest(unittest.TestCase): def test_basic(self): @@ -1862,6 +1876,7 @@ broken_unicode_with_stateful = [ "unicode_internal" ] + class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): def test_basics(self): s = "abc123" # all codecs should be able to encode these @@ -2024,6 +2039,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): self.check_state_handling_decode(encoding, u, u.encode(encoding)) self.check_state_handling_encode(encoding, u, u.encode(encoding)) + class CharmapTest(unittest.TestCase): def test_decode_with_string_map(self): self.assertEqual( @@ -2274,6 +2290,7 @@ class WithStmtTest(unittest.TestCase): info.streamwriter, 'strict') as srw: self.assertEqual(srw.read(), "\xfc") + class TypesTest(unittest.TestCase): def test_decode_unicode(self): # Most decoders don't accept unicode input @@ -2564,6 +2581,7 @@ else: bytes_transform_encodings.append("bz2_codec") transform_aliases["bz2_codec"] = ["bz2"] + class TransformCodecTest(unittest.TestCase): def test_basics(self): @@ -3041,5 +3059,19 @@ class CodePageTest(unittest.TestCase): self.assertEqual(decoded, ('abc', 3)) +class ASCIITest(unittest.TestCase): + def test_decode(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode('ascii', error_handler), + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0709789..b8840e1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6644,6 +6644,28 @@ PyUnicode_AsLatin1String(PyObject *unicode) /* --- 7-bit ASCII Codec -------------------------------------------------- */ +typedef enum { + _Py_ERROR_UNKNOWN=0, + _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_REPLACE, + _Py_ERROR_IGNORE, + _Py_ERROR_OTHER +} _Py_error_handler; + +static _Py_error_handler +get_error_handler(const char *errors) +{ + if (errors == NULL) + return _Py_ERROR_OTHER; + if (strcmp(errors, "surrogateescape") == 0) + return _Py_ERROR_SURROGATEESCAPE; + if (strcmp(errors, "ignore") == 0) + return _Py_ERROR_IGNORE; + if (strcmp(errors, "replace") == 0) + return _Py_ERROR_REPLACE; + return _Py_ERROR_OTHER; +} + PyObject * PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, @@ -6657,8 +6679,9 @@ PyUnicode_DecodeASCII(const char *s, Py_ssize_t endinpos; Py_ssize_t outpos; const char *e; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) _Py_RETURN_UNICODE_EMPTY(); @@ -6687,12 +6710,45 @@ PyUnicode_DecodeASCII(const char *s, PyUnicode_WRITE(kind, data, writer.pos, c); writer.pos++; ++s; + continue; } - else { + + /* byte outsize range 0x00..0x7f: call the error handler */ + + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) + { + case _Py_ERROR_REPLACE: + case _Py_ERROR_SURROGATEESCAPE: + /* Fast-path: the error handler only writes one character, + but we must switch to UCS2 at the first write */ + if (kind < PyUnicode_2BYTE_KIND) { + if (_PyUnicodeWriter_Prepare(&writer, size - writer.pos, + 0xffff) < 0) + return NULL; + kind = writer.kind; + data = writer.data; + } + + if (error_handler == _Py_ERROR_REPLACE) + PyUnicode_WRITE(kind, data, writer.pos, 0xfffd); + else + PyUnicode_WRITE(kind, data, writer.pos, c + 0xdc00); + writer.pos++; + ++s; + break; + + case _Py_ERROR_IGNORE: + ++s; + break; + + default: startinpos = s-starts; endinpos = startinpos + 1; if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, + errors, &error_handler_obj, "ascii", "ordinal not in range(128)", &starts, &e, &startinpos, &endinpos, &exc, &s, &writer)) @@ -6701,13 +6757,13 @@ PyUnicode_DecodeASCII(const char *s, data = writer.data; } } - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: _PyUnicodeWriter_Dealloc(&writer); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; } -- cgit v0.12 From 5014920cb72768bc54924e55e7004e79fcad94f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 00:26:54 +0200 Subject: Issue #24870: Reuse the new _Py_error_handler enum Factorize code with the new get_error_handler() function. Add some empty lines for readability. --- Objects/unicodeobject.c | 164 +++++++++++++++++++++++------------------------- 1 file changed, 77 insertions(+), 87 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index b8840e1..f5f2d48 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -293,6 +293,34 @@ static unsigned char ascii_linebreak[] = { #include "clinic/unicodeobject.c.h" +typedef enum { + _Py_ERROR_UNKNOWN=0, + _Py_ERROR_STRICT, + _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_REPLACE, + _Py_ERROR_IGNORE, + _Py_ERROR_XMLCHARREFREPLACE, + _Py_ERROR_OTHER +} _Py_error_handler; + +static _Py_error_handler +get_error_handler(const char *errors) +{ + if (errors == NULL) + return _Py_ERROR_STRICT; + if (strcmp(errors, "strict") == 0) + return _Py_ERROR_STRICT; + if (strcmp(errors, "surrogateescape") == 0) + return _Py_ERROR_SURROGATEESCAPE; + if (strcmp(errors, "ignore") == 0) + return _Py_ERROR_IGNORE; + if (strcmp(errors, "replace") == 0) + return _Py_ERROR_REPLACE; + if (strcmp(errors, "xmlcharrefreplace") == 0) + return _Py_ERROR_XMLCHARREFREPLACE; + return _Py_ERROR_OTHER; +} + /* The max unicode value is always 0x10FFFF while using the PEP-393 API. This function is kept for backward compatibility with the old API. */ Py_UNICODE @@ -3163,24 +3191,22 @@ wcstombs_errorpos(const wchar_t *wstr) static int locale_error_handler(const char *errors, int *surrogateescape) { - if (errors == NULL) { - *surrogateescape = 0; - return 0; - } - - if (strcmp(errors, "strict") == 0) { + _Py_error_handler error_handler = get_error_handler(errors); + switch (error_handler) + { + case _Py_ERROR_STRICT: *surrogateescape = 0; return 0; - } - if (strcmp(errors, "surrogateescape") == 0) { + case _Py_ERROR_SURROGATEESCAPE: *surrogateescape = 1; return 0; + default: + PyErr_Format(PyExc_ValueError, + "only 'strict' and 'surrogateescape' error handlers " + "are supported, not '%s'", + errors); + return -1; } - PyErr_Format(PyExc_ValueError, - "only 'strict' and 'surrogateescape' error handlers " - "are supported, not '%s'", - errors); - return -1; } PyObject * @@ -6403,11 +6429,9 @@ unicode_encode_ucs1(PyObject *unicode, Py_ssize_t ressize; const char *encoding = (limit == 256) ? "latin-1" : "ascii"; const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)"; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; - /* the following variable is used for caching string comparisons - * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ - int known_errorHandler = -1; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (PyUnicode_READY(unicode) == -1) return NULL; @@ -6441,32 +6465,28 @@ unicode_encode_ucs1(PyObject *unicode, Py_ssize_t collstart = pos; Py_ssize_t collend = pos; /* find all unecodable characters */ + while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit)) ++collend; + /* cache callback name lookup (if not done yet, i.e. it's the first error) */ - if (known_errorHandler==-1) { - if ((errors==NULL) || (!strcmp(errors, "strict"))) - known_errorHandler = 1; - else if (!strcmp(errors, "replace")) - known_errorHandler = 2; - else if (!strcmp(errors, "ignore")) - known_errorHandler = 3; - else if (!strcmp(errors, "xmlcharrefreplace")) - known_errorHandler = 4; - else - known_errorHandler = 0; - } - switch (known_errorHandler) { - case 1: /* strict */ + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) { + case _Py_ERROR_STRICT: raise_encode_exception(&exc, encoding, unicode, collstart, collend, reason); goto onError; - case 2: /* replace */ + + case _Py_ERROR_REPLACE: while (collstart++ < collend) - *str++ = '?'; /* fall through */ - case 3: /* ignore */ + *str++ = '?'; + /* fall through */ + case _Py_ERROR_IGNORE: pos = collend; break; - case 4: /* xmlcharrefreplace */ + + case _Py_ERROR_XMLCHARREFREPLACE: respos = str - PyBytes_AS_STRING(res); requiredsize = respos; /* determine replacement size */ @@ -6510,8 +6530,9 @@ unicode_encode_ucs1(PyObject *unicode, } pos = collend; break; + default: - repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, + repunicode = unicode_encode_call_errorhandler(errors, &error_handler_obj, encoding, reason, unicode, &exc, collstart, collend, &newpos); if (repunicode == NULL || (PyUnicode_Check(repunicode) && @@ -6587,7 +6608,7 @@ unicode_encode_ucs1(PyObject *unicode, goto onError; } - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return res; @@ -6597,7 +6618,7 @@ unicode_encode_ucs1(PyObject *unicode, onError: Py_XDECREF(res); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; } @@ -6644,28 +6665,6 @@ PyUnicode_AsLatin1String(PyObject *unicode) /* --- 7-bit ASCII Codec -------------------------------------------------- */ -typedef enum { - _Py_ERROR_UNKNOWN=0, - _Py_ERROR_SURROGATEESCAPE, - _Py_ERROR_REPLACE, - _Py_ERROR_IGNORE, - _Py_ERROR_OTHER -} _Py_error_handler; - -static _Py_error_handler -get_error_handler(const char *errors) -{ - if (errors == NULL) - return _Py_ERROR_OTHER; - if (strcmp(errors, "surrogateescape") == 0) - return _Py_ERROR_SURROGATEESCAPE; - if (strcmp(errors, "ignore") == 0) - return _Py_ERROR_IGNORE; - if (strcmp(errors, "replace") == 0) - return _Py_ERROR_REPLACE; - return _Py_ERROR_OTHER; -} - PyObject * PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, @@ -8129,7 +8128,7 @@ static int charmap_encoding_error( PyObject *unicode, Py_ssize_t *inpos, PyObject *mapping, PyObject **exceptionObject, - int *known_errorHandler, PyObject **errorHandler, const char *errors, + _Py_error_handler *error_handler, PyObject **error_handler_obj, const char *errors, PyObject **res, Py_ssize_t *respos) { PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ @@ -8176,23 +8175,15 @@ charmap_encoding_error( } /* cache callback name lookup * (if not done yet, i.e. it's the first error) */ - if (*known_errorHandler==-1) { - if ((errors==NULL) || (!strcmp(errors, "strict"))) - *known_errorHandler = 1; - else if (!strcmp(errors, "replace")) - *known_errorHandler = 2; - else if (!strcmp(errors, "ignore")) - *known_errorHandler = 3; - else if (!strcmp(errors, "xmlcharrefreplace")) - *known_errorHandler = 4; - else - *known_errorHandler = 0; - } - switch (*known_errorHandler) { - case 1: /* strict */ + if (*error_handler == _Py_ERROR_UNKNOWN) + *error_handler = get_error_handler(errors); + + switch (*error_handler) { + case _Py_ERROR_STRICT: raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; - case 2: /* replace */ + + case _Py_ERROR_REPLACE: for (collpos = collstartpos; collpos Date: Tue, 22 Sep 2015 00:58:32 +0200 Subject: Issue #24870: Add _PyUnicodeWriter_PrepareKind() macro Add a macro which ensures that the writer has at least the requested kind. --- Include/unicodeobject.h | 17 +++++++++++++++++ Objects/unicodeobject.c | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 33e8f19..d0e0142 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -942,6 +942,23 @@ PyAPI_FUNC(int) _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar); +/* Prepare the buffer to have at least the kind KIND. + For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will + support characters in range U+000-U+FFFF. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ + (assert((KIND) != PyUnicode_WCHAR_KIND), \ + (KIND) <= (WRITER)->kind \ + ? 0 \ + : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) + +/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() + macro instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + enum PyUnicode_Kind kind); + /* Append a Unicode character. Return 0 on success, raise an exception and return -1 on error. */ PyAPI_FUNC(int) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f5f2d48..7c079e0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6722,14 +6722,11 @@ PyUnicode_DecodeASCII(const char *s, case _Py_ERROR_REPLACE: case _Py_ERROR_SURROGATEESCAPE: /* Fast-path: the error handler only writes one character, - but we must switch to UCS2 at the first write */ - if (kind < PyUnicode_2BYTE_KIND) { - if (_PyUnicodeWriter_Prepare(&writer, size - writer.pos, - 0xffff) < 0) - return NULL; - kind = writer.kind; - data = writer.data; - } + but we may switch to UCS2 at the first write */ + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + kind = writer.kind; + data = writer.data; if (error_handler == _Py_ERROR_REPLACE) PyUnicode_WRITE(kind, data, writer.pos, 0xfffd); @@ -13309,7 +13306,8 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t newlen; PyObject *newbuffer; - assert(length > 0); + /* ensure that the _PyUnicodeWriter_Prepare macro was used */ + assert(maxchar > writer->maxchar || length > 0); if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); @@ -13375,6 +13373,28 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, #undef OVERALLOCATE_FACTOR } +int +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + enum PyUnicode_Kind kind) +{ + Py_UCS4 maxchar; + + /* ensure that the _PyUnicodeWriter_PrepareKind macro was used */ + assert(writer->kind < kind); + + switch (kind) + { + case PyUnicode_1BYTE_KIND: maxchar = 0xff; break; + case PyUnicode_2BYTE_KIND: maxchar = 0xffff; break; + case PyUnicode_4BYTE_KIND: maxchar = 0x10ffff; break; + default: + assert(0 && "invalid kind"); + return -1; + } + + return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar); +} + Py_LOCAL_INLINE(int) _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) { -- cgit v0.12 From 6174474bea9fe6f5f12f05a16004eabb817ce721 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 01:01:17 +0200 Subject: _PyUnicodeWriter_PrepareInternal(): make the assertion more strict --- Objects/unicodeobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7c079e0..d0b285a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13307,7 +13307,8 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, PyObject *newbuffer; /* ensure that the _PyUnicodeWriter_Prepare macro was used */ - assert(maxchar > writer->maxchar || length > 0); + assert((maxchar > writer->maxchar && length >= 0) + || length > 0); if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); -- cgit v0.12 From 5ebae876281828c17f139ec063dae43a39fd7741 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 01:29:33 +0200 Subject: Issue #25207, #14626: Fix my commit. It doesn't work to use #define XXX defined(YYY)" and then "#ifdef XXX" to check YYY. --- Modules/posixmodule.c | 6 +++--- Objects/unicodeobject.c | 52 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e5b4792..467ce2c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4605,9 +4605,9 @@ utime_fd(utime_t *ut, int fd) #define PATH_UTIME_HAVE_FD 0 #endif - -#define UTIME_HAVE_NOFOLLOW_SYMLINKS \ - (defined(HAVE_UTIMENSAT) || defined(HAVE_LUTIMES)) +#if defined(HAVE_UTIMENSAT) || defined(HAVE_LUTIMES) +# define UTIME_HAVE_NOFOLLOW_SYMLINKS +#endif #ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d0b285a..63a627f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4709,8 +4709,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; const char *errmsg = ""; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) { if (consumed) @@ -4773,24 +4774,57 @@ PyUnicode_DecodeUTF8Stateful(const char *s, continue; } - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; + /* undecodable byte: call the error handler */ + + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) + { + case _Py_ERROR_REPLACE: + case _Py_ERROR_SURROGATEESCAPE: + { + unsigned char ch = (unsigned char)*s; + + /* Fast-path: the error handler only writes one character, + but we may switch to UCS2 at the first write */ + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + kind = writer.kind; + + if (error_handler == _Py_ERROR_REPLACE) + PyUnicode_WRITE(kind, writer.data, writer.pos, 0xfffd); + else + PyUnicode_WRITE(kind, writer.data, writer.pos, ch + 0xdc00); + writer.pos++; + ++s; + break; + } + + case _Py_ERROR_IGNORE: + s++; + break; + + default: + if (unicode_decode_call_errorhandler_writer( + errors, &error_handler_obj, + "utf-8", errmsg, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &writer)) + goto onError; + } } End: if (consumed) *consumed = s - starts; - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); _PyUnicodeWriter_Dealloc(&writer); return NULL; -- cgit v0.12 From d3d2b2c50c8e85e1d3f9429964e080d0ea5f8634 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 21 Sep 2015 23:41:56 -0700 Subject: Minor consistency improvements to negative value comparisons. --- Modules/_collectionsmodule.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 3e2c69f..7dbf755 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -209,7 +209,7 @@ deque_pop(dequeobject *deque, PyObject *unused) Py_SIZE(deque)--; deque->state++; - if (deque->rightindex == -1) { + if (deque->rightindex < 0) { if (Py_SIZE(deque)) { prevblock = deque->rightblock->leftlink; assert(deque->leftblock != deque->rightblock); @@ -715,7 +715,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) *(dest++) = *(src++); } while (--m); } - if (rightindex == -1) { + if (rightindex < 0) { assert(leftblock != rightblock); assert(b == NULL); b = rightblock; @@ -827,7 +827,7 @@ deque_reverse(dequeobject *deque, PyObject *unused) /* Step backwards with the right block/index pair */ rightindex--; - if (rightindex == -1) { + if (rightindex < 0) { rightblock = rightblock->leftlink; rightindex = BLOCKLEN - 1; } @@ -1234,7 +1234,7 @@ deque_copy(PyObject *deque) Py_DECREF(new_deque); return NULL; } - if (old_deque->maxlen == -1) + if (old_deque->maxlen < 0) return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); else return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", @@ -1258,12 +1258,12 @@ deque_reduce(dequeobject *deque) return NULL; } if (dict == NULL) { - if (deque->maxlen == -1) + if (deque->maxlen < 0) result = Py_BuildValue("O(O)", Py_TYPE(deque), aslist); else result = Py_BuildValue("O(On)", Py_TYPE(deque), aslist, deque->maxlen); } else { - if (deque->maxlen == -1) + if (deque->maxlen < 0) result = Py_BuildValue("O(OO)O", Py_TYPE(deque), aslist, Py_None, dict); else result = Py_BuildValue("O(On)O", Py_TYPE(deque), aslist, deque->maxlen, dict); @@ -1354,7 +1354,7 @@ deque_richcompare(PyObject *v, PyObject *w, int op) } Py_DECREF(x); Py_DECREF(y); - if (b == -1) + if (b < 0) goto done; } /* We reached the end of one deque or both */ @@ -1437,7 +1437,7 @@ deque_bool(dequeobject *deque) static PyObject * deque_get_maxlen(dequeobject *deque) { - if (deque->maxlen == -1) + if (deque->maxlen < 0) Py_RETURN_NONE; return PyLong_FromSsize_t(deque->maxlen); } @@ -1778,7 +1778,7 @@ dequereviter_next(dequeiterobject *it) item = it->b->data[it->index]; it->index--; it->counter--; - if (it->index == -1 && it->counter > 0) { + if (it->index < 0 && it->counter > 0) { CHECK_NOT_END(it->b->leftlink); it->b = it->b->leftlink; it->index = BLOCKLEN - 1; -- cgit v0.12 From 7a237230d1a98a369de7e3c6581a42d31465430a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 22 Sep 2015 01:20:36 -0700 Subject: Eliminate unnecessary variable --- Modules/_collectionsmodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 7dbf755..4eb31bf 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -804,10 +804,9 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_ssize_t leftindex = deque->leftindex; Py_ssize_t rightindex = deque->rightindex; Py_ssize_t n = Py_SIZE(deque) >> 1; - Py_ssize_t i; PyObject *tmp; - for (i=0 ; i 0) { /* Validate that pointers haven't met in the middle */ assert(leftblock != rightblock || leftindex < rightindex); CHECK_NOT_END(leftblock); -- cgit v0.12 From 54385b206d5a91e17eaf7d54cadb049847fa1c8a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 10:46:52 +0200 Subject: Issue #24870: revert unwanted change Sorry, I pushed the patch on the UTF-8 decoder by mistake :-( --- Objects/unicodeobject.c | 52 +++++++++---------------------------------------- 1 file changed, 9 insertions(+), 43 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 63a627f..d0b285a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4709,9 +4709,8 @@ PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; const char *errmsg = ""; - PyObject *error_handler_obj = NULL; + PyObject *errorHandler = NULL; PyObject *exc = NULL; - _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) { if (consumed) @@ -4774,57 +4773,24 @@ PyUnicode_DecodeUTF8Stateful(const char *s, continue; } - /* undecodable byte: call the error handler */ - - if (error_handler == _Py_ERROR_UNKNOWN) - error_handler = get_error_handler(errors); - - switch (error_handler) - { - case _Py_ERROR_REPLACE: - case _Py_ERROR_SURROGATEESCAPE: - { - unsigned char ch = (unsigned char)*s; - - /* Fast-path: the error handler only writes one character, - but we may switch to UCS2 at the first write */ - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) - goto onError; - kind = writer.kind; - - if (error_handler == _Py_ERROR_REPLACE) - PyUnicode_WRITE(kind, writer.data, writer.pos, 0xfffd); - else - PyUnicode_WRITE(kind, writer.data, writer.pos, ch + 0xdc00); - writer.pos++; - ++s; - break; - } - - case _Py_ERROR_IGNORE: - s++; - break; - - default: - if (unicode_decode_call_errorhandler_writer( - errors, &error_handler_obj, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - } + if (unicode_decode_call_errorhandler_writer( + errors, &errorHandler, + "utf-8", errmsg, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &writer)) + goto onError; } End: if (consumed) *consumed = s - starts; - Py_XDECREF(error_handler_obj); + Py_XDECREF(errorHandler); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: - Py_XDECREF(error_handler_obj); + Py_XDECREF(errorHandler); Py_XDECREF(exc); _PyUnicodeWriter_Dealloc(&writer); return NULL; -- cgit v0.12 From 3a1a8d04244cb5d98e8bbeaff131a705f84b8521 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 23 Sep 2015 02:42:02 -0700 Subject: Eliminate unnecessary variables --- Modules/_collectionsmodule.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 4eb31bf..be6c90c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -843,13 +843,12 @@ deque_count(dequeobject *deque, PyObject *v) block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; Py_ssize_t n = Py_SIZE(deque); - Py_ssize_t i; Py_ssize_t count = 0; size_t start_state = deque->state; PyObject *item; int cmp; - for (i=0 ; idata[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); @@ -883,12 +882,11 @@ deque_contains(dequeobject *deque, PyObject *v) block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; Py_ssize_t n = Py_SIZE(deque); - Py_ssize_t i; size_t start_state = deque->state; PyObject *item; int cmp; - for (i=0 ; idata[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); -- cgit v0.12 From 1d44c41b0cc183603b31ac943ef452456bef089f Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 07:49:00 -0400 Subject: Move f-string compilation of the expression earlier, before the conversion character and format_spec are checked. This allows for error messages that more closely match what a user would expect. --- Lib/test/test_fstring.py | 9 +++++++ Python/ast.c | 66 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index a6ff9cf..2a1a3cd 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -287,6 +287,15 @@ f'{a * x()}'""" "f' { } '", r"f'{\n}'", r"f'{\n \n}'", + + # Catch the empty expression before the + # invalid conversion. + "f'{!x}'", + "f'{ !xr}'", + "f'{!x:}'", + "f'{!x:a}'", + "f'{ !xr:}'", + "f'{ !xr:a}'", ]) def test_parens_in_expressions(self): diff --git a/Python/ast.c b/Python/ast.c index 4b2dca4..83e4f00 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4007,13 +4007,14 @@ decode_unicode(struct compiling *c, const char *s, size_t len, const char *encod expression. This is to allow strings with embedded newlines, for example. */ static expr_ty -fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, - Py_ssize_t expr_end, PyArena *arena) +fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, + Py_ssize_t expr_end, PyArena *arena) { PyCompilerFlags cf; mod_ty mod; char *utf_expr; Py_ssize_t i; + Py_UCS4 end_ch = -1; int all_whitespace; PyObject *sub = NULL; @@ -4023,6 +4024,16 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, assert(str); + assert(expr_start >= 0 && expr_start < PyUnicode_GET_LENGTH(str)); + assert(expr_end >= 0 && expr_end < PyUnicode_GET_LENGTH(str)); + assert(expr_end >= expr_start); + + /* There has to be at least on character on each side of the + expression inside this str. This will have been caught before + we're called. */ + assert(expr_start >= 1); + assert(expr_end <= PyUnicode_GET_LENGTH(str)-1); + /* If the substring is all whitespace, it's an error. We need to catch this here, and not when we call PyParser_ASTFromString, because turning the expression '' in to '()' would go from @@ -4049,10 +4060,17 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, string directly. */ if (expr_start-1 == 0 && expr_end+1 == PyUnicode_GET_LENGTH(str)) { - /* No need to actually remember these characters, because we - know they must be braces. */ + /* If str is well formed, then the first and last chars must + be '{' and '}', respectively. But, if there's a syntax + error, for example f'{3!', then the last char won't be a + closing brace. So, remember the last character we read in + order for us to restore it. */ + end_ch = PyUnicode_ReadChar(str, expr_end-expr_start+1); + assert(end_ch != (Py_UCS4)-1); + + /* In all cases, however, start_ch must be '{'. */ assert(PyUnicode_ReadChar(str, 0) == '{'); - assert(PyUnicode_ReadChar(str, expr_end-expr_start+1) == '}'); + sub = str; } else { /* Create a substring object. It must be a new object, with @@ -4064,21 +4082,23 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, decref_sub = 1; /* Remember to deallocate it on error. */ } + /* Put () around the expression. */ if (PyUnicode_WriteChar(sub, 0, '(') < 0 || PyUnicode_WriteChar(sub, expr_end-expr_start+1, ')') < 0) goto error; - cf.cf_flags = PyCF_ONLY_AST; - /* No need to free the memory returned here: it's managed by the string. */ utf_expr = PyUnicode_AsUTF8(sub); if (!utf_expr) goto error; + + cf.cf_flags = PyCF_ONLY_AST; mod = PyParser_ASTFromString(utf_expr, "", Py_eval_input, &cf, arena); if (!mod) goto error; + if (sub != str) /* Clear instead of decref in case we ever modify this code to change the error handling: this is safest because the XDECREF won't try @@ -4089,9 +4109,10 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, Py_CLEAR(sub); else { assert(!decref_sub); + assert(end_ch != (Py_UCS4)-1); /* Restore str, which we earlier modified directly. */ if (PyUnicode_WriteChar(str, 0, '{') < 0 || - PyUnicode_WriteChar(str, expr_end-expr_start+1, '}') < 0) + PyUnicode_WriteChar(str, expr_end-expr_start+1, end_ch) < 0) goto error; } return mod->v.Expression.body; @@ -4100,6 +4121,18 @@ error: /* Only decref sub if it was the result of a call to SubString. */ if (decref_sub) Py_XDECREF(sub); + + if (end_ch != (Py_UCS4)-1) { + /* We only get here if we modified str. Make sure that's the + case: str will be equal to sub. */ + if (str == sub) { + /* Don't check the error, because we've already set the + error state (that's why we're in 'error', after + all). */ + PyUnicode_WriteChar(str, 0, '{'); + PyUnicode_WriteChar(str, expr_end-expr_start+1, end_ch); + } + } return NULL; } @@ -4331,9 +4364,18 @@ fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, return -1; } - /* Check for a conversion char, if present. */ if (*ofs >= PyUnicode_GET_LENGTH(str)) goto unexpected_end_of_string; + + /* Compile the expression as soon as possible, so we show errors + related to the expression before errors related to the + conversion or format_spec. */ + simple_expression = fstring_compile_expr(str, expr_start, expr_end, + c->c_arena); + if (!simple_expression) + return -1; + + /* Check for a conversion char, if present. */ if (PyUnicode_READ(kind, data, *ofs) == '!') { *ofs += 1; if (*ofs >= PyUnicode_GET_LENGTH(str)) @@ -4374,12 +4416,6 @@ fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, assert(PyUnicode_READ(kind, data, *ofs) == '}'); *ofs += 1; - /* Compile the expression. */ - simple_expression = fstring_expression_compile(str, expr_start, expr_end, - c->c_arena); - if (!simple_expression) - return -1; - /* And now create the FormattedValue node that represents this entire expression with the conversion and format spec. */ *expression = FormattedValue(simple_expression, (int)conversion, -- cgit v0.12 From 548c4d31784048d64eb11ab13652ca0d4abb6d39 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 08:00:01 -0400 Subject: Added more f-string test for empty expressions. --- Lib/test/test_fstring.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 2a1a3cd..10c5849 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -296,6 +296,9 @@ f'{a * x()}'""" "f'{!x:a}'", "f'{ !xr:}'", "f'{ !xr:a}'", + + "f'{!}'", + "f'{:}'", ]) def test_parens_in_expressions(self): -- cgit v0.12 From b2080f65544093c8f07b4ecc43fe035aeb08c8f0 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 10:24:43 -0400 Subject: f-strings: More tests for empty expressions along with missing closing braces. --- Lib/test/test_fstring.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 10c5849..d6f781c 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -299,6 +299,13 @@ f'{a * x()}'""" "f'{!}'", "f'{:}'", + + # We find the empty expression before the + # missing closing brace. + "f'{!'", + "f'{!s:'", + "f'{:'", + "f'{:x'", ]) def test_parens_in_expressions(self): -- cgit v0.12 From 7b92abf0e972158d20585cc7ef5c72aec56811cd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2015 23:04:18 +0200 Subject: Issue #25220: Create Lib/test/libregrtest/ Start to split regrtest.py into smaller parts with the creation of Lib/test/libregrtest/cmdline.py. --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 340 +++++++++++++++++++++++++++++++++++++++ Lib/test/regrtest.py | 332 +------------------------------------- Lib/test/test_regrtest.py | 4 +- 4 files changed, 345 insertions(+), 332 deletions(-) create mode 100644 Lib/test/libregrtest/__init__.py create mode 100644 Lib/test/libregrtest/cmdline.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py new file mode 100644 index 0000000..554aeed --- /dev/null +++ b/Lib/test/libregrtest/__init__.py @@ -0,0 +1 @@ +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py new file mode 100644 index 0000000..9464996 --- /dev/null +++ b/Lib/test/libregrtest/cmdline.py @@ -0,0 +1,340 @@ +import argparse +import faulthandler +import os + +from test import support + +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b49e66b..51e9e18 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,119 +6,6 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - # We import importlib *ASAP* in order to test #15386 import importlib @@ -153,6 +40,8 @@ try: except ImportError: multiprocessing = None +from test.libregrtest import _parse_args + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -198,9 +87,6 @@ CHILD_ERROR = -5 # error in a child process from test import support -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -210,220 +96,6 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns - - def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a398a4f..cf4b84f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support +from test import regrtest, support, libregrtest class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(regrtest.RESOURCE_NAMES) + expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v0.12 From 0cca00b67edc10bbbf6eae00cb79756fa365033e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2015 23:16:47 +0200 Subject: Issue #25220: Backed out changeset eaf9a99b6bb8 --- Lib/test/libregrtest/__init__.py | 1 - Lib/test/libregrtest/cmdline.py | 340 --------------------------------------- Lib/test/regrtest.py | 332 +++++++++++++++++++++++++++++++++++++- Lib/test/test_regrtest.py | 4 +- 4 files changed, 332 insertions(+), 345 deletions(-) delete mode 100644 Lib/test/libregrtest/__init__.py delete mode 100644 Lib/test/libregrtest/cmdline.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py deleted file mode 100644 index 554aeed..0000000 --- a/Lib/test/libregrtest/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py deleted file mode 100644 index 9464996..0000000 --- a/Lib/test/libregrtest/cmdline.py +++ /dev/null @@ -1,340 +0,0 @@ -import argparse -import faulthandler -import os - -from test import support - -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - - -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 51e9e18..b49e66b 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,6 +6,119 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + # We import importlib *ASAP* in order to test #15386 import importlib @@ -40,8 +153,6 @@ try: except ImportError: multiprocessing = None -from test.libregrtest import _parse_args - # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -87,6 +198,9 @@ CHILD_ERROR = -5 # error in a child process from test import support +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -96,6 +210,220 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns + + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index cf4b84f..a398a4f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support, libregrtest +from test import regrtest, support class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.RESOURCE_NAMES) + expected = list(regrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v0.12 From 2b0d646b751e6e5bdb466c05ffd2824dd46c7d60 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 23 Sep 2015 19:15:44 -0700 Subject: Replace an unpredictable branch with a simple addition. --- Modules/_collectionsmodule.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index be6c90c..8d753c3 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -852,10 +852,9 @@ deque_count(dequeobject *deque, PyObject *v) CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); - if (cmp > 0) - count++; - else if (cmp < 0) + if (cmp < 0) return NULL; + count += cmp; if (start_state != deque->state) { PyErr_SetString(PyExc_RuntimeError, -- cgit v0.12 From 1e5fcc3dea4263a5c01ecc4cd9b5d755fab5ee6a Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Thu, 24 Sep 2015 08:52:04 -0400 Subject: Fixed error creation if the problem is an empty expression in an f-string: use ast_error instead of PyErr_SetString. --- Python/ast.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 83e4f00..7eab3c0 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4008,7 +4008,8 @@ decode_unicode(struct compiling *c, const char *s, size_t len, const char *encod example. */ static expr_ty fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, - Py_ssize_t expr_end, PyArena *arena) + Py_ssize_t expr_end, struct compiling *c, const node *n) + { PyCompilerFlags cf; mod_ty mod; @@ -4048,8 +4049,7 @@ fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, } } if (all_whitespace) { - PyErr_SetString(PyExc_SyntaxError, "f-string: empty expression " - "not allowed"); + ast_error(c, n, "f-string: empty expression not allowed"); goto error; } @@ -4095,7 +4095,7 @@ fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, cf.cf_flags = PyCF_ONLY_AST; mod = PyParser_ASTFromString(utf_expr, "", - Py_eval_input, &cf, arena); + Py_eval_input, &cf, c->c_arena); if (!mod) goto error; @@ -4370,8 +4370,7 @@ fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, /* Compile the expression as soon as possible, so we show errors related to the expression before errors related to the conversion or format_spec. */ - simple_expression = fstring_compile_expr(str, expr_start, expr_end, - c->c_arena); + simple_expression = fstring_compile_expr(str, expr_start, expr_end, c, n); if (!simple_expression) return -1; -- cgit v0.12 From 0030cd52dacdd95d2017a0947d661feb737449af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Sep 2015 14:45:00 +0200 Subject: Issue #25227: Cleanup unicode_encode_ucs1() error handler * Change limit type from unsigned int to Py_UCS4, to use the same type than the "ch" variable (an Unicode character). * Reuse ch variable for _Py_ERROR_XMLCHARREFREPLACE * Add some newlines for readability --- Objects/unicodeobject.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d0b285a..da2aac7 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6415,7 +6415,7 @@ unicode_encode_call_errorhandler(const char *errors, static PyObject * unicode_encode_ucs1(PyObject *unicode, const char *errors, - unsigned int limit) + const Py_UCS4 limit) { /* input state */ Py_ssize_t pos=0, size; @@ -6449,12 +6449,12 @@ unicode_encode_ucs1(PyObject *unicode, ressize = size; while (pos < size) { - Py_UCS4 c = PyUnicode_READ(kind, data, pos); + Py_UCS4 ch = PyUnicode_READ(kind, data, pos); /* can we encode this? */ - if (c0; ++i, ++str) { - c = PyUnicode_READ_CHAR(repunicode, i); - if (c >= limit) { + ch = PyUnicode_READ_CHAR(repunicode, i); + if (ch >= limit) { raise_encode_exception(&exc, encoding, unicode, pos, pos+1, reason); Py_DECREF(repunicode); goto onError; } - *str = (char)c; + *str = (char)ch; } pos = newpos; Py_DECREF(repunicode); -- cgit v0.12 From 1dae0c68dd98fca70de121a61d28493a3e92a3ac Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 25 Sep 2015 13:05:13 -0700 Subject: Issue #25186: Remove duplicated function from importlib._bootstrap_external --- Lib/importlib/_bootstrap_external.py | 48 +- Python/importlib_external.h | 4977 +++++++++++++++++----------------- 2 files changed, 2502 insertions(+), 2523 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 3508ce9..abde6d9 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -360,14 +360,6 @@ def _calc_mode(path): return mode -def _verbose_message(message, *args, verbosity=1): - """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - if sys.flags.verbose >= verbosity: - if not message.startswith(('#', 'import ')): - message = '# ' + message - print(message.format(*args), file=sys.stderr) - - def _check_name(method): """Decorator to verify that the module being requested matches the one the loader can handle. @@ -437,15 +429,15 @@ def _validate_bytecode_header(data, source_stats=None, name=None, path=None): raw_size = data[8:12] if magic != MAGIC_NUMBER: message = 'bad magic number in {!r}: {!r}'.format(name, magic) - _verbose_message(message) + _bootstrap._verbose_message(message) raise ImportError(message, **exc_details) elif len(raw_timestamp) != 4: message = 'reached EOF while reading timestamp in {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise EOFError(message) elif len(raw_size) != 4: message = 'reached EOF while reading size of source in {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise EOFError(message) if source_stats is not None: try: @@ -455,7 +447,7 @@ def _validate_bytecode_header(data, source_stats=None, name=None, path=None): else: if _r_long(raw_timestamp) != source_mtime: message = 'bytecode is stale for {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise ImportError(message, **exc_details) try: source_size = source_stats['size'] & 0xFFFFFFFF @@ -472,7 +464,7 @@ def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): """Compile bytecode as returned by _validate_bytecode_header().""" code = marshal.loads(data) if isinstance(code, _code_type): - _verbose_message('code object from {!r}', bytecode_path) + _bootstrap._verbose_message('code object from {!r}', bytecode_path) if source_path is not None: _imp._fix_co_filename(code, source_path) return code @@ -755,21 +747,21 @@ class SourceLoader(_LoaderBasics): except (ImportError, EOFError): pass else: - _verbose_message('{} matches {}', bytecode_path, - source_path) + _bootstrap._verbose_message('{} matches {}', bytecode_path, + source_path) return _compile_bytecode(bytes_data, name=fullname, bytecode_path=bytecode_path, source_path=source_path) source_bytes = self.get_data(source_path) code_object = self.source_to_code(source_bytes, source_path) - _verbose_message('code object from {}', source_path) + _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): data = _code_to_bytecode(code_object, source_mtime, len(source_bytes)) try: self._cache_bytecode(source_path, bytecode_path, data) - _verbose_message('wrote {!r}', bytecode_path) + _bootstrap._verbose_message('wrote {!r}', bytecode_path) except NotImplementedError: pass return code_object @@ -849,14 +841,16 @@ class SourceFileLoader(FileLoader, SourceLoader): except OSError as exc: # Could be a permission error, read-only filesystem: just forget # about writing the data. - _verbose_message('could not create {!r}: {!r}', parent, exc) + _bootstrap._verbose_message('could not create {!r}: {!r}', + parent, exc) return try: _write_atomic(path, data, _mode) - _verbose_message('created {!r}', path) + _bootstrap._verbose_message('created {!r}', path) except OSError as exc: # Same as above: just don't write the bytecode. - _verbose_message('could not create {!r}: {!r}', path, exc) + _bootstrap._verbose_message('could not create {!r}: {!r}', path, + exc) class SourcelessFileLoader(FileLoader, _LoaderBasics): @@ -901,14 +895,14 @@ class ExtensionFileLoader(FileLoader, _LoaderBasics): """Create an unitialized extension module""" module = _bootstrap._call_with_frames_removed( _imp.create_dynamic, spec) - _verbose_message('extension module {!r} loaded from {!r}', + _bootstrap._verbose_message('extension module {!r} loaded from {!r}', spec.name, self.path) return module def exec_module(self, module): """Initialize an extension module""" _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) - _verbose_message('extension module {!r} executed from {!r}', + _bootstrap._verbose_message('extension module {!r} executed from {!r}', self.name, self.path) def is_package(self, fullname): @@ -1023,7 +1017,8 @@ class _NamespaceLoader: """ # The import system never calls this method. - _verbose_message('namespace module loaded with path {!r}', self._path) + _bootstrap._verbose_message('namespace module loaded with path {!r}', + self._path) return _bootstrap._load_module_shim(self, fullname) @@ -1243,12 +1238,13 @@ class FileFinder: # Check for a file w/ a proper suffix exists. for suffix, loader_class in self._loaders: full_path = _path_join(self.path, tail_module + suffix) - _verbose_message('trying {}'.format(full_path), verbosity=2) + _bootstrap._verbose_message('trying {}', full_path, verbosity=2) if cache_module + suffix in cache: if _path_isfile(full_path): - return self._get_spec(loader_class, fullname, full_path, None, target) + return self._get_spec(loader_class, fullname, full_path, + None, target) if is_namespace: - _verbose_message('possible namespace for {}'.format(base_path)) + _bootstrap._verbose_message('possible namespace for {}', base_path) spec = _bootstrap.ModuleSpec(fullname, None) spec.submodule_search_locations = [base_path] return spec diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 48d6be5..f4f5ac4 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1,8 +1,8 @@ /* Auto-generated by Programs/_freeze_importlib.c */ const unsigned char _Py_M__importlib_external[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0, - 0,64,0,0,0,115,228,2,0,0,100,0,0,90,0,0, - 100,96,0,90,1,0,100,4,0,100,5,0,132,0,0,90, + 0,64,0,0,0,115,210,2,0,0,100,0,0,90,0,0, + 100,92,0,90,1,0,100,4,0,100,5,0,132,0,0,90, 2,0,100,6,0,100,7,0,132,0,0,90,3,0,100,8, 0,100,9,0,132,0,0,90,4,0,100,10,0,100,11,0, 132,0,0,90,5,0,100,12,0,100,13,0,132,0,0,90, @@ -20,563 +20,541 @@ const unsigned char _Py_M__importlib_external[] = { 1,1,90,26,0,100,37,0,100,38,0,132,0,0,90,27, 0,100,39,0,100,40,0,132,0,0,90,28,0,100,41,0, 100,42,0,132,0,0,90,29,0,100,43,0,100,44,0,132, - 0,0,90,30,0,100,45,0,100,46,0,100,47,0,100,48, - 0,132,0,1,90,31,0,100,49,0,100,50,0,132,0,0, - 90,32,0,100,51,0,100,52,0,132,0,0,90,33,0,100, - 33,0,100,33,0,100,33,0,100,53,0,100,54,0,132,3, - 0,90,34,0,100,33,0,100,33,0,100,33,0,100,55,0, - 100,56,0,132,3,0,90,35,0,100,57,0,100,57,0,100, - 58,0,100,59,0,132,2,0,90,36,0,100,60,0,100,61, - 0,132,0,0,90,37,0,101,38,0,131,0,0,90,39,0, - 100,33,0,100,62,0,100,33,0,100,63,0,101,39,0,100, - 64,0,100,65,0,132,1,2,90,40,0,71,100,66,0,100, - 67,0,132,0,0,100,67,0,131,2,0,90,41,0,71,100, - 68,0,100,69,0,132,0,0,100,69,0,131,2,0,90,42, - 0,71,100,70,0,100,71,0,132,0,0,100,71,0,101,42, - 0,131,3,0,90,43,0,71,100,72,0,100,73,0,132,0, - 0,100,73,0,131,2,0,90,44,0,71,100,74,0,100,75, - 0,132,0,0,100,75,0,101,44,0,101,43,0,131,4,0, - 90,45,0,71,100,76,0,100,77,0,132,0,0,100,77,0, - 101,44,0,101,42,0,131,4,0,90,46,0,103,0,0,90, - 47,0,71,100,78,0,100,79,0,132,0,0,100,79,0,101, - 44,0,101,42,0,131,4,0,90,48,0,71,100,80,0,100, - 81,0,132,0,0,100,81,0,131,2,0,90,49,0,71,100, - 82,0,100,83,0,132,0,0,100,83,0,131,2,0,90,50, - 0,71,100,84,0,100,85,0,132,0,0,100,85,0,131,2, - 0,90,51,0,71,100,86,0,100,87,0,132,0,0,100,87, - 0,131,2,0,90,52,0,100,33,0,100,88,0,100,89,0, - 132,1,0,90,53,0,100,90,0,100,91,0,132,0,0,90, - 54,0,100,92,0,100,93,0,132,0,0,90,55,0,100,94, - 0,100,95,0,132,0,0,90,56,0,100,33,0,83,41,97, - 97,94,1,0,0,67,111,114,101,32,105,109,112,108,101,109, - 101,110,116,97,116,105,111,110,32,111,102,32,112,97,116,104, - 45,98,97,115,101,100,32,105,109,112,111,114,116,46,10,10, - 84,104,105,115,32,109,111,100,117,108,101,32,105,115,32,78, - 79,84,32,109,101,97,110,116,32,116,111,32,98,101,32,100, - 105,114,101,99,116,108,121,32,105,109,112,111,114,116,101,100, - 33,32,73,116,32,104,97,115,32,98,101,101,110,32,100,101, - 115,105,103,110,101,100,32,115,117,99,104,10,116,104,97,116, - 32,105,116,32,99,97,110,32,98,101,32,98,111,111,116,115, - 116,114,97,112,112,101,100,32,105,110,116,111,32,80,121,116, - 104,111,110,32,97,115,32,116,104,101,32,105,109,112,108,101, - 109,101,110,116,97,116,105,111,110,32,111,102,32,105,109,112, - 111,114,116,46,32,65,115,10,115,117,99,104,32,105,116,32, - 114,101,113,117,105,114,101,115,32,116,104,101,32,105,110,106, - 101,99,116,105,111,110,32,111,102,32,115,112,101,99,105,102, - 105,99,32,109,111,100,117,108,101,115,32,97,110,100,32,97, - 116,116,114,105,98,117,116,101,115,32,105,110,32,111,114,100, - 101,114,32,116,111,10,119,111,114,107,46,32,79,110,101,32, - 115,104,111,117,108,100,32,117,115,101,32,105,109,112,111,114, - 116,108,105,98,32,97,115,32,116,104,101,32,112,117,98,108, - 105,99,45,102,97,99,105,110,103,32,118,101,114,115,105,111, - 110,32,111,102,32,116,104,105,115,32,109,111,100,117,108,101, - 46,10,10,218,3,119,105,110,218,6,99,121,103,119,105,110, - 218,6,100,97,114,119,105,110,99,0,0,0,0,0,0,0, - 0,1,0,0,0,2,0,0,0,67,0,0,0,115,49,0, - 0,0,116,0,0,106,1,0,106,2,0,116,3,0,131,1, - 0,114,33,0,100,1,0,100,2,0,132,0,0,125,0,0, - 110,12,0,100,3,0,100,2,0,132,0,0,125,0,0,124, - 0,0,83,41,4,78,99,0,0,0,0,0,0,0,0,0, - 0,0,0,2,0,0,0,83,0,0,0,115,13,0,0,0, - 100,1,0,116,0,0,106,1,0,107,6,0,83,41,2,122, - 53,84,114,117,101,32,105,102,32,102,105,108,101,110,97,109, - 101,115,32,109,117,115,116,32,98,101,32,99,104,101,99,107, - 101,100,32,99,97,115,101,45,105,110,115,101,110,115,105,116, - 105,118,101,108,121,46,115,12,0,0,0,80,89,84,72,79, - 78,67,65,83,69,79,75,41,2,218,3,95,111,115,90,7, - 101,110,118,105,114,111,110,169,0,114,4,0,0,0,114,4, - 0,0,0,250,38,60,102,114,111,122,101,110,32,105,109,112, - 111,114,116,108,105,98,46,95,98,111,111,116,115,116,114,97, - 112,95,101,120,116,101,114,110,97,108,62,218,11,95,114,101, - 108,97,120,95,99,97,115,101,30,0,0,0,115,2,0,0, - 0,0,2,122,37,95,109,97,107,101,95,114,101,108,97,120, - 95,99,97,115,101,46,60,108,111,99,97,108,115,62,46,95, - 114,101,108,97,120,95,99,97,115,101,99,0,0,0,0,0, - 0,0,0,0,0,0,0,1,0,0,0,83,0,0,0,115, - 4,0,0,0,100,1,0,83,41,2,122,53,84,114,117,101, - 32,105,102,32,102,105,108,101,110,97,109,101,115,32,109,117, - 115,116,32,98,101,32,99,104,101,99,107,101,100,32,99,97, - 115,101,45,105,110,115,101,110,115,105,116,105,118,101,108,121, - 46,70,114,4,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,6,0,0,0, - 34,0,0,0,115,2,0,0,0,0,2,41,4,218,3,115, - 121,115,218,8,112,108,97,116,102,111,114,109,218,10,115,116, - 97,114,116,115,119,105,116,104,218,27,95,67,65,83,69,95, - 73,78,83,69,78,83,73,84,73,86,69,95,80,76,65,84, - 70,79,82,77,83,41,1,114,6,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,16,95,109,97, - 107,101,95,114,101,108,97,120,95,99,97,115,101,28,0,0, - 0,115,8,0,0,0,0,1,18,1,15,4,12,3,114,11, + 0,0,90,30,0,100,45,0,100,46,0,132,0,0,90,31, + 0,100,47,0,100,48,0,132,0,0,90,32,0,100,33,0, + 100,33,0,100,33,0,100,49,0,100,50,0,132,3,0,90, + 33,0,100,33,0,100,33,0,100,33,0,100,51,0,100,52, + 0,132,3,0,90,34,0,100,53,0,100,53,0,100,54,0, + 100,55,0,132,2,0,90,35,0,100,56,0,100,57,0,132, + 0,0,90,36,0,101,37,0,131,0,0,90,38,0,100,33, + 0,100,58,0,100,33,0,100,59,0,101,38,0,100,60,0, + 100,61,0,132,1,2,90,39,0,71,100,62,0,100,63,0, + 132,0,0,100,63,0,131,2,0,90,40,0,71,100,64,0, + 100,65,0,132,0,0,100,65,0,131,2,0,90,41,0,71, + 100,66,0,100,67,0,132,0,0,100,67,0,101,41,0,131, + 3,0,90,42,0,71,100,68,0,100,69,0,132,0,0,100, + 69,0,131,2,0,90,43,0,71,100,70,0,100,71,0,132, + 0,0,100,71,0,101,43,0,101,42,0,131,4,0,90,44, + 0,71,100,72,0,100,73,0,132,0,0,100,73,0,101,43, + 0,101,41,0,131,4,0,90,45,0,103,0,0,90,46,0, + 71,100,74,0,100,75,0,132,0,0,100,75,0,101,43,0, + 101,41,0,131,4,0,90,47,0,71,100,76,0,100,77,0, + 132,0,0,100,77,0,131,2,0,90,48,0,71,100,78,0, + 100,79,0,132,0,0,100,79,0,131,2,0,90,49,0,71, + 100,80,0,100,81,0,132,0,0,100,81,0,131,2,0,90, + 50,0,71,100,82,0,100,83,0,132,0,0,100,83,0,131, + 2,0,90,51,0,100,33,0,100,84,0,100,85,0,132,1, + 0,90,52,0,100,86,0,100,87,0,132,0,0,90,53,0, + 100,88,0,100,89,0,132,0,0,90,54,0,100,90,0,100, + 91,0,132,0,0,90,55,0,100,33,0,83,41,93,97,94, + 1,0,0,67,111,114,101,32,105,109,112,108,101,109,101,110, + 116,97,116,105,111,110,32,111,102,32,112,97,116,104,45,98, + 97,115,101,100,32,105,109,112,111,114,116,46,10,10,84,104, + 105,115,32,109,111,100,117,108,101,32,105,115,32,78,79,84, + 32,109,101,97,110,116,32,116,111,32,98,101,32,100,105,114, + 101,99,116,108,121,32,105,109,112,111,114,116,101,100,33,32, + 73,116,32,104,97,115,32,98,101,101,110,32,100,101,115,105, + 103,110,101,100,32,115,117,99,104,10,116,104,97,116,32,105, + 116,32,99,97,110,32,98,101,32,98,111,111,116,115,116,114, + 97,112,112,101,100,32,105,110,116,111,32,80,121,116,104,111, + 110,32,97,115,32,116,104,101,32,105,109,112,108,101,109,101, + 110,116,97,116,105,111,110,32,111,102,32,105,109,112,111,114, + 116,46,32,65,115,10,115,117,99,104,32,105,116,32,114,101, + 113,117,105,114,101,115,32,116,104,101,32,105,110,106,101,99, + 116,105,111,110,32,111,102,32,115,112,101,99,105,102,105,99, + 32,109,111,100,117,108,101,115,32,97,110,100,32,97,116,116, + 114,105,98,117,116,101,115,32,105,110,32,111,114,100,101,114, + 32,116,111,10,119,111,114,107,46,32,79,110,101,32,115,104, + 111,117,108,100,32,117,115,101,32,105,109,112,111,114,116,108, + 105,98,32,97,115,32,116,104,101,32,112,117,98,108,105,99, + 45,102,97,99,105,110,103,32,118,101,114,115,105,111,110,32, + 111,102,32,116,104,105,115,32,109,111,100,117,108,101,46,10, + 10,218,3,119,105,110,218,6,99,121,103,119,105,110,218,6, + 100,97,114,119,105,110,99,0,0,0,0,0,0,0,0,1, + 0,0,0,2,0,0,0,67,0,0,0,115,49,0,0,0, + 116,0,0,106,1,0,106,2,0,116,3,0,131,1,0,114, + 33,0,100,1,0,100,2,0,132,0,0,125,0,0,110,12, + 0,100,3,0,100,2,0,132,0,0,125,0,0,124,0,0, + 83,41,4,78,99,0,0,0,0,0,0,0,0,0,0,0, + 0,2,0,0,0,83,0,0,0,115,13,0,0,0,100,1, + 0,116,0,0,106,1,0,107,6,0,83,41,2,122,53,84, + 114,117,101,32,105,102,32,102,105,108,101,110,97,109,101,115, + 32,109,117,115,116,32,98,101,32,99,104,101,99,107,101,100, + 32,99,97,115,101,45,105,110,115,101,110,115,105,116,105,118, + 101,108,121,46,115,12,0,0,0,80,89,84,72,79,78,67, + 65,83,69,79,75,41,2,218,3,95,111,115,90,7,101,110, + 118,105,114,111,110,169,0,114,4,0,0,0,114,4,0,0, + 0,250,38,60,102,114,111,122,101,110,32,105,109,112,111,114, + 116,108,105,98,46,95,98,111,111,116,115,116,114,97,112,95, + 101,120,116,101,114,110,97,108,62,218,11,95,114,101,108,97, + 120,95,99,97,115,101,30,0,0,0,115,2,0,0,0,0, + 2,122,37,95,109,97,107,101,95,114,101,108,97,120,95,99, + 97,115,101,46,60,108,111,99,97,108,115,62,46,95,114,101, + 108,97,120,95,99,97,115,101,99,0,0,0,0,0,0,0, + 0,0,0,0,0,1,0,0,0,83,0,0,0,115,4,0, + 0,0,100,1,0,83,41,2,122,53,84,114,117,101,32,105, + 102,32,102,105,108,101,110,97,109,101,115,32,109,117,115,116, + 32,98,101,32,99,104,101,99,107,101,100,32,99,97,115,101, + 45,105,110,115,101,110,115,105,116,105,118,101,108,121,46,70, + 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,6,0,0,0,34,0, + 0,0,115,2,0,0,0,0,2,41,4,218,3,115,121,115, + 218,8,112,108,97,116,102,111,114,109,218,10,115,116,97,114, + 116,115,119,105,116,104,218,27,95,67,65,83,69,95,73,78, + 83,69,78,83,73,84,73,86,69,95,80,76,65,84,70,79, + 82,77,83,41,1,114,6,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,16,95,109,97,107,101, + 95,114,101,108,97,120,95,99,97,115,101,28,0,0,0,115, + 8,0,0,0,0,1,18,1,15,4,12,3,114,11,0,0, + 0,99,1,0,0,0,0,0,0,0,1,0,0,0,3,0, + 0,0,67,0,0,0,115,26,0,0,0,116,0,0,124,0, + 0,131,1,0,100,1,0,64,106,1,0,100,2,0,100,3, + 0,131,2,0,83,41,4,122,42,67,111,110,118,101,114,116, + 32,97,32,51,50,45,98,105,116,32,105,110,116,101,103,101, + 114,32,116,111,32,108,105,116,116,108,101,45,101,110,100,105, + 97,110,46,108,3,0,0,0,255,127,255,127,3,0,233,4, + 0,0,0,218,6,108,105,116,116,108,101,41,2,218,3,105, + 110,116,218,8,116,111,95,98,121,116,101,115,41,1,218,1, + 120,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 218,7,95,119,95,108,111,110,103,40,0,0,0,115,2,0, + 0,0,0,2,114,17,0,0,0,99,1,0,0,0,0,0, + 0,0,1,0,0,0,3,0,0,0,67,0,0,0,115,16, + 0,0,0,116,0,0,106,1,0,124,0,0,100,1,0,131, + 2,0,83,41,2,122,47,67,111,110,118,101,114,116,32,52, + 32,98,121,116,101,115,32,105,110,32,108,105,116,116,108,101, + 45,101,110,100,105,97,110,32,116,111,32,97,110,32,105,110, + 116,101,103,101,114,46,114,13,0,0,0,41,2,114,14,0, + 0,0,218,10,102,114,111,109,95,98,121,116,101,115,41,1, + 90,9,105,110,116,95,98,121,116,101,115,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,7,95,114,95,108, + 111,110,103,45,0,0,0,115,2,0,0,0,0,2,114,19, + 0,0,0,99,0,0,0,0,0,0,0,0,1,0,0,0, + 3,0,0,0,71,0,0,0,115,26,0,0,0,116,0,0, + 106,1,0,100,1,0,100,2,0,132,0,0,124,0,0,68, + 131,1,0,131,1,0,83,41,3,122,31,82,101,112,108,97, + 99,101,109,101,110,116,32,102,111,114,32,111,115,46,112,97, + 116,104,46,106,111,105,110,40,41,46,99,1,0,0,0,0, + 0,0,0,2,0,0,0,4,0,0,0,83,0,0,0,115, + 37,0,0,0,103,0,0,124,0,0,93,27,0,125,1,0, + 124,1,0,114,6,0,124,1,0,106,0,0,116,1,0,131, + 1,0,145,2,0,113,6,0,83,114,4,0,0,0,41,2, + 218,6,114,115,116,114,105,112,218,15,112,97,116,104,95,115, + 101,112,97,114,97,116,111,114,115,41,2,218,2,46,48,218, + 4,112,97,114,116,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,250,10,60,108,105,115,116,99,111,109,112,62, + 52,0,0,0,115,2,0,0,0,9,1,122,30,95,112,97, + 116,104,95,106,111,105,110,46,60,108,111,99,97,108,115,62, + 46,60,108,105,115,116,99,111,109,112,62,41,2,218,8,112, + 97,116,104,95,115,101,112,218,4,106,111,105,110,41,1,218, + 10,112,97,116,104,95,112,97,114,116,115,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,10,95,112,97,116, + 104,95,106,111,105,110,50,0,0,0,115,4,0,0,0,0, + 2,15,1,114,28,0,0,0,99,1,0,0,0,0,0,0, + 0,5,0,0,0,5,0,0,0,67,0,0,0,115,134,0, + 0,0,116,0,0,116,1,0,131,1,0,100,1,0,107,2, + 0,114,52,0,124,0,0,106,2,0,116,3,0,131,1,0, + 92,3,0,125,1,0,125,2,0,125,3,0,124,1,0,124, + 3,0,102,2,0,83,120,69,0,116,4,0,124,0,0,131, + 1,0,68,93,55,0,125,4,0,124,4,0,116,1,0,107, + 6,0,114,65,0,124,0,0,106,5,0,124,4,0,100,2, + 0,100,1,0,131,1,1,92,2,0,125,1,0,125,3,0, + 124,1,0,124,3,0,102,2,0,83,113,65,0,87,100,3, + 0,124,0,0,102,2,0,83,41,4,122,32,82,101,112,108, + 97,99,101,109,101,110,116,32,102,111,114,32,111,115,46,112, + 97,116,104,46,115,112,108,105,116,40,41,46,233,1,0,0, + 0,90,8,109,97,120,115,112,108,105,116,218,0,41,6,218, + 3,108,101,110,114,21,0,0,0,218,10,114,112,97,114,116, + 105,116,105,111,110,114,25,0,0,0,218,8,114,101,118,101, + 114,115,101,100,218,6,114,115,112,108,105,116,41,5,218,4, + 112,97,116,104,90,5,102,114,111,110,116,218,1,95,218,4, + 116,97,105,108,114,16,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,11,95,112,97,116,104,95, + 115,112,108,105,116,56,0,0,0,115,16,0,0,0,0,2, + 18,1,24,1,10,1,19,1,12,1,27,1,14,1,114,38, 0,0,0,99,1,0,0,0,0,0,0,0,1,0,0,0, - 3,0,0,0,67,0,0,0,115,26,0,0,0,116,0,0, - 124,0,0,131,1,0,100,1,0,64,106,1,0,100,2,0, - 100,3,0,131,2,0,83,41,4,122,42,67,111,110,118,101, - 114,116,32,97,32,51,50,45,98,105,116,32,105,110,116,101, - 103,101,114,32,116,111,32,108,105,116,116,108,101,45,101,110, - 100,105,97,110,46,108,3,0,0,0,255,127,255,127,3,0, - 233,4,0,0,0,218,6,108,105,116,116,108,101,41,2,218, - 3,105,110,116,218,8,116,111,95,98,121,116,101,115,41,1, - 218,1,120,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,7,95,119,95,108,111,110,103,40,0,0,0,115, - 2,0,0,0,0,2,114,17,0,0,0,99,1,0,0,0, - 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, - 115,16,0,0,0,116,0,0,106,1,0,124,0,0,100,1, - 0,131,2,0,83,41,2,122,47,67,111,110,118,101,114,116, - 32,52,32,98,121,116,101,115,32,105,110,32,108,105,116,116, - 108,101,45,101,110,100,105,97,110,32,116,111,32,97,110,32, - 105,110,116,101,103,101,114,46,114,13,0,0,0,41,2,114, - 14,0,0,0,218,10,102,114,111,109,95,98,121,116,101,115, - 41,1,90,9,105,110,116,95,98,121,116,101,115,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,7,95,114, - 95,108,111,110,103,45,0,0,0,115,2,0,0,0,0,2, - 114,19,0,0,0,99,0,0,0,0,0,0,0,0,1,0, - 0,0,3,0,0,0,71,0,0,0,115,26,0,0,0,116, - 0,0,106,1,0,100,1,0,100,2,0,132,0,0,124,0, - 0,68,131,1,0,131,1,0,83,41,3,122,31,82,101,112, + 2,0,0,0,67,0,0,0,115,13,0,0,0,116,0,0, + 106,1,0,124,0,0,131,1,0,83,41,1,122,126,83,116, + 97,116,32,116,104,101,32,112,97,116,104,46,10,10,32,32, + 32,32,77,97,100,101,32,97,32,115,101,112,97,114,97,116, + 101,32,102,117,110,99,116,105,111,110,32,116,111,32,109,97, + 107,101,32,105,116,32,101,97,115,105,101,114,32,116,111,32, + 111,118,101,114,114,105,100,101,32,105,110,32,101,120,112,101, + 114,105,109,101,110,116,115,10,32,32,32,32,40,101,46,103, + 46,32,99,97,99,104,101,32,115,116,97,116,32,114,101,115, + 117,108,116,115,41,46,10,10,32,32,32,32,41,2,114,3, + 0,0,0,90,4,115,116,97,116,41,1,114,35,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,218, + 10,95,112,97,116,104,95,115,116,97,116,68,0,0,0,115, + 2,0,0,0,0,7,114,39,0,0,0,99,2,0,0,0, + 0,0,0,0,3,0,0,0,11,0,0,0,67,0,0,0, + 115,58,0,0,0,121,16,0,116,0,0,124,0,0,131,1, + 0,125,2,0,87,110,22,0,4,116,1,0,107,10,0,114, + 40,0,1,1,1,100,1,0,83,89,110,1,0,88,124,2, + 0,106,2,0,100,2,0,64,124,1,0,107,2,0,83,41, + 3,122,49,84,101,115,116,32,119,104,101,116,104,101,114,32, + 116,104,101,32,112,97,116,104,32,105,115,32,116,104,101,32, + 115,112,101,99,105,102,105,101,100,32,109,111,100,101,32,116, + 121,112,101,46,70,105,0,240,0,0,41,3,114,39,0,0, + 0,218,7,79,83,69,114,114,111,114,218,7,115,116,95,109, + 111,100,101,41,3,114,35,0,0,0,218,4,109,111,100,101, + 90,9,115,116,97,116,95,105,110,102,111,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,18,95,112,97,116, + 104,95,105,115,95,109,111,100,101,95,116,121,112,101,78,0, + 0,0,115,10,0,0,0,0,2,3,1,16,1,13,1,9, + 1,114,43,0,0,0,99,1,0,0,0,0,0,0,0,1, + 0,0,0,3,0,0,0,67,0,0,0,115,13,0,0,0, + 116,0,0,124,0,0,100,1,0,131,2,0,83,41,2,122, + 31,82,101,112,108,97,99,101,109,101,110,116,32,102,111,114, + 32,111,115,46,112,97,116,104,46,105,115,102,105,108,101,46, + 105,0,128,0,0,41,1,114,43,0,0,0,41,1,114,35, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,12,95,112,97,116,104,95,105,115,102,105,108,101, + 87,0,0,0,115,2,0,0,0,0,2,114,44,0,0,0, + 99,1,0,0,0,0,0,0,0,1,0,0,0,3,0,0, + 0,67,0,0,0,115,31,0,0,0,124,0,0,115,18,0, + 116,0,0,106,1,0,131,0,0,125,0,0,116,2,0,124, + 0,0,100,1,0,131,2,0,83,41,2,122,30,82,101,112, 108,97,99,101,109,101,110,116,32,102,111,114,32,111,115,46, - 112,97,116,104,46,106,111,105,110,40,41,46,99,1,0,0, - 0,0,0,0,0,2,0,0,0,4,0,0,0,83,0,0, - 0,115,37,0,0,0,103,0,0,124,0,0,93,27,0,125, - 1,0,124,1,0,114,6,0,124,1,0,106,0,0,116,1, - 0,131,1,0,145,2,0,113,6,0,83,114,4,0,0,0, - 41,2,218,6,114,115,116,114,105,112,218,15,112,97,116,104, - 95,115,101,112,97,114,97,116,111,114,115,41,2,218,2,46, - 48,218,4,112,97,114,116,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,250,10,60,108,105,115,116,99,111,109, - 112,62,52,0,0,0,115,2,0,0,0,9,1,122,30,95, - 112,97,116,104,95,106,111,105,110,46,60,108,111,99,97,108, - 115,62,46,60,108,105,115,116,99,111,109,112,62,41,2,218, - 8,112,97,116,104,95,115,101,112,218,4,106,111,105,110,41, - 1,218,10,112,97,116,104,95,112,97,114,116,115,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,10,95,112, - 97,116,104,95,106,111,105,110,50,0,0,0,115,4,0,0, - 0,0,2,15,1,114,28,0,0,0,99,1,0,0,0,0, - 0,0,0,5,0,0,0,5,0,0,0,67,0,0,0,115, - 134,0,0,0,116,0,0,116,1,0,131,1,0,100,1,0, - 107,2,0,114,52,0,124,0,0,106,2,0,116,3,0,131, - 1,0,92,3,0,125,1,0,125,2,0,125,3,0,124,1, - 0,124,3,0,102,2,0,83,120,69,0,116,4,0,124,0, - 0,131,1,0,68,93,55,0,125,4,0,124,4,0,116,1, - 0,107,6,0,114,65,0,124,0,0,106,5,0,124,4,0, - 100,2,0,100,1,0,131,1,1,92,2,0,125,1,0,125, - 3,0,124,1,0,124,3,0,102,2,0,83,113,65,0,87, - 100,3,0,124,0,0,102,2,0,83,41,4,122,32,82,101, - 112,108,97,99,101,109,101,110,116,32,102,111,114,32,111,115, - 46,112,97,116,104,46,115,112,108,105,116,40,41,46,233,1, - 0,0,0,90,8,109,97,120,115,112,108,105,116,218,0,41, - 6,218,3,108,101,110,114,21,0,0,0,218,10,114,112,97, - 114,116,105,116,105,111,110,114,25,0,0,0,218,8,114,101, - 118,101,114,115,101,100,218,6,114,115,112,108,105,116,41,5, - 218,4,112,97,116,104,90,5,102,114,111,110,116,218,1,95, - 218,4,116,97,105,108,114,16,0,0,0,114,4,0,0,0, + 112,97,116,104,46,105,115,100,105,114,46,105,0,64,0,0, + 41,3,114,3,0,0,0,218,6,103,101,116,99,119,100,114, + 43,0,0,0,41,1,114,35,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,11,95,112,97,116, - 104,95,115,112,108,105,116,56,0,0,0,115,16,0,0,0, - 0,2,18,1,24,1,10,1,19,1,12,1,27,1,14,1, - 114,38,0,0,0,99,1,0,0,0,0,0,0,0,1,0, - 0,0,2,0,0,0,67,0,0,0,115,13,0,0,0,116, - 0,0,106,1,0,124,0,0,131,1,0,83,41,1,122,126, - 83,116,97,116,32,116,104,101,32,112,97,116,104,46,10,10, - 32,32,32,32,77,97,100,101,32,97,32,115,101,112,97,114, - 97,116,101,32,102,117,110,99,116,105,111,110,32,116,111,32, - 109,97,107,101,32,105,116,32,101,97,115,105,101,114,32,116, - 111,32,111,118,101,114,114,105,100,101,32,105,110,32,101,120, - 112,101,114,105,109,101,110,116,115,10,32,32,32,32,40,101, - 46,103,46,32,99,97,99,104,101,32,115,116,97,116,32,114, - 101,115,117,108,116,115,41,46,10,10,32,32,32,32,41,2, - 114,3,0,0,0,90,4,115,116,97,116,41,1,114,35,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,10,95,112,97,116,104,95,115,116,97,116,68,0,0, - 0,115,2,0,0,0,0,7,114,39,0,0,0,99,2,0, - 0,0,0,0,0,0,3,0,0,0,11,0,0,0,67,0, - 0,0,115,58,0,0,0,121,16,0,116,0,0,124,0,0, - 131,1,0,125,2,0,87,110,22,0,4,116,1,0,107,10, - 0,114,40,0,1,1,1,100,1,0,83,89,110,1,0,88, - 124,2,0,106,2,0,100,2,0,64,124,1,0,107,2,0, - 83,41,3,122,49,84,101,115,116,32,119,104,101,116,104,101, - 114,32,116,104,101,32,112,97,116,104,32,105,115,32,116,104, - 101,32,115,112,101,99,105,102,105,101,100,32,109,111,100,101, - 32,116,121,112,101,46,70,105,0,240,0,0,41,3,114,39, - 0,0,0,218,7,79,83,69,114,114,111,114,218,7,115,116, - 95,109,111,100,101,41,3,114,35,0,0,0,218,4,109,111, - 100,101,90,9,115,116,97,116,95,105,110,102,111,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,18,95,112, - 97,116,104,95,105,115,95,109,111,100,101,95,116,121,112,101, - 78,0,0,0,115,10,0,0,0,0,2,3,1,16,1,13, - 1,9,1,114,43,0,0,0,99,1,0,0,0,0,0,0, - 0,1,0,0,0,3,0,0,0,67,0,0,0,115,13,0, - 0,0,116,0,0,124,0,0,100,1,0,131,2,0,83,41, - 2,122,31,82,101,112,108,97,99,101,109,101,110,116,32,102, - 111,114,32,111,115,46,112,97,116,104,46,105,115,102,105,108, - 101,46,105,0,128,0,0,41,1,114,43,0,0,0,41,1, - 114,35,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,12,95,112,97,116,104,95,105,115,102,105, - 108,101,87,0,0,0,115,2,0,0,0,0,2,114,44,0, - 0,0,99,1,0,0,0,0,0,0,0,1,0,0,0,3, - 0,0,0,67,0,0,0,115,31,0,0,0,124,0,0,115, - 18,0,116,0,0,106,1,0,131,0,0,125,0,0,116,2, - 0,124,0,0,100,1,0,131,2,0,83,41,2,122,30,82, - 101,112,108,97,99,101,109,101,110,116,32,102,111,114,32,111, - 115,46,112,97,116,104,46,105,115,100,105,114,46,105,0,64, - 0,0,41,3,114,3,0,0,0,218,6,103,101,116,99,119, - 100,114,43,0,0,0,41,1,114,35,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,112, - 97,116,104,95,105,115,100,105,114,92,0,0,0,115,6,0, - 0,0,0,2,6,1,12,1,114,46,0,0,0,105,182,1, - 0,0,99,3,0,0,0,0,0,0,0,6,0,0,0,17, - 0,0,0,67,0,0,0,115,193,0,0,0,100,1,0,106, - 0,0,124,0,0,116,1,0,124,0,0,131,1,0,131,2, - 0,125,3,0,116,2,0,106,3,0,124,3,0,116,2,0, - 106,4,0,116,2,0,106,5,0,66,116,2,0,106,6,0, - 66,124,2,0,100,2,0,64,131,3,0,125,4,0,121,61, - 0,116,7,0,106,8,0,124,4,0,100,3,0,131,2,0, - 143,20,0,125,5,0,124,5,0,106,9,0,124,1,0,131, - 1,0,1,87,100,4,0,81,82,88,116,2,0,106,10,0, - 124,3,0,124,0,0,131,2,0,1,87,110,59,0,4,116, - 11,0,107,10,0,114,188,0,1,1,1,121,17,0,116,2, - 0,106,12,0,124,3,0,131,1,0,1,87,110,18,0,4, - 116,11,0,107,10,0,114,180,0,1,1,1,89,110,1,0, - 88,130,0,0,89,110,1,0,88,100,4,0,83,41,5,122, - 162,66,101,115,116,45,101,102,102,111,114,116,32,102,117,110, - 99,116,105,111,110,32,116,111,32,119,114,105,116,101,32,100, - 97,116,97,32,116,111,32,97,32,112,97,116,104,32,97,116, - 111,109,105,99,97,108,108,121,46,10,32,32,32,32,66,101, - 32,112,114,101,112,97,114,101,100,32,116,111,32,104,97,110, - 100,108,101,32,97,32,70,105,108,101,69,120,105,115,116,115, - 69,114,114,111,114,32,105,102,32,99,111,110,99,117,114,114, - 101,110,116,32,119,114,105,116,105,110,103,32,111,102,32,116, - 104,101,10,32,32,32,32,116,101,109,112,111,114,97,114,121, - 32,102,105,108,101,32,105,115,32,97,116,116,101,109,112,116, - 101,100,46,122,5,123,125,46,123,125,105,182,1,0,0,90, - 2,119,98,78,41,13,218,6,102,111,114,109,97,116,218,2, - 105,100,114,3,0,0,0,90,4,111,112,101,110,90,6,79, - 95,69,88,67,76,90,7,79,95,67,82,69,65,84,90,8, - 79,95,87,82,79,78,76,89,218,3,95,105,111,218,6,70, - 105,108,101,73,79,218,5,119,114,105,116,101,218,7,114,101, - 112,108,97,99,101,114,40,0,0,0,90,6,117,110,108,105, - 110,107,41,6,114,35,0,0,0,218,4,100,97,116,97,114, - 42,0,0,0,90,8,112,97,116,104,95,116,109,112,90,2, - 102,100,218,4,102,105,108,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,13,95,119,114,105,116,101,95, - 97,116,111,109,105,99,99,0,0,0,115,26,0,0,0,0, - 5,24,1,9,1,33,1,3,3,21,1,20,1,20,1,13, - 1,3,1,17,1,13,1,5,1,114,55,0,0,0,105,22, - 13,0,0,233,2,0,0,0,114,13,0,0,0,115,2,0, - 0,0,13,10,90,11,95,95,112,121,99,97,99,104,101,95, - 95,122,4,111,112,116,45,122,3,46,112,121,122,4,46,112, - 121,99,78,218,12,111,112,116,105,109,105,122,97,116,105,111, - 110,99,2,0,0,0,1,0,0,0,11,0,0,0,6,0, - 0,0,67,0,0,0,115,87,1,0,0,124,1,0,100,1, - 0,107,9,0,114,76,0,116,0,0,106,1,0,100,2,0, - 116,2,0,131,2,0,1,124,2,0,100,1,0,107,9,0, - 114,58,0,100,3,0,125,3,0,116,3,0,124,3,0,131, - 1,0,130,1,0,124,1,0,114,70,0,100,4,0,110,3, - 0,100,5,0,125,2,0,116,4,0,124,0,0,131,1,0, - 92,2,0,125,4,0,125,5,0,124,5,0,106,5,0,100, - 6,0,131,1,0,92,3,0,125,6,0,125,7,0,125,8, - 0,116,6,0,106,7,0,106,8,0,125,9,0,124,9,0, - 100,1,0,107,8,0,114,154,0,116,9,0,100,7,0,131, - 1,0,130,1,0,100,4,0,106,10,0,124,6,0,114,172, - 0,124,6,0,110,3,0,124,8,0,124,7,0,124,9,0, - 103,3,0,131,1,0,125,10,0,124,2,0,100,1,0,107, - 8,0,114,241,0,116,6,0,106,11,0,106,12,0,100,8, - 0,107,2,0,114,229,0,100,4,0,125,2,0,110,12,0, - 116,6,0,106,11,0,106,12,0,125,2,0,116,13,0,124, - 2,0,131,1,0,125,2,0,124,2,0,100,4,0,107,3, - 0,114,63,1,124,2,0,106,14,0,131,0,0,115,42,1, - 116,15,0,100,9,0,106,16,0,124,2,0,131,1,0,131, - 1,0,130,1,0,100,10,0,106,16,0,124,10,0,116,17, - 0,124,2,0,131,3,0,125,10,0,116,18,0,124,4,0, - 116,19,0,124,10,0,116,20,0,100,8,0,25,23,131,3, - 0,83,41,11,97,254,2,0,0,71,105,118,101,110,32,116, - 104,101,32,112,97,116,104,32,116,111,32,97,32,46,112,121, - 32,102,105,108,101,44,32,114,101,116,117,114,110,32,116,104, - 101,32,112,97,116,104,32,116,111,32,105,116,115,32,46,112, - 121,99,32,102,105,108,101,46,10,10,32,32,32,32,84,104, - 101,32,46,112,121,32,102,105,108,101,32,100,111,101,115,32, - 110,111,116,32,110,101,101,100,32,116,111,32,101,120,105,115, - 116,59,32,116,104,105,115,32,115,105,109,112,108,121,32,114, - 101,116,117,114,110,115,32,116,104,101,32,112,97,116,104,32, - 116,111,32,116,104,101,10,32,32,32,32,46,112,121,99,32, - 102,105,108,101,32,99,97,108,99,117,108,97,116,101,100,32, - 97,115,32,105,102,32,116,104,101,32,46,112,121,32,102,105, - 108,101,32,119,101,114,101,32,105,109,112,111,114,116,101,100, - 46,10,10,32,32,32,32,84,104,101,32,39,111,112,116,105, - 109,105,122,97,116,105,111,110,39,32,112,97,114,97,109,101, - 116,101,114,32,99,111,110,116,114,111,108,115,32,116,104,101, - 32,112,114,101,115,117,109,101,100,32,111,112,116,105,109,105, - 122,97,116,105,111,110,32,108,101,118,101,108,32,111,102,10, - 32,32,32,32,116,104,101,32,98,121,116,101,99,111,100,101, - 32,102,105,108,101,46,32,73,102,32,39,111,112,116,105,109, - 105,122,97,116,105,111,110,39,32,105,115,32,110,111,116,32, - 78,111,110,101,44,32,116,104,101,32,115,116,114,105,110,103, - 32,114,101,112,114,101,115,101,110,116,97,116,105,111,110,10, - 32,32,32,32,111,102,32,116,104,101,32,97,114,103,117,109, - 101,110,116,32,105,115,32,116,97,107,101,110,32,97,110,100, - 32,118,101,114,105,102,105,101,100,32,116,111,32,98,101,32, - 97,108,112,104,97,110,117,109,101,114,105,99,32,40,101,108, - 115,101,32,86,97,108,117,101,69,114,114,111,114,10,32,32, - 32,32,105,115,32,114,97,105,115,101,100,41,46,10,10,32, - 32,32,32,84,104,101,32,100,101,98,117,103,95,111,118,101, - 114,114,105,100,101,32,112,97,114,97,109,101,116,101,114,32, - 105,115,32,100,101,112,114,101,99,97,116,101,100,46,32,73, - 102,32,100,101,98,117,103,95,111,118,101,114,114,105,100,101, - 32,105,115,32,110,111,116,32,78,111,110,101,44,10,32,32, - 32,32,97,32,84,114,117,101,32,118,97,108,117,101,32,105, - 115,32,116,104,101,32,115,97,109,101,32,97,115,32,115,101, - 116,116,105,110,103,32,39,111,112,116,105,109,105,122,97,116, - 105,111,110,39,32,116,111,32,116,104,101,32,101,109,112,116, - 121,32,115,116,114,105,110,103,10,32,32,32,32,119,104,105, - 108,101,32,97,32,70,97,108,115,101,32,118,97,108,117,101, - 32,105,115,32,101,113,117,105,118,97,108,101,110,116,32,116, - 111,32,115,101,116,116,105,110,103,32,39,111,112,116,105,109, - 105,122,97,116,105,111,110,39,32,116,111,32,39,49,39,46, - 10,10,32,32,32,32,73,102,32,115,121,115,46,105,109,112, - 108,101,109,101,110,116,97,116,105,111,110,46,99,97,99,104, - 101,95,116,97,103,32,105,115,32,78,111,110,101,32,116,104, - 101,110,32,78,111,116,73,109,112,108,101,109,101,110,116,101, - 100,69,114,114,111,114,32,105,115,32,114,97,105,115,101,100, - 46,10,10,32,32,32,32,78,122,70,116,104,101,32,100,101, - 98,117,103,95,111,118,101,114,114,105,100,101,32,112,97,114, - 97,109,101,116,101,114,32,105,115,32,100,101,112,114,101,99, - 97,116,101,100,59,32,117,115,101,32,39,111,112,116,105,109, - 105,122,97,116,105,111,110,39,32,105,110,115,116,101,97,100, - 122,50,100,101,98,117,103,95,111,118,101,114,114,105,100,101, - 32,111,114,32,111,112,116,105,109,105,122,97,116,105,111,110, - 32,109,117,115,116,32,98,101,32,115,101,116,32,116,111,32, - 78,111,110,101,114,30,0,0,0,114,29,0,0,0,218,1, - 46,122,36,115,121,115,46,105,109,112,108,101,109,101,110,116, - 97,116,105,111,110,46,99,97,99,104,101,95,116,97,103,32, - 105,115,32,78,111,110,101,233,0,0,0,0,122,24,123,33, - 114,125,32,105,115,32,110,111,116,32,97,108,112,104,97,110, - 117,109,101,114,105,99,122,7,123,125,46,123,125,123,125,41, - 21,218,9,95,119,97,114,110,105,110,103,115,218,4,119,97, - 114,110,218,18,68,101,112,114,101,99,97,116,105,111,110,87, - 97,114,110,105,110,103,218,9,84,121,112,101,69,114,114,111, - 114,114,38,0,0,0,114,32,0,0,0,114,7,0,0,0, - 218,14,105,109,112,108,101,109,101,110,116,97,116,105,111,110, - 218,9,99,97,99,104,101,95,116,97,103,218,19,78,111,116, - 73,109,112,108,101,109,101,110,116,101,100,69,114,114,111,114, - 114,26,0,0,0,218,5,102,108,97,103,115,218,8,111,112, - 116,105,109,105,122,101,218,3,115,116,114,218,7,105,115,97, - 108,110,117,109,218,10,86,97,108,117,101,69,114,114,111,114, - 114,47,0,0,0,218,4,95,79,80,84,114,28,0,0,0, - 218,8,95,80,89,67,65,67,72,69,218,17,66,89,84,69, - 67,79,68,69,95,83,85,70,70,73,88,69,83,41,11,114, - 35,0,0,0,90,14,100,101,98,117,103,95,111,118,101,114, - 114,105,100,101,114,57,0,0,0,218,7,109,101,115,115,97, - 103,101,218,4,104,101,97,100,114,37,0,0,0,90,4,98, - 97,115,101,218,3,115,101,112,218,4,114,101,115,116,90,3, - 116,97,103,90,15,97,108,109,111,115,116,95,102,105,108,101, - 110,97,109,101,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,17,99,97,99,104,101,95,102,114,111,109,95, - 115,111,117,114,99,101,243,0,0,0,115,46,0,0,0,0, - 18,12,1,9,1,7,1,12,1,6,1,12,1,18,1,18, - 1,24,1,12,1,12,1,12,1,36,1,12,1,18,1,9, - 2,12,1,12,1,12,1,12,1,21,1,21,1,114,79,0, - 0,0,99,1,0,0,0,0,0,0,0,8,0,0,0,5, - 0,0,0,67,0,0,0,115,62,1,0,0,116,0,0,106, - 1,0,106,2,0,100,1,0,107,8,0,114,30,0,116,3, - 0,100,2,0,131,1,0,130,1,0,116,4,0,124,0,0, - 131,1,0,92,2,0,125,1,0,125,2,0,116,4,0,124, - 1,0,131,1,0,92,2,0,125,1,0,125,3,0,124,3, - 0,116,5,0,107,3,0,114,102,0,116,6,0,100,3,0, - 106,7,0,116,5,0,124,0,0,131,2,0,131,1,0,130, - 1,0,124,2,0,106,8,0,100,4,0,131,1,0,125,4, - 0,124,4,0,100,11,0,107,7,0,114,153,0,116,6,0, - 100,7,0,106,7,0,124,2,0,131,1,0,131,1,0,130, - 1,0,110,125,0,124,4,0,100,6,0,107,2,0,114,22, - 1,124,2,0,106,9,0,100,4,0,100,5,0,131,2,0, - 100,12,0,25,125,5,0,124,5,0,106,10,0,116,11,0, - 131,1,0,115,223,0,116,6,0,100,8,0,106,7,0,116, - 11,0,131,1,0,131,1,0,130,1,0,124,5,0,116,12, - 0,116,11,0,131,1,0,100,1,0,133,2,0,25,125,6, - 0,124,6,0,106,13,0,131,0,0,115,22,1,116,6,0, - 100,9,0,106,7,0,124,5,0,131,1,0,131,1,0,130, - 1,0,124,2,0,106,14,0,100,4,0,131,1,0,100,10, - 0,25,125,7,0,116,15,0,124,1,0,124,7,0,116,16, - 0,100,10,0,25,23,131,2,0,83,41,13,97,110,1,0, - 0,71,105,118,101,110,32,116,104,101,32,112,97,116,104,32, - 116,111,32,97,32,46,112,121,99,46,32,102,105,108,101,44, - 32,114,101,116,117,114,110,32,116,104,101,32,112,97,116,104, - 32,116,111,32,105,116,115,32,46,112,121,32,102,105,108,101, - 46,10,10,32,32,32,32,84,104,101,32,46,112,121,99,32, - 102,105,108,101,32,100,111,101,115,32,110,111,116,32,110,101, - 101,100,32,116,111,32,101,120,105,115,116,59,32,116,104,105, - 115,32,115,105,109,112,108,121,32,114,101,116,117,114,110,115, - 32,116,104,101,32,112,97,116,104,32,116,111,10,32,32,32, - 32,116,104,101,32,46,112,121,32,102,105,108,101,32,99,97, - 108,99,117,108,97,116,101,100,32,116,111,32,99,111,114,114, - 101,115,112,111,110,100,32,116,111,32,116,104,101,32,46,112, - 121,99,32,102,105,108,101,46,32,32,73,102,32,112,97,116, - 104,32,100,111,101,115,10,32,32,32,32,110,111,116,32,99, - 111,110,102,111,114,109,32,116,111,32,80,69,80,32,51,49, - 52,55,47,52,56,56,32,102,111,114,109,97,116,44,32,86, - 97,108,117,101,69,114,114,111,114,32,119,105,108,108,32,98, - 101,32,114,97,105,115,101,100,46,32,73,102,10,32,32,32, - 32,115,121,115,46,105,109,112,108,101,109,101,110,116,97,116, + 104,95,105,115,100,105,114,92,0,0,0,115,6,0,0,0, + 0,2,6,1,12,1,114,46,0,0,0,105,182,1,0,0, + 99,3,0,0,0,0,0,0,0,6,0,0,0,17,0,0, + 0,67,0,0,0,115,193,0,0,0,100,1,0,106,0,0, + 124,0,0,116,1,0,124,0,0,131,1,0,131,2,0,125, + 3,0,116,2,0,106,3,0,124,3,0,116,2,0,106,4, + 0,116,2,0,106,5,0,66,116,2,0,106,6,0,66,124, + 2,0,100,2,0,64,131,3,0,125,4,0,121,61,0,116, + 7,0,106,8,0,124,4,0,100,3,0,131,2,0,143,20, + 0,125,5,0,124,5,0,106,9,0,124,1,0,131,1,0, + 1,87,100,4,0,81,82,88,116,2,0,106,10,0,124,3, + 0,124,0,0,131,2,0,1,87,110,59,0,4,116,11,0, + 107,10,0,114,188,0,1,1,1,121,17,0,116,2,0,106, + 12,0,124,3,0,131,1,0,1,87,110,18,0,4,116,11, + 0,107,10,0,114,180,0,1,1,1,89,110,1,0,88,130, + 0,0,89,110,1,0,88,100,4,0,83,41,5,122,162,66, + 101,115,116,45,101,102,102,111,114,116,32,102,117,110,99,116, + 105,111,110,32,116,111,32,119,114,105,116,101,32,100,97,116, + 97,32,116,111,32,97,32,112,97,116,104,32,97,116,111,109, + 105,99,97,108,108,121,46,10,32,32,32,32,66,101,32,112, + 114,101,112,97,114,101,100,32,116,111,32,104,97,110,100,108, + 101,32,97,32,70,105,108,101,69,120,105,115,116,115,69,114, + 114,111,114,32,105,102,32,99,111,110,99,117,114,114,101,110, + 116,32,119,114,105,116,105,110,103,32,111,102,32,116,104,101, + 10,32,32,32,32,116,101,109,112,111,114,97,114,121,32,102, + 105,108,101,32,105,115,32,97,116,116,101,109,112,116,101,100, + 46,122,5,123,125,46,123,125,105,182,1,0,0,90,2,119, + 98,78,41,13,218,6,102,111,114,109,97,116,218,2,105,100, + 114,3,0,0,0,90,4,111,112,101,110,90,6,79,95,69, + 88,67,76,90,7,79,95,67,82,69,65,84,90,8,79,95, + 87,82,79,78,76,89,218,3,95,105,111,218,6,70,105,108, + 101,73,79,218,5,119,114,105,116,101,218,7,114,101,112,108, + 97,99,101,114,40,0,0,0,90,6,117,110,108,105,110,107, + 41,6,114,35,0,0,0,218,4,100,97,116,97,114,42,0, + 0,0,90,8,112,97,116,104,95,116,109,112,90,2,102,100, + 218,4,102,105,108,101,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,13,95,119,114,105,116,101,95,97,116, + 111,109,105,99,99,0,0,0,115,26,0,0,0,0,5,24, + 1,9,1,33,1,3,3,21,1,20,1,20,1,13,1,3, + 1,17,1,13,1,5,1,114,55,0,0,0,105,22,13,0, + 0,233,2,0,0,0,114,13,0,0,0,115,2,0,0,0, + 13,10,90,11,95,95,112,121,99,97,99,104,101,95,95,122, + 4,111,112,116,45,122,3,46,112,121,122,4,46,112,121,99, + 78,218,12,111,112,116,105,109,105,122,97,116,105,111,110,99, + 2,0,0,0,1,0,0,0,11,0,0,0,6,0,0,0, + 67,0,0,0,115,87,1,0,0,124,1,0,100,1,0,107, + 9,0,114,76,0,116,0,0,106,1,0,100,2,0,116,2, + 0,131,2,0,1,124,2,0,100,1,0,107,9,0,114,58, + 0,100,3,0,125,3,0,116,3,0,124,3,0,131,1,0, + 130,1,0,124,1,0,114,70,0,100,4,0,110,3,0,100, + 5,0,125,2,0,116,4,0,124,0,0,131,1,0,92,2, + 0,125,4,0,125,5,0,124,5,0,106,5,0,100,6,0, + 131,1,0,92,3,0,125,6,0,125,7,0,125,8,0,116, + 6,0,106,7,0,106,8,0,125,9,0,124,9,0,100,1, + 0,107,8,0,114,154,0,116,9,0,100,7,0,131,1,0, + 130,1,0,100,4,0,106,10,0,124,6,0,114,172,0,124, + 6,0,110,3,0,124,8,0,124,7,0,124,9,0,103,3, + 0,131,1,0,125,10,0,124,2,0,100,1,0,107,8,0, + 114,241,0,116,6,0,106,11,0,106,12,0,100,8,0,107, + 2,0,114,229,0,100,4,0,125,2,0,110,12,0,116,6, + 0,106,11,0,106,12,0,125,2,0,116,13,0,124,2,0, + 131,1,0,125,2,0,124,2,0,100,4,0,107,3,0,114, + 63,1,124,2,0,106,14,0,131,0,0,115,42,1,116,15, + 0,100,9,0,106,16,0,124,2,0,131,1,0,131,1,0, + 130,1,0,100,10,0,106,16,0,124,10,0,116,17,0,124, + 2,0,131,3,0,125,10,0,116,18,0,124,4,0,116,19, + 0,124,10,0,116,20,0,100,8,0,25,23,131,3,0,83, + 41,11,97,254,2,0,0,71,105,118,101,110,32,116,104,101, + 32,112,97,116,104,32,116,111,32,97,32,46,112,121,32,102, + 105,108,101,44,32,114,101,116,117,114,110,32,116,104,101,32, + 112,97,116,104,32,116,111,32,105,116,115,32,46,112,121,99, + 32,102,105,108,101,46,10,10,32,32,32,32,84,104,101,32, + 46,112,121,32,102,105,108,101,32,100,111,101,115,32,110,111, + 116,32,110,101,101,100,32,116,111,32,101,120,105,115,116,59, + 32,116,104,105,115,32,115,105,109,112,108,121,32,114,101,116, + 117,114,110,115,32,116,104,101,32,112,97,116,104,32,116,111, + 32,116,104,101,10,32,32,32,32,46,112,121,99,32,102,105, + 108,101,32,99,97,108,99,117,108,97,116,101,100,32,97,115, + 32,105,102,32,116,104,101,32,46,112,121,32,102,105,108,101, + 32,119,101,114,101,32,105,109,112,111,114,116,101,100,46,10, + 10,32,32,32,32,84,104,101,32,39,111,112,116,105,109,105, + 122,97,116,105,111,110,39,32,112,97,114,97,109,101,116,101, + 114,32,99,111,110,116,114,111,108,115,32,116,104,101,32,112, + 114,101,115,117,109,101,100,32,111,112,116,105,109,105,122,97, + 116,105,111,110,32,108,101,118,101,108,32,111,102,10,32,32, + 32,32,116,104,101,32,98,121,116,101,99,111,100,101,32,102, + 105,108,101,46,32,73,102,32,39,111,112,116,105,109,105,122, + 97,116,105,111,110,39,32,105,115,32,110,111,116,32,78,111, + 110,101,44,32,116,104,101,32,115,116,114,105,110,103,32,114, + 101,112,114,101,115,101,110,116,97,116,105,111,110,10,32,32, + 32,32,111,102,32,116,104,101,32,97,114,103,117,109,101,110, + 116,32,105,115,32,116,97,107,101,110,32,97,110,100,32,118, + 101,114,105,102,105,101,100,32,116,111,32,98,101,32,97,108, + 112,104,97,110,117,109,101,114,105,99,32,40,101,108,115,101, + 32,86,97,108,117,101,69,114,114,111,114,10,32,32,32,32, + 105,115,32,114,97,105,115,101,100,41,46,10,10,32,32,32, + 32,84,104,101,32,100,101,98,117,103,95,111,118,101,114,114, + 105,100,101,32,112,97,114,97,109,101,116,101,114,32,105,115, + 32,100,101,112,114,101,99,97,116,101,100,46,32,73,102,32, + 100,101,98,117,103,95,111,118,101,114,114,105,100,101,32,105, + 115,32,110,111,116,32,78,111,110,101,44,10,32,32,32,32, + 97,32,84,114,117,101,32,118,97,108,117,101,32,105,115,32, + 116,104,101,32,115,97,109,101,32,97,115,32,115,101,116,116, + 105,110,103,32,39,111,112,116,105,109,105,122,97,116,105,111, + 110,39,32,116,111,32,116,104,101,32,101,109,112,116,121,32, + 115,116,114,105,110,103,10,32,32,32,32,119,104,105,108,101, + 32,97,32,70,97,108,115,101,32,118,97,108,117,101,32,105, + 115,32,101,113,117,105,118,97,108,101,110,116,32,116,111,32, + 115,101,116,116,105,110,103,32,39,111,112,116,105,109,105,122, + 97,116,105,111,110,39,32,116,111,32,39,49,39,46,10,10, + 32,32,32,32,73,102,32,115,121,115,46,105,109,112,108,101, + 109,101,110,116,97,116,105,111,110,46,99,97,99,104,101,95, + 116,97,103,32,105,115,32,78,111,110,101,32,116,104,101,110, + 32,78,111,116,73,109,112,108,101,109,101,110,116,101,100,69, + 114,114,111,114,32,105,115,32,114,97,105,115,101,100,46,10, + 10,32,32,32,32,78,122,70,116,104,101,32,100,101,98,117, + 103,95,111,118,101,114,114,105,100,101,32,112,97,114,97,109, + 101,116,101,114,32,105,115,32,100,101,112,114,101,99,97,116, + 101,100,59,32,117,115,101,32,39,111,112,116,105,109,105,122, + 97,116,105,111,110,39,32,105,110,115,116,101,97,100,122,50, + 100,101,98,117,103,95,111,118,101,114,114,105,100,101,32,111, + 114,32,111,112,116,105,109,105,122,97,116,105,111,110,32,109, + 117,115,116,32,98,101,32,115,101,116,32,116,111,32,78,111, + 110,101,114,30,0,0,0,114,29,0,0,0,218,1,46,122, + 36,115,121,115,46,105,109,112,108,101,109,101,110,116,97,116, 105,111,110,46,99,97,99,104,101,95,116,97,103,32,105,115, - 32,78,111,110,101,32,116,104,101,110,32,78,111,116,73,109, - 112,108,101,109,101,110,116,101,100,69,114,114,111,114,32,105, - 115,32,114,97,105,115,101,100,46,10,10,32,32,32,32,78, - 122,36,115,121,115,46,105,109,112,108,101,109,101,110,116,97, - 116,105,111,110,46,99,97,99,104,101,95,116,97,103,32,105, - 115,32,78,111,110,101,122,37,123,125,32,110,111,116,32,98, - 111,116,116,111,109,45,108,101,118,101,108,32,100,105,114,101, - 99,116,111,114,121,32,105,110,32,123,33,114,125,114,58,0, - 0,0,114,56,0,0,0,233,3,0,0,0,122,33,101,120, - 112,101,99,116,101,100,32,111,110,108,121,32,50,32,111,114, - 32,51,32,100,111,116,115,32,105,110,32,123,33,114,125,122, - 57,111,112,116,105,109,105,122,97,116,105,111,110,32,112,111, - 114,116,105,111,110,32,111,102,32,102,105,108,101,110,97,109, - 101,32,100,111,101,115,32,110,111,116,32,115,116,97,114,116, - 32,119,105,116,104,32,123,33,114,125,122,52,111,112,116,105, - 109,105,122,97,116,105,111,110,32,108,101,118,101,108,32,123, - 33,114,125,32,105,115,32,110,111,116,32,97,110,32,97,108, - 112,104,97,110,117,109,101,114,105,99,32,118,97,108,117,101, - 114,59,0,0,0,62,2,0,0,0,114,56,0,0,0,114, - 80,0,0,0,233,254,255,255,255,41,17,114,7,0,0,0, - 114,64,0,0,0,114,65,0,0,0,114,66,0,0,0,114, - 38,0,0,0,114,73,0,0,0,114,71,0,0,0,114,47, - 0,0,0,218,5,99,111,117,110,116,114,34,0,0,0,114, - 9,0,0,0,114,72,0,0,0,114,31,0,0,0,114,70, - 0,0,0,218,9,112,97,114,116,105,116,105,111,110,114,28, - 0,0,0,218,15,83,79,85,82,67,69,95,83,85,70,70, - 73,88,69,83,41,8,114,35,0,0,0,114,76,0,0,0, - 90,16,112,121,99,97,99,104,101,95,102,105,108,101,110,97, - 109,101,90,7,112,121,99,97,99,104,101,90,9,100,111,116, - 95,99,111,117,110,116,114,57,0,0,0,90,9,111,112,116, - 95,108,101,118,101,108,90,13,98,97,115,101,95,102,105,108, - 101,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,17,115,111,117,114,99,101,95,102,114,111, - 109,95,99,97,99,104,101,31,1,0,0,115,44,0,0,0, - 0,9,18,1,12,1,18,1,18,1,12,1,9,1,15,1, - 15,1,12,1,9,1,15,1,12,1,22,1,15,1,9,1, - 12,1,22,1,12,1,9,1,12,1,19,1,114,85,0,0, - 0,99,1,0,0,0,0,0,0,0,5,0,0,0,12,0, - 0,0,67,0,0,0,115,164,0,0,0,116,0,0,124,0, - 0,131,1,0,100,1,0,107,2,0,114,22,0,100,2,0, - 83,124,0,0,106,1,0,100,3,0,131,1,0,92,3,0, - 125,1,0,125,2,0,125,3,0,124,1,0,12,115,81,0, - 124,3,0,106,2,0,131,0,0,100,7,0,100,8,0,133, - 2,0,25,100,6,0,107,3,0,114,85,0,124,0,0,83, - 121,16,0,116,3,0,124,0,0,131,1,0,125,4,0,87, - 110,40,0,4,116,4,0,116,5,0,102,2,0,107,10,0, - 114,143,0,1,1,1,124,0,0,100,2,0,100,9,0,133, - 2,0,25,125,4,0,89,110,1,0,88,116,6,0,124,4, - 0,131,1,0,114,160,0,124,4,0,83,124,0,0,83,41, - 10,122,188,67,111,110,118,101,114,116,32,97,32,98,121,116, - 101,99,111,100,101,32,102,105,108,101,32,112,97,116,104,32, - 116,111,32,97,32,115,111,117,114,99,101,32,112,97,116,104, - 32,40,105,102,32,112,111,115,115,105,98,108,101,41,46,10, - 10,32,32,32,32,84,104,105,115,32,102,117,110,99,116,105, - 111,110,32,101,120,105,115,116,115,32,112,117,114,101,108,121, - 32,102,111,114,32,98,97,99,107,119,97,114,100,115,45,99, - 111,109,112,97,116,105,98,105,108,105,116,121,32,102,111,114, - 10,32,32,32,32,80,121,73,109,112,111,114,116,95,69,120, - 101,99,67,111,100,101,77,111,100,117,108,101,87,105,116,104, - 70,105,108,101,110,97,109,101,115,40,41,32,105,110,32,116, - 104,101,32,67,32,65,80,73,46,10,10,32,32,32,32,114, - 59,0,0,0,78,114,58,0,0,0,114,80,0,0,0,114, - 29,0,0,0,90,2,112,121,233,253,255,255,255,233,255,255, - 255,255,114,87,0,0,0,41,7,114,31,0,0,0,114,32, - 0,0,0,218,5,108,111,119,101,114,114,85,0,0,0,114, - 66,0,0,0,114,71,0,0,0,114,44,0,0,0,41,5, - 218,13,98,121,116,101,99,111,100,101,95,112,97,116,104,114, - 78,0,0,0,114,36,0,0,0,90,9,101,120,116,101,110, - 115,105,111,110,218,11,115,111,117,114,99,101,95,112,97,116, - 104,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,15,95,103,101,116,95,115,111,117,114,99,101,102,105,108, - 101,64,1,0,0,115,20,0,0,0,0,7,18,1,4,1, - 24,1,35,1,4,1,3,1,16,1,19,1,21,1,114,91, - 0,0,0,99,1,0,0,0,0,0,0,0,1,0,0,0, - 11,0,0,0,67,0,0,0,115,92,0,0,0,124,0,0, - 106,0,0,116,1,0,116,2,0,131,1,0,131,1,0,114, - 59,0,121,14,0,116,3,0,124,0,0,131,1,0,83,87, - 113,88,0,4,116,4,0,107,10,0,114,55,0,1,1,1, - 89,113,88,0,88,110,29,0,124,0,0,106,0,0,116,1, - 0,116,5,0,131,1,0,131,1,0,114,84,0,124,0,0, - 83,100,0,0,83,100,0,0,83,41,1,78,41,6,218,8, - 101,110,100,115,119,105,116,104,218,5,116,117,112,108,101,114, - 84,0,0,0,114,79,0,0,0,114,66,0,0,0,114,74, - 0,0,0,41,1,218,8,102,105,108,101,110,97,109,101,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 95,103,101,116,95,99,97,99,104,101,100,83,1,0,0,115, - 16,0,0,0,0,1,21,1,3,1,14,1,13,1,8,1, - 21,1,4,2,114,95,0,0,0,99,1,0,0,0,0,0, - 0,0,2,0,0,0,11,0,0,0,67,0,0,0,115,60, - 0,0,0,121,19,0,116,0,0,124,0,0,131,1,0,106, - 1,0,125,1,0,87,110,24,0,4,116,2,0,107,10,0, - 114,45,0,1,1,1,100,1,0,125,1,0,89,110,1,0, - 88,124,1,0,100,2,0,79,125,1,0,124,1,0,83,41, - 3,122,51,67,97,108,99,117,108,97,116,101,32,116,104,101, - 32,109,111,100,101,32,112,101,114,109,105,115,115,105,111,110, - 115,32,102,111,114,32,97,32,98,121,116,101,99,111,100,101, - 32,102,105,108,101,46,105,182,1,0,0,233,128,0,0,0, - 41,3,114,39,0,0,0,114,41,0,0,0,114,40,0,0, - 0,41,2,114,35,0,0,0,114,42,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,10,95,99, - 97,108,99,95,109,111,100,101,95,1,0,0,115,12,0,0, - 0,0,2,3,1,19,1,13,1,11,3,10,1,114,97,0, - 0,0,218,9,118,101,114,98,111,115,105,116,121,114,29,0, - 0,0,99,1,0,0,0,1,0,0,0,3,0,0,0,4, - 0,0,0,71,0,0,0,115,75,0,0,0,116,0,0,106, - 1,0,106,2,0,124,1,0,107,5,0,114,71,0,124,0, - 0,106,3,0,100,6,0,131,1,0,115,43,0,100,3,0, - 124,0,0,23,125,0,0,116,4,0,124,0,0,106,5,0, - 124,2,0,140,0,0,100,4,0,116,0,0,106,6,0,131, - 1,1,1,100,5,0,83,41,7,122,61,80,114,105,110,116, - 32,116,104,101,32,109,101,115,115,97,103,101,32,116,111,32, - 115,116,100,101,114,114,32,105,102,32,45,118,47,80,89,84, - 72,79,78,86,69,82,66,79,83,69,32,105,115,32,116,117, - 114,110,101,100,32,111,110,46,250,1,35,250,7,105,109,112, - 111,114,116,32,122,2,35,32,114,54,0,0,0,78,41,2, - 114,99,0,0,0,114,100,0,0,0,41,7,114,7,0,0, - 0,114,67,0,0,0,218,7,118,101,114,98,111,115,101,114, - 9,0,0,0,218,5,112,114,105,110,116,114,47,0,0,0, - 218,6,115,116,100,101,114,114,41,3,114,75,0,0,0,114, - 98,0,0,0,218,4,97,114,103,115,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,16,95,118,101,114,98, - 111,115,101,95,109,101,115,115,97,103,101,107,1,0,0,115, - 8,0,0,0,0,2,18,1,15,1,10,1,114,105,0,0, - 0,99,1,0,0,0,0,0,0,0,3,0,0,0,11,0, - 0,0,3,0,0,0,115,84,0,0,0,100,1,0,135,0, - 0,102,1,0,100,2,0,100,3,0,134,1,0,125,1,0, - 121,13,0,116,0,0,106,1,0,125,2,0,87,110,30,0, - 4,116,2,0,107,10,0,114,66,0,1,1,1,100,4,0, - 100,5,0,132,0,0,125,2,0,89,110,1,0,88,124,2, - 0,124,1,0,136,0,0,131,2,0,1,124,1,0,83,41, - 6,122,252,68,101,99,111,114,97,116,111,114,32,116,111,32, - 118,101,114,105,102,121,32,116,104,97,116,32,116,104,101,32, - 109,111,100,117,108,101,32,98,101,105,110,103,32,114,101,113, - 117,101,115,116,101,100,32,109,97,116,99,104,101,115,32,116, - 104,101,32,111,110,101,32,116,104,101,10,32,32,32,32,108, - 111,97,100,101,114,32,99,97,110,32,104,97,110,100,108,101, - 46,10,10,32,32,32,32,84,104,101,32,102,105,114,115,116, - 32,97,114,103,117,109,101,110,116,32,40,115,101,108,102,41, - 32,109,117,115,116,32,100,101,102,105,110,101,32,95,110,97, - 109,101,32,119,104,105,99,104,32,116,104,101,32,115,101,99, - 111,110,100,32,97,114,103,117,109,101,110,116,32,105,115,10, - 32,32,32,32,99,111,109,112,97,114,101,100,32,97,103,97, - 105,110,115,116,46,32,73,102,32,116,104,101,32,99,111,109, - 112,97,114,105,115,111,110,32,102,97,105,108,115,32,116,104, - 101,110,32,73,109,112,111,114,116,69,114,114,111,114,32,105, - 115,32,114,97,105,115,101,100,46,10,10,32,32,32,32,78, - 99,2,0,0,0,0,0,0,0,4,0,0,0,5,0,0, - 0,31,0,0,0,115,89,0,0,0,124,1,0,100,0,0, - 107,8,0,114,24,0,124,0,0,106,0,0,125,1,0,110, - 46,0,124,0,0,106,0,0,124,1,0,107,3,0,114,70, - 0,116,1,0,100,1,0,124,0,0,106,0,0,124,1,0, - 102,2,0,22,100,2,0,124,1,0,131,1,1,130,1,0, - 136,0,0,124,0,0,124,1,0,124,2,0,124,3,0,142, - 2,0,83,41,3,78,122,30,108,111,97,100,101,114,32,102, - 111,114,32,37,115,32,99,97,110,110,111,116,32,104,97,110, - 100,108,101,32,37,115,218,4,110,97,109,101,41,2,114,106, - 0,0,0,218,11,73,109,112,111,114,116,69,114,114,111,114, - 41,4,218,4,115,101,108,102,114,106,0,0,0,114,104,0, - 0,0,90,6,107,119,97,114,103,115,41,1,218,6,109,101, + 32,78,111,110,101,233,0,0,0,0,122,24,123,33,114,125, + 32,105,115,32,110,111,116,32,97,108,112,104,97,110,117,109, + 101,114,105,99,122,7,123,125,46,123,125,123,125,41,21,218, + 9,95,119,97,114,110,105,110,103,115,218,4,119,97,114,110, + 218,18,68,101,112,114,101,99,97,116,105,111,110,87,97,114, + 110,105,110,103,218,9,84,121,112,101,69,114,114,111,114,114, + 38,0,0,0,114,32,0,0,0,114,7,0,0,0,218,14, + 105,109,112,108,101,109,101,110,116,97,116,105,111,110,218,9, + 99,97,99,104,101,95,116,97,103,218,19,78,111,116,73,109, + 112,108,101,109,101,110,116,101,100,69,114,114,111,114,114,26, + 0,0,0,218,5,102,108,97,103,115,218,8,111,112,116,105, + 109,105,122,101,218,3,115,116,114,218,7,105,115,97,108,110, + 117,109,218,10,86,97,108,117,101,69,114,114,111,114,114,47, + 0,0,0,218,4,95,79,80,84,114,28,0,0,0,218,8, + 95,80,89,67,65,67,72,69,218,17,66,89,84,69,67,79, + 68,69,95,83,85,70,70,73,88,69,83,41,11,114,35,0, + 0,0,90,14,100,101,98,117,103,95,111,118,101,114,114,105, + 100,101,114,57,0,0,0,218,7,109,101,115,115,97,103,101, + 218,4,104,101,97,100,114,37,0,0,0,90,4,98,97,115, + 101,218,3,115,101,112,218,4,114,101,115,116,90,3,116,97, + 103,90,15,97,108,109,111,115,116,95,102,105,108,101,110,97, + 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,17,99,97,99,104,101,95,102,114,111,109,95,115,111, + 117,114,99,101,243,0,0,0,115,46,0,0,0,0,18,12, + 1,9,1,7,1,12,1,6,1,12,1,18,1,18,1,24, + 1,12,1,12,1,12,1,36,1,12,1,18,1,9,2,12, + 1,12,1,12,1,12,1,21,1,21,1,114,79,0,0,0, + 99,1,0,0,0,0,0,0,0,8,0,0,0,5,0,0, + 0,67,0,0,0,115,62,1,0,0,116,0,0,106,1,0, + 106,2,0,100,1,0,107,8,0,114,30,0,116,3,0,100, + 2,0,131,1,0,130,1,0,116,4,0,124,0,0,131,1, + 0,92,2,0,125,1,0,125,2,0,116,4,0,124,1,0, + 131,1,0,92,2,0,125,1,0,125,3,0,124,3,0,116, + 5,0,107,3,0,114,102,0,116,6,0,100,3,0,106,7, + 0,116,5,0,124,0,0,131,2,0,131,1,0,130,1,0, + 124,2,0,106,8,0,100,4,0,131,1,0,125,4,0,124, + 4,0,100,11,0,107,7,0,114,153,0,116,6,0,100,7, + 0,106,7,0,124,2,0,131,1,0,131,1,0,130,1,0, + 110,125,0,124,4,0,100,6,0,107,2,0,114,22,1,124, + 2,0,106,9,0,100,4,0,100,5,0,131,2,0,100,12, + 0,25,125,5,0,124,5,0,106,10,0,116,11,0,131,1, + 0,115,223,0,116,6,0,100,8,0,106,7,0,116,11,0, + 131,1,0,131,1,0,130,1,0,124,5,0,116,12,0,116, + 11,0,131,1,0,100,1,0,133,2,0,25,125,6,0,124, + 6,0,106,13,0,131,0,0,115,22,1,116,6,0,100,9, + 0,106,7,0,124,5,0,131,1,0,131,1,0,130,1,0, + 124,2,0,106,14,0,100,4,0,131,1,0,100,10,0,25, + 125,7,0,116,15,0,124,1,0,124,7,0,116,16,0,100, + 10,0,25,23,131,2,0,83,41,13,97,110,1,0,0,71, + 105,118,101,110,32,116,104,101,32,112,97,116,104,32,116,111, + 32,97,32,46,112,121,99,46,32,102,105,108,101,44,32,114, + 101,116,117,114,110,32,116,104,101,32,112,97,116,104,32,116, + 111,32,105,116,115,32,46,112,121,32,102,105,108,101,46,10, + 10,32,32,32,32,84,104,101,32,46,112,121,99,32,102,105, + 108,101,32,100,111,101,115,32,110,111,116,32,110,101,101,100, + 32,116,111,32,101,120,105,115,116,59,32,116,104,105,115,32, + 115,105,109,112,108,121,32,114,101,116,117,114,110,115,32,116, + 104,101,32,112,97,116,104,32,116,111,10,32,32,32,32,116, + 104,101,32,46,112,121,32,102,105,108,101,32,99,97,108,99, + 117,108,97,116,101,100,32,116,111,32,99,111,114,114,101,115, + 112,111,110,100,32,116,111,32,116,104,101,32,46,112,121,99, + 32,102,105,108,101,46,32,32,73,102,32,112,97,116,104,32, + 100,111,101,115,10,32,32,32,32,110,111,116,32,99,111,110, + 102,111,114,109,32,116,111,32,80,69,80,32,51,49,52,55, + 47,52,56,56,32,102,111,114,109,97,116,44,32,86,97,108, + 117,101,69,114,114,111,114,32,119,105,108,108,32,98,101,32, + 114,97,105,115,101,100,46,32,73,102,10,32,32,32,32,115, + 121,115,46,105,109,112,108,101,109,101,110,116,97,116,105,111, + 110,46,99,97,99,104,101,95,116,97,103,32,105,115,32,78, + 111,110,101,32,116,104,101,110,32,78,111,116,73,109,112,108, + 101,109,101,110,116,101,100,69,114,114,111,114,32,105,115,32, + 114,97,105,115,101,100,46,10,10,32,32,32,32,78,122,36, + 115,121,115,46,105,109,112,108,101,109,101,110,116,97,116,105, + 111,110,46,99,97,99,104,101,95,116,97,103,32,105,115,32, + 78,111,110,101,122,37,123,125,32,110,111,116,32,98,111,116, + 116,111,109,45,108,101,118,101,108,32,100,105,114,101,99,116, + 111,114,121,32,105,110,32,123,33,114,125,114,58,0,0,0, + 114,56,0,0,0,233,3,0,0,0,122,33,101,120,112,101, + 99,116,101,100,32,111,110,108,121,32,50,32,111,114,32,51, + 32,100,111,116,115,32,105,110,32,123,33,114,125,122,57,111, + 112,116,105,109,105,122,97,116,105,111,110,32,112,111,114,116, + 105,111,110,32,111,102,32,102,105,108,101,110,97,109,101,32, + 100,111,101,115,32,110,111,116,32,115,116,97,114,116,32,119, + 105,116,104,32,123,33,114,125,122,52,111,112,116,105,109,105, + 122,97,116,105,111,110,32,108,101,118,101,108,32,123,33,114, + 125,32,105,115,32,110,111,116,32,97,110,32,97,108,112,104, + 97,110,117,109,101,114,105,99,32,118,97,108,117,101,114,59, + 0,0,0,62,2,0,0,0,114,56,0,0,0,114,80,0, + 0,0,233,254,255,255,255,41,17,114,7,0,0,0,114,64, + 0,0,0,114,65,0,0,0,114,66,0,0,0,114,38,0, + 0,0,114,73,0,0,0,114,71,0,0,0,114,47,0,0, + 0,218,5,99,111,117,110,116,114,34,0,0,0,114,9,0, + 0,0,114,72,0,0,0,114,31,0,0,0,114,70,0,0, + 0,218,9,112,97,114,116,105,116,105,111,110,114,28,0,0, + 0,218,15,83,79,85,82,67,69,95,83,85,70,70,73,88, + 69,83,41,8,114,35,0,0,0,114,76,0,0,0,90,16, + 112,121,99,97,99,104,101,95,102,105,108,101,110,97,109,101, + 90,7,112,121,99,97,99,104,101,90,9,100,111,116,95,99, + 111,117,110,116,114,57,0,0,0,90,9,111,112,116,95,108, + 101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,110, + 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,95, + 99,97,99,104,101,31,1,0,0,115,44,0,0,0,0,9, + 18,1,12,1,18,1,18,1,12,1,9,1,15,1,15,1, + 12,1,9,1,15,1,12,1,22,1,15,1,9,1,12,1, + 22,1,12,1,9,1,12,1,19,1,114,85,0,0,0,99, + 1,0,0,0,0,0,0,0,5,0,0,0,12,0,0,0, + 67,0,0,0,115,164,0,0,0,116,0,0,124,0,0,131, + 1,0,100,1,0,107,2,0,114,22,0,100,2,0,83,124, + 0,0,106,1,0,100,3,0,131,1,0,92,3,0,125,1, + 0,125,2,0,125,3,0,124,1,0,12,115,81,0,124,3, + 0,106,2,0,131,0,0,100,7,0,100,8,0,133,2,0, + 25,100,6,0,107,3,0,114,85,0,124,0,0,83,121,16, + 0,116,3,0,124,0,0,131,1,0,125,4,0,87,110,40, + 0,4,116,4,0,116,5,0,102,2,0,107,10,0,114,143, + 0,1,1,1,124,0,0,100,2,0,100,9,0,133,2,0, + 25,125,4,0,89,110,1,0,88,116,6,0,124,4,0,131, + 1,0,114,160,0,124,4,0,83,124,0,0,83,41,10,122, + 188,67,111,110,118,101,114,116,32,97,32,98,121,116,101,99, + 111,100,101,32,102,105,108,101,32,112,97,116,104,32,116,111, + 32,97,32,115,111,117,114,99,101,32,112,97,116,104,32,40, + 105,102,32,112,111,115,115,105,98,108,101,41,46,10,10,32, + 32,32,32,84,104,105,115,32,102,117,110,99,116,105,111,110, + 32,101,120,105,115,116,115,32,112,117,114,101,108,121,32,102, + 111,114,32,98,97,99,107,119,97,114,100,115,45,99,111,109, + 112,97,116,105,98,105,108,105,116,121,32,102,111,114,10,32, + 32,32,32,80,121,73,109,112,111,114,116,95,69,120,101,99, + 67,111,100,101,77,111,100,117,108,101,87,105,116,104,70,105, + 108,101,110,97,109,101,115,40,41,32,105,110,32,116,104,101, + 32,67,32,65,80,73,46,10,10,32,32,32,32,114,59,0, + 0,0,78,114,58,0,0,0,114,80,0,0,0,114,29,0, + 0,0,90,2,112,121,233,253,255,255,255,233,255,255,255,255, + 114,87,0,0,0,41,7,114,31,0,0,0,114,32,0,0, + 0,218,5,108,111,119,101,114,114,85,0,0,0,114,66,0, + 0,0,114,71,0,0,0,114,44,0,0,0,41,5,218,13, + 98,121,116,101,99,111,100,101,95,112,97,116,104,114,78,0, + 0,0,114,36,0,0,0,90,9,101,120,116,101,110,115,105, + 111,110,218,11,115,111,117,114,99,101,95,112,97,116,104,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,15, + 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,64, + 1,0,0,115,20,0,0,0,0,7,18,1,4,1,24,1, + 35,1,4,1,3,1,16,1,19,1,21,1,114,91,0,0, + 0,99,1,0,0,0,0,0,0,0,1,0,0,0,11,0, + 0,0,67,0,0,0,115,92,0,0,0,124,0,0,106,0, + 0,116,1,0,116,2,0,131,1,0,131,1,0,114,59,0, + 121,14,0,116,3,0,124,0,0,131,1,0,83,87,113,88, + 0,4,116,4,0,107,10,0,114,55,0,1,1,1,89,113, + 88,0,88,110,29,0,124,0,0,106,0,0,116,1,0,116, + 5,0,131,1,0,131,1,0,114,84,0,124,0,0,83,100, + 0,0,83,100,0,0,83,41,1,78,41,6,218,8,101,110, + 100,115,119,105,116,104,218,5,116,117,112,108,101,114,84,0, + 0,0,114,79,0,0,0,114,66,0,0,0,114,74,0,0, + 0,41,1,218,8,102,105,108,101,110,97,109,101,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,103, + 101,116,95,99,97,99,104,101,100,83,1,0,0,115,16,0, + 0,0,0,1,21,1,3,1,14,1,13,1,8,1,21,1, + 4,2,114,95,0,0,0,99,1,0,0,0,0,0,0,0, + 2,0,0,0,11,0,0,0,67,0,0,0,115,60,0,0, + 0,121,19,0,116,0,0,124,0,0,131,1,0,106,1,0, + 125,1,0,87,110,24,0,4,116,2,0,107,10,0,114,45, + 0,1,1,1,100,1,0,125,1,0,89,110,1,0,88,124, + 1,0,100,2,0,79,125,1,0,124,1,0,83,41,3,122, + 51,67,97,108,99,117,108,97,116,101,32,116,104,101,32,109, + 111,100,101,32,112,101,114,109,105,115,115,105,111,110,115,32, + 102,111,114,32,97,32,98,121,116,101,99,111,100,101,32,102, + 105,108,101,46,105,182,1,0,0,233,128,0,0,0,41,3, + 114,39,0,0,0,114,41,0,0,0,114,40,0,0,0,41, + 2,114,35,0,0,0,114,42,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,10,95,99,97,108, + 99,95,109,111,100,101,95,1,0,0,115,12,0,0,0,0, + 2,3,1,19,1,13,1,11,3,10,1,114,97,0,0,0, + 99,1,0,0,0,0,0,0,0,3,0,0,0,11,0,0, + 0,3,0,0,0,115,84,0,0,0,100,1,0,135,0,0, + 102,1,0,100,2,0,100,3,0,134,1,0,125,1,0,121, + 13,0,116,0,0,106,1,0,125,2,0,87,110,30,0,4, + 116,2,0,107,10,0,114,66,0,1,1,1,100,4,0,100, + 5,0,132,0,0,125,2,0,89,110,1,0,88,124,2,0, + 124,1,0,136,0,0,131,2,0,1,124,1,0,83,41,6, + 122,252,68,101,99,111,114,97,116,111,114,32,116,111,32,118, + 101,114,105,102,121,32,116,104,97,116,32,116,104,101,32,109, + 111,100,117,108,101,32,98,101,105,110,103,32,114,101,113,117, + 101,115,116,101,100,32,109,97,116,99,104,101,115,32,116,104, + 101,32,111,110,101,32,116,104,101,10,32,32,32,32,108,111, + 97,100,101,114,32,99,97,110,32,104,97,110,100,108,101,46, + 10,10,32,32,32,32,84,104,101,32,102,105,114,115,116,32, + 97,114,103,117,109,101,110,116,32,40,115,101,108,102,41,32, + 109,117,115,116,32,100,101,102,105,110,101,32,95,110,97,109, + 101,32,119,104,105,99,104,32,116,104,101,32,115,101,99,111, + 110,100,32,97,114,103,117,109,101,110,116,32,105,115,10,32, + 32,32,32,99,111,109,112,97,114,101,100,32,97,103,97,105, + 110,115,116,46,32,73,102,32,116,104,101,32,99,111,109,112, + 97,114,105,115,111,110,32,102,97,105,108,115,32,116,104,101, + 110,32,73,109,112,111,114,116,69,114,114,111,114,32,105,115, + 32,114,97,105,115,101,100,46,10,10,32,32,32,32,78,99, + 2,0,0,0,0,0,0,0,4,0,0,0,5,0,0,0, + 31,0,0,0,115,89,0,0,0,124,1,0,100,0,0,107, + 8,0,114,24,0,124,0,0,106,0,0,125,1,0,110,46, + 0,124,0,0,106,0,0,124,1,0,107,3,0,114,70,0, + 116,1,0,100,1,0,124,0,0,106,0,0,124,1,0,102, + 2,0,22,100,2,0,124,1,0,131,1,1,130,1,0,136, + 0,0,124,0,0,124,1,0,124,2,0,124,3,0,142,2, + 0,83,41,3,78,122,30,108,111,97,100,101,114,32,102,111, + 114,32,37,115,32,99,97,110,110,111,116,32,104,97,110,100, + 108,101,32,37,115,218,4,110,97,109,101,41,2,114,98,0, + 0,0,218,11,73,109,112,111,114,116,69,114,114,111,114,41, + 4,218,4,115,101,108,102,114,98,0,0,0,218,4,97,114, + 103,115,90,6,107,119,97,114,103,115,41,1,218,6,109,101, 116,104,111,100,114,4,0,0,0,114,5,0,0,0,218,19, 95,99,104,101,99,107,95,110,97,109,101,95,119,114,97,112, - 112,101,114,123,1,0,0,115,12,0,0,0,0,1,12,1, + 112,101,114,115,1,0,0,115,12,0,0,0,0,1,12,1, 12,1,15,1,6,1,25,1,122,40,95,99,104,101,99,107, 95,110,97,109,101,46,60,108,111,99,97,108,115,62,46,95, 99,104,101,99,107,95,110,97,109,101,95,119,114,97,112,112, @@ -595,17 +573,17 @@ const unsigned char _Py_M__importlib_external[] = { 116,97,116,116,114,218,8,95,95,100,105,99,116,95,95,218, 6,117,112,100,97,116,101,41,3,90,3,110,101,119,90,3, 111,108,100,114,52,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,5,95,119,114,97,112,134,1, + 0,0,114,5,0,0,0,218,5,95,119,114,97,112,126,1, 0,0,115,8,0,0,0,0,1,25,1,15,1,29,1,122, 26,95,99,104,101,99,107,95,110,97,109,101,46,60,108,111, 99,97,108,115,62,46,95,119,114,97,112,41,3,218,10,95, - 98,111,111,116,115,116,114,97,112,114,120,0,0,0,218,9, - 78,97,109,101,69,114,114,111,114,41,3,114,109,0,0,0, - 114,110,0,0,0,114,120,0,0,0,114,4,0,0,0,41, - 1,114,109,0,0,0,114,5,0,0,0,218,11,95,99,104, - 101,99,107,95,110,97,109,101,115,1,0,0,115,14,0,0, + 98,111,111,116,115,116,114,97,112,114,113,0,0,0,218,9, + 78,97,109,101,69,114,114,111,114,41,3,114,102,0,0,0, + 114,103,0,0,0,114,113,0,0,0,114,4,0,0,0,41, + 1,114,102,0,0,0,114,5,0,0,0,218,11,95,99,104, + 101,99,107,95,110,97,109,101,107,1,0,0,115,14,0,0, 0,0,8,21,7,3,1,13,1,13,2,17,5,13,1,114, - 123,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, + 116,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, 0,4,0,0,0,67,0,0,0,115,84,0,0,0,124,0, 0,106,0,0,124,1,0,131,1,0,92,2,0,125,2,0, 125,3,0,124,2,0,100,1,0,107,8,0,114,80,0,116, @@ -628,14 +606,14 @@ const unsigned char _Py_M__importlib_external[] = { 114,59,0,0,0,41,6,218,11,102,105,110,100,95,108,111, 97,100,101,114,114,31,0,0,0,114,60,0,0,0,114,61, 0,0,0,114,47,0,0,0,218,13,73,109,112,111,114,116, - 87,97,114,110,105,110,103,41,5,114,108,0,0,0,218,8, + 87,97,114,110,105,110,103,41,5,114,100,0,0,0,218,8, 102,117,108,108,110,97,109,101,218,6,108,111,97,100,101,114, 218,8,112,111,114,116,105,111,110,115,218,3,109,115,103,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,17, 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, - 109,143,1,0,0,115,10,0,0,0,0,10,21,1,24,1, - 6,1,29,1,114,130,0,0,0,99,4,0,0,0,0,0, - 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,228, + 109,135,1,0,0,115,10,0,0,0,0,10,21,1,24,1, + 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, + 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,240, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, 9,0,114,31,0,124,2,0,124,4,0,100,2,0,60,110, 6,0,100,3,0,125,2,0,124,3,0,100,1,0,107,9, @@ -643,1954 +621,1959 @@ const unsigned char _Py_M__importlib_external[] = { 0,100,1,0,100,5,0,133,2,0,25,125,5,0,124,0, 0,100,5,0,100,6,0,133,2,0,25,125,6,0,124,0, 0,100,6,0,100,7,0,133,2,0,25,125,7,0,124,5, - 0,116,0,0,107,3,0,114,165,0,100,8,0,106,1,0, - 124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,124, - 8,0,131,1,0,1,116,3,0,124,8,0,124,4,0,141, - 1,0,130,1,0,110,113,0,116,4,0,124,6,0,131,1, - 0,100,5,0,107,3,0,114,223,0,100,9,0,106,1,0, - 124,2,0,131,1,0,125,8,0,116,2,0,124,8,0,131, - 1,0,1,116,5,0,124,8,0,131,1,0,130,1,0,110, - 55,0,116,4,0,124,7,0,131,1,0,100,5,0,107,3, - 0,114,22,1,100,10,0,106,1,0,124,2,0,131,1,0, - 125,8,0,116,2,0,124,8,0,131,1,0,1,116,5,0, - 124,8,0,131,1,0,130,1,0,124,1,0,100,1,0,107, - 9,0,114,214,1,121,20,0,116,6,0,124,1,0,100,11, - 0,25,131,1,0,125,9,0,87,110,18,0,4,116,7,0, - 107,10,0,114,74,1,1,1,1,89,110,59,0,88,116,8, - 0,124,6,0,131,1,0,124,9,0,107,3,0,114,133,1, - 100,12,0,106,1,0,124,2,0,131,1,0,125,8,0,116, - 2,0,124,8,0,131,1,0,1,116,3,0,124,8,0,124, - 4,0,141,1,0,130,1,0,121,18,0,124,1,0,100,13, - 0,25,100,14,0,64,125,10,0,87,110,18,0,4,116,7, - 0,107,10,0,114,171,1,1,1,1,89,110,43,0,88,116, - 8,0,124,7,0,131,1,0,124,10,0,107,3,0,114,214, - 1,116,3,0,100,12,0,106,1,0,124,2,0,131,1,0, - 124,4,0,141,1,0,130,1,0,124,0,0,100,7,0,100, - 1,0,133,2,0,25,83,41,15,97,122,1,0,0,86,97, - 108,105,100,97,116,101,32,116,104,101,32,104,101,97,100,101, - 114,32,111,102,32,116,104,101,32,112,97,115,115,101,100,45, - 105,110,32,98,121,116,101,99,111,100,101,32,97,103,97,105, - 110,115,116,32,115,111,117,114,99,101,95,115,116,97,116,115, - 32,40,105,102,10,32,32,32,32,103,105,118,101,110,41,32, - 97,110,100,32,114,101,116,117,114,110,105,110,103,32,116,104, - 101,32,98,121,116,101,99,111,100,101,32,116,104,97,116,32, - 99,97,110,32,98,101,32,99,111,109,112,105,108,101,100,32, - 98,121,32,99,111,109,112,105,108,101,40,41,46,10,10,32, - 32,32,32,65,108,108,32,111,116,104,101,114,32,97,114,103, - 117,109,101,110,116,115,32,97,114,101,32,117,115,101,100,32, - 116,111,32,101,110,104,97,110,99,101,32,101,114,114,111,114, - 32,114,101,112,111,114,116,105,110,103,46,10,10,32,32,32, - 32,73,109,112,111,114,116,69,114,114,111,114,32,105,115,32, - 114,97,105,115,101,100,32,119,104,101,110,32,116,104,101,32, - 109,97,103,105,99,32,110,117,109,98,101,114,32,105,115,32, - 105,110,99,111,114,114,101,99,116,32,111,114,32,116,104,101, - 32,98,121,116,101,99,111,100,101,32,105,115,10,32,32,32, - 32,102,111,117,110,100,32,116,111,32,98,101,32,115,116,97, - 108,101,46,32,69,79,70,69,114,114,111,114,32,105,115,32, - 114,97,105,115,101,100,32,119,104,101,110,32,116,104,101,32, - 100,97,116,97,32,105,115,32,102,111,117,110,100,32,116,111, - 32,98,101,10,32,32,32,32,116,114,117,110,99,97,116,101, - 100,46,10,10,32,32,32,32,78,114,106,0,0,0,122,10, - 60,98,121,116,101,99,111,100,101,62,114,35,0,0,0,114, - 12,0,0,0,233,8,0,0,0,233,12,0,0,0,122,30, - 98,97,100,32,109,97,103,105,99,32,110,117,109,98,101,114, - 32,105,110,32,123,33,114,125,58,32,123,33,114,125,122,43, - 114,101,97,99,104,101,100,32,69,79,70,32,119,104,105,108, - 101,32,114,101,97,100,105,110,103,32,116,105,109,101,115,116, - 97,109,112,32,105,110,32,123,33,114,125,122,48,114,101,97, - 99,104,101,100,32,69,79,70,32,119,104,105,108,101,32,114, - 101,97,100,105,110,103,32,115,105,122,101,32,111,102,32,115, - 111,117,114,99,101,32,105,110,32,123,33,114,125,218,5,109, - 116,105,109,101,122,26,98,121,116,101,99,111,100,101,32,105, - 115,32,115,116,97,108,101,32,102,111,114,32,123,33,114,125, - 218,4,115,105,122,101,108,3,0,0,0,255,127,255,127,3, - 0,41,9,218,12,77,65,71,73,67,95,78,85,77,66,69, - 82,114,47,0,0,0,114,105,0,0,0,114,107,0,0,0, - 114,31,0,0,0,218,8,69,79,70,69,114,114,111,114,114, - 14,0,0,0,218,8,75,101,121,69,114,114,111,114,114,19, - 0,0,0,41,11,114,53,0,0,0,218,12,115,111,117,114, - 99,101,95,115,116,97,116,115,114,106,0,0,0,114,35,0, - 0,0,90,11,101,120,99,95,100,101,116,97,105,108,115,90, - 5,109,97,103,105,99,90,13,114,97,119,95,116,105,109,101, - 115,116,97,109,112,90,8,114,97,119,95,115,105,122,101,114, - 75,0,0,0,218,12,115,111,117,114,99,101,95,109,116,105, - 109,101,218,11,115,111,117,114,99,101,95,115,105,122,101,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,25, - 95,118,97,108,105,100,97,116,101,95,98,121,116,101,99,111, - 100,101,95,104,101,97,100,101,114,160,1,0,0,115,76,0, - 0,0,0,11,6,1,12,1,13,3,6,1,12,1,10,1, - 16,1,16,1,16,1,12,1,18,1,10,1,18,1,18,1, - 15,1,10,1,15,1,18,1,15,1,10,1,12,1,12,1, - 3,1,20,1,13,1,5,2,18,1,15,1,10,1,15,1, - 3,1,18,1,13,1,5,2,18,1,15,1,9,1,114,141, - 0,0,0,99,4,0,0,0,0,0,0,0,5,0,0,0, - 6,0,0,0,67,0,0,0,115,112,0,0,0,116,0,0, - 106,1,0,124,0,0,131,1,0,125,4,0,116,2,0,124, - 4,0,116,3,0,131,2,0,114,75,0,116,4,0,100,1, - 0,124,2,0,131,2,0,1,124,3,0,100,2,0,107,9, - 0,114,71,0,116,5,0,106,6,0,124,4,0,124,3,0, - 131,2,0,1,124,4,0,83,116,7,0,100,3,0,106,8, - 0,124,2,0,131,1,0,100,4,0,124,1,0,100,5,0, - 124,2,0,131,1,2,130,1,0,100,2,0,83,41,6,122, - 60,67,111,109,112,105,108,101,32,98,121,116,101,99,111,100, - 101,32,97,115,32,114,101,116,117,114,110,101,100,32,98,121, - 32,95,118,97,108,105,100,97,116,101,95,98,121,116,101,99, - 111,100,101,95,104,101,97,100,101,114,40,41,46,122,21,99, - 111,100,101,32,111,98,106,101,99,116,32,102,114,111,109,32, - 123,33,114,125,78,122,23,78,111,110,45,99,111,100,101,32, - 111,98,106,101,99,116,32,105,110,32,123,33,114,125,114,106, - 0,0,0,114,35,0,0,0,41,9,218,7,109,97,114,115, - 104,97,108,90,5,108,111,97,100,115,218,10,105,115,105,110, - 115,116,97,110,99,101,218,10,95,99,111,100,101,95,116,121, - 112,101,114,105,0,0,0,218,4,95,105,109,112,90,16,95, - 102,105,120,95,99,111,95,102,105,108,101,110,97,109,101,114, - 107,0,0,0,114,47,0,0,0,41,5,114,53,0,0,0, - 114,106,0,0,0,114,89,0,0,0,114,90,0,0,0,218, - 4,99,111,100,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,17,95,99,111,109,112,105,108,101,95,98, - 121,116,101,99,111,100,101,215,1,0,0,115,16,0,0,0, - 0,2,15,1,15,1,13,1,12,1,16,1,4,2,18,1, - 114,147,0,0,0,114,59,0,0,0,99,3,0,0,0,0, - 0,0,0,4,0,0,0,3,0,0,0,67,0,0,0,115, - 76,0,0,0,116,0,0,116,1,0,131,1,0,125,3,0, - 124,3,0,106,2,0,116,3,0,124,1,0,131,1,0,131, - 1,0,1,124,3,0,106,2,0,116,3,0,124,2,0,131, - 1,0,131,1,0,1,124,3,0,106,2,0,116,4,0,106, - 5,0,124,0,0,131,1,0,131,1,0,1,124,3,0,83, - 41,1,122,80,67,111,109,112,105,108,101,32,97,32,99,111, - 100,101,32,111,98,106,101,99,116,32,105,110,116,111,32,98, - 121,116,101,99,111,100,101,32,102,111,114,32,119,114,105,116, - 105,110,103,32,111,117,116,32,116,111,32,97,32,98,121,116, - 101,45,99,111,109,112,105,108,101,100,10,32,32,32,32,102, - 105,108,101,46,41,6,218,9,98,121,116,101,97,114,114,97, - 121,114,135,0,0,0,218,6,101,120,116,101,110,100,114,17, - 0,0,0,114,142,0,0,0,90,5,100,117,109,112,115,41, - 4,114,146,0,0,0,114,133,0,0,0,114,140,0,0,0, - 114,53,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,17,95,99,111,100,101,95,116,111,95,98, - 121,116,101,99,111,100,101,227,1,0,0,115,10,0,0,0, - 0,3,12,1,19,1,19,1,22,1,114,150,0,0,0,99, - 1,0,0,0,0,0,0,0,5,0,0,0,4,0,0,0, - 67,0,0,0,115,89,0,0,0,100,1,0,100,2,0,108, - 0,0,125,1,0,116,1,0,106,2,0,124,0,0,131,1, - 0,106,3,0,125,2,0,124,1,0,106,4,0,124,2,0, - 131,1,0,125,3,0,116,1,0,106,5,0,100,2,0,100, - 3,0,131,2,0,125,4,0,124,4,0,106,6,0,124,0, - 0,106,6,0,124,3,0,100,1,0,25,131,1,0,131,1, - 0,83,41,4,122,121,68,101,99,111,100,101,32,98,121,116, - 101,115,32,114,101,112,114,101,115,101,110,116,105,110,103,32, - 115,111,117,114,99,101,32,99,111,100,101,32,97,110,100,32, - 114,101,116,117,114,110,32,116,104,101,32,115,116,114,105,110, - 103,46,10,10,32,32,32,32,85,110,105,118,101,114,115,97, - 108,32,110,101,119,108,105,110,101,32,115,117,112,112,111,114, - 116,32,105,115,32,117,115,101,100,32,105,110,32,116,104,101, - 32,100,101,99,111,100,105,110,103,46,10,32,32,32,32,114, - 59,0,0,0,78,84,41,7,218,8,116,111,107,101,110,105, - 122,101,114,49,0,0,0,90,7,66,121,116,101,115,73,79, - 90,8,114,101,97,100,108,105,110,101,90,15,100,101,116,101, - 99,116,95,101,110,99,111,100,105,110,103,90,25,73,110,99, - 114,101,109,101,110,116,97,108,78,101,119,108,105,110,101,68, - 101,99,111,100,101,114,218,6,100,101,99,111,100,101,41,5, - 218,12,115,111,117,114,99,101,95,98,121,116,101,115,114,151, - 0,0,0,90,21,115,111,117,114,99,101,95,98,121,116,101, - 115,95,114,101,97,100,108,105,110,101,218,8,101,110,99,111, - 100,105,110,103,90,15,110,101,119,108,105,110,101,95,100,101, - 99,111,100,101,114,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,13,100,101,99,111,100,101,95,115,111,117, - 114,99,101,237,1,0,0,115,10,0,0,0,0,5,12,1, - 18,1,15,1,18,1,114,155,0,0,0,114,127,0,0,0, - 218,26,115,117,98,109,111,100,117,108,101,95,115,101,97,114, - 99,104,95,108,111,99,97,116,105,111,110,115,99,2,0,0, - 0,2,0,0,0,9,0,0,0,19,0,0,0,67,0,0, - 0,115,89,1,0,0,124,1,0,100,1,0,107,8,0,114, - 73,0,100,2,0,125,1,0,116,0,0,124,2,0,100,3, - 0,131,2,0,114,73,0,121,19,0,124,2,0,106,1,0, - 124,0,0,131,1,0,125,1,0,87,110,18,0,4,116,2, - 0,107,10,0,114,72,0,1,1,1,89,110,1,0,88,116, - 3,0,106,4,0,124,0,0,124,2,0,100,4,0,124,1, - 0,131,2,1,125,4,0,100,5,0,124,4,0,95,5,0, - 124,2,0,100,1,0,107,8,0,114,194,0,120,73,0,116, - 6,0,131,0,0,68,93,58,0,92,2,0,125,5,0,125, - 6,0,124,1,0,106,7,0,116,8,0,124,6,0,131,1, - 0,131,1,0,114,128,0,124,5,0,124,0,0,124,1,0, - 131,2,0,125,2,0,124,2,0,124,4,0,95,9,0,80, - 113,128,0,87,100,1,0,83,124,3,0,116,10,0,107,8, - 0,114,23,1,116,0,0,124,2,0,100,6,0,131,2,0, - 114,32,1,121,19,0,124,2,0,106,11,0,124,0,0,131, - 1,0,125,7,0,87,110,18,0,4,116,2,0,107,10,0, - 114,4,1,1,1,1,89,113,32,1,88,124,7,0,114,32, - 1,103,0,0,124,4,0,95,12,0,110,9,0,124,3,0, - 124,4,0,95,12,0,124,4,0,106,12,0,103,0,0,107, - 2,0,114,85,1,124,1,0,114,85,1,116,13,0,124,1, - 0,131,1,0,100,7,0,25,125,8,0,124,4,0,106,12, - 0,106,14,0,124,8,0,131,1,0,1,124,4,0,83,41, - 8,97,61,1,0,0,82,101,116,117,114,110,32,97,32,109, - 111,100,117,108,101,32,115,112,101,99,32,98,97,115,101,100, - 32,111,110,32,97,32,102,105,108,101,32,108,111,99,97,116, - 105,111,110,46,10,10,32,32,32,32,84,111,32,105,110,100, - 105,99,97,116,101,32,116,104,97,116,32,116,104,101,32,109, - 111,100,117,108,101,32,105,115,32,97,32,112,97,99,107,97, - 103,101,44,32,115,101,116,10,32,32,32,32,115,117,98,109, - 111,100,117,108,101,95,115,101,97,114,99,104,95,108,111,99, - 97,116,105,111,110,115,32,116,111,32,97,32,108,105,115,116, - 32,111,102,32,100,105,114,101,99,116,111,114,121,32,112,97, - 116,104,115,46,32,32,65,110,10,32,32,32,32,101,109,112, - 116,121,32,108,105,115,116,32,105,115,32,115,117,102,102,105, - 99,105,101,110,116,44,32,116,104,111,117,103,104,32,105,116, - 115,32,110,111,116,32,111,116,104,101,114,119,105,115,101,32, - 117,115,101,102,117,108,32,116,111,32,116,104,101,10,32,32, - 32,32,105,109,112,111,114,116,32,115,121,115,116,101,109,46, - 10,10,32,32,32,32,84,104,101,32,108,111,97,100,101,114, - 32,109,117,115,116,32,116,97,107,101,32,97,32,115,112,101, - 99,32,97,115,32,105,116,115,32,111,110,108,121,32,95,95, - 105,110,105,116,95,95,40,41,32,97,114,103,46,10,10,32, - 32,32,32,78,122,9,60,117,110,107,110,111,119,110,62,218, - 12,103,101,116,95,102,105,108,101,110,97,109,101,218,6,111, - 114,105,103,105,110,84,218,10,105,115,95,112,97,99,107,97, - 103,101,114,59,0,0,0,41,15,114,115,0,0,0,114,157, - 0,0,0,114,107,0,0,0,114,121,0,0,0,218,10,77, - 111,100,117,108,101,83,112,101,99,90,13,95,115,101,116,95, - 102,105,108,101,97,116,116,114,218,27,95,103,101,116,95,115, - 117,112,112,111,114,116,101,100,95,102,105,108,101,95,108,111, - 97,100,101,114,115,114,92,0,0,0,114,93,0,0,0,114, - 127,0,0,0,218,9,95,80,79,80,85,76,65,84,69,114, - 159,0,0,0,114,156,0,0,0,114,38,0,0,0,218,6, - 97,112,112,101,110,100,41,9,114,106,0,0,0,90,8,108, - 111,99,97,116,105,111,110,114,127,0,0,0,114,156,0,0, - 0,218,4,115,112,101,99,218,12,108,111,97,100,101,114,95, - 99,108,97,115,115,218,8,115,117,102,102,105,120,101,115,114, - 159,0,0,0,90,7,100,105,114,110,97,109,101,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,23,115,112, - 101,99,95,102,114,111,109,95,102,105,108,101,95,108,111,99, - 97,116,105,111,110,254,1,0,0,115,60,0,0,0,0,12, - 12,4,6,1,15,2,3,1,19,1,13,1,5,8,24,1, - 9,3,12,1,22,1,21,1,15,1,9,1,5,2,4,3, - 12,2,15,1,3,1,19,1,13,1,5,2,6,1,12,2, - 9,1,15,1,6,1,16,1,16,2,114,167,0,0,0,99, - 0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0, - 64,0,0,0,115,121,0,0,0,101,0,0,90,1,0,100, - 0,0,90,2,0,100,1,0,90,3,0,100,2,0,90,4, - 0,100,3,0,90,5,0,100,4,0,90,6,0,101,7,0, - 100,5,0,100,6,0,132,0,0,131,1,0,90,8,0,101, - 7,0,100,7,0,100,8,0,132,0,0,131,1,0,90,9, - 0,101,7,0,100,9,0,100,9,0,100,10,0,100,11,0, - 132,2,0,131,1,0,90,10,0,101,7,0,100,9,0,100, - 12,0,100,13,0,132,1,0,131,1,0,90,11,0,100,9, - 0,83,41,14,218,21,87,105,110,100,111,119,115,82,101,103, - 105,115,116,114,121,70,105,110,100,101,114,122,62,77,101,116, - 97,32,112,97,116,104,32,102,105,110,100,101,114,32,102,111, - 114,32,109,111,100,117,108,101,115,32,100,101,99,108,97,114, - 101,100,32,105,110,32,116,104,101,32,87,105,110,100,111,119, - 115,32,114,101,103,105,115,116,114,121,46,122,59,83,111,102, - 116,119,97,114,101,92,80,121,116,104,111,110,92,80,121,116, - 104,111,110,67,111,114,101,92,123,115,121,115,95,118,101,114, - 115,105,111,110,125,92,77,111,100,117,108,101,115,92,123,102, - 117,108,108,110,97,109,101,125,122,65,83,111,102,116,119,97, - 114,101,92,80,121,116,104,111,110,92,80,121,116,104,111,110, - 67,111,114,101,92,123,115,121,115,95,118,101,114,115,105,111, - 110,125,92,77,111,100,117,108,101,115,92,123,102,117,108,108, - 110,97,109,101,125,92,68,101,98,117,103,70,99,2,0,0, - 0,0,0,0,0,2,0,0,0,11,0,0,0,67,0,0, - 0,115,67,0,0,0,121,23,0,116,0,0,106,1,0,116, - 0,0,106,2,0,124,1,0,131,2,0,83,87,110,37,0, - 4,116,3,0,107,10,0,114,62,0,1,1,1,116,0,0, - 106,1,0,116,0,0,106,4,0,124,1,0,131,2,0,83, - 89,110,1,0,88,100,0,0,83,41,1,78,41,5,218,7, - 95,119,105,110,114,101,103,90,7,79,112,101,110,75,101,121, - 90,17,72,75,69,89,95,67,85,82,82,69,78,84,95,85, - 83,69,82,114,40,0,0,0,90,18,72,75,69,89,95,76, - 79,67,65,76,95,77,65,67,72,73,78,69,41,2,218,3, - 99,108,115,218,3,107,101,121,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,14,95,111,112,101,110,95,114, - 101,103,105,115,116,114,121,76,2,0,0,115,8,0,0,0, - 0,2,3,1,23,1,13,1,122,36,87,105,110,100,111,119, - 115,82,101,103,105,115,116,114,121,70,105,110,100,101,114,46, - 95,111,112,101,110,95,114,101,103,105,115,116,114,121,99,2, - 0,0,0,0,0,0,0,6,0,0,0,16,0,0,0,67, - 0,0,0,115,143,0,0,0,124,0,0,106,0,0,114,21, - 0,124,0,0,106,1,0,125,2,0,110,9,0,124,0,0, - 106,2,0,125,2,0,124,2,0,106,3,0,100,1,0,124, - 1,0,100,2,0,116,4,0,106,5,0,100,0,0,100,3, - 0,133,2,0,25,131,0,2,125,3,0,121,47,0,124,0, - 0,106,6,0,124,3,0,131,1,0,143,25,0,125,4,0, - 116,7,0,106,8,0,124,4,0,100,4,0,131,2,0,125, - 5,0,87,100,0,0,81,82,88,87,110,22,0,4,116,9, - 0,107,10,0,114,138,0,1,1,1,100,0,0,83,89,110, - 1,0,88,124,5,0,83,41,5,78,114,126,0,0,0,90, - 11,115,121,115,95,118,101,114,115,105,111,110,114,80,0,0, - 0,114,30,0,0,0,41,10,218,11,68,69,66,85,71,95, - 66,85,73,76,68,218,18,82,69,71,73,83,84,82,89,95, - 75,69,89,95,68,69,66,85,71,218,12,82,69,71,73,83, - 84,82,89,95,75,69,89,114,47,0,0,0,114,7,0,0, - 0,218,7,118,101,114,115,105,111,110,114,172,0,0,0,114, - 169,0,0,0,90,10,81,117,101,114,121,86,97,108,117,101, - 114,40,0,0,0,41,6,114,170,0,0,0,114,126,0,0, - 0,90,12,114,101,103,105,115,116,114,121,95,107,101,121,114, - 171,0,0,0,90,4,104,107,101,121,218,8,102,105,108,101, - 112,97,116,104,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,16,95,115,101,97,114,99,104,95,114,101,103, - 105,115,116,114,121,83,2,0,0,115,22,0,0,0,0,2, - 9,1,12,2,9,1,15,1,22,1,3,1,18,1,29,1, - 13,1,9,1,122,38,87,105,110,100,111,119,115,82,101,103, - 105,115,116,114,121,70,105,110,100,101,114,46,95,115,101,97, - 114,99,104,95,114,101,103,105,115,116,114,121,78,99,4,0, - 0,0,0,0,0,0,8,0,0,0,14,0,0,0,67,0, - 0,0,115,158,0,0,0,124,0,0,106,0,0,124,1,0, - 131,1,0,125,4,0,124,4,0,100,0,0,107,8,0,114, - 31,0,100,0,0,83,121,14,0,116,1,0,124,4,0,131, - 1,0,1,87,110,22,0,4,116,2,0,107,10,0,114,69, - 0,1,1,1,100,0,0,83,89,110,1,0,88,120,81,0, - 116,3,0,131,0,0,68,93,70,0,92,2,0,125,5,0, - 125,6,0,124,4,0,106,4,0,116,5,0,124,6,0,131, - 1,0,131,1,0,114,80,0,116,6,0,106,7,0,124,1, - 0,124,5,0,124,1,0,124,4,0,131,2,0,100,1,0, - 124,4,0,131,2,1,125,7,0,124,7,0,83,113,80,0, - 87,100,0,0,83,41,2,78,114,158,0,0,0,41,8,114, - 178,0,0,0,114,39,0,0,0,114,40,0,0,0,114,161, - 0,0,0,114,92,0,0,0,114,93,0,0,0,114,121,0, - 0,0,218,16,115,112,101,99,95,102,114,111,109,95,108,111, - 97,100,101,114,41,8,114,170,0,0,0,114,126,0,0,0, - 114,35,0,0,0,218,6,116,97,114,103,101,116,114,177,0, - 0,0,114,127,0,0,0,114,166,0,0,0,114,164,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,9,102,105,110,100,95,115,112,101,99,98,2,0,0,115, - 26,0,0,0,0,2,15,1,12,1,4,1,3,1,14,1, - 13,1,9,1,22,1,21,1,9,1,15,1,9,1,122,31, + 0,116,0,0,107,3,0,114,168,0,100,8,0,106,1,0, + 124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,106, + 3,0,124,8,0,131,1,0,1,116,4,0,124,8,0,124, + 4,0,141,1,0,130,1,0,110,119,0,116,5,0,124,6, + 0,131,1,0,100,5,0,107,3,0,114,229,0,100,9,0, + 106,1,0,124,2,0,131,1,0,125,8,0,116,2,0,106, + 3,0,124,8,0,131,1,0,1,116,6,0,124,8,0,131, + 1,0,130,1,0,110,58,0,116,5,0,124,7,0,131,1, + 0,100,5,0,107,3,0,114,31,1,100,10,0,106,1,0, + 124,2,0,131,1,0,125,8,0,116,2,0,106,3,0,124, + 8,0,131,1,0,1,116,6,0,124,8,0,131,1,0,130, + 1,0,124,1,0,100,1,0,107,9,0,114,226,1,121,20, + 0,116,7,0,124,1,0,100,11,0,25,131,1,0,125,9, + 0,87,110,18,0,4,116,8,0,107,10,0,114,83,1,1, + 1,1,89,110,62,0,88,116,9,0,124,6,0,131,1,0, + 124,9,0,107,3,0,114,145,1,100,12,0,106,1,0,124, + 2,0,131,1,0,125,8,0,116,2,0,106,3,0,124,8, + 0,131,1,0,1,116,4,0,124,8,0,124,4,0,141,1, + 0,130,1,0,121,18,0,124,1,0,100,13,0,25,100,14, + 0,64,125,10,0,87,110,18,0,4,116,8,0,107,10,0, + 114,183,1,1,1,1,89,110,43,0,88,116,9,0,124,7, + 0,131,1,0,124,10,0,107,3,0,114,226,1,116,4,0, + 100,12,0,106,1,0,124,2,0,131,1,0,124,4,0,141, + 1,0,130,1,0,124,0,0,100,7,0,100,1,0,133,2, + 0,25,83,41,15,97,122,1,0,0,86,97,108,105,100,97, + 116,101,32,116,104,101,32,104,101,97,100,101,114,32,111,102, + 32,116,104,101,32,112,97,115,115,101,100,45,105,110,32,98, + 121,116,101,99,111,100,101,32,97,103,97,105,110,115,116,32, + 115,111,117,114,99,101,95,115,116,97,116,115,32,40,105,102, + 10,32,32,32,32,103,105,118,101,110,41,32,97,110,100,32, + 114,101,116,117,114,110,105,110,103,32,116,104,101,32,98,121, + 116,101,99,111,100,101,32,116,104,97,116,32,99,97,110,32, + 98,101,32,99,111,109,112,105,108,101,100,32,98,121,32,99, + 111,109,112,105,108,101,40,41,46,10,10,32,32,32,32,65, + 108,108,32,111,116,104,101,114,32,97,114,103,117,109,101,110, + 116,115,32,97,114,101,32,117,115,101,100,32,116,111,32,101, + 110,104,97,110,99,101,32,101,114,114,111,114,32,114,101,112, + 111,114,116,105,110,103,46,10,10,32,32,32,32,73,109,112, + 111,114,116,69,114,114,111,114,32,105,115,32,114,97,105,115, + 101,100,32,119,104,101,110,32,116,104,101,32,109,97,103,105, + 99,32,110,117,109,98,101,114,32,105,115,32,105,110,99,111, + 114,114,101,99,116,32,111,114,32,116,104,101,32,98,121,116, + 101,99,111,100,101,32,105,115,10,32,32,32,32,102,111,117, + 110,100,32,116,111,32,98,101,32,115,116,97,108,101,46,32, + 69,79,70,69,114,114,111,114,32,105,115,32,114,97,105,115, + 101,100,32,119,104,101,110,32,116,104,101,32,100,97,116,97, + 32,105,115,32,102,111,117,110,100,32,116,111,32,98,101,10, + 32,32,32,32,116,114,117,110,99,97,116,101,100,46,10,10, + 32,32,32,32,78,114,98,0,0,0,122,10,60,98,121,116, + 101,99,111,100,101,62,114,35,0,0,0,114,12,0,0,0, + 233,8,0,0,0,233,12,0,0,0,122,30,98,97,100,32, + 109,97,103,105,99,32,110,117,109,98,101,114,32,105,110,32, + 123,33,114,125,58,32,123,33,114,125,122,43,114,101,97,99, + 104,101,100,32,69,79,70,32,119,104,105,108,101,32,114,101, + 97,100,105,110,103,32,116,105,109,101,115,116,97,109,112,32, + 105,110,32,123,33,114,125,122,48,114,101,97,99,104,101,100, + 32,69,79,70,32,119,104,105,108,101,32,114,101,97,100,105, + 110,103,32,115,105,122,101,32,111,102,32,115,111,117,114,99, + 101,32,105,110,32,123,33,114,125,218,5,109,116,105,109,101, + 122,26,98,121,116,101,99,111,100,101,32,105,115,32,115,116, + 97,108,101,32,102,111,114,32,123,33,114,125,218,4,115,105, + 122,101,108,3,0,0,0,255,127,255,127,3,0,41,10,218, + 12,77,65,71,73,67,95,78,85,77,66,69,82,114,47,0, + 0,0,114,114,0,0,0,218,16,95,118,101,114,98,111,115, + 101,95,109,101,115,115,97,103,101,114,99,0,0,0,114,31, + 0,0,0,218,8,69,79,70,69,114,114,111,114,114,14,0, + 0,0,218,8,75,101,121,69,114,114,111,114,114,19,0,0, + 0,41,11,114,53,0,0,0,218,12,115,111,117,114,99,101, + 95,115,116,97,116,115,114,98,0,0,0,114,35,0,0,0, + 90,11,101,120,99,95,100,101,116,97,105,108,115,90,5,109, + 97,103,105,99,90,13,114,97,119,95,116,105,109,101,115,116, + 97,109,112,90,8,114,97,119,95,115,105,122,101,114,75,0, + 0,0,218,12,115,111,117,114,99,101,95,109,116,105,109,101, + 218,11,115,111,117,114,99,101,95,115,105,122,101,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,25,95,118, + 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, + 95,104,101,97,100,101,114,152,1,0,0,115,76,0,0,0, + 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, + 16,1,16,1,12,1,18,1,13,1,18,1,18,1,15,1, + 13,1,15,1,18,1,15,1,13,1,12,1,12,1,3,1, + 20,1,13,1,5,2,18,1,15,1,13,1,15,1,3,1, + 18,1,13,1,5,2,18,1,15,1,9,1,114,135,0,0, + 0,99,4,0,0,0,0,0,0,0,5,0,0,0,6,0, + 0,0,67,0,0,0,115,115,0,0,0,116,0,0,106,1, + 0,124,0,0,131,1,0,125,4,0,116,2,0,124,4,0, + 116,3,0,131,2,0,114,78,0,116,4,0,106,5,0,100, + 1,0,124,2,0,131,2,0,1,124,3,0,100,2,0,107, + 9,0,114,74,0,116,6,0,106,7,0,124,4,0,124,3, + 0,131,2,0,1,124,4,0,83,116,8,0,100,3,0,106, + 9,0,124,2,0,131,1,0,100,4,0,124,1,0,100,5, + 0,124,2,0,131,1,2,130,1,0,100,2,0,83,41,6, + 122,60,67,111,109,112,105,108,101,32,98,121,116,101,99,111, + 100,101,32,97,115,32,114,101,116,117,114,110,101,100,32,98, + 121,32,95,118,97,108,105,100,97,116,101,95,98,121,116,101, + 99,111,100,101,95,104,101,97,100,101,114,40,41,46,122,21, + 99,111,100,101,32,111,98,106,101,99,116,32,102,114,111,109, + 32,123,33,114,125,78,122,23,78,111,110,45,99,111,100,101, + 32,111,98,106,101,99,116,32,105,110,32,123,33,114,125,114, + 98,0,0,0,114,35,0,0,0,41,10,218,7,109,97,114, + 115,104,97,108,90,5,108,111,97,100,115,218,10,105,115,105, + 110,115,116,97,110,99,101,218,10,95,99,111,100,101,95,116, + 121,112,101,114,114,0,0,0,114,129,0,0,0,218,4,95, + 105,109,112,90,16,95,102,105,120,95,99,111,95,102,105,108, + 101,110,97,109,101,114,99,0,0,0,114,47,0,0,0,41, + 5,114,53,0,0,0,114,98,0,0,0,114,89,0,0,0, + 114,90,0,0,0,218,4,99,111,100,101,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,109, + 112,105,108,101,95,98,121,116,101,99,111,100,101,207,1,0, + 0,115,16,0,0,0,0,2,15,1,15,1,16,1,12,1, + 16,1,4,2,18,1,114,141,0,0,0,114,59,0,0,0, + 99,3,0,0,0,0,0,0,0,4,0,0,0,3,0,0, + 0,67,0,0,0,115,76,0,0,0,116,0,0,116,1,0, + 131,1,0,125,3,0,124,3,0,106,2,0,116,3,0,124, + 1,0,131,1,0,131,1,0,1,124,3,0,106,2,0,116, + 3,0,124,2,0,131,1,0,131,1,0,1,124,3,0,106, + 2,0,116,4,0,106,5,0,124,0,0,131,1,0,131,1, + 0,1,124,3,0,83,41,1,122,80,67,111,109,112,105,108, + 101,32,97,32,99,111,100,101,32,111,98,106,101,99,116,32, + 105,110,116,111,32,98,121,116,101,99,111,100,101,32,102,111, + 114,32,119,114,105,116,105,110,103,32,111,117,116,32,116,111, + 32,97,32,98,121,116,101,45,99,111,109,112,105,108,101,100, + 10,32,32,32,32,102,105,108,101,46,41,6,218,9,98,121, + 116,101,97,114,114,97,121,114,128,0,0,0,218,6,101,120, + 116,101,110,100,114,17,0,0,0,114,136,0,0,0,90,5, + 100,117,109,112,115,41,4,114,140,0,0,0,114,126,0,0, + 0,114,134,0,0,0,114,53,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,100, + 101,95,116,111,95,98,121,116,101,99,111,100,101,219,1,0, + 0,115,10,0,0,0,0,3,12,1,19,1,19,1,22,1, + 114,144,0,0,0,99,1,0,0,0,0,0,0,0,5,0, + 0,0,4,0,0,0,67,0,0,0,115,89,0,0,0,100, + 1,0,100,2,0,108,0,0,125,1,0,116,1,0,106,2, + 0,124,0,0,131,1,0,106,3,0,125,2,0,124,1,0, + 106,4,0,124,2,0,131,1,0,125,3,0,116,1,0,106, + 5,0,100,2,0,100,3,0,131,2,0,125,4,0,124,4, + 0,106,6,0,124,0,0,106,6,0,124,3,0,100,1,0, + 25,131,1,0,131,1,0,83,41,4,122,121,68,101,99,111, + 100,101,32,98,121,116,101,115,32,114,101,112,114,101,115,101, + 110,116,105,110,103,32,115,111,117,114,99,101,32,99,111,100, + 101,32,97,110,100,32,114,101,116,117,114,110,32,116,104,101, + 32,115,116,114,105,110,103,46,10,10,32,32,32,32,85,110, + 105,118,101,114,115,97,108,32,110,101,119,108,105,110,101,32, + 115,117,112,112,111,114,116,32,105,115,32,117,115,101,100,32, + 105,110,32,116,104,101,32,100,101,99,111,100,105,110,103,46, + 10,32,32,32,32,114,59,0,0,0,78,84,41,7,218,8, + 116,111,107,101,110,105,122,101,114,49,0,0,0,90,7,66, + 121,116,101,115,73,79,90,8,114,101,97,100,108,105,110,101, + 90,15,100,101,116,101,99,116,95,101,110,99,111,100,105,110, + 103,90,25,73,110,99,114,101,109,101,110,116,97,108,78,101, + 119,108,105,110,101,68,101,99,111,100,101,114,218,6,100,101, + 99,111,100,101,41,5,218,12,115,111,117,114,99,101,95,98, + 121,116,101,115,114,145,0,0,0,90,21,115,111,117,114,99, + 101,95,98,121,116,101,115,95,114,101,97,100,108,105,110,101, + 218,8,101,110,99,111,100,105,110,103,90,15,110,101,119,108, + 105,110,101,95,100,101,99,111,100,101,114,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,13,100,101,99,111, + 100,101,95,115,111,117,114,99,101,229,1,0,0,115,10,0, + 0,0,0,5,12,1,18,1,15,1,18,1,114,149,0,0, + 0,114,120,0,0,0,218,26,115,117,98,109,111,100,117,108, + 101,95,115,101,97,114,99,104,95,108,111,99,97,116,105,111, + 110,115,99,2,0,0,0,2,0,0,0,9,0,0,0,19, + 0,0,0,67,0,0,0,115,89,1,0,0,124,1,0,100, + 1,0,107,8,0,114,73,0,100,2,0,125,1,0,116,0, + 0,124,2,0,100,3,0,131,2,0,114,73,0,121,19,0, + 124,2,0,106,1,0,124,0,0,131,1,0,125,1,0,87, + 110,18,0,4,116,2,0,107,10,0,114,72,0,1,1,1, + 89,110,1,0,88,116,3,0,106,4,0,124,0,0,124,2, + 0,100,4,0,124,1,0,131,2,1,125,4,0,100,5,0, + 124,4,0,95,5,0,124,2,0,100,1,0,107,8,0,114, + 194,0,120,73,0,116,6,0,131,0,0,68,93,58,0,92, + 2,0,125,5,0,125,6,0,124,1,0,106,7,0,116,8, + 0,124,6,0,131,1,0,131,1,0,114,128,0,124,5,0, + 124,0,0,124,1,0,131,2,0,125,2,0,124,2,0,124, + 4,0,95,9,0,80,113,128,0,87,100,1,0,83,124,3, + 0,116,10,0,107,8,0,114,23,1,116,0,0,124,2,0, + 100,6,0,131,2,0,114,32,1,121,19,0,124,2,0,106, + 11,0,124,0,0,131,1,0,125,7,0,87,110,18,0,4, + 116,2,0,107,10,0,114,4,1,1,1,1,89,113,32,1, + 88,124,7,0,114,32,1,103,0,0,124,4,0,95,12,0, + 110,9,0,124,3,0,124,4,0,95,12,0,124,4,0,106, + 12,0,103,0,0,107,2,0,114,85,1,124,1,0,114,85, + 1,116,13,0,124,1,0,131,1,0,100,7,0,25,125,8, + 0,124,4,0,106,12,0,106,14,0,124,8,0,131,1,0, + 1,124,4,0,83,41,8,97,61,1,0,0,82,101,116,117, + 114,110,32,97,32,109,111,100,117,108,101,32,115,112,101,99, + 32,98,97,115,101,100,32,111,110,32,97,32,102,105,108,101, + 32,108,111,99,97,116,105,111,110,46,10,10,32,32,32,32, + 84,111,32,105,110,100,105,99,97,116,101,32,116,104,97,116, + 32,116,104,101,32,109,111,100,117,108,101,32,105,115,32,97, + 32,112,97,99,107,97,103,101,44,32,115,101,116,10,32,32, + 32,32,115,117,98,109,111,100,117,108,101,95,115,101,97,114, + 99,104,95,108,111,99,97,116,105,111,110,115,32,116,111,32, + 97,32,108,105,115,116,32,111,102,32,100,105,114,101,99,116, + 111,114,121,32,112,97,116,104,115,46,32,32,65,110,10,32, + 32,32,32,101,109,112,116,121,32,108,105,115,116,32,105,115, + 32,115,117,102,102,105,99,105,101,110,116,44,32,116,104,111, + 117,103,104,32,105,116,115,32,110,111,116,32,111,116,104,101, + 114,119,105,115,101,32,117,115,101,102,117,108,32,116,111,32, + 116,104,101,10,32,32,32,32,105,109,112,111,114,116,32,115, + 121,115,116,101,109,46,10,10,32,32,32,32,84,104,101,32, + 108,111,97,100,101,114,32,109,117,115,116,32,116,97,107,101, + 32,97,32,115,112,101,99,32,97,115,32,105,116,115,32,111, + 110,108,121,32,95,95,105,110,105,116,95,95,40,41,32,97, + 114,103,46,10,10,32,32,32,32,78,122,9,60,117,110,107, + 110,111,119,110,62,218,12,103,101,116,95,102,105,108,101,110, + 97,109,101,218,6,111,114,105,103,105,110,84,218,10,105,115, + 95,112,97,99,107,97,103,101,114,59,0,0,0,41,15,114, + 108,0,0,0,114,151,0,0,0,114,99,0,0,0,114,114, + 0,0,0,218,10,77,111,100,117,108,101,83,112,101,99,90, + 13,95,115,101,116,95,102,105,108,101,97,116,116,114,218,27, + 95,103,101,116,95,115,117,112,112,111,114,116,101,100,95,102, + 105,108,101,95,108,111,97,100,101,114,115,114,92,0,0,0, + 114,93,0,0,0,114,120,0,0,0,218,9,95,80,79,80, + 85,76,65,84,69,114,153,0,0,0,114,150,0,0,0,114, + 38,0,0,0,218,6,97,112,112,101,110,100,41,9,114,98, + 0,0,0,90,8,108,111,99,97,116,105,111,110,114,120,0, + 0,0,114,150,0,0,0,218,4,115,112,101,99,218,12,108, + 111,97,100,101,114,95,99,108,97,115,115,218,8,115,117,102, + 102,105,120,101,115,114,153,0,0,0,90,7,100,105,114,110, + 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,23,115,112,101,99,95,102,114,111,109,95,102,105, + 108,101,95,108,111,99,97,116,105,111,110,246,1,0,0,115, + 60,0,0,0,0,12,12,4,6,1,15,2,3,1,19,1, + 13,1,5,8,24,1,9,3,12,1,22,1,21,1,15,1, + 9,1,5,2,4,3,12,2,15,1,3,1,19,1,13,1, + 5,2,6,1,12,2,9,1,15,1,6,1,16,1,16,2, + 114,161,0,0,0,99,0,0,0,0,0,0,0,0,0,0, + 0,0,5,0,0,0,64,0,0,0,115,121,0,0,0,101, + 0,0,90,1,0,100,0,0,90,2,0,100,1,0,90,3, + 0,100,2,0,90,4,0,100,3,0,90,5,0,100,4,0, + 90,6,0,101,7,0,100,5,0,100,6,0,132,0,0,131, + 1,0,90,8,0,101,7,0,100,7,0,100,8,0,132,0, + 0,131,1,0,90,9,0,101,7,0,100,9,0,100,9,0, + 100,10,0,100,11,0,132,2,0,131,1,0,90,10,0,101, + 7,0,100,9,0,100,12,0,100,13,0,132,1,0,131,1, + 0,90,11,0,100,9,0,83,41,14,218,21,87,105,110,100, + 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, + 114,122,62,77,101,116,97,32,112,97,116,104,32,102,105,110, + 100,101,114,32,102,111,114,32,109,111,100,117,108,101,115,32, + 100,101,99,108,97,114,101,100,32,105,110,32,116,104,101,32, + 87,105,110,100,111,119,115,32,114,101,103,105,115,116,114,121, + 46,122,59,83,111,102,116,119,97,114,101,92,80,121,116,104, + 111,110,92,80,121,116,104,111,110,67,111,114,101,92,123,115, + 121,115,95,118,101,114,115,105,111,110,125,92,77,111,100,117, + 108,101,115,92,123,102,117,108,108,110,97,109,101,125,122,65, + 83,111,102,116,119,97,114,101,92,80,121,116,104,111,110,92, + 80,121,116,104,111,110,67,111,114,101,92,123,115,121,115,95, + 118,101,114,115,105,111,110,125,92,77,111,100,117,108,101,115, + 92,123,102,117,108,108,110,97,109,101,125,92,68,101,98,117, + 103,70,99,2,0,0,0,0,0,0,0,2,0,0,0,11, + 0,0,0,67,0,0,0,115,67,0,0,0,121,23,0,116, + 0,0,106,1,0,116,0,0,106,2,0,124,1,0,131,2, + 0,83,87,110,37,0,4,116,3,0,107,10,0,114,62,0, + 1,1,1,116,0,0,106,1,0,116,0,0,106,4,0,124, + 1,0,131,2,0,83,89,110,1,0,88,100,0,0,83,41, + 1,78,41,5,218,7,95,119,105,110,114,101,103,90,7,79, + 112,101,110,75,101,121,90,17,72,75,69,89,95,67,85,82, + 82,69,78,84,95,85,83,69,82,114,40,0,0,0,90,18, + 72,75,69,89,95,76,79,67,65,76,95,77,65,67,72,73, + 78,69,41,2,218,3,99,108,115,218,3,107,101,121,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,218,14,95, + 111,112,101,110,95,114,101,103,105,115,116,114,121,68,2,0, + 0,115,8,0,0,0,0,2,3,1,23,1,13,1,122,36, 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, - 105,110,100,101,114,46,102,105,110,100,95,115,112,101,99,99, - 3,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, - 67,0,0,0,115,45,0,0,0,124,0,0,106,0,0,124, - 1,0,124,2,0,131,2,0,125,3,0,124,3,0,100,1, - 0,107,9,0,114,37,0,124,3,0,106,1,0,83,100,1, - 0,83,100,1,0,83,41,2,122,108,70,105,110,100,32,109, - 111,100,117,108,101,32,110,97,109,101,100,32,105,110,32,116, - 104,101,32,114,101,103,105,115,116,114,121,46,10,10,32,32, - 32,32,32,32,32,32,84,104,105,115,32,109,101,116,104,111, - 100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46, - 32,32,85,115,101,32,101,120,101,99,95,109,111,100,117,108, - 101,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32, - 32,32,32,32,32,32,78,41,2,114,181,0,0,0,114,127, - 0,0,0,41,4,114,170,0,0,0,114,126,0,0,0,114, - 35,0,0,0,114,164,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,218,11,102,105,110,100,95,109, - 111,100,117,108,101,114,2,0,0,115,8,0,0,0,0,7, - 18,1,12,1,7,2,122,33,87,105,110,100,111,119,115,82, - 101,103,105,115,116,114,121,70,105,110,100,101,114,46,102,105, - 110,100,95,109,111,100,117,108,101,41,12,114,112,0,0,0, - 114,111,0,0,0,114,113,0,0,0,114,114,0,0,0,114, - 175,0,0,0,114,174,0,0,0,114,173,0,0,0,218,11, - 99,108,97,115,115,109,101,116,104,111,100,114,172,0,0,0, - 114,178,0,0,0,114,181,0,0,0,114,182,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,168,0,0,0,64,2,0,0,115,20,0,0, - 0,12,2,6,3,6,3,6,2,6,2,18,7,18,15,3, - 1,21,15,3,1,114,168,0,0,0,99,0,0,0,0,0, - 0,0,0,0,0,0,0,2,0,0,0,64,0,0,0,115, - 70,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, - 100,1,0,90,3,0,100,2,0,100,3,0,132,0,0,90, - 4,0,100,4,0,100,5,0,132,0,0,90,5,0,100,6, - 0,100,7,0,132,0,0,90,6,0,100,8,0,100,9,0, - 132,0,0,90,7,0,100,10,0,83,41,11,218,13,95,76, - 111,97,100,101,114,66,97,115,105,99,115,122,83,66,97,115, - 101,32,99,108,97,115,115,32,111,102,32,99,111,109,109,111, - 110,32,99,111,100,101,32,110,101,101,100,101,100,32,98,121, - 32,98,111,116,104,32,83,111,117,114,99,101,76,111,97,100, - 101,114,32,97,110,100,10,32,32,32,32,83,111,117,114,99, - 101,108,101,115,115,70,105,108,101,76,111,97,100,101,114,46, - 99,2,0,0,0,0,0,0,0,5,0,0,0,3,0,0, - 0,67,0,0,0,115,88,0,0,0,116,0,0,124,0,0, - 106,1,0,124,1,0,131,1,0,131,1,0,100,1,0,25, - 125,2,0,124,2,0,106,2,0,100,2,0,100,1,0,131, - 2,0,100,3,0,25,125,3,0,124,1,0,106,3,0,100, - 2,0,131,1,0,100,4,0,25,125,4,0,124,3,0,100, - 5,0,107,2,0,111,87,0,124,4,0,100,5,0,107,3, - 0,83,41,6,122,141,67,111,110,99,114,101,116,101,32,105, - 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, - 32,73,110,115,112,101,99,116,76,111,97,100,101,114,46,105, - 115,95,112,97,99,107,97,103,101,32,98,121,32,99,104,101, - 99,107,105,110,103,32,105,102,10,32,32,32,32,32,32,32, - 32,116,104,101,32,112,97,116,104,32,114,101,116,117,114,110, - 101,100,32,98,121,32,103,101,116,95,102,105,108,101,110,97, - 109,101,32,104,97,115,32,97,32,102,105,108,101,110,97,109, - 101,32,111,102,32,39,95,95,105,110,105,116,95,95,46,112, - 121,39,46,114,29,0,0,0,114,58,0,0,0,114,59,0, - 0,0,114,56,0,0,0,218,8,95,95,105,110,105,116,95, - 95,41,4,114,38,0,0,0,114,157,0,0,0,114,34,0, - 0,0,114,32,0,0,0,41,5,114,108,0,0,0,114,126, - 0,0,0,114,94,0,0,0,90,13,102,105,108,101,110,97, - 109,101,95,98,97,115,101,90,9,116,97,105,108,95,110,97, - 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,159,0,0,0,133,2,0,0,115,8,0,0,0,0, - 3,25,1,22,1,19,1,122,24,95,76,111,97,100,101,114, - 66,97,115,105,99,115,46,105,115,95,112,97,99,107,97,103, - 101,99,2,0,0,0,0,0,0,0,2,0,0,0,1,0, - 0,0,67,0,0,0,115,4,0,0,0,100,1,0,83,41, - 2,122,42,85,115,101,32,100,101,102,97,117,108,116,32,115, - 101,109,97,110,116,105,99,115,32,102,111,114,32,109,111,100, - 117,108,101,32,99,114,101,97,116,105,111,110,46,78,114,4, - 0,0,0,41,2,114,108,0,0,0,114,164,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,13, - 99,114,101,97,116,101,95,109,111,100,117,108,101,141,2,0, - 0,115,0,0,0,0,122,27,95,76,111,97,100,101,114,66, - 97,115,105,99,115,46,99,114,101,97,116,101,95,109,111,100, - 117,108,101,99,2,0,0,0,0,0,0,0,3,0,0,0, - 4,0,0,0,67,0,0,0,115,80,0,0,0,124,0,0, - 106,0,0,124,1,0,106,1,0,131,1,0,125,2,0,124, - 2,0,100,1,0,107,8,0,114,54,0,116,2,0,100,2, - 0,106,3,0,124,1,0,106,1,0,131,1,0,131,1,0, - 130,1,0,116,4,0,106,5,0,116,6,0,124,2,0,124, - 1,0,106,7,0,131,3,0,1,100,1,0,83,41,3,122, - 19,69,120,101,99,117,116,101,32,116,104,101,32,109,111,100, - 117,108,101,46,78,122,52,99,97,110,110,111,116,32,108,111, - 97,100,32,109,111,100,117,108,101,32,123,33,114,125,32,119, - 104,101,110,32,103,101,116,95,99,111,100,101,40,41,32,114, - 101,116,117,114,110,115,32,78,111,110,101,41,8,218,8,103, - 101,116,95,99,111,100,101,114,112,0,0,0,114,107,0,0, - 0,114,47,0,0,0,114,121,0,0,0,218,25,95,99,97, - 108,108,95,119,105,116,104,95,102,114,97,109,101,115,95,114, - 101,109,111,118,101,100,218,4,101,120,101,99,114,118,0,0, - 0,41,3,114,108,0,0,0,218,6,109,111,100,117,108,101, - 114,146,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,11,101,120,101,99,95,109,111,100,117,108, - 101,144,2,0,0,115,10,0,0,0,0,2,18,1,12,1, - 9,1,15,1,122,25,95,76,111,97,100,101,114,66,97,115, - 105,99,115,46,101,120,101,99,95,109,111,100,117,108,101,99, - 2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, - 67,0,0,0,115,16,0,0,0,116,0,0,106,1,0,124, - 0,0,124,1,0,131,2,0,83,41,1,78,41,2,114,121, - 0,0,0,218,17,95,108,111,97,100,95,109,111,100,117,108, - 101,95,115,104,105,109,41,2,114,108,0,0,0,114,126,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,11,108,111,97,100,95,109,111,100,117,108,101,152,2, - 0,0,115,2,0,0,0,0,1,122,25,95,76,111,97,100, - 101,114,66,97,115,105,99,115,46,108,111,97,100,95,109,111, - 100,117,108,101,78,41,8,114,112,0,0,0,114,111,0,0, - 0,114,113,0,0,0,114,114,0,0,0,114,159,0,0,0, - 114,186,0,0,0,114,191,0,0,0,114,193,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,184,0,0,0,128,2,0,0,115,10,0,0, - 0,12,3,6,2,12,8,12,3,12,8,114,184,0,0,0, - 99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0, - 0,64,0,0,0,115,106,0,0,0,101,0,0,90,1,0, - 100,0,0,90,2,0,100,1,0,100,2,0,132,0,0,90, - 3,0,100,3,0,100,4,0,132,0,0,90,4,0,100,5, - 0,100,6,0,132,0,0,90,5,0,100,7,0,100,8,0, - 132,0,0,90,6,0,100,9,0,100,10,0,132,0,0,90, - 7,0,100,11,0,100,18,0,100,13,0,100,14,0,132,0, - 1,90,8,0,100,15,0,100,16,0,132,0,0,90,9,0, - 100,17,0,83,41,19,218,12,83,111,117,114,99,101,76,111, - 97,100,101,114,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,10,0,0,0,116,0, - 0,130,1,0,100,1,0,83,41,2,122,178,79,112,116,105, - 111,110,97,108,32,109,101,116,104,111,100,32,116,104,97,116, - 32,114,101,116,117,114,110,115,32,116,104,101,32,109,111,100, - 105,102,105,99,97,116,105,111,110,32,116,105,109,101,32,40, - 97,110,32,105,110,116,41,32,102,111,114,32,116,104,101,10, - 32,32,32,32,32,32,32,32,115,112,101,99,105,102,105,101, - 100,32,112,97,116,104,44,32,119,104,101,114,101,32,112,97, - 116,104,32,105,115,32,97,32,115,116,114,46,10,10,32,32, - 32,32,32,32,32,32,82,97,105,115,101,115,32,73,79,69, - 114,114,111,114,32,119,104,101,110,32,116,104,101,32,112,97, - 116,104,32,99,97,110,110,111,116,32,98,101,32,104,97,110, - 100,108,101,100,46,10,32,32,32,32,32,32,32,32,78,41, - 1,218,7,73,79,69,114,114,111,114,41,2,114,108,0,0, - 0,114,35,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,10,112,97,116,104,95,109,116,105,109, - 101,158,2,0,0,115,2,0,0,0,0,6,122,23,83,111, - 117,114,99,101,76,111,97,100,101,114,46,112,97,116,104,95, - 109,116,105,109,101,99,2,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,19,0,0,0,100, - 1,0,124,0,0,106,0,0,124,1,0,131,1,0,105,1, - 0,83,41,2,97,170,1,0,0,79,112,116,105,111,110,97, - 108,32,109,101,116,104,111,100,32,114,101,116,117,114,110,105, - 110,103,32,97,32,109,101,116,97,100,97,116,97,32,100,105, - 99,116,32,102,111,114,32,116,104,101,32,115,112,101,99,105, - 102,105,101,100,32,112,97,116,104,10,32,32,32,32,32,32, - 32,32,116,111,32,98,121,32,116,104,101,32,112,97,116,104, - 32,40,115,116,114,41,46,10,32,32,32,32,32,32,32,32, - 80,111,115,115,105,98,108,101,32,107,101,121,115,58,10,32, - 32,32,32,32,32,32,32,45,32,39,109,116,105,109,101,39, - 32,40,109,97,110,100,97,116,111,114,121,41,32,105,115,32, - 116,104,101,32,110,117,109,101,114,105,99,32,116,105,109,101, - 115,116,97,109,112,32,111,102,32,108,97,115,116,32,115,111, - 117,114,99,101,10,32,32,32,32,32,32,32,32,32,32,99, - 111,100,101,32,109,111,100,105,102,105,99,97,116,105,111,110, - 59,10,32,32,32,32,32,32,32,32,45,32,39,115,105,122, - 101,39,32,40,111,112,116,105,111,110,97,108,41,32,105,115, - 32,116,104,101,32,115,105,122,101,32,105,110,32,98,121,116, - 101,115,32,111,102,32,116,104,101,32,115,111,117,114,99,101, - 32,99,111,100,101,46,10,10,32,32,32,32,32,32,32,32, - 73,109,112,108,101,109,101,110,116,105,110,103,32,116,104,105, - 115,32,109,101,116,104,111,100,32,97,108,108,111,119,115,32, - 116,104,101,32,108,111,97,100,101,114,32,116,111,32,114,101, - 97,100,32,98,121,116,101,99,111,100,101,32,102,105,108,101, - 115,46,10,32,32,32,32,32,32,32,32,82,97,105,115,101, - 115,32,73,79,69,114,114,111,114,32,119,104,101,110,32,116, - 104,101,32,112,97,116,104,32,99,97,110,110,111,116,32,98, - 101,32,104,97,110,100,108,101,100,46,10,32,32,32,32,32, - 32,32,32,114,133,0,0,0,41,1,114,196,0,0,0,41, - 2,114,108,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,10,112,97,116,104, - 95,115,116,97,116,115,166,2,0,0,115,2,0,0,0,0, - 11,122,23,83,111,117,114,99,101,76,111,97,100,101,114,46, - 112,97,116,104,95,115,116,97,116,115,99,4,0,0,0,0, - 0,0,0,4,0,0,0,3,0,0,0,67,0,0,0,115, - 16,0,0,0,124,0,0,106,0,0,124,2,0,124,3,0, - 131,2,0,83,41,1,122,228,79,112,116,105,111,110,97,108, - 32,109,101,116,104,111,100,32,119,104,105,99,104,32,119,114, - 105,116,101,115,32,100,97,116,97,32,40,98,121,116,101,115, - 41,32,116,111,32,97,32,102,105,108,101,32,112,97,116,104, - 32,40,97,32,115,116,114,41,46,10,10,32,32,32,32,32, - 32,32,32,73,109,112,108,101,109,101,110,116,105,110,103,32, - 116,104,105,115,32,109,101,116,104,111,100,32,97,108,108,111, - 119,115,32,102,111,114,32,116,104,101,32,119,114,105,116,105, - 110,103,32,111,102,32,98,121,116,101,99,111,100,101,32,102, - 105,108,101,115,46,10,10,32,32,32,32,32,32,32,32,84, - 104,101,32,115,111,117,114,99,101,32,112,97,116,104,32,105, - 115,32,110,101,101,100,101,100,32,105,110,32,111,114,100,101, - 114,32,116,111,32,99,111,114,114,101,99,116,108,121,32,116, - 114,97,110,115,102,101,114,32,112,101,114,109,105,115,115,105, - 111,110,115,10,32,32,32,32,32,32,32,32,41,1,218,8, - 115,101,116,95,100,97,116,97,41,4,114,108,0,0,0,114, - 90,0,0,0,90,10,99,97,99,104,101,95,112,97,116,104, - 114,53,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,15,95,99,97,99,104,101,95,98,121,116, - 101,99,111,100,101,179,2,0,0,115,2,0,0,0,0,8, - 122,28,83,111,117,114,99,101,76,111,97,100,101,114,46,95, - 99,97,99,104,101,95,98,121,116,101,99,111,100,101,99,3, - 0,0,0,0,0,0,0,3,0,0,0,1,0,0,0,67, - 0,0,0,115,4,0,0,0,100,1,0,83,41,2,122,150, - 79,112,116,105,111,110,97,108,32,109,101,116,104,111,100,32, - 119,104,105,99,104,32,119,114,105,116,101,115,32,100,97,116, - 97,32,40,98,121,116,101,115,41,32,116,111,32,97,32,102, - 105,108,101,32,112,97,116,104,32,40,97,32,115,116,114,41, - 46,10,10,32,32,32,32,32,32,32,32,73,109,112,108,101, - 109,101,110,116,105,110,103,32,116,104,105,115,32,109,101,116, - 104,111,100,32,97,108,108,111,119,115,32,102,111,114,32,116, - 104,101,32,119,114,105,116,105,110,103,32,111,102,32,98,121, - 116,101,99,111,100,101,32,102,105,108,101,115,46,10,32,32, - 32,32,32,32,32,32,78,114,4,0,0,0,41,3,114,108, - 0,0,0,114,35,0,0,0,114,53,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,198,0,0, - 0,189,2,0,0,115,0,0,0,0,122,21,83,111,117,114, - 99,101,76,111,97,100,101,114,46,115,101,116,95,100,97,116, - 97,99,2,0,0,0,0,0,0,0,5,0,0,0,16,0, - 0,0,67,0,0,0,115,105,0,0,0,124,0,0,106,0, - 0,124,1,0,131,1,0,125,2,0,121,19,0,124,0,0, - 106,1,0,124,2,0,131,1,0,125,3,0,87,110,58,0, - 4,116,2,0,107,10,0,114,94,0,1,125,4,0,1,122, - 26,0,116,3,0,100,1,0,100,2,0,124,1,0,131,1, - 1,124,4,0,130,2,0,87,89,100,3,0,100,3,0,125, - 4,0,126,4,0,88,110,1,0,88,116,4,0,124,3,0, - 131,1,0,83,41,4,122,52,67,111,110,99,114,101,116,101, - 32,105,109,112,108,101,109,101,110,116,97,116,105,111,110,32, - 111,102,32,73,110,115,112,101,99,116,76,111,97,100,101,114, - 46,103,101,116,95,115,111,117,114,99,101,46,122,39,115,111, - 117,114,99,101,32,110,111,116,32,97,118,97,105,108,97,98, - 108,101,32,116,104,114,111,117,103,104,32,103,101,116,95,100, - 97,116,97,40,41,114,106,0,0,0,78,41,5,114,157,0, - 0,0,218,8,103,101,116,95,100,97,116,97,114,40,0,0, - 0,114,107,0,0,0,114,155,0,0,0,41,5,114,108,0, - 0,0,114,126,0,0,0,114,35,0,0,0,114,153,0,0, - 0,218,3,101,120,99,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,10,103,101,116,95,115,111,117,114,99, - 101,196,2,0,0,115,14,0,0,0,0,2,15,1,3,1, - 19,1,18,1,9,1,31,1,122,23,83,111,117,114,99,101, - 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, - 101,218,9,95,111,112,116,105,109,105,122,101,114,29,0,0, - 0,99,3,0,0,0,1,0,0,0,4,0,0,0,9,0, - 0,0,67,0,0,0,115,34,0,0,0,116,0,0,106,1, - 0,116,2,0,124,1,0,124,2,0,100,1,0,100,2,0, - 100,3,0,100,4,0,124,3,0,131,4,2,83,41,5,122, - 130,82,101,116,117,114,110,32,116,104,101,32,99,111,100,101, - 32,111,98,106,101,99,116,32,99,111,109,112,105,108,101,100, - 32,102,114,111,109,32,115,111,117,114,99,101,46,10,10,32, - 32,32,32,32,32,32,32,84,104,101,32,39,100,97,116,97, - 39,32,97,114,103,117,109,101,110,116,32,99,97,110,32,98, - 101,32,97,110,121,32,111,98,106,101,99,116,32,116,121,112, - 101,32,116,104,97,116,32,99,111,109,112,105,108,101,40,41, - 32,115,117,112,112,111,114,116,115,46,10,32,32,32,32,32, - 32,32,32,114,189,0,0,0,218,12,100,111,110,116,95,105, - 110,104,101,114,105,116,84,114,68,0,0,0,41,3,114,121, - 0,0,0,114,188,0,0,0,218,7,99,111,109,112,105,108, - 101,41,4,114,108,0,0,0,114,53,0,0,0,114,35,0, - 0,0,114,203,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,14,115,111,117,114,99,101,95,116, - 111,95,99,111,100,101,206,2,0,0,115,4,0,0,0,0, - 5,21,1,122,27,83,111,117,114,99,101,76,111,97,100,101, - 114,46,115,111,117,114,99,101,95,116,111,95,99,111,100,101, - 99,2,0,0,0,0,0,0,0,10,0,0,0,43,0,0, - 0,67,0,0,0,115,174,1,0,0,124,0,0,106,0,0, - 124,1,0,131,1,0,125,2,0,100,1,0,125,3,0,121, - 16,0,116,1,0,124,2,0,131,1,0,125,4,0,87,110, - 24,0,4,116,2,0,107,10,0,114,63,0,1,1,1,100, - 1,0,125,4,0,89,110,202,0,88,121,19,0,124,0,0, - 106,3,0,124,2,0,131,1,0,125,5,0,87,110,18,0, - 4,116,4,0,107,10,0,114,103,0,1,1,1,89,110,162, - 0,88,116,5,0,124,5,0,100,2,0,25,131,1,0,125, - 3,0,121,19,0,124,0,0,106,6,0,124,4,0,131,1, - 0,125,6,0,87,110,18,0,4,116,7,0,107,10,0,114, - 159,0,1,1,1,89,110,106,0,88,121,34,0,116,8,0, - 124,6,0,100,3,0,124,5,0,100,4,0,124,1,0,100, - 5,0,124,4,0,131,1,3,125,7,0,87,110,24,0,4, - 116,9,0,116,10,0,102,2,0,107,10,0,114,220,0,1, - 1,1,89,110,45,0,88,116,11,0,100,6,0,124,4,0, - 124,2,0,131,3,0,1,116,12,0,124,7,0,100,4,0, - 124,1,0,100,7,0,124,4,0,100,8,0,124,2,0,131, - 1,3,83,124,0,0,106,6,0,124,2,0,131,1,0,125, - 8,0,124,0,0,106,13,0,124,8,0,124,2,0,131,2, - 0,125,9,0,116,11,0,100,9,0,124,2,0,131,2,0, - 1,116,14,0,106,15,0,12,114,170,1,124,4,0,100,1, - 0,107,9,0,114,170,1,124,3,0,100,1,0,107,9,0, - 114,170,1,116,16,0,124,9,0,124,3,0,116,17,0,124, - 8,0,131,1,0,131,3,0,125,6,0,121,36,0,124,0, - 0,106,18,0,124,2,0,124,4,0,124,6,0,131,3,0, - 1,116,11,0,100,10,0,124,4,0,131,2,0,1,87,110, - 18,0,4,116,2,0,107,10,0,114,169,1,1,1,1,89, - 110,1,0,88,124,9,0,83,41,11,122,190,67,111,110,99, + 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, + 115,116,114,121,99,2,0,0,0,0,0,0,0,6,0,0, + 0,16,0,0,0,67,0,0,0,115,143,0,0,0,124,0, + 0,106,0,0,114,21,0,124,0,0,106,1,0,125,2,0, + 110,9,0,124,0,0,106,2,0,125,2,0,124,2,0,106, + 3,0,100,1,0,124,1,0,100,2,0,116,4,0,106,5, + 0,100,0,0,100,3,0,133,2,0,25,131,0,2,125,3, + 0,121,47,0,124,0,0,106,6,0,124,3,0,131,1,0, + 143,25,0,125,4,0,116,7,0,106,8,0,124,4,0,100, + 4,0,131,2,0,125,5,0,87,100,0,0,81,82,88,87, + 110,22,0,4,116,9,0,107,10,0,114,138,0,1,1,1, + 100,0,0,83,89,110,1,0,88,124,5,0,83,41,5,78, + 114,119,0,0,0,90,11,115,121,115,95,118,101,114,115,105, + 111,110,114,80,0,0,0,114,30,0,0,0,41,10,218,11, + 68,69,66,85,71,95,66,85,73,76,68,218,18,82,69,71, + 73,83,84,82,89,95,75,69,89,95,68,69,66,85,71,218, + 12,82,69,71,73,83,84,82,89,95,75,69,89,114,47,0, + 0,0,114,7,0,0,0,218,7,118,101,114,115,105,111,110, + 114,166,0,0,0,114,163,0,0,0,90,10,81,117,101,114, + 121,86,97,108,117,101,114,40,0,0,0,41,6,114,164,0, + 0,0,114,119,0,0,0,90,12,114,101,103,105,115,116,114, + 121,95,107,101,121,114,165,0,0,0,90,4,104,107,101,121, + 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, + 99,104,95,114,101,103,105,115,116,114,121,75,2,0,0,115, + 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, + 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, + 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, + 114,46,95,115,101,97,114,99,104,95,114,101,103,105,115,116, + 114,121,78,99,4,0,0,0,0,0,0,0,8,0,0,0, + 14,0,0,0,67,0,0,0,115,158,0,0,0,124,0,0, + 106,0,0,124,1,0,131,1,0,125,4,0,124,4,0,100, + 0,0,107,8,0,114,31,0,100,0,0,83,121,14,0,116, + 1,0,124,4,0,131,1,0,1,87,110,22,0,4,116,2, + 0,107,10,0,114,69,0,1,1,1,100,0,0,83,89,110, + 1,0,88,120,81,0,116,3,0,131,0,0,68,93,70,0, + 92,2,0,125,5,0,125,6,0,124,4,0,106,4,0,116, + 5,0,124,6,0,131,1,0,131,1,0,114,80,0,116,6, + 0,106,7,0,124,1,0,124,5,0,124,1,0,124,4,0, + 131,2,0,100,1,0,124,4,0,131,2,1,125,7,0,124, + 7,0,83,113,80,0,87,100,0,0,83,41,2,78,114,152, + 0,0,0,41,8,114,172,0,0,0,114,39,0,0,0,114, + 40,0,0,0,114,155,0,0,0,114,92,0,0,0,114,93, + 0,0,0,114,114,0,0,0,218,16,115,112,101,99,95,102, + 114,111,109,95,108,111,97,100,101,114,41,8,114,164,0,0, + 0,114,119,0,0,0,114,35,0,0,0,218,6,116,97,114, + 103,101,116,114,171,0,0,0,114,120,0,0,0,114,160,0, + 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,9,102,105,110,100,95,115,112,101, + 99,90,2,0,0,115,26,0,0,0,0,2,15,1,12,1, + 4,1,3,1,14,1,13,1,9,1,22,1,21,1,9,1, + 15,1,9,1,122,31,87,105,110,100,111,119,115,82,101,103, + 105,115,116,114,121,70,105,110,100,101,114,46,102,105,110,100, + 95,115,112,101,99,99,3,0,0,0,0,0,0,0,4,0, + 0,0,3,0,0,0,67,0,0,0,115,45,0,0,0,124, + 0,0,106,0,0,124,1,0,124,2,0,131,2,0,125,3, + 0,124,3,0,100,1,0,107,9,0,114,37,0,124,3,0, + 106,1,0,83,100,1,0,83,100,1,0,83,41,2,122,108, + 70,105,110,100,32,109,111,100,117,108,101,32,110,97,109,101, + 100,32,105,110,32,116,104,101,32,114,101,103,105,115,116,114, + 121,46,10,10,32,32,32,32,32,32,32,32,84,104,105,115, + 32,109,101,116,104,111,100,32,105,115,32,100,101,112,114,101, + 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, + 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, + 100,46,10,10,32,32,32,32,32,32,32,32,78,41,2,114, + 175,0,0,0,114,120,0,0,0,41,4,114,164,0,0,0, + 114,119,0,0,0,114,35,0,0,0,114,158,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, + 102,105,110,100,95,109,111,100,117,108,101,106,2,0,0,115, + 8,0,0,0,0,7,18,1,12,1,7,2,122,33,87,105, + 110,100,111,119,115,82,101,103,105,115,116,114,121,70,105,110, + 100,101,114,46,102,105,110,100,95,109,111,100,117,108,101,41, + 12,114,105,0,0,0,114,104,0,0,0,114,106,0,0,0, + 114,107,0,0,0,114,169,0,0,0,114,168,0,0,0,114, + 167,0,0,0,218,11,99,108,97,115,115,109,101,116,104,111, + 100,114,166,0,0,0,114,172,0,0,0,114,175,0,0,0, + 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,162,0,0,0,56,2, + 0,0,115,20,0,0,0,12,2,6,3,6,3,6,2,6, + 2,18,7,18,15,3,1,21,15,3,1,114,162,0,0,0, + 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, + 0,64,0,0,0,115,70,0,0,0,101,0,0,90,1,0, + 100,0,0,90,2,0,100,1,0,90,3,0,100,2,0,100, + 3,0,132,0,0,90,4,0,100,4,0,100,5,0,132,0, + 0,90,5,0,100,6,0,100,7,0,132,0,0,90,6,0, + 100,8,0,100,9,0,132,0,0,90,7,0,100,10,0,83, + 41,11,218,13,95,76,111,97,100,101,114,66,97,115,105,99, + 115,122,83,66,97,115,101,32,99,108,97,115,115,32,111,102, + 32,99,111,109,109,111,110,32,99,111,100,101,32,110,101,101, + 100,101,100,32,98,121,32,98,111,116,104,32,83,111,117,114, + 99,101,76,111,97,100,101,114,32,97,110,100,10,32,32,32, + 32,83,111,117,114,99,101,108,101,115,115,70,105,108,101,76, + 111,97,100,101,114,46,99,2,0,0,0,0,0,0,0,5, + 0,0,0,3,0,0,0,67,0,0,0,115,88,0,0,0, + 116,0,0,124,0,0,106,1,0,124,1,0,131,1,0,131, + 1,0,100,1,0,25,125,2,0,124,2,0,106,2,0,100, + 2,0,100,1,0,131,2,0,100,3,0,25,125,3,0,124, + 1,0,106,3,0,100,2,0,131,1,0,100,4,0,25,125, + 4,0,124,3,0,100,5,0,107,2,0,111,87,0,124,4, + 0,100,5,0,107,3,0,83,41,6,122,141,67,111,110,99, 114,101,116,101,32,105,109,112,108,101,109,101,110,116,97,116, 105,111,110,32,111,102,32,73,110,115,112,101,99,116,76,111, - 97,100,101,114,46,103,101,116,95,99,111,100,101,46,10,10, - 32,32,32,32,32,32,32,32,82,101,97,100,105,110,103,32, - 111,102,32,98,121,116,101,99,111,100,101,32,114,101,113,117, - 105,114,101,115,32,112,97,116,104,95,115,116,97,116,115,32, - 116,111,32,98,101,32,105,109,112,108,101,109,101,110,116,101, - 100,46,32,84,111,32,119,114,105,116,101,10,32,32,32,32, - 32,32,32,32,98,121,116,101,99,111,100,101,44,32,115,101, - 116,95,100,97,116,97,32,109,117,115,116,32,97,108,115,111, - 32,98,101,32,105,109,112,108,101,109,101,110,116,101,100,46, - 10,10,32,32,32,32,32,32,32,32,78,114,133,0,0,0, - 114,138,0,0,0,114,106,0,0,0,114,35,0,0,0,122, - 13,123,125,32,109,97,116,99,104,101,115,32,123,125,114,89, - 0,0,0,114,90,0,0,0,122,19,99,111,100,101,32,111, - 98,106,101,99,116,32,102,114,111,109,32,123,125,122,10,119, - 114,111,116,101,32,123,33,114,125,41,19,114,157,0,0,0, - 114,79,0,0,0,114,66,0,0,0,114,197,0,0,0,114, - 195,0,0,0,114,14,0,0,0,114,200,0,0,0,114,40, - 0,0,0,114,141,0,0,0,114,107,0,0,0,114,136,0, - 0,0,114,105,0,0,0,114,147,0,0,0,114,206,0,0, - 0,114,7,0,0,0,218,19,100,111,110,116,95,119,114,105, - 116,101,95,98,121,116,101,99,111,100,101,114,150,0,0,0, - 114,31,0,0,0,114,199,0,0,0,41,10,114,108,0,0, - 0,114,126,0,0,0,114,90,0,0,0,114,139,0,0,0, - 114,89,0,0,0,218,2,115,116,114,53,0,0,0,218,10, - 98,121,116,101,115,95,100,97,116,97,114,153,0,0,0,90, - 11,99,111,100,101,95,111,98,106,101,99,116,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,187,0,0,0, - 214,2,0,0,115,78,0,0,0,0,7,15,1,6,1,3, - 1,16,1,13,1,11,2,3,1,19,1,13,1,5,2,16, - 1,3,1,19,1,13,1,5,2,3,1,9,1,12,1,13, - 1,19,1,5,2,9,1,7,1,15,1,6,1,7,1,15, - 1,18,1,13,1,22,1,12,1,9,1,15,1,3,1,19, - 1,17,1,13,1,5,1,122,21,83,111,117,114,99,101,76, - 111,97,100,101,114,46,103,101,116,95,99,111,100,101,78,114, - 87,0,0,0,41,10,114,112,0,0,0,114,111,0,0,0, - 114,113,0,0,0,114,196,0,0,0,114,197,0,0,0,114, - 199,0,0,0,114,198,0,0,0,114,202,0,0,0,114,206, - 0,0,0,114,187,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,194,0,0, - 0,156,2,0,0,115,14,0,0,0,12,2,12,8,12,13, - 12,10,12,7,12,10,18,8,114,194,0,0,0,99,0,0, - 0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0, - 0,0,115,112,0,0,0,101,0,0,90,1,0,100,0,0, - 90,2,0,100,1,0,90,3,0,100,2,0,100,3,0,132, - 0,0,90,4,0,100,4,0,100,5,0,132,0,0,90,5, - 0,100,6,0,100,7,0,132,0,0,90,6,0,101,7,0, - 135,0,0,102,1,0,100,8,0,100,9,0,134,0,0,131, - 1,0,90,8,0,101,7,0,100,10,0,100,11,0,132,0, - 0,131,1,0,90,9,0,100,12,0,100,13,0,132,0,0, - 90,10,0,135,0,0,83,41,14,218,10,70,105,108,101,76, - 111,97,100,101,114,122,103,66,97,115,101,32,102,105,108,101, - 32,108,111,97,100,101,114,32,99,108,97,115,115,32,119,104, - 105,99,104,32,105,109,112,108,101,109,101,110,116,115,32,116, - 104,101,32,108,111,97,100,101,114,32,112,114,111,116,111,99, - 111,108,32,109,101,116,104,111,100,115,32,116,104,97,116,10, - 32,32,32,32,114,101,113,117,105,114,101,32,102,105,108,101, - 32,115,121,115,116,101,109,32,117,115,97,103,101,46,99,3, - 0,0,0,0,0,0,0,3,0,0,0,2,0,0,0,67, - 0,0,0,115,22,0,0,0,124,1,0,124,0,0,95,0, - 0,124,2,0,124,0,0,95,1,0,100,1,0,83,41,2, - 122,75,67,97,99,104,101,32,116,104,101,32,109,111,100,117, - 108,101,32,110,97,109,101,32,97,110,100,32,116,104,101,32, - 112,97,116,104,32,116,111,32,116,104,101,32,102,105,108,101, - 32,102,111,117,110,100,32,98,121,32,116,104,101,10,32,32, - 32,32,32,32,32,32,102,105,110,100,101,114,46,78,41,2, - 114,106,0,0,0,114,35,0,0,0,41,3,114,108,0,0, - 0,114,126,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,185,0,0,0,15, - 3,0,0,115,4,0,0,0,0,3,9,1,122,19,70,105, - 108,101,76,111,97,100,101,114,46,95,95,105,110,105,116,95, - 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, - 0,0,67,0,0,0,115,34,0,0,0,124,0,0,106,0, - 0,124,1,0,106,0,0,107,2,0,111,33,0,124,0,0, - 106,1,0,124,1,0,106,1,0,107,2,0,83,41,1,78, - 41,2,218,9,95,95,99,108,97,115,115,95,95,114,118,0, - 0,0,41,2,114,108,0,0,0,218,5,111,116,104,101,114, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,218, - 6,95,95,101,113,95,95,21,3,0,0,115,4,0,0,0, - 0,1,18,1,122,17,70,105,108,101,76,111,97,100,101,114, - 46,95,95,101,113,95,95,99,1,0,0,0,0,0,0,0, - 1,0,0,0,3,0,0,0,67,0,0,0,115,26,0,0, - 0,116,0,0,124,0,0,106,1,0,131,1,0,116,0,0, - 124,0,0,106,2,0,131,1,0,65,83,41,1,78,41,3, - 218,4,104,97,115,104,114,106,0,0,0,114,35,0,0,0, - 41,1,114,108,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,8,95,95,104,97,115,104,95,95, - 25,3,0,0,115,2,0,0,0,0,1,122,19,70,105,108, - 101,76,111,97,100,101,114,46,95,95,104,97,115,104,95,95, - 99,2,0,0,0,0,0,0,0,2,0,0,0,3,0,0, - 0,3,0,0,0,115,22,0,0,0,116,0,0,116,1,0, - 124,0,0,131,2,0,106,2,0,124,1,0,131,1,0,83, - 41,1,122,100,76,111,97,100,32,97,32,109,111,100,117,108, - 101,32,102,114,111,109,32,97,32,102,105,108,101,46,10,10, - 32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116, - 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, - 100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100, - 117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10, - 32,32,32,32,32,32,32,32,41,3,218,5,115,117,112,101, - 114,114,210,0,0,0,114,193,0,0,0,41,2,114,108,0, - 0,0,114,126,0,0,0,41,1,114,211,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,193,0,0,0,28,3,0, - 0,115,2,0,0,0,0,10,122,22,70,105,108,101,76,111, - 97,100,101,114,46,108,111,97,100,95,109,111,100,117,108,101, - 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, - 0,67,0,0,0,115,7,0,0,0,124,0,0,106,0,0, - 83,41,1,122,58,82,101,116,117,114,110,32,116,104,101,32, - 112,97,116,104,32,116,111,32,116,104,101,32,115,111,117,114, - 99,101,32,102,105,108,101,32,97,115,32,102,111,117,110,100, - 32,98,121,32,116,104,101,32,102,105,110,100,101,114,46,41, - 1,114,35,0,0,0,41,2,114,108,0,0,0,114,126,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,157,0,0,0,40,3,0,0,115,2,0,0,0,0, - 3,122,23,70,105,108,101,76,111,97,100,101,114,46,103,101, - 116,95,102,105,108,101,110,97,109,101,99,2,0,0,0,0, - 0,0,0,3,0,0,0,9,0,0,0,67,0,0,0,115, - 42,0,0,0,116,0,0,106,1,0,124,1,0,100,1,0, - 131,2,0,143,17,0,125,2,0,124,2,0,106,2,0,131, - 0,0,83,87,100,2,0,81,82,88,100,2,0,83,41,3, - 122,39,82,101,116,117,114,110,32,116,104,101,32,100,97,116, - 97,32,102,114,111,109,32,112,97,116,104,32,97,115,32,114, - 97,119,32,98,121,116,101,115,46,218,1,114,78,41,3,114, - 49,0,0,0,114,50,0,0,0,90,4,114,101,97,100,41, - 3,114,108,0,0,0,114,35,0,0,0,114,54,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 200,0,0,0,45,3,0,0,115,4,0,0,0,0,2,21, - 1,122,19,70,105,108,101,76,111,97,100,101,114,46,103,101, - 116,95,100,97,116,97,41,11,114,112,0,0,0,114,111,0, - 0,0,114,113,0,0,0,114,114,0,0,0,114,185,0,0, - 0,114,213,0,0,0,114,215,0,0,0,114,123,0,0,0, - 114,193,0,0,0,114,157,0,0,0,114,200,0,0,0,114, - 4,0,0,0,114,4,0,0,0,41,1,114,211,0,0,0, - 114,5,0,0,0,114,210,0,0,0,10,3,0,0,115,14, - 0,0,0,12,3,6,2,12,6,12,4,12,3,24,12,18, - 5,114,210,0,0,0,99,0,0,0,0,0,0,0,0,0, - 0,0,0,4,0,0,0,64,0,0,0,115,64,0,0,0, - 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, - 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, - 0,100,5,0,132,0,0,90,5,0,100,6,0,100,7,0, - 100,8,0,100,9,0,132,0,1,90,6,0,100,10,0,83, - 41,11,218,16,83,111,117,114,99,101,70,105,108,101,76,111, - 97,100,101,114,122,62,67,111,110,99,114,101,116,101,32,105, - 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, - 32,83,111,117,114,99,101,76,111,97,100,101,114,32,117,115, - 105,110,103,32,116,104,101,32,102,105,108,101,32,115,121,115, - 116,101,109,46,99,2,0,0,0,0,0,0,0,3,0,0, - 0,4,0,0,0,67,0,0,0,115,34,0,0,0,116,0, - 0,124,1,0,131,1,0,125,2,0,100,1,0,124,2,0, - 106,1,0,100,2,0,124,2,0,106,2,0,105,2,0,83, - 41,3,122,33,82,101,116,117,114,110,32,116,104,101,32,109, - 101,116,97,100,97,116,97,32,102,111,114,32,116,104,101,32, - 112,97,116,104,46,114,133,0,0,0,114,134,0,0,0,41, - 3,114,39,0,0,0,218,8,115,116,95,109,116,105,109,101, - 90,7,115,116,95,115,105,122,101,41,3,114,108,0,0,0, - 114,35,0,0,0,114,208,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,197,0,0,0,55,3, - 0,0,115,4,0,0,0,0,2,12,1,122,27,83,111,117, - 114,99,101,70,105,108,101,76,111,97,100,101,114,46,112,97, - 116,104,95,115,116,97,116,115,99,4,0,0,0,0,0,0, - 0,5,0,0,0,5,0,0,0,67,0,0,0,115,34,0, - 0,0,116,0,0,124,1,0,131,1,0,125,4,0,124,0, - 0,106,1,0,124,2,0,124,3,0,100,1,0,124,4,0, - 131,2,1,83,41,2,78,218,5,95,109,111,100,101,41,2, - 114,97,0,0,0,114,198,0,0,0,41,5,114,108,0,0, - 0,114,90,0,0,0,114,89,0,0,0,114,53,0,0,0, - 114,42,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,199,0,0,0,60,3,0,0,115,4,0, - 0,0,0,2,12,1,122,32,83,111,117,114,99,101,70,105, - 108,101,76,111,97,100,101,114,46,95,99,97,99,104,101,95, - 98,121,116,101,99,111,100,101,114,220,0,0,0,105,182,1, - 0,0,99,3,0,0,0,1,0,0,0,9,0,0,0,17, - 0,0,0,67,0,0,0,115,53,1,0,0,116,0,0,124, - 1,0,131,1,0,92,2,0,125,4,0,125,5,0,103,0, - 0,125,6,0,120,54,0,124,4,0,114,80,0,116,1,0, - 124,4,0,131,1,0,12,114,80,0,116,0,0,124,4,0, - 131,1,0,92,2,0,125,4,0,125,7,0,124,6,0,106, - 2,0,124,7,0,131,1,0,1,113,27,0,87,120,132,0, - 116,3,0,124,6,0,131,1,0,68,93,118,0,125,7,0, - 116,4,0,124,4,0,124,7,0,131,2,0,125,4,0,121, - 17,0,116,5,0,106,6,0,124,4,0,131,1,0,1,87, - 113,94,0,4,116,7,0,107,10,0,114,155,0,1,1,1, - 119,94,0,89,113,94,0,4,116,8,0,107,10,0,114,211, - 0,1,125,8,0,1,122,25,0,116,9,0,100,1,0,124, - 4,0,124,8,0,131,3,0,1,100,2,0,83,87,89,100, - 2,0,100,2,0,125,8,0,126,8,0,88,113,94,0,88, - 113,94,0,87,121,33,0,116,10,0,124,1,0,124,2,0, - 124,3,0,131,3,0,1,116,9,0,100,3,0,124,1,0, - 131,2,0,1,87,110,53,0,4,116,8,0,107,10,0,114, - 48,1,1,125,8,0,1,122,21,0,116,9,0,100,1,0, - 124,1,0,124,8,0,131,3,0,1,87,89,100,2,0,100, - 2,0,125,8,0,126,8,0,88,110,1,0,88,100,2,0, - 83,41,4,122,27,87,114,105,116,101,32,98,121,116,101,115, - 32,100,97,116,97,32,116,111,32,97,32,102,105,108,101,46, - 122,27,99,111,117,108,100,32,110,111,116,32,99,114,101,97, - 116,101,32,123,33,114,125,58,32,123,33,114,125,78,122,12, - 99,114,101,97,116,101,100,32,123,33,114,125,41,11,114,38, - 0,0,0,114,46,0,0,0,114,163,0,0,0,114,33,0, - 0,0,114,28,0,0,0,114,3,0,0,0,90,5,109,107, - 100,105,114,218,15,70,105,108,101,69,120,105,115,116,115,69, - 114,114,111,114,114,40,0,0,0,114,105,0,0,0,114,55, - 0,0,0,41,9,114,108,0,0,0,114,35,0,0,0,114, - 53,0,0,0,114,220,0,0,0,218,6,112,97,114,101,110, - 116,114,94,0,0,0,114,27,0,0,0,114,23,0,0,0, - 114,201,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,198,0,0,0,65,3,0,0,115,38,0, - 0,0,0,2,18,1,6,2,22,1,18,1,17,2,19,1, - 15,1,3,1,17,1,13,2,7,1,18,3,16,1,27,1, - 3,1,16,1,17,1,18,2,122,25,83,111,117,114,99,101, - 70,105,108,101,76,111,97,100,101,114,46,115,101,116,95,100, - 97,116,97,78,41,7,114,112,0,0,0,114,111,0,0,0, - 114,113,0,0,0,114,114,0,0,0,114,197,0,0,0,114, - 199,0,0,0,114,198,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,218,0, - 0,0,51,3,0,0,115,8,0,0,0,12,2,6,2,12, - 5,12,5,114,218,0,0,0,99,0,0,0,0,0,0,0, - 0,0,0,0,0,2,0,0,0,64,0,0,0,115,46,0, - 0,0,101,0,0,90,1,0,100,0,0,90,2,0,100,1, - 0,90,3,0,100,2,0,100,3,0,132,0,0,90,4,0, - 100,4,0,100,5,0,132,0,0,90,5,0,100,6,0,83, - 41,7,218,20,83,111,117,114,99,101,108,101,115,115,70,105, - 108,101,76,111,97,100,101,114,122,45,76,111,97,100,101,114, - 32,119,104,105,99,104,32,104,97,110,100,108,101,115,32,115, - 111,117,114,99,101,108,101,115,115,32,102,105,108,101,32,105, - 109,112,111,114,116,115,46,99,2,0,0,0,0,0,0,0, - 5,0,0,0,6,0,0,0,67,0,0,0,115,76,0,0, - 0,124,0,0,106,0,0,124,1,0,131,1,0,125,2,0, - 124,0,0,106,1,0,124,2,0,131,1,0,125,3,0,116, - 2,0,124,3,0,100,1,0,124,1,0,100,2,0,124,2, - 0,131,1,2,125,4,0,116,3,0,124,4,0,100,1,0, - 124,1,0,100,3,0,124,2,0,131,1,2,83,41,4,78, - 114,106,0,0,0,114,35,0,0,0,114,89,0,0,0,41, - 4,114,157,0,0,0,114,200,0,0,0,114,141,0,0,0, - 114,147,0,0,0,41,5,114,108,0,0,0,114,126,0,0, - 0,114,35,0,0,0,114,53,0,0,0,114,209,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 187,0,0,0,98,3,0,0,115,8,0,0,0,0,1,15, - 1,15,1,24,1,122,29,83,111,117,114,99,101,108,101,115, - 115,70,105,108,101,76,111,97,100,101,114,46,103,101,116,95, - 99,111,100,101,99,2,0,0,0,0,0,0,0,2,0,0, + 97,100,101,114,46,105,115,95,112,97,99,107,97,103,101,32, + 98,121,32,99,104,101,99,107,105,110,103,32,105,102,10,32, + 32,32,32,32,32,32,32,116,104,101,32,112,97,116,104,32, + 114,101,116,117,114,110,101,100,32,98,121,32,103,101,116,95, + 102,105,108,101,110,97,109,101,32,104,97,115,32,97,32,102, + 105,108,101,110,97,109,101,32,111,102,32,39,95,95,105,110, + 105,116,95,95,46,112,121,39,46,114,29,0,0,0,114,58, + 0,0,0,114,59,0,0,0,114,56,0,0,0,218,8,95, + 95,105,110,105,116,95,95,41,4,114,38,0,0,0,114,151, + 0,0,0,114,34,0,0,0,114,32,0,0,0,41,5,114, + 100,0,0,0,114,119,0,0,0,114,94,0,0,0,90,13, + 102,105,108,101,110,97,109,101,95,98,97,115,101,90,9,116, + 97,105,108,95,110,97,109,101,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,153,0,0,0,125,2,0,0, + 115,8,0,0,0,0,3,25,1,22,1,19,1,122,24,95, + 76,111,97,100,101,114,66,97,115,105,99,115,46,105,115,95, + 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, + 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, + 0,100,1,0,83,41,2,122,42,85,115,101,32,100,101,102, + 97,117,108,116,32,115,101,109,97,110,116,105,99,115,32,102, + 111,114,32,109,111,100,117,108,101,32,99,114,101,97,116,105, + 111,110,46,78,114,4,0,0,0,41,2,114,100,0,0,0, + 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,218,13,99,114,101,97,116,101,95,109,111,100, + 117,108,101,133,2,0,0,115,0,0,0,0,122,27,95,76, + 111,97,100,101,114,66,97,115,105,99,115,46,99,114,101,97, + 116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,0, + 0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,80, + 0,0,0,124,0,0,106,0,0,124,1,0,106,1,0,131, + 1,0,125,2,0,124,2,0,100,1,0,107,8,0,114,54, + 0,116,2,0,100,2,0,106,3,0,124,1,0,106,1,0, + 131,1,0,131,1,0,130,1,0,116,4,0,106,5,0,116, + 6,0,124,2,0,124,1,0,106,7,0,131,3,0,1,100, + 1,0,83,41,3,122,19,69,120,101,99,117,116,101,32,116, + 104,101,32,109,111,100,117,108,101,46,78,122,52,99,97,110, + 110,111,116,32,108,111,97,100,32,109,111,100,117,108,101,32, + 123,33,114,125,32,119,104,101,110,32,103,101,116,95,99,111, + 100,101,40,41,32,114,101,116,117,114,110,115,32,78,111,110, + 101,41,8,218,8,103,101,116,95,99,111,100,101,114,105,0, + 0,0,114,99,0,0,0,114,47,0,0,0,114,114,0,0, + 0,218,25,95,99,97,108,108,95,119,105,116,104,95,102,114, + 97,109,101,115,95,114,101,109,111,118,101,100,218,4,101,120, + 101,99,114,111,0,0,0,41,3,114,100,0,0,0,218,6, + 109,111,100,117,108,101,114,140,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,11,101,120,101,99, + 95,109,111,100,117,108,101,136,2,0,0,115,10,0,0,0, + 0,2,18,1,12,1,9,1,15,1,122,25,95,76,111,97, + 100,101,114,66,97,115,105,99,115,46,101,120,101,99,95,109, + 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, + 0,0,3,0,0,0,67,0,0,0,115,16,0,0,0,116, + 0,0,106,1,0,124,0,0,124,1,0,131,2,0,83,41, + 1,78,41,2,114,114,0,0,0,218,17,95,108,111,97,100, + 95,109,111,100,117,108,101,95,115,104,105,109,41,2,114,100, + 0,0,0,114,119,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,11,108,111,97,100,95,109,111, + 100,117,108,101,144,2,0,0,115,2,0,0,0,0,1,122, + 25,95,76,111,97,100,101,114,66,97,115,105,99,115,46,108, + 111,97,100,95,109,111,100,117,108,101,78,41,8,114,105,0, + 0,0,114,104,0,0,0,114,106,0,0,0,114,107,0,0, + 0,114,153,0,0,0,114,180,0,0,0,114,185,0,0,0, + 114,187,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,178,0,0,0,120,2, + 0,0,115,10,0,0,0,12,3,6,2,12,8,12,3,12, + 8,114,178,0,0,0,99,0,0,0,0,0,0,0,0,0, + 0,0,0,4,0,0,0,64,0,0,0,115,106,0,0,0, + 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,100, + 2,0,132,0,0,90,3,0,100,3,0,100,4,0,132,0, + 0,90,4,0,100,5,0,100,6,0,132,0,0,90,5,0, + 100,7,0,100,8,0,132,0,0,90,6,0,100,9,0,100, + 10,0,132,0,0,90,7,0,100,11,0,100,18,0,100,13, + 0,100,14,0,132,0,1,90,8,0,100,15,0,100,16,0, + 132,0,0,90,9,0,100,17,0,83,41,19,218,12,83,111, + 117,114,99,101,76,111,97,100,101,114,99,2,0,0,0,0, + 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, + 10,0,0,0,116,0,0,130,1,0,100,1,0,83,41,2, + 122,178,79,112,116,105,111,110,97,108,32,109,101,116,104,111, + 100,32,116,104,97,116,32,114,101,116,117,114,110,115,32,116, + 104,101,32,109,111,100,105,102,105,99,97,116,105,111,110,32, + 116,105,109,101,32,40,97,110,32,105,110,116,41,32,102,111, + 114,32,116,104,101,10,32,32,32,32,32,32,32,32,115,112, + 101,99,105,102,105,101,100,32,112,97,116,104,44,32,119,104, + 101,114,101,32,112,97,116,104,32,105,115,32,97,32,115,116, + 114,46,10,10,32,32,32,32,32,32,32,32,82,97,105,115, + 101,115,32,73,79,69,114,114,111,114,32,119,104,101,110,32, + 116,104,101,32,112,97,116,104,32,99,97,110,110,111,116,32, + 98,101,32,104,97,110,100,108,101,100,46,10,32,32,32,32, + 32,32,32,32,78,41,1,218,7,73,79,69,114,114,111,114, + 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, + 104,95,109,116,105,109,101,150,2,0,0,115,2,0,0,0, + 0,6,122,23,83,111,117,114,99,101,76,111,97,100,101,114, + 46,112,97,116,104,95,109,116,105,109,101,99,2,0,0,0, + 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, + 115,19,0,0,0,100,1,0,124,0,0,106,0,0,124,1, + 0,131,1,0,105,1,0,83,41,2,97,170,1,0,0,79, + 112,116,105,111,110,97,108,32,109,101,116,104,111,100,32,114, + 101,116,117,114,110,105,110,103,32,97,32,109,101,116,97,100, + 97,116,97,32,100,105,99,116,32,102,111,114,32,116,104,101, + 32,115,112,101,99,105,102,105,101,100,32,112,97,116,104,10, + 32,32,32,32,32,32,32,32,116,111,32,98,121,32,116,104, + 101,32,112,97,116,104,32,40,115,116,114,41,46,10,32,32, + 32,32,32,32,32,32,80,111,115,115,105,98,108,101,32,107, + 101,121,115,58,10,32,32,32,32,32,32,32,32,45,32,39, + 109,116,105,109,101,39,32,40,109,97,110,100,97,116,111,114, + 121,41,32,105,115,32,116,104,101,32,110,117,109,101,114,105, + 99,32,116,105,109,101,115,116,97,109,112,32,111,102,32,108, + 97,115,116,32,115,111,117,114,99,101,10,32,32,32,32,32, + 32,32,32,32,32,99,111,100,101,32,109,111,100,105,102,105, + 99,97,116,105,111,110,59,10,32,32,32,32,32,32,32,32, + 45,32,39,115,105,122,101,39,32,40,111,112,116,105,111,110, + 97,108,41,32,105,115,32,116,104,101,32,115,105,122,101,32, + 105,110,32,98,121,116,101,115,32,111,102,32,116,104,101,32, + 115,111,117,114,99,101,32,99,111,100,101,46,10,10,32,32, + 32,32,32,32,32,32,73,109,112,108,101,109,101,110,116,105, + 110,103,32,116,104,105,115,32,109,101,116,104,111,100,32,97, + 108,108,111,119,115,32,116,104,101,32,108,111,97,100,101,114, + 32,116,111,32,114,101,97,100,32,98,121,116,101,99,111,100, + 101,32,102,105,108,101,115,46,10,32,32,32,32,32,32,32, + 32,82,97,105,115,101,115,32,73,79,69,114,114,111,114,32, + 119,104,101,110,32,116,104,101,32,112,97,116,104,32,99,97, + 110,110,111,116,32,98,101,32,104,97,110,100,108,101,100,46, + 10,32,32,32,32,32,32,32,32,114,126,0,0,0,41,1, + 114,190,0,0,0,41,2,114,100,0,0,0,114,35,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 218,10,112,97,116,104,95,115,116,97,116,115,158,2,0,0, + 115,2,0,0,0,0,11,122,23,83,111,117,114,99,101,76, + 111,97,100,101,114,46,112,97,116,104,95,115,116,97,116,115, + 99,4,0,0,0,0,0,0,0,4,0,0,0,3,0,0, + 0,67,0,0,0,115,16,0,0,0,124,0,0,106,0,0, + 124,2,0,124,3,0,131,2,0,83,41,1,122,228,79,112, + 116,105,111,110,97,108,32,109,101,116,104,111,100,32,119,104, + 105,99,104,32,119,114,105,116,101,115,32,100,97,116,97,32, + 40,98,121,116,101,115,41,32,116,111,32,97,32,102,105,108, + 101,32,112,97,116,104,32,40,97,32,115,116,114,41,46,10, + 10,32,32,32,32,32,32,32,32,73,109,112,108,101,109,101, + 110,116,105,110,103,32,116,104,105,115,32,109,101,116,104,111, + 100,32,97,108,108,111,119,115,32,102,111,114,32,116,104,101, + 32,119,114,105,116,105,110,103,32,111,102,32,98,121,116,101, + 99,111,100,101,32,102,105,108,101,115,46,10,10,32,32,32, + 32,32,32,32,32,84,104,101,32,115,111,117,114,99,101,32, + 112,97,116,104,32,105,115,32,110,101,101,100,101,100,32,105, + 110,32,111,114,100,101,114,32,116,111,32,99,111,114,114,101, + 99,116,108,121,32,116,114,97,110,115,102,101,114,32,112,101, + 114,109,105,115,115,105,111,110,115,10,32,32,32,32,32,32, + 32,32,41,1,218,8,115,101,116,95,100,97,116,97,41,4, + 114,100,0,0,0,114,90,0,0,0,90,10,99,97,99,104, + 101,95,112,97,116,104,114,53,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,15,95,99,97,99, + 104,101,95,98,121,116,101,99,111,100,101,171,2,0,0,115, + 2,0,0,0,0,8,122,28,83,111,117,114,99,101,76,111, + 97,100,101,114,46,95,99,97,99,104,101,95,98,121,116,101, + 99,111,100,101,99,3,0,0,0,0,0,0,0,3,0,0, 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,122,39,82,101,116,117,114,110,32,78,111,110, - 101,32,97,115,32,116,104,101,114,101,32,105,115,32,110,111, - 32,115,111,117,114,99,101,32,99,111,100,101,46,78,114,4, - 0,0,0,41,2,114,108,0,0,0,114,126,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,202, - 0,0,0,104,3,0,0,115,2,0,0,0,0,2,122,31, - 83,111,117,114,99,101,108,101,115,115,70,105,108,101,76,111, - 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,78, - 41,6,114,112,0,0,0,114,111,0,0,0,114,113,0,0, - 0,114,114,0,0,0,114,187,0,0,0,114,202,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,223,0,0,0,94,3,0,0,115,6,0, - 0,0,12,2,6,2,12,6,114,223,0,0,0,99,0,0, - 0,0,0,0,0,0,0,0,0,0,3,0,0,0,64,0, - 0,0,115,136,0,0,0,101,0,0,90,1,0,100,0,0, - 90,2,0,100,1,0,90,3,0,100,2,0,100,3,0,132, - 0,0,90,4,0,100,4,0,100,5,0,132,0,0,90,5, - 0,100,6,0,100,7,0,132,0,0,90,6,0,100,8,0, - 100,9,0,132,0,0,90,7,0,100,10,0,100,11,0,132, - 0,0,90,8,0,100,12,0,100,13,0,132,0,0,90,9, - 0,100,14,0,100,15,0,132,0,0,90,10,0,100,16,0, - 100,17,0,132,0,0,90,11,0,101,12,0,100,18,0,100, - 19,0,132,0,0,131,1,0,90,13,0,100,20,0,83,41, - 21,218,19,69,120,116,101,110,115,105,111,110,70,105,108,101, - 76,111,97,100,101,114,122,93,76,111,97,100,101,114,32,102, - 111,114,32,101,120,116,101,110,115,105,111,110,32,109,111,100, - 117,108,101,115,46,10,10,32,32,32,32,84,104,101,32,99, - 111,110,115,116,114,117,99,116,111,114,32,105,115,32,100,101, - 115,105,103,110,101,100,32,116,111,32,119,111,114,107,32,119, - 105,116,104,32,70,105,108,101,70,105,110,100,101,114,46,10, - 10,32,32,32,32,99,3,0,0,0,0,0,0,0,3,0, - 0,0,2,0,0,0,67,0,0,0,115,22,0,0,0,124, - 1,0,124,0,0,95,0,0,124,2,0,124,0,0,95,1, - 0,100,0,0,83,41,1,78,41,2,114,106,0,0,0,114, - 35,0,0,0,41,3,114,108,0,0,0,114,106,0,0,0, - 114,35,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,185,0,0,0,121,3,0,0,115,4,0, - 0,0,0,1,9,1,122,28,69,120,116,101,110,115,105,111, - 110,70,105,108,101,76,111,97,100,101,114,46,95,95,105,110, - 105,116,95,95,99,2,0,0,0,0,0,0,0,2,0,0, - 0,2,0,0,0,67,0,0,0,115,34,0,0,0,124,0, - 0,106,0,0,124,1,0,106,0,0,107,2,0,111,33,0, - 124,0,0,106,1,0,124,1,0,106,1,0,107,2,0,83, - 41,1,78,41,2,114,211,0,0,0,114,118,0,0,0,41, - 2,114,108,0,0,0,114,212,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,213,0,0,0,125, - 3,0,0,115,4,0,0,0,0,1,18,1,122,26,69,120, - 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, - 114,46,95,95,101,113,95,95,99,1,0,0,0,0,0,0, - 0,1,0,0,0,3,0,0,0,67,0,0,0,115,26,0, - 0,0,116,0,0,124,0,0,106,1,0,131,1,0,116,0, - 0,124,0,0,106,2,0,131,1,0,65,83,41,1,78,41, - 3,114,214,0,0,0,114,106,0,0,0,114,35,0,0,0, - 41,1,114,108,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,215,0,0,0,129,3,0,0,115, - 2,0,0,0,0,1,122,28,69,120,116,101,110,115,105,111, - 110,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, - 115,104,95,95,99,2,0,0,0,0,0,0,0,3,0,0, - 0,4,0,0,0,67,0,0,0,115,47,0,0,0,116,0, - 0,106,1,0,116,2,0,106,3,0,124,1,0,131,2,0, - 125,2,0,116,4,0,100,1,0,124,1,0,106,5,0,124, - 0,0,106,6,0,131,3,0,1,124,2,0,83,41,2,122, - 38,67,114,101,97,116,101,32,97,110,32,117,110,105,116,105, - 97,108,105,122,101,100,32,101,120,116,101,110,115,105,111,110, - 32,109,111,100,117,108,101,122,38,101,120,116,101,110,115,105, - 111,110,32,109,111,100,117,108,101,32,123,33,114,125,32,108, - 111,97,100,101,100,32,102,114,111,109,32,123,33,114,125,41, - 7,114,121,0,0,0,114,188,0,0,0,114,145,0,0,0, - 90,14,99,114,101,97,116,101,95,100,121,110,97,109,105,99, - 114,105,0,0,0,114,106,0,0,0,114,35,0,0,0,41, - 3,114,108,0,0,0,114,164,0,0,0,114,190,0,0,0, + 0,83,41,2,122,150,79,112,116,105,111,110,97,108,32,109, + 101,116,104,111,100,32,119,104,105,99,104,32,119,114,105,116, + 101,115,32,100,97,116,97,32,40,98,121,116,101,115,41,32, + 116,111,32,97,32,102,105,108,101,32,112,97,116,104,32,40, + 97,32,115,116,114,41,46,10,10,32,32,32,32,32,32,32, + 32,73,109,112,108,101,109,101,110,116,105,110,103,32,116,104, + 105,115,32,109,101,116,104,111,100,32,97,108,108,111,119,115, + 32,102,111,114,32,116,104,101,32,119,114,105,116,105,110,103, + 32,111,102,32,98,121,116,101,99,111,100,101,32,102,105,108, + 101,115,46,10,32,32,32,32,32,32,32,32,78,114,4,0, + 0,0,41,3,114,100,0,0,0,114,35,0,0,0,114,53, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,114,192,0,0,0,181,2,0,0,115,0,0,0,0, + 122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,115, + 101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0, + 5,0,0,0,16,0,0,0,67,0,0,0,115,105,0,0, + 0,124,0,0,106,0,0,124,1,0,131,1,0,125,2,0, + 121,19,0,124,0,0,106,1,0,124,2,0,131,1,0,125, + 3,0,87,110,58,0,4,116,2,0,107,10,0,114,94,0, + 1,125,4,0,1,122,26,0,116,3,0,100,1,0,100,2, + 0,124,1,0,131,1,1,124,4,0,130,2,0,87,89,100, + 3,0,100,3,0,125,4,0,126,4,0,88,110,1,0,88, + 116,4,0,124,3,0,131,1,0,83,41,4,122,52,67,111, + 110,99,114,101,116,101,32,105,109,112,108,101,109,101,110,116, + 97,116,105,111,110,32,111,102,32,73,110,115,112,101,99,116, + 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, + 101,46,122,39,115,111,117,114,99,101,32,110,111,116,32,97, + 118,97,105,108,97,98,108,101,32,116,104,114,111,117,103,104, + 32,103,101,116,95,100,97,116,97,40,41,114,98,0,0,0, + 78,41,5,114,151,0,0,0,218,8,103,101,116,95,100,97, + 116,97,114,40,0,0,0,114,99,0,0,0,114,149,0,0, + 0,41,5,114,100,0,0,0,114,119,0,0,0,114,35,0, + 0,0,114,147,0,0,0,218,3,101,120,99,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,10,103,101,116, + 95,115,111,117,114,99,101,188,2,0,0,115,14,0,0,0, + 0,2,15,1,3,1,19,1,18,1,9,1,31,1,122,23, + 83,111,117,114,99,101,76,111,97,100,101,114,46,103,101,116, + 95,115,111,117,114,99,101,218,9,95,111,112,116,105,109,105, + 122,101,114,29,0,0,0,99,3,0,0,0,1,0,0,0, + 4,0,0,0,9,0,0,0,67,0,0,0,115,34,0,0, + 0,116,0,0,106,1,0,116,2,0,124,1,0,124,2,0, + 100,1,0,100,2,0,100,3,0,100,4,0,124,3,0,131, + 4,2,83,41,5,122,130,82,101,116,117,114,110,32,116,104, + 101,32,99,111,100,101,32,111,98,106,101,99,116,32,99,111, + 109,112,105,108,101,100,32,102,114,111,109,32,115,111,117,114, + 99,101,46,10,10,32,32,32,32,32,32,32,32,84,104,101, + 32,39,100,97,116,97,39,32,97,114,103,117,109,101,110,116, + 32,99,97,110,32,98,101,32,97,110,121,32,111,98,106,101, + 99,116,32,116,121,112,101,32,116,104,97,116,32,99,111,109, + 112,105,108,101,40,41,32,115,117,112,112,111,114,116,115,46, + 10,32,32,32,32,32,32,32,32,114,183,0,0,0,218,12, + 100,111,110,116,95,105,110,104,101,114,105,116,84,114,68,0, + 0,0,41,3,114,114,0,0,0,114,182,0,0,0,218,7, + 99,111,109,112,105,108,101,41,4,114,100,0,0,0,114,53, + 0,0,0,114,35,0,0,0,114,197,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,14,115,111, + 117,114,99,101,95,116,111,95,99,111,100,101,198,2,0,0, + 115,4,0,0,0,0,5,21,1,122,27,83,111,117,114,99, + 101,76,111,97,100,101,114,46,115,111,117,114,99,101,95,116, + 111,95,99,111,100,101,99,2,0,0,0,0,0,0,0,10, + 0,0,0,43,0,0,0,67,0,0,0,115,183,1,0,0, + 124,0,0,106,0,0,124,1,0,131,1,0,125,2,0,100, + 1,0,125,3,0,121,16,0,116,1,0,124,2,0,131,1, + 0,125,4,0,87,110,24,0,4,116,2,0,107,10,0,114, + 63,0,1,1,1,100,1,0,125,4,0,89,110,205,0,88, + 121,19,0,124,0,0,106,3,0,124,2,0,131,1,0,125, + 5,0,87,110,18,0,4,116,4,0,107,10,0,114,103,0, + 1,1,1,89,110,165,0,88,116,5,0,124,5,0,100,2, + 0,25,131,1,0,125,3,0,121,19,0,124,0,0,106,6, + 0,124,4,0,131,1,0,125,6,0,87,110,18,0,4,116, + 7,0,107,10,0,114,159,0,1,1,1,89,110,109,0,88, + 121,34,0,116,8,0,124,6,0,100,3,0,124,5,0,100, + 4,0,124,1,0,100,5,0,124,4,0,131,1,3,125,7, + 0,87,110,24,0,4,116,9,0,116,10,0,102,2,0,107, + 10,0,114,220,0,1,1,1,89,110,48,0,88,116,11,0, + 106,12,0,100,6,0,124,4,0,124,2,0,131,3,0,1, + 116,13,0,124,7,0,100,4,0,124,1,0,100,7,0,124, + 4,0,100,8,0,124,2,0,131,1,3,83,124,0,0,106, + 6,0,124,2,0,131,1,0,125,8,0,124,0,0,106,14, + 0,124,8,0,124,2,0,131,2,0,125,9,0,116,11,0, + 106,12,0,100,9,0,124,2,0,131,2,0,1,116,15,0, + 106,16,0,12,114,179,1,124,4,0,100,1,0,107,9,0, + 114,179,1,124,3,0,100,1,0,107,9,0,114,179,1,116, + 17,0,124,9,0,124,3,0,116,18,0,124,8,0,131,1, + 0,131,3,0,125,6,0,121,39,0,124,0,0,106,19,0, + 124,2,0,124,4,0,124,6,0,131,3,0,1,116,11,0, + 106,12,0,100,10,0,124,4,0,131,2,0,1,87,110,18, + 0,4,116,2,0,107,10,0,114,178,1,1,1,1,89,110, + 1,0,88,124,9,0,83,41,11,122,190,67,111,110,99,114, + 101,116,101,32,105,109,112,108,101,109,101,110,116,97,116,105, + 111,110,32,111,102,32,73,110,115,112,101,99,116,76,111,97, + 100,101,114,46,103,101,116,95,99,111,100,101,46,10,10,32, + 32,32,32,32,32,32,32,82,101,97,100,105,110,103,32,111, + 102,32,98,121,116,101,99,111,100,101,32,114,101,113,117,105, + 114,101,115,32,112,97,116,104,95,115,116,97,116,115,32,116, + 111,32,98,101,32,105,109,112,108,101,109,101,110,116,101,100, + 46,32,84,111,32,119,114,105,116,101,10,32,32,32,32,32, + 32,32,32,98,121,116,101,99,111,100,101,44,32,115,101,116, + 95,100,97,116,97,32,109,117,115,116,32,97,108,115,111,32, + 98,101,32,105,109,112,108,101,109,101,110,116,101,100,46,10, + 10,32,32,32,32,32,32,32,32,78,114,126,0,0,0,114, + 132,0,0,0,114,98,0,0,0,114,35,0,0,0,122,13, + 123,125,32,109,97,116,99,104,101,115,32,123,125,114,89,0, + 0,0,114,90,0,0,0,122,19,99,111,100,101,32,111,98, + 106,101,99,116,32,102,114,111,109,32,123,125,122,10,119,114, + 111,116,101,32,123,33,114,125,41,20,114,151,0,0,0,114, + 79,0,0,0,114,66,0,0,0,114,191,0,0,0,114,189, + 0,0,0,114,14,0,0,0,114,194,0,0,0,114,40,0, + 0,0,114,135,0,0,0,114,99,0,0,0,114,130,0,0, + 0,114,114,0,0,0,114,129,0,0,0,114,141,0,0,0, + 114,200,0,0,0,114,7,0,0,0,218,19,100,111,110,116, + 95,119,114,105,116,101,95,98,121,116,101,99,111,100,101,114, + 144,0,0,0,114,31,0,0,0,114,193,0,0,0,41,10, + 114,100,0,0,0,114,119,0,0,0,114,90,0,0,0,114, + 133,0,0,0,114,89,0,0,0,218,2,115,116,114,53,0, + 0,0,218,10,98,121,116,101,115,95,100,97,116,97,114,147, + 0,0,0,90,11,99,111,100,101,95,111,98,106,101,99,116, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 186,0,0,0,132,3,0,0,115,10,0,0,0,0,2,6, - 1,15,1,6,1,16,1,122,33,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,46,99,114,101, - 97,116,101,95,109,111,100,117,108,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,4,0,0,0,67,0,0,0,115, - 45,0,0,0,116,0,0,106,1,0,116,2,0,106,3,0, - 124,1,0,131,2,0,1,116,4,0,100,1,0,124,0,0, - 106,5,0,124,0,0,106,6,0,131,3,0,1,100,2,0, - 83,41,3,122,30,73,110,105,116,105,97,108,105,122,101,32, - 97,110,32,101,120,116,101,110,115,105,111,110,32,109,111,100, - 117,108,101,122,40,101,120,116,101,110,115,105,111,110,32,109, - 111,100,117,108,101,32,123,33,114,125,32,101,120,101,99,117, - 116,101,100,32,102,114,111,109,32,123,33,114,125,78,41,7, - 114,121,0,0,0,114,188,0,0,0,114,145,0,0,0,90, - 12,101,120,101,99,95,100,121,110,97,109,105,99,114,105,0, - 0,0,114,106,0,0,0,114,35,0,0,0,41,2,114,108, - 0,0,0,114,190,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,191,0,0,0,140,3,0,0, - 115,6,0,0,0,0,2,19,1,6,1,122,31,69,120,116, - 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, - 46,101,120,101,99,95,109,111,100,117,108,101,99,2,0,0, - 0,0,0,0,0,2,0,0,0,4,0,0,0,3,0,0, - 0,115,48,0,0,0,116,0,0,124,0,0,106,1,0,131, - 1,0,100,1,0,25,137,0,0,116,2,0,135,0,0,102, - 1,0,100,2,0,100,3,0,134,0,0,116,3,0,68,131, - 1,0,131,1,0,83,41,4,122,49,82,101,116,117,114,110, - 32,84,114,117,101,32,105,102,32,116,104,101,32,101,120,116, - 101,110,115,105,111,110,32,109,111,100,117,108,101,32,105,115, - 32,97,32,112,97,99,107,97,103,101,46,114,29,0,0,0, - 99,1,0,0,0,0,0,0,0,2,0,0,0,4,0,0, - 0,51,0,0,0,115,31,0,0,0,124,0,0,93,21,0, - 125,1,0,136,0,0,100,0,0,124,1,0,23,107,2,0, - 86,1,113,3,0,100,1,0,83,41,2,114,185,0,0,0, - 78,114,4,0,0,0,41,2,114,22,0,0,0,218,6,115, - 117,102,102,105,120,41,1,218,9,102,105,108,101,95,110,97, - 109,101,114,4,0,0,0,114,5,0,0,0,250,9,60,103, - 101,110,101,120,112,114,62,149,3,0,0,115,2,0,0,0, - 6,1,122,49,69,120,116,101,110,115,105,111,110,70,105,108, - 101,76,111,97,100,101,114,46,105,115,95,112,97,99,107,97, - 103,101,46,60,108,111,99,97,108,115,62,46,60,103,101,110, - 101,120,112,114,62,41,4,114,38,0,0,0,114,35,0,0, - 0,218,3,97,110,121,218,18,69,88,84,69,78,83,73,79, - 78,95,83,85,70,70,73,88,69,83,41,2,114,108,0,0, - 0,114,126,0,0,0,114,4,0,0,0,41,1,114,226,0, - 0,0,114,5,0,0,0,114,159,0,0,0,146,3,0,0, - 115,6,0,0,0,0,2,19,1,18,1,122,30,69,120,116, - 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, - 46,105,115,95,112,97,99,107,97,103,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,1,0,83,41,2,122,63,82,101,116, - 117,114,110,32,78,111,110,101,32,97,115,32,97,110,32,101, - 120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,32, - 99,97,110,110,111,116,32,99,114,101,97,116,101,32,97,32, - 99,111,100,101,32,111,98,106,101,99,116,46,78,114,4,0, - 0,0,41,2,114,108,0,0,0,114,126,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,187,0, - 0,0,152,3,0,0,115,2,0,0,0,0,2,122,28,69, - 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, - 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,1,0,83,41,2,122,53,82,101,116, - 117,114,110,32,78,111,110,101,32,97,115,32,101,120,116,101, - 110,115,105,111,110,32,109,111,100,117,108,101,115,32,104,97, - 118,101,32,110,111,32,115,111,117,114,99,101,32,99,111,100, - 101,46,78,114,4,0,0,0,41,2,114,108,0,0,0,114, - 126,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,202,0,0,0,156,3,0,0,115,2,0,0, - 0,0,2,122,30,69,120,116,101,110,115,105,111,110,70,105, - 108,101,76,111,97,100,101,114,46,103,101,116,95,115,111,117, - 114,99,101,99,2,0,0,0,0,0,0,0,2,0,0,0, - 1,0,0,0,67,0,0,0,115,7,0,0,0,124,0,0, - 106,0,0,83,41,1,122,58,82,101,116,117,114,110,32,116, - 104,101,32,112,97,116,104,32,116,111,32,116,104,101,32,115, - 111,117,114,99,101,32,102,105,108,101,32,97,115,32,102,111, - 117,110,100,32,98,121,32,116,104,101,32,102,105,110,100,101, - 114,46,41,1,114,35,0,0,0,41,2,114,108,0,0,0, - 114,126,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,157,0,0,0,160,3,0,0,115,2,0, - 0,0,0,3,122,32,69,120,116,101,110,115,105,111,110,70, - 105,108,101,76,111,97,100,101,114,46,103,101,116,95,102,105, - 108,101,110,97,109,101,78,41,14,114,112,0,0,0,114,111, - 0,0,0,114,113,0,0,0,114,114,0,0,0,114,185,0, - 0,0,114,213,0,0,0,114,215,0,0,0,114,186,0,0, - 0,114,191,0,0,0,114,159,0,0,0,114,187,0,0,0, - 114,202,0,0,0,114,123,0,0,0,114,157,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,224,0,0,0,113,3,0,0,115,20,0,0, - 0,12,6,6,2,12,4,12,4,12,3,12,8,12,6,12, - 6,12,4,12,4,114,224,0,0,0,99,0,0,0,0,0, - 0,0,0,0,0,0,0,2,0,0,0,64,0,0,0,115, - 130,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, + 181,0,0,0,206,2,0,0,115,78,0,0,0,0,7,15, + 1,6,1,3,1,16,1,13,1,11,2,3,1,19,1,13, + 1,5,2,16,1,3,1,19,1,13,1,5,2,3,1,9, + 1,12,1,13,1,19,1,5,2,12,1,7,1,15,1,6, + 1,7,1,15,1,18,1,16,1,22,1,12,1,9,1,15, + 1,3,1,19,1,20,1,13,1,5,1,122,21,83,111,117, + 114,99,101,76,111,97,100,101,114,46,103,101,116,95,99,111, + 100,101,78,114,87,0,0,0,41,10,114,105,0,0,0,114, + 104,0,0,0,114,106,0,0,0,114,190,0,0,0,114,191, + 0,0,0,114,193,0,0,0,114,192,0,0,0,114,196,0, + 0,0,114,200,0,0,0,114,181,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,188,0,0,0,148,2,0,0,115,14,0,0,0,12,2, + 12,8,12,13,12,10,12,7,12,10,18,8,114,188,0,0, + 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, + 0,0,0,0,0,0,115,112,0,0,0,101,0,0,90,1, + 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, + 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, + 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, + 0,101,7,0,135,0,0,102,1,0,100,8,0,100,9,0, + 134,0,0,131,1,0,90,8,0,101,7,0,100,10,0,100, + 11,0,132,0,0,131,1,0,90,9,0,100,12,0,100,13, + 0,132,0,0,90,10,0,135,0,0,83,41,14,218,10,70, + 105,108,101,76,111,97,100,101,114,122,103,66,97,115,101,32, + 102,105,108,101,32,108,111,97,100,101,114,32,99,108,97,115, + 115,32,119,104,105,99,104,32,105,109,112,108,101,109,101,110, + 116,115,32,116,104,101,32,108,111,97,100,101,114,32,112,114, + 111,116,111,99,111,108,32,109,101,116,104,111,100,115,32,116, + 104,97,116,10,32,32,32,32,114,101,113,117,105,114,101,32, + 102,105,108,101,32,115,121,115,116,101,109,32,117,115,97,103, + 101,46,99,3,0,0,0,0,0,0,0,3,0,0,0,2, + 0,0,0,67,0,0,0,115,22,0,0,0,124,1,0,124, + 0,0,95,0,0,124,2,0,124,0,0,95,1,0,100,1, + 0,83,41,2,122,75,67,97,99,104,101,32,116,104,101,32, + 109,111,100,117,108,101,32,110,97,109,101,32,97,110,100,32, + 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, + 102,105,108,101,32,102,111,117,110,100,32,98,121,32,116,104, + 101,10,32,32,32,32,32,32,32,32,102,105,110,100,101,114, + 46,78,41,2,114,98,0,0,0,114,35,0,0,0,41,3, + 114,100,0,0,0,114,119,0,0,0,114,35,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,179, + 0,0,0,7,3,0,0,115,4,0,0,0,0,3,9,1, + 122,19,70,105,108,101,76,111,97,100,101,114,46,95,95,105, + 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, + 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, + 0,0,106,0,0,124,1,0,106,0,0,107,2,0,111,33, + 0,124,0,0,106,1,0,124,1,0,106,1,0,107,2,0, + 83,41,1,78,41,2,218,9,95,95,99,108,97,115,115,95, + 95,114,111,0,0,0,41,2,114,100,0,0,0,218,5,111, + 116,104,101,114,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,6,95,95,101,113,95,95,13,3,0,0,115, + 4,0,0,0,0,1,18,1,122,17,70,105,108,101,76,111, + 97,100,101,114,46,95,95,101,113,95,95,99,1,0,0,0, + 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, + 115,26,0,0,0,116,0,0,124,0,0,106,1,0,131,1, + 0,116,0,0,124,0,0,106,2,0,131,1,0,65,83,41, + 1,78,41,3,218,4,104,97,115,104,114,98,0,0,0,114, + 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,8,95,95,104,97, + 115,104,95,95,17,3,0,0,115,2,0,0,0,0,1,122, + 19,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, + 115,104,95,95,99,2,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,3,0,0,0,115,22,0,0,0,116,0, + 0,116,1,0,124,0,0,131,2,0,106,2,0,124,1,0, + 131,1,0,83,41,1,122,100,76,111,97,100,32,97,32,109, + 111,100,117,108,101,32,102,114,111,109,32,97,32,102,105,108, + 101,46,10,10,32,32,32,32,32,32,32,32,84,104,105,115, + 32,109,101,116,104,111,100,32,105,115,32,100,101,112,114,101, + 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, + 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, + 100,46,10,10,32,32,32,32,32,32,32,32,41,3,218,5, + 115,117,112,101,114,114,204,0,0,0,114,187,0,0,0,41, + 2,114,100,0,0,0,114,119,0,0,0,41,1,114,205,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,187,0,0, + 0,20,3,0,0,115,2,0,0,0,0,10,122,22,70,105, + 108,101,76,111,97,100,101,114,46,108,111,97,100,95,109,111, + 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, + 0,1,0,0,0,67,0,0,0,115,7,0,0,0,124,0, + 0,106,0,0,83,41,1,122,58,82,101,116,117,114,110,32, + 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, + 115,111,117,114,99,101,32,102,105,108,101,32,97,115,32,102, + 111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100, + 101,114,46,41,1,114,35,0,0,0,41,2,114,100,0,0, + 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,151,0,0,0,32,3,0,0,115,2, + 0,0,0,0,3,122,23,70,105,108,101,76,111,97,100,101, + 114,46,103,101,116,95,102,105,108,101,110,97,109,101,99,2, + 0,0,0,0,0,0,0,3,0,0,0,9,0,0,0,67, + 0,0,0,115,42,0,0,0,116,0,0,106,1,0,124,1, + 0,100,1,0,131,2,0,143,17,0,125,2,0,124,2,0, + 106,2,0,131,0,0,83,87,100,2,0,81,82,88,100,2, + 0,83,41,3,122,39,82,101,116,117,114,110,32,116,104,101, + 32,100,97,116,97,32,102,114,111,109,32,112,97,116,104,32, + 97,115,32,114,97,119,32,98,121,116,101,115,46,218,1,114, + 78,41,3,114,49,0,0,0,114,50,0,0,0,90,4,114, + 101,97,100,41,3,114,100,0,0,0,114,35,0,0,0,114, + 54,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,194,0,0,0,37,3,0,0,115,4,0,0, + 0,0,2,21,1,122,19,70,105,108,101,76,111,97,100,101, + 114,46,103,101,116,95,100,97,116,97,41,11,114,105,0,0, + 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, + 114,179,0,0,0,114,207,0,0,0,114,209,0,0,0,114, + 116,0,0,0,114,187,0,0,0,114,151,0,0,0,114,194, + 0,0,0,114,4,0,0,0,114,4,0,0,0,41,1,114, + 205,0,0,0,114,5,0,0,0,114,204,0,0,0,2,3, + 0,0,115,14,0,0,0,12,3,6,2,12,6,12,4,12, + 3,24,12,18,5,114,204,0,0,0,99,0,0,0,0,0, + 0,0,0,0,0,0,0,4,0,0,0,64,0,0,0,115, + 64,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, 100,1,0,90,3,0,100,2,0,100,3,0,132,0,0,90, 4,0,100,4,0,100,5,0,132,0,0,90,5,0,100,6, - 0,100,7,0,132,0,0,90,6,0,100,8,0,100,9,0, - 132,0,0,90,7,0,100,10,0,100,11,0,132,0,0,90, - 8,0,100,12,0,100,13,0,132,0,0,90,9,0,100,14, - 0,100,15,0,132,0,0,90,10,0,100,16,0,100,17,0, - 132,0,0,90,11,0,100,18,0,100,19,0,132,0,0,90, - 12,0,100,20,0,83,41,21,218,14,95,78,97,109,101,115, - 112,97,99,101,80,97,116,104,97,38,1,0,0,82,101,112, - 114,101,115,101,110,116,115,32,97,32,110,97,109,101,115,112, - 97,99,101,32,112,97,99,107,97,103,101,39,115,32,112,97, - 116,104,46,32,32,73,116,32,117,115,101,115,32,116,104,101, - 32,109,111,100,117,108,101,32,110,97,109,101,10,32,32,32, - 32,116,111,32,102,105,110,100,32,105,116,115,32,112,97,114, - 101,110,116,32,109,111,100,117,108,101,44,32,97,110,100,32, - 102,114,111,109,32,116,104,101,114,101,32,105,116,32,108,111, - 111,107,115,32,117,112,32,116,104,101,32,112,97,114,101,110, - 116,39,115,10,32,32,32,32,95,95,112,97,116,104,95,95, - 46,32,32,87,104,101,110,32,116,104,105,115,32,99,104,97, - 110,103,101,115,44,32,116,104,101,32,109,111,100,117,108,101, - 39,115,32,111,119,110,32,112,97,116,104,32,105,115,32,114, - 101,99,111,109,112,117,116,101,100,44,10,32,32,32,32,117, - 115,105,110,103,32,112,97,116,104,95,102,105,110,100,101,114, - 46,32,32,70,111,114,32,116,111,112,45,108,101,118,101,108, - 32,109,111,100,117,108,101,115,44,32,116,104,101,32,112,97, - 114,101,110,116,32,109,111,100,117,108,101,39,115,32,112,97, - 116,104,10,32,32,32,32,105,115,32,115,121,115,46,112,97, - 116,104,46,99,4,0,0,0,0,0,0,0,4,0,0,0, - 2,0,0,0,67,0,0,0,115,52,0,0,0,124,1,0, - 124,0,0,95,0,0,124,2,0,124,0,0,95,1,0,116, - 2,0,124,0,0,106,3,0,131,0,0,131,1,0,124,0, - 0,95,4,0,124,3,0,124,0,0,95,5,0,100,0,0, - 83,41,1,78,41,6,218,5,95,110,97,109,101,218,5,95, - 112,97,116,104,114,93,0,0,0,218,16,95,103,101,116,95, - 112,97,114,101,110,116,95,112,97,116,104,218,17,95,108,97, - 115,116,95,112,97,114,101,110,116,95,112,97,116,104,218,12, - 95,112,97,116,104,95,102,105,110,100,101,114,41,4,114,108, - 0,0,0,114,106,0,0,0,114,35,0,0,0,218,11,112, - 97,116,104,95,102,105,110,100,101,114,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,185,0,0,0,173,3, - 0,0,115,8,0,0,0,0,1,9,1,9,1,21,1,122, - 23,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, - 95,95,105,110,105,116,95,95,99,1,0,0,0,0,0,0, - 0,4,0,0,0,3,0,0,0,67,0,0,0,115,53,0, - 0,0,124,0,0,106,0,0,106,1,0,100,1,0,131,1, - 0,92,3,0,125,1,0,125,2,0,125,3,0,124,2,0, - 100,2,0,107,2,0,114,43,0,100,6,0,83,124,1,0, - 100,5,0,102,2,0,83,41,7,122,62,82,101,116,117,114, - 110,115,32,97,32,116,117,112,108,101,32,111,102,32,40,112, - 97,114,101,110,116,45,109,111,100,117,108,101,45,110,97,109, - 101,44,32,112,97,114,101,110,116,45,112,97,116,104,45,97, - 116,116,114,45,110,97,109,101,41,114,58,0,0,0,114,30, - 0,0,0,114,7,0,0,0,114,35,0,0,0,90,8,95, - 95,112,97,116,104,95,95,41,2,122,3,115,121,115,122,4, - 112,97,116,104,41,2,114,231,0,0,0,114,32,0,0,0, - 41,4,114,108,0,0,0,114,222,0,0,0,218,3,100,111, - 116,90,2,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,23,95,102,105,110,100,95,112,97,114,101, - 110,116,95,112,97,116,104,95,110,97,109,101,115,179,3,0, - 0,115,8,0,0,0,0,2,27,1,12,2,4,3,122,38, - 95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,95, - 102,105,110,100,95,112,97,114,101,110,116,95,112,97,116,104, - 95,110,97,109,101,115,99,1,0,0,0,0,0,0,0,3, - 0,0,0,3,0,0,0,67,0,0,0,115,38,0,0,0, - 124,0,0,106,0,0,131,0,0,92,2,0,125,1,0,125, - 2,0,116,1,0,116,2,0,106,3,0,124,1,0,25,124, - 2,0,131,2,0,83,41,1,78,41,4,114,238,0,0,0, - 114,117,0,0,0,114,7,0,0,0,218,7,109,111,100,117, - 108,101,115,41,3,114,108,0,0,0,90,18,112,97,114,101, - 110,116,95,109,111,100,117,108,101,95,110,97,109,101,90,14, - 112,97,116,104,95,97,116,116,114,95,110,97,109,101,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,233,0, - 0,0,189,3,0,0,115,4,0,0,0,0,1,18,1,122, - 31,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, - 95,103,101,116,95,112,97,114,101,110,116,95,112,97,116,104, - 99,1,0,0,0,0,0,0,0,3,0,0,0,3,0,0, - 0,67,0,0,0,115,118,0,0,0,116,0,0,124,0,0, - 106,1,0,131,0,0,131,1,0,125,1,0,124,1,0,124, - 0,0,106,2,0,107,3,0,114,111,0,124,0,0,106,3, - 0,124,0,0,106,4,0,124,1,0,131,2,0,125,2,0, - 124,2,0,100,0,0,107,9,0,114,102,0,124,2,0,106, - 5,0,100,0,0,107,8,0,114,102,0,124,2,0,106,6, - 0,114,102,0,124,2,0,106,6,0,124,0,0,95,7,0, - 124,1,0,124,0,0,95,2,0,124,0,0,106,7,0,83, - 41,1,78,41,8,114,93,0,0,0,114,233,0,0,0,114, - 234,0,0,0,114,235,0,0,0,114,231,0,0,0,114,127, - 0,0,0,114,156,0,0,0,114,232,0,0,0,41,3,114, - 108,0,0,0,90,11,112,97,114,101,110,116,95,112,97,116, - 104,114,164,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,12,95,114,101,99,97,108,99,117,108, - 97,116,101,193,3,0,0,115,16,0,0,0,0,2,18,1, - 15,1,21,3,27,1,9,1,12,1,9,1,122,27,95,78, - 97,109,101,115,112,97,99,101,80,97,116,104,46,95,114,101, - 99,97,108,99,117,108,97,116,101,99,1,0,0,0,0,0, - 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, - 0,0,0,116,0,0,124,0,0,106,1,0,131,0,0,131, - 1,0,83,41,1,78,41,2,218,4,105,116,101,114,114,240, - 0,0,0,41,1,114,108,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,8,95,95,105,116,101, - 114,95,95,206,3,0,0,115,2,0,0,0,0,1,122,23, - 95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,95, - 95,105,116,101,114,95,95,99,1,0,0,0,0,0,0,0, - 1,0,0,0,2,0,0,0,67,0,0,0,115,16,0,0, - 0,116,0,0,124,0,0,106,1,0,131,0,0,131,1,0, - 83,41,1,78,41,2,114,31,0,0,0,114,240,0,0,0, - 41,1,114,108,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,7,95,95,108,101,110,95,95,209, - 3,0,0,115,2,0,0,0,0,1,122,22,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,95,108,101,110, - 95,95,99,1,0,0,0,0,0,0,0,1,0,0,0,2, - 0,0,0,67,0,0,0,115,16,0,0,0,100,1,0,106, - 0,0,124,0,0,106,1,0,131,1,0,83,41,2,78,122, - 20,95,78,97,109,101,115,112,97,99,101,80,97,116,104,40, - 123,33,114,125,41,41,2,114,47,0,0,0,114,232,0,0, - 0,41,1,114,108,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,8,95,95,114,101,112,114,95, - 95,212,3,0,0,115,2,0,0,0,0,1,122,23,95,78, - 97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,114, - 101,112,114,95,95,99,2,0,0,0,0,0,0,0,2,0, - 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,124, - 1,0,124,0,0,106,0,0,131,0,0,107,6,0,83,41, - 1,78,41,1,114,240,0,0,0,41,2,114,108,0,0,0, - 218,4,105,116,101,109,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,12,95,95,99,111,110,116,97,105,110, - 115,95,95,215,3,0,0,115,2,0,0,0,0,1,122,27, - 95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,95, - 95,99,111,110,116,97,105,110,115,95,95,99,2,0,0,0, - 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, - 115,20,0,0,0,124,0,0,106,0,0,106,1,0,124,1, - 0,131,1,0,1,100,0,0,83,41,1,78,41,2,114,232, - 0,0,0,114,163,0,0,0,41,2,114,108,0,0,0,114, - 245,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,163,0,0,0,218,3,0,0,115,2,0,0, - 0,0,1,122,21,95,78,97,109,101,115,112,97,99,101,80, - 97,116,104,46,97,112,112,101,110,100,78,41,13,114,112,0, - 0,0,114,111,0,0,0,114,113,0,0,0,114,114,0,0, - 0,114,185,0,0,0,114,238,0,0,0,114,233,0,0,0, - 114,240,0,0,0,114,242,0,0,0,114,243,0,0,0,114, - 244,0,0,0,114,246,0,0,0,114,163,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,230,0,0,0,166,3,0,0,115,20,0,0,0, - 12,5,6,2,12,6,12,10,12,4,12,13,12,3,12,3, - 12,3,12,3,114,230,0,0,0,99,0,0,0,0,0,0, - 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,118, - 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, - 1,0,100,2,0,132,0,0,90,3,0,101,4,0,100,3, - 0,100,4,0,132,0,0,131,1,0,90,5,0,100,5,0, - 100,6,0,132,0,0,90,6,0,100,7,0,100,8,0,132, - 0,0,90,7,0,100,9,0,100,10,0,132,0,0,90,8, - 0,100,11,0,100,12,0,132,0,0,90,9,0,100,13,0, - 100,14,0,132,0,0,90,10,0,100,15,0,100,16,0,132, - 0,0,90,11,0,100,17,0,83,41,18,218,16,95,78,97, - 109,101,115,112,97,99,101,76,111,97,100,101,114,99,4,0, - 0,0,0,0,0,0,4,0,0,0,4,0,0,0,67,0, - 0,0,115,25,0,0,0,116,0,0,124,1,0,124,2,0, - 124,3,0,131,3,0,124,0,0,95,1,0,100,0,0,83, - 41,1,78,41,2,114,230,0,0,0,114,232,0,0,0,41, - 4,114,108,0,0,0,114,106,0,0,0,114,35,0,0,0, - 114,236,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,185,0,0,0,224,3,0,0,115,2,0, - 0,0,0,1,122,25,95,78,97,109,101,115,112,97,99,101, - 76,111,97,100,101,114,46,95,95,105,110,105,116,95,95,99, - 2,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0, - 67,0,0,0,115,16,0,0,0,100,1,0,106,0,0,124, - 1,0,106,1,0,131,1,0,83,41,2,122,115,82,101,116, - 117,114,110,32,114,101,112,114,32,102,111,114,32,116,104,101, - 32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,32, - 32,32,84,104,101,32,109,101,116,104,111,100,32,105,115,32, - 100,101,112,114,101,99,97,116,101,100,46,32,32,84,104,101, - 32,105,109,112,111,114,116,32,109,97,99,104,105,110,101,114, - 121,32,100,111,101,115,32,116,104,101,32,106,111,98,32,105, - 116,115,101,108,102,46,10,10,32,32,32,32,32,32,32,32, - 122,25,60,109,111,100,117,108,101,32,123,33,114,125,32,40, - 110,97,109,101,115,112,97,99,101,41,62,41,2,114,47,0, - 0,0,114,112,0,0,0,41,2,114,170,0,0,0,114,190, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,11,109,111,100,117,108,101,95,114,101,112,114,227, - 3,0,0,115,2,0,0,0,0,7,122,28,95,78,97,109, - 101,115,112,97,99,101,76,111,97,100,101,114,46,109,111,100, - 117,108,101,95,114,101,112,114,99,2,0,0,0,0,0,0, - 0,2,0,0,0,1,0,0,0,67,0,0,0,115,4,0, - 0,0,100,1,0,83,41,2,78,84,114,4,0,0,0,41, - 2,114,108,0,0,0,114,126,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,159,0,0,0,236, - 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, - 101,115,112,97,99,101,76,111,97,100,101,114,46,105,115,95, + 0,100,7,0,100,8,0,100,9,0,132,0,1,90,6,0, + 100,10,0,83,41,11,218,16,83,111,117,114,99,101,70,105, + 108,101,76,111,97,100,101,114,122,62,67,111,110,99,114,101, + 116,101,32,105,109,112,108,101,109,101,110,116,97,116,105,111, + 110,32,111,102,32,83,111,117,114,99,101,76,111,97,100,101, + 114,32,117,115,105,110,103,32,116,104,101,32,102,105,108,101, + 32,115,121,115,116,101,109,46,99,2,0,0,0,0,0,0, + 0,3,0,0,0,4,0,0,0,67,0,0,0,115,34,0, + 0,0,116,0,0,124,1,0,131,1,0,125,2,0,100,1, + 0,124,2,0,106,1,0,100,2,0,124,2,0,106,2,0, + 105,2,0,83,41,3,122,33,82,101,116,117,114,110,32,116, + 104,101,32,109,101,116,97,100,97,116,97,32,102,111,114,32, + 116,104,101,32,112,97,116,104,46,114,126,0,0,0,114,127, + 0,0,0,41,3,114,39,0,0,0,218,8,115,116,95,109, + 116,105,109,101,90,7,115,116,95,115,105,122,101,41,3,114, + 100,0,0,0,114,35,0,0,0,114,202,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,191,0, + 0,0,47,3,0,0,115,4,0,0,0,0,2,12,1,122, + 27,83,111,117,114,99,101,70,105,108,101,76,111,97,100,101, + 114,46,112,97,116,104,95,115,116,97,116,115,99,4,0,0, + 0,0,0,0,0,5,0,0,0,5,0,0,0,67,0,0, + 0,115,34,0,0,0,116,0,0,124,1,0,131,1,0,125, + 4,0,124,0,0,106,1,0,124,2,0,124,3,0,100,1, + 0,124,4,0,131,2,1,83,41,2,78,218,5,95,109,111, + 100,101,41,2,114,97,0,0,0,114,192,0,0,0,41,5, + 114,100,0,0,0,114,90,0,0,0,114,89,0,0,0,114, + 53,0,0,0,114,42,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,193,0,0,0,52,3,0, + 0,115,4,0,0,0,0,2,12,1,122,32,83,111,117,114, + 99,101,70,105,108,101,76,111,97,100,101,114,46,95,99,97, + 99,104,101,95,98,121,116,101,99,111,100,101,114,214,0,0, + 0,105,182,1,0,0,99,3,0,0,0,1,0,0,0,9, + 0,0,0,17,0,0,0,67,0,0,0,115,62,1,0,0, + 116,0,0,124,1,0,131,1,0,92,2,0,125,4,0,125, + 5,0,103,0,0,125,6,0,120,54,0,124,4,0,114,80, + 0,116,1,0,124,4,0,131,1,0,12,114,80,0,116,0, + 0,124,4,0,131,1,0,92,2,0,125,4,0,125,7,0, + 124,6,0,106,2,0,124,7,0,131,1,0,1,113,27,0, + 87,120,135,0,116,3,0,124,6,0,131,1,0,68,93,121, + 0,125,7,0,116,4,0,124,4,0,124,7,0,131,2,0, + 125,4,0,121,17,0,116,5,0,106,6,0,124,4,0,131, + 1,0,1,87,113,94,0,4,116,7,0,107,10,0,114,155, + 0,1,1,1,119,94,0,89,113,94,0,4,116,8,0,107, + 10,0,114,214,0,1,125,8,0,1,122,28,0,116,9,0, + 106,10,0,100,1,0,124,4,0,124,8,0,131,3,0,1, + 100,2,0,83,87,89,100,2,0,100,2,0,125,8,0,126, + 8,0,88,113,94,0,88,113,94,0,87,121,36,0,116,11, + 0,124,1,0,124,2,0,124,3,0,131,3,0,1,116,9, + 0,106,10,0,100,3,0,124,1,0,131,2,0,1,87,110, + 56,0,4,116,8,0,107,10,0,114,57,1,1,125,8,0, + 1,122,24,0,116,9,0,106,10,0,100,1,0,124,1,0, + 124,8,0,131,3,0,1,87,89,100,2,0,100,2,0,125, + 8,0,126,8,0,88,110,1,0,88,100,2,0,83,41,4, + 122,27,87,114,105,116,101,32,98,121,116,101,115,32,100,97, + 116,97,32,116,111,32,97,32,102,105,108,101,46,122,27,99, + 111,117,108,100,32,110,111,116,32,99,114,101,97,116,101,32, + 123,33,114,125,58,32,123,33,114,125,78,122,12,99,114,101, + 97,116,101,100,32,123,33,114,125,41,12,114,38,0,0,0, + 114,46,0,0,0,114,157,0,0,0,114,33,0,0,0,114, + 28,0,0,0,114,3,0,0,0,90,5,109,107,100,105,114, + 218,15,70,105,108,101,69,120,105,115,116,115,69,114,114,111, + 114,114,40,0,0,0,114,114,0,0,0,114,129,0,0,0, + 114,55,0,0,0,41,9,114,100,0,0,0,114,35,0,0, + 0,114,53,0,0,0,114,214,0,0,0,218,6,112,97,114, + 101,110,116,114,94,0,0,0,114,27,0,0,0,114,23,0, + 0,0,114,195,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,192,0,0,0,57,3,0,0,115, + 42,0,0,0,0,2,18,1,6,2,22,1,18,1,17,2, + 19,1,15,1,3,1,17,1,13,2,7,1,18,3,9,1, + 10,1,27,1,3,1,16,1,20,1,18,2,12,1,122,25, + 83,111,117,114,99,101,70,105,108,101,76,111,97,100,101,114, + 46,115,101,116,95,100,97,116,97,78,41,7,114,105,0,0, + 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, + 114,191,0,0,0,114,193,0,0,0,114,192,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,212,0,0,0,43,3,0,0,115,8,0,0, + 0,12,2,6,2,12,5,12,5,114,212,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, + 0,0,0,115,46,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,90,3,0,100,2,0,100,3,0, + 132,0,0,90,4,0,100,4,0,100,5,0,132,0,0,90, + 5,0,100,6,0,83,41,7,218,20,83,111,117,114,99,101, + 108,101,115,115,70,105,108,101,76,111,97,100,101,114,122,45, + 76,111,97,100,101,114,32,119,104,105,99,104,32,104,97,110, + 100,108,101,115,32,115,111,117,114,99,101,108,101,115,115,32, + 102,105,108,101,32,105,109,112,111,114,116,115,46,99,2,0, + 0,0,0,0,0,0,5,0,0,0,6,0,0,0,67,0, + 0,0,115,76,0,0,0,124,0,0,106,0,0,124,1,0, + 131,1,0,125,2,0,124,0,0,106,1,0,124,2,0,131, + 1,0,125,3,0,116,2,0,124,3,0,100,1,0,124,1, + 0,100,2,0,124,2,0,131,1,2,125,4,0,116,3,0, + 124,4,0,100,1,0,124,1,0,100,3,0,124,2,0,131, + 1,2,83,41,4,78,114,98,0,0,0,114,35,0,0,0, + 114,89,0,0,0,41,4,114,151,0,0,0,114,194,0,0, + 0,114,135,0,0,0,114,141,0,0,0,41,5,114,100,0, + 0,0,114,119,0,0,0,114,35,0,0,0,114,53,0,0, + 0,114,203,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,181,0,0,0,92,3,0,0,115,8, + 0,0,0,0,1,15,1,15,1,24,1,122,29,83,111,117, + 114,99,101,108,101,115,115,70,105,108,101,76,111,97,100,101, + 114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0, + 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, + 4,0,0,0,100,1,0,83,41,2,122,39,82,101,116,117, + 114,110,32,78,111,110,101,32,97,115,32,116,104,101,114,101, + 32,105,115,32,110,111,32,115,111,117,114,99,101,32,99,111, + 100,101,46,78,114,4,0,0,0,41,2,114,100,0,0,0, + 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,196,0,0,0,98,3,0,0,115,2,0, + 0,0,0,2,122,31,83,111,117,114,99,101,108,101,115,115, + 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,115, + 111,117,114,99,101,78,41,6,114,105,0,0,0,114,104,0, + 0,0,114,106,0,0,0,114,107,0,0,0,114,181,0,0, + 0,114,196,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,88, + 3,0,0,115,6,0,0,0,12,2,6,2,12,6,114,217, + 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, + 3,0,0,0,64,0,0,0,115,136,0,0,0,101,0,0, + 90,1,0,100,0,0,90,2,0,100,1,0,90,3,0,100, + 2,0,100,3,0,132,0,0,90,4,0,100,4,0,100,5, + 0,132,0,0,90,5,0,100,6,0,100,7,0,132,0,0, + 90,6,0,100,8,0,100,9,0,132,0,0,90,7,0,100, + 10,0,100,11,0,132,0,0,90,8,0,100,12,0,100,13, + 0,132,0,0,90,9,0,100,14,0,100,15,0,132,0,0, + 90,10,0,100,16,0,100,17,0,132,0,0,90,11,0,101, + 12,0,100,18,0,100,19,0,132,0,0,131,1,0,90,13, + 0,100,20,0,83,41,21,218,19,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,122,93,76,111, + 97,100,101,114,32,102,111,114,32,101,120,116,101,110,115,105, + 111,110,32,109,111,100,117,108,101,115,46,10,10,32,32,32, + 32,84,104,101,32,99,111,110,115,116,114,117,99,116,111,114, + 32,105,115,32,100,101,115,105,103,110,101,100,32,116,111,32, + 119,111,114,107,32,119,105,116,104,32,70,105,108,101,70,105, + 110,100,101,114,46,10,10,32,32,32,32,99,3,0,0,0, + 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, + 115,22,0,0,0,124,1,0,124,0,0,95,0,0,124,2, + 0,124,0,0,95,1,0,100,0,0,83,41,1,78,41,2, + 114,98,0,0,0,114,35,0,0,0,41,3,114,100,0,0, + 0,114,98,0,0,0,114,35,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,115, + 3,0,0,115,4,0,0,0,0,1,9,1,122,28,69,120, + 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, + 114,46,95,95,105,110,105,116,95,95,99,2,0,0,0,0, + 0,0,0,2,0,0,0,2,0,0,0,67,0,0,0,115, + 34,0,0,0,124,0,0,106,0,0,124,1,0,106,0,0, + 107,2,0,111,33,0,124,0,0,106,1,0,124,1,0,106, + 1,0,107,2,0,83,41,1,78,41,2,114,205,0,0,0, + 114,111,0,0,0,41,2,114,100,0,0,0,114,206,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,207,0,0,0,119,3,0,0,115,4,0,0,0,0,1, + 18,1,122,26,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,46,95,95,101,113,95,95,99,1, + 0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,67, + 0,0,0,115,26,0,0,0,116,0,0,124,0,0,106,1, + 0,131,1,0,116,0,0,124,0,0,106,2,0,131,1,0, + 65,83,41,1,78,41,3,114,208,0,0,0,114,98,0,0, + 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,209,0,0, + 0,123,3,0,0,115,2,0,0,0,0,1,122,28,69,120, + 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, + 114,46,95,95,104,97,115,104,95,95,99,2,0,0,0,0, + 0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115, + 50,0,0,0,116,0,0,106,1,0,116,2,0,106,3,0, + 124,1,0,131,2,0,125,2,0,116,0,0,106,4,0,100, + 1,0,124,1,0,106,5,0,124,0,0,106,6,0,131,3, + 0,1,124,2,0,83,41,2,122,38,67,114,101,97,116,101, + 32,97,110,32,117,110,105,116,105,97,108,105,122,101,100,32, + 101,120,116,101,110,115,105,111,110,32,109,111,100,117,108,101, + 122,38,101,120,116,101,110,115,105,111,110,32,109,111,100,117, + 108,101,32,123,33,114,125,32,108,111,97,100,101,100,32,102, + 114,111,109,32,123,33,114,125,41,7,114,114,0,0,0,114, + 182,0,0,0,114,139,0,0,0,90,14,99,114,101,97,116, + 101,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, + 0,0,0,114,35,0,0,0,41,3,114,100,0,0,0,114, + 158,0,0,0,114,184,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,180,0,0,0,126,3,0, + 0,115,10,0,0,0,0,2,6,1,15,1,9,1,16,1, + 122,33,69,120,116,101,110,115,105,111,110,70,105,108,101,76, + 111,97,100,101,114,46,99,114,101,97,116,101,95,109,111,100, + 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 4,0,0,0,67,0,0,0,115,48,0,0,0,116,0,0, + 106,1,0,116,2,0,106,3,0,124,1,0,131,2,0,1, + 116,0,0,106,4,0,100,1,0,124,0,0,106,5,0,124, + 0,0,106,6,0,131,3,0,1,100,2,0,83,41,3,122, + 30,73,110,105,116,105,97,108,105,122,101,32,97,110,32,101, + 120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,122, + 40,101,120,116,101,110,115,105,111,110,32,109,111,100,117,108, + 101,32,123,33,114,125,32,101,120,101,99,117,116,101,100,32, + 102,114,111,109,32,123,33,114,125,78,41,7,114,114,0,0, + 0,114,182,0,0,0,114,139,0,0,0,90,12,101,120,101, + 99,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, + 0,0,0,114,35,0,0,0,41,2,114,100,0,0,0,114, + 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,185,0,0,0,134,3,0,0,115,6,0,0, + 0,0,2,19,1,9,1,122,31,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,101,120,101, + 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, + 0,2,0,0,0,4,0,0,0,3,0,0,0,115,48,0, + 0,0,116,0,0,124,0,0,106,1,0,131,1,0,100,1, + 0,25,137,0,0,116,2,0,135,0,0,102,1,0,100,2, + 0,100,3,0,134,0,0,116,3,0,68,131,1,0,131,1, + 0,83,41,4,122,49,82,101,116,117,114,110,32,84,114,117, + 101,32,105,102,32,116,104,101,32,101,120,116,101,110,115,105, + 111,110,32,109,111,100,117,108,101,32,105,115,32,97,32,112, + 97,99,107,97,103,101,46,114,29,0,0,0,99,1,0,0, + 0,0,0,0,0,2,0,0,0,4,0,0,0,51,0,0, + 0,115,31,0,0,0,124,0,0,93,21,0,125,1,0,136, + 0,0,100,0,0,124,1,0,23,107,2,0,86,1,113,3, + 0,100,1,0,83,41,2,114,179,0,0,0,78,114,4,0, + 0,0,41,2,114,22,0,0,0,218,6,115,117,102,102,105, + 120,41,1,218,9,102,105,108,101,95,110,97,109,101,114,4, + 0,0,0,114,5,0,0,0,250,9,60,103,101,110,101,120, + 112,114,62,143,3,0,0,115,2,0,0,0,6,1,122,49, + 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, + 100,101,114,46,105,115,95,112,97,99,107,97,103,101,46,60, + 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, + 62,41,4,114,38,0,0,0,114,35,0,0,0,218,3,97, + 110,121,218,18,69,88,84,69,78,83,73,79,78,95,83,85, + 70,70,73,88,69,83,41,2,114,100,0,0,0,114,119,0, + 0,0,114,4,0,0,0,41,1,114,220,0,0,0,114,5, + 0,0,0,114,153,0,0,0,140,3,0,0,115,6,0,0, + 0,0,2,19,1,18,1,122,30,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, - 0,100,1,0,83,41,2,78,114,30,0,0,0,114,4,0, - 0,0,41,2,114,108,0,0,0,114,126,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,202,0, - 0,0,239,3,0,0,115,2,0,0,0,0,1,122,27,95, - 78,97,109,101,115,112,97,99,101,76,111,97,100,101,114,46, - 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,6,0,0,0,67,0,0,0,115, - 22,0,0,0,116,0,0,100,1,0,100,2,0,100,3,0, - 100,4,0,100,5,0,131,3,1,83,41,6,78,114,30,0, - 0,0,122,8,60,115,116,114,105,110,103,62,114,189,0,0, - 0,114,204,0,0,0,84,41,1,114,205,0,0,0,41,2, - 114,108,0,0,0,114,126,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,187,0,0,0,242,3, - 0,0,115,2,0,0,0,0,1,122,25,95,78,97,109,101, - 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, - 99,111,100,101,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,122,42,85,115,101,32,100,101,102,97,117,108, - 116,32,115,101,109,97,110,116,105,99,115,32,102,111,114,32, - 109,111,100,117,108,101,32,99,114,101,97,116,105,111,110,46, - 78,114,4,0,0,0,41,2,114,108,0,0,0,114,164,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,186,0,0,0,245,3,0,0,115,0,0,0,0,122, - 30,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, - 114,46,99,114,101,97,116,101,95,109,111,100,117,108,101,99, + 0,100,1,0,83,41,2,122,63,82,101,116,117,114,110,32, + 78,111,110,101,32,97,115,32,97,110,32,101,120,116,101,110, + 115,105,111,110,32,109,111,100,117,108,101,32,99,97,110,110, + 111,116,32,99,114,101,97,116,101,32,97,32,99,111,100,101, + 32,111,98,106,101,99,116,46,78,114,4,0,0,0,41,2, + 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,181,0,0,0,146,3, + 0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,110, + 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,103, + 101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,0, + 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, + 0,100,1,0,83,41,2,122,53,82,101,116,117,114,110,32, + 78,111,110,101,32,97,115,32,101,120,116,101,110,115,105,111, + 110,32,109,111,100,117,108,101,115,32,104,97,118,101,32,110, + 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, + 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, + 196,0,0,0,150,3,0,0,115,2,0,0,0,0,2,122, + 30,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, + 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, - 67,0,0,0,115,4,0,0,0,100,0,0,83,41,1,78, - 114,4,0,0,0,41,2,114,108,0,0,0,114,190,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,191,0,0,0,248,3,0,0,115,2,0,0,0,0,1, - 122,28,95,78,97,109,101,115,112,97,99,101,76,111,97,100, - 101,114,46,101,120,101,99,95,109,111,100,117,108,101,99,2, - 0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,67, - 0,0,0,115,32,0,0,0,116,0,0,100,1,0,124,0, - 0,106,1,0,131,2,0,1,116,2,0,106,3,0,124,0, - 0,124,1,0,131,2,0,83,41,2,122,98,76,111,97,100, - 32,97,32,110,97,109,101,115,112,97,99,101,32,109,111,100, - 117,108,101,46,10,10,32,32,32,32,32,32,32,32,84,104, - 105,115,32,109,101,116,104,111,100,32,105,115,32,100,101,112, - 114,101,99,97,116,101,100,46,32,32,85,115,101,32,101,120, - 101,99,95,109,111,100,117,108,101,40,41,32,105,110,115,116, - 101,97,100,46,10,10,32,32,32,32,32,32,32,32,122,38, - 110,97,109,101,115,112,97,99,101,32,109,111,100,117,108,101, - 32,108,111,97,100,101,100,32,119,105,116,104,32,112,97,116, - 104,32,123,33,114,125,41,4,114,105,0,0,0,114,232,0, - 0,0,114,121,0,0,0,114,192,0,0,0,41,2,114,108, - 0,0,0,114,126,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,193,0,0,0,251,3,0,0, - 115,4,0,0,0,0,7,16,1,122,28,95,78,97,109,101, - 115,112,97,99,101,76,111,97,100,101,114,46,108,111,97,100, - 95,109,111,100,117,108,101,78,41,12,114,112,0,0,0,114, - 111,0,0,0,114,113,0,0,0,114,185,0,0,0,114,183, - 0,0,0,114,248,0,0,0,114,159,0,0,0,114,202,0, - 0,0,114,187,0,0,0,114,186,0,0,0,114,191,0,0, - 0,114,193,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,247,0,0,0,223, - 3,0,0,115,16,0,0,0,12,1,12,3,18,9,12,3, - 12,3,12,3,12,3,12,3,114,247,0,0,0,99,0,0, - 0,0,0,0,0,0,0,0,0,0,5,0,0,0,64,0, - 0,0,115,160,0,0,0,101,0,0,90,1,0,100,0,0, - 90,2,0,100,1,0,90,3,0,101,4,0,100,2,0,100, - 3,0,132,0,0,131,1,0,90,5,0,101,4,0,100,4, - 0,100,5,0,132,0,0,131,1,0,90,6,0,101,4,0, - 100,6,0,100,7,0,132,0,0,131,1,0,90,7,0,101, - 4,0,100,8,0,100,9,0,132,0,0,131,1,0,90,8, - 0,101,4,0,100,10,0,100,11,0,100,12,0,132,1,0, - 131,1,0,90,9,0,101,4,0,100,10,0,100,10,0,100, - 13,0,100,14,0,132,2,0,131,1,0,90,10,0,101,4, - 0,100,10,0,100,15,0,100,16,0,132,1,0,131,1,0, - 90,11,0,100,10,0,83,41,17,218,10,80,97,116,104,70, - 105,110,100,101,114,122,62,77,101,116,97,32,112,97,116,104, - 32,102,105,110,100,101,114,32,102,111,114,32,115,121,115,46, - 112,97,116,104,32,97,110,100,32,112,97,99,107,97,103,101, - 32,95,95,112,97,116,104,95,95,32,97,116,116,114,105,98, - 117,116,101,115,46,99,1,0,0,0,0,0,0,0,2,0, - 0,0,4,0,0,0,67,0,0,0,115,55,0,0,0,120, - 48,0,116,0,0,106,1,0,106,2,0,131,0,0,68,93, - 31,0,125,1,0,116,3,0,124,1,0,100,1,0,131,2, - 0,114,16,0,124,1,0,106,4,0,131,0,0,1,113,16, - 0,87,100,2,0,83,41,3,122,125,67,97,108,108,32,116, - 104,101,32,105,110,118,97,108,105,100,97,116,101,95,99,97, - 99,104,101,115,40,41,32,109,101,116,104,111,100,32,111,110, - 32,97,108,108,32,112,97,116,104,32,101,110,116,114,121,32, - 102,105,110,100,101,114,115,10,32,32,32,32,32,32,32,32, - 115,116,111,114,101,100,32,105,110,32,115,121,115,46,112,97, - 116,104,95,105,109,112,111,114,116,101,114,95,99,97,99,104, - 101,115,32,40,119,104,101,114,101,32,105,109,112,108,101,109, - 101,110,116,101,100,41,46,218,17,105,110,118,97,108,105,100, - 97,116,101,95,99,97,99,104,101,115,78,41,5,114,7,0, - 0,0,218,19,112,97,116,104,95,105,109,112,111,114,116,101, - 114,95,99,97,99,104,101,218,6,118,97,108,117,101,115,114, - 115,0,0,0,114,250,0,0,0,41,2,114,170,0,0,0, - 218,6,102,105,110,100,101,114,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,250,0,0,0,12,4,0,0, - 115,6,0,0,0,0,4,22,1,15,1,122,28,80,97,116, - 104,70,105,110,100,101,114,46,105,110,118,97,108,105,100,97, - 116,101,95,99,97,99,104,101,115,99,2,0,0,0,0,0, - 0,0,3,0,0,0,12,0,0,0,67,0,0,0,115,107, - 0,0,0,116,0,0,106,1,0,100,1,0,107,9,0,114, - 41,0,116,0,0,106,1,0,12,114,41,0,116,2,0,106, - 3,0,100,2,0,116,4,0,131,2,0,1,120,59,0,116, - 0,0,106,1,0,68,93,44,0,125,2,0,121,14,0,124, - 2,0,124,1,0,131,1,0,83,87,113,51,0,4,116,5, - 0,107,10,0,114,94,0,1,1,1,119,51,0,89,113,51, - 0,88,113,51,0,87,100,1,0,83,100,1,0,83,41,3, - 122,113,83,101,97,114,99,104,32,115,101,113,117,101,110,99, - 101,32,111,102,32,104,111,111,107,115,32,102,111,114,32,97, - 32,102,105,110,100,101,114,32,102,111,114,32,39,112,97,116, - 104,39,46,10,10,32,32,32,32,32,32,32,32,73,102,32, - 39,104,111,111,107,115,39,32,105,115,32,102,97,108,115,101, - 32,116,104,101,110,32,117,115,101,32,115,121,115,46,112,97, - 116,104,95,104,111,111,107,115,46,10,10,32,32,32,32,32, - 32,32,32,78,122,23,115,121,115,46,112,97,116,104,95,104, - 111,111,107,115,32,105,115,32,101,109,112,116,121,41,6,114, - 7,0,0,0,218,10,112,97,116,104,95,104,111,111,107,115, - 114,60,0,0,0,114,61,0,0,0,114,125,0,0,0,114, - 107,0,0,0,41,3,114,170,0,0,0,114,35,0,0,0, - 90,4,104,111,111,107,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,11,95,112,97,116,104,95,104,111,111, - 107,115,20,4,0,0,115,16,0,0,0,0,7,25,1,16, - 1,16,1,3,1,14,1,13,1,12,2,122,22,80,97,116, - 104,70,105,110,100,101,114,46,95,112,97,116,104,95,104,111, - 111,107,115,99,2,0,0,0,0,0,0,0,3,0,0,0, - 19,0,0,0,67,0,0,0,115,123,0,0,0,124,1,0, - 100,1,0,107,2,0,114,53,0,121,16,0,116,0,0,106, - 1,0,131,0,0,125,1,0,87,110,22,0,4,116,2,0, - 107,10,0,114,52,0,1,1,1,100,2,0,83,89,110,1, - 0,88,121,17,0,116,3,0,106,4,0,124,1,0,25,125, - 2,0,87,110,46,0,4,116,5,0,107,10,0,114,118,0, - 1,1,1,124,0,0,106,6,0,124,1,0,131,1,0,125, - 2,0,124,2,0,116,3,0,106,4,0,124,1,0,60,89, - 110,1,0,88,124,2,0,83,41,3,122,210,71,101,116,32, - 116,104,101,32,102,105,110,100,101,114,32,102,111,114,32,116, - 104,101,32,112,97,116,104,32,101,110,116,114,121,32,102,114, - 111,109,32,115,121,115,46,112,97,116,104,95,105,109,112,111, - 114,116,101,114,95,99,97,99,104,101,46,10,10,32,32,32, - 32,32,32,32,32,73,102,32,116,104,101,32,112,97,116,104, - 32,101,110,116,114,121,32,105,115,32,110,111,116,32,105,110, - 32,116,104,101,32,99,97,99,104,101,44,32,102,105,110,100, - 32,116,104,101,32,97,112,112,114,111,112,114,105,97,116,101, - 32,102,105,110,100,101,114,10,32,32,32,32,32,32,32,32, - 97,110,100,32,99,97,99,104,101,32,105,116,46,32,73,102, - 32,110,111,32,102,105,110,100,101,114,32,105,115,32,97,118, - 97,105,108,97,98,108,101,44,32,115,116,111,114,101,32,78, - 111,110,101,46,10,10,32,32,32,32,32,32,32,32,114,30, - 0,0,0,78,41,7,114,3,0,0,0,114,45,0,0,0, - 218,17,70,105,108,101,78,111,116,70,111,117,110,100,69,114, - 114,111,114,114,7,0,0,0,114,251,0,0,0,114,137,0, - 0,0,114,255,0,0,0,41,3,114,170,0,0,0,114,35, - 0,0,0,114,253,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,20,95,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,37,4,0, - 0,115,22,0,0,0,0,8,12,1,3,1,16,1,13,3, - 9,1,3,1,17,1,13,1,15,1,18,1,122,31,80,97, - 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,99,3,0, - 0,0,0,0,0,0,6,0,0,0,3,0,0,0,67,0, - 0,0,115,119,0,0,0,116,0,0,124,2,0,100,1,0, - 131,2,0,114,39,0,124,2,0,106,1,0,124,1,0,131, - 1,0,92,2,0,125,3,0,125,4,0,110,21,0,124,2, - 0,106,2,0,124,1,0,131,1,0,125,3,0,103,0,0, - 125,4,0,124,3,0,100,0,0,107,9,0,114,88,0,116, - 3,0,106,4,0,124,1,0,124,3,0,131,2,0,83,116, - 3,0,106,5,0,124,1,0,100,0,0,131,2,0,125,5, - 0,124,4,0,124,5,0,95,6,0,124,5,0,83,41,2, - 78,114,124,0,0,0,41,7,114,115,0,0,0,114,124,0, - 0,0,114,182,0,0,0,114,121,0,0,0,114,179,0,0, - 0,114,160,0,0,0,114,156,0,0,0,41,6,114,170,0, - 0,0,114,126,0,0,0,114,253,0,0,0,114,127,0,0, - 0,114,128,0,0,0,114,164,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,16,95,108,101,103, - 97,99,121,95,103,101,116,95,115,112,101,99,59,4,0,0, - 115,18,0,0,0,0,4,15,1,24,2,15,1,6,1,12, - 1,16,1,18,1,9,1,122,27,80,97,116,104,70,105,110, - 100,101,114,46,95,108,101,103,97,99,121,95,103,101,116,95, - 115,112,101,99,78,99,4,0,0,0,0,0,0,0,9,0, - 0,0,5,0,0,0,67,0,0,0,115,243,0,0,0,103, - 0,0,125,4,0,120,230,0,124,2,0,68,93,191,0,125, - 5,0,116,0,0,124,5,0,116,1,0,116,2,0,102,2, - 0,131,2,0,115,43,0,113,13,0,124,0,0,106,3,0, - 124,5,0,131,1,0,125,6,0,124,6,0,100,1,0,107, - 9,0,114,13,0,116,4,0,124,6,0,100,2,0,131,2, - 0,114,106,0,124,6,0,106,5,0,124,1,0,124,3,0, - 131,2,0,125,7,0,110,18,0,124,0,0,106,6,0,124, - 1,0,124,6,0,131,2,0,125,7,0,124,7,0,100,1, - 0,107,8,0,114,139,0,113,13,0,124,7,0,106,7,0, - 100,1,0,107,9,0,114,158,0,124,7,0,83,124,7,0, - 106,8,0,125,8,0,124,8,0,100,1,0,107,8,0,114, - 191,0,116,9,0,100,3,0,131,1,0,130,1,0,124,4, - 0,106,10,0,124,8,0,131,1,0,1,113,13,0,87,116, - 11,0,106,12,0,124,1,0,100,1,0,131,2,0,125,7, - 0,124,4,0,124,7,0,95,8,0,124,7,0,83,100,1, - 0,83,41,4,122,63,70,105,110,100,32,116,104,101,32,108, - 111,97,100,101,114,32,111,114,32,110,97,109,101,115,112,97, - 99,101,95,112,97,116,104,32,102,111,114,32,116,104,105,115, - 32,109,111,100,117,108,101,47,112,97,99,107,97,103,101,32, - 110,97,109,101,46,78,114,181,0,0,0,122,19,115,112,101, - 99,32,109,105,115,115,105,110,103,32,108,111,97,100,101,114, - 41,13,114,143,0,0,0,114,69,0,0,0,218,5,98,121, - 116,101,115,114,1,1,0,0,114,115,0,0,0,114,181,0, - 0,0,114,2,1,0,0,114,127,0,0,0,114,156,0,0, - 0,114,107,0,0,0,114,149,0,0,0,114,121,0,0,0, - 114,160,0,0,0,41,9,114,170,0,0,0,114,126,0,0, - 0,114,35,0,0,0,114,180,0,0,0,218,14,110,97,109, - 101,115,112,97,99,101,95,112,97,116,104,90,5,101,110,116, - 114,121,114,253,0,0,0,114,164,0,0,0,114,128,0,0, + 67,0,0,0,115,7,0,0,0,124,0,0,106,0,0,83, + 41,1,122,58,82,101,116,117,114,110,32,116,104,101,32,112, + 97,116,104,32,116,111,32,116,104,101,32,115,111,117,114,99, + 101,32,102,105,108,101,32,97,115,32,102,111,117,110,100,32, + 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, + 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,9,95,103,101,116,95,115,112,101,99,74,4,0,0,115, - 40,0,0,0,0,5,6,1,13,1,21,1,3,1,15,1, - 12,1,15,1,21,2,18,1,12,1,3,1,15,1,4,1, - 9,1,12,1,12,5,17,2,18,1,9,1,122,20,80,97, - 116,104,70,105,110,100,101,114,46,95,103,101,116,95,115,112, - 101,99,99,4,0,0,0,0,0,0,0,6,0,0,0,4, - 0,0,0,67,0,0,0,115,140,0,0,0,124,2,0,100, - 1,0,107,8,0,114,21,0,116,0,0,106,1,0,125,2, - 0,124,0,0,106,2,0,124,1,0,124,2,0,124,3,0, - 131,3,0,125,4,0,124,4,0,100,1,0,107,8,0,114, - 58,0,100,1,0,83,124,4,0,106,3,0,100,1,0,107, - 8,0,114,132,0,124,4,0,106,4,0,125,5,0,124,5, - 0,114,125,0,100,2,0,124,4,0,95,5,0,116,6,0, - 124,1,0,124,5,0,124,0,0,106,2,0,131,3,0,124, - 4,0,95,4,0,124,4,0,83,100,1,0,83,110,4,0, - 124,4,0,83,100,1,0,83,41,3,122,98,102,105,110,100, - 32,116,104,101,32,109,111,100,117,108,101,32,111,110,32,115, - 121,115,46,112,97,116,104,32,111,114,32,39,112,97,116,104, - 39,32,98,97,115,101,100,32,111,110,32,115,121,115,46,112, - 97,116,104,95,104,111,111,107,115,32,97,110,100,10,32,32, - 32,32,32,32,32,32,115,121,115,46,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,46,78,90, - 9,110,97,109,101,115,112,97,99,101,41,7,114,7,0,0, - 0,114,35,0,0,0,114,5,1,0,0,114,127,0,0,0, - 114,156,0,0,0,114,158,0,0,0,114,230,0,0,0,41, - 6,114,170,0,0,0,114,126,0,0,0,114,35,0,0,0, - 114,180,0,0,0,114,164,0,0,0,114,4,1,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,181, - 0,0,0,106,4,0,0,115,26,0,0,0,0,4,12,1, - 9,1,21,1,12,1,4,1,15,1,9,1,6,3,9,1, - 24,1,4,2,7,2,122,20,80,97,116,104,70,105,110,100, - 101,114,46,102,105,110,100,95,115,112,101,99,99,3,0,0, - 0,0,0,0,0,4,0,0,0,3,0,0,0,67,0,0, - 0,115,41,0,0,0,124,0,0,106,0,0,124,1,0,124, - 2,0,131,2,0,125,3,0,124,3,0,100,1,0,107,8, - 0,114,34,0,100,1,0,83,124,3,0,106,1,0,83,41, - 2,122,170,102,105,110,100,32,116,104,101,32,109,111,100,117, - 108,101,32,111,110,32,115,121,115,46,112,97,116,104,32,111, - 114,32,39,112,97,116,104,39,32,98,97,115,101,100,32,111, - 110,32,115,121,115,46,112,97,116,104,95,104,111,111,107,115, - 32,97,110,100,10,32,32,32,32,32,32,32,32,115,121,115, - 46,112,97,116,104,95,105,109,112,111,114,116,101,114,95,99, - 97,99,104,101,46,10,10,32,32,32,32,32,32,32,32,84, - 104,105,115,32,109,101,116,104,111,100,32,105,115,32,100,101, - 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,102, - 105,110,100,95,115,112,101,99,40,41,32,105,110,115,116,101, - 97,100,46,10,10,32,32,32,32,32,32,32,32,78,41,2, - 114,181,0,0,0,114,127,0,0,0,41,4,114,170,0,0, - 0,114,126,0,0,0,114,35,0,0,0,114,164,0,0,0, + 114,151,0,0,0,154,3,0,0,115,2,0,0,0,0,3, + 122,32,69,120,116,101,110,115,105,111,110,70,105,108,101,76, + 111,97,100,101,114,46,103,101,116,95,102,105,108,101,110,97, + 109,101,78,41,14,114,105,0,0,0,114,104,0,0,0,114, + 106,0,0,0,114,107,0,0,0,114,179,0,0,0,114,207, + 0,0,0,114,209,0,0,0,114,180,0,0,0,114,185,0, + 0,0,114,153,0,0,0,114,181,0,0,0,114,196,0,0, + 0,114,116,0,0,0,114,151,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 182,0,0,0,128,4,0,0,115,8,0,0,0,0,8,18, - 1,12,1,4,1,122,22,80,97,116,104,70,105,110,100,101, - 114,46,102,105,110,100,95,109,111,100,117,108,101,41,12,114, - 112,0,0,0,114,111,0,0,0,114,113,0,0,0,114,114, - 0,0,0,114,183,0,0,0,114,250,0,0,0,114,255,0, - 0,0,114,1,1,0,0,114,2,1,0,0,114,5,1,0, - 0,114,181,0,0,0,114,182,0,0,0,114,4,0,0,0, + 218,0,0,0,107,3,0,0,115,20,0,0,0,12,6,6, + 2,12,4,12,4,12,3,12,8,12,6,12,6,12,4,12, + 4,114,218,0,0,0,99,0,0,0,0,0,0,0,0,0, + 0,0,0,2,0,0,0,64,0,0,0,115,130,0,0,0, + 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, + 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, + 0,100,5,0,132,0,0,90,5,0,100,6,0,100,7,0, + 132,0,0,90,6,0,100,8,0,100,9,0,132,0,0,90, + 7,0,100,10,0,100,11,0,132,0,0,90,8,0,100,12, + 0,100,13,0,132,0,0,90,9,0,100,14,0,100,15,0, + 132,0,0,90,10,0,100,16,0,100,17,0,132,0,0,90, + 11,0,100,18,0,100,19,0,132,0,0,90,12,0,100,20, + 0,83,41,21,218,14,95,78,97,109,101,115,112,97,99,101, + 80,97,116,104,97,38,1,0,0,82,101,112,114,101,115,101, + 110,116,115,32,97,32,110,97,109,101,115,112,97,99,101,32, + 112,97,99,107,97,103,101,39,115,32,112,97,116,104,46,32, + 32,73,116,32,117,115,101,115,32,116,104,101,32,109,111,100, + 117,108,101,32,110,97,109,101,10,32,32,32,32,116,111,32, + 102,105,110,100,32,105,116,115,32,112,97,114,101,110,116,32, + 109,111,100,117,108,101,44,32,97,110,100,32,102,114,111,109, + 32,116,104,101,114,101,32,105,116,32,108,111,111,107,115,32, + 117,112,32,116,104,101,32,112,97,114,101,110,116,39,115,10, + 32,32,32,32,95,95,112,97,116,104,95,95,46,32,32,87, + 104,101,110,32,116,104,105,115,32,99,104,97,110,103,101,115, + 44,32,116,104,101,32,109,111,100,117,108,101,39,115,32,111, + 119,110,32,112,97,116,104,32,105,115,32,114,101,99,111,109, + 112,117,116,101,100,44,10,32,32,32,32,117,115,105,110,103, + 32,112,97,116,104,95,102,105,110,100,101,114,46,32,32,70, + 111,114,32,116,111,112,45,108,101,118,101,108,32,109,111,100, + 117,108,101,115,44,32,116,104,101,32,112,97,114,101,110,116, + 32,109,111,100,117,108,101,39,115,32,112,97,116,104,10,32, + 32,32,32,105,115,32,115,121,115,46,112,97,116,104,46,99, + 4,0,0,0,0,0,0,0,4,0,0,0,2,0,0,0, + 67,0,0,0,115,52,0,0,0,124,1,0,124,0,0,95, + 0,0,124,2,0,124,0,0,95,1,0,116,2,0,124,0, + 0,106,3,0,131,0,0,131,1,0,124,0,0,95,4,0, + 124,3,0,124,0,0,95,5,0,100,0,0,83,41,1,78, + 41,6,218,5,95,110,97,109,101,218,5,95,112,97,116,104, + 114,93,0,0,0,218,16,95,103,101,116,95,112,97,114,101, + 110,116,95,112,97,116,104,218,17,95,108,97,115,116,95,112, + 97,114,101,110,116,95,112,97,116,104,218,12,95,112,97,116, + 104,95,102,105,110,100,101,114,41,4,114,100,0,0,0,114, + 98,0,0,0,114,35,0,0,0,218,11,112,97,116,104,95, + 102,105,110,100,101,114,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,179,0,0,0,167,3,0,0,115,8, + 0,0,0,0,1,9,1,9,1,21,1,122,23,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,110, + 105,116,95,95,99,1,0,0,0,0,0,0,0,4,0,0, + 0,3,0,0,0,67,0,0,0,115,53,0,0,0,124,0, + 0,106,0,0,106,1,0,100,1,0,131,1,0,92,3,0, + 125,1,0,125,2,0,125,3,0,124,2,0,100,2,0,107, + 2,0,114,43,0,100,6,0,83,124,1,0,100,5,0,102, + 2,0,83,41,7,122,62,82,101,116,117,114,110,115,32,97, + 32,116,117,112,108,101,32,111,102,32,40,112,97,114,101,110, + 116,45,109,111,100,117,108,101,45,110,97,109,101,44,32,112, + 97,114,101,110,116,45,112,97,116,104,45,97,116,116,114,45, + 110,97,109,101,41,114,58,0,0,0,114,30,0,0,0,114, + 7,0,0,0,114,35,0,0,0,90,8,95,95,112,97,116, + 104,95,95,41,2,122,3,115,121,115,122,4,112,97,116,104, + 41,2,114,225,0,0,0,114,32,0,0,0,41,4,114,100, + 0,0,0,114,216,0,0,0,218,3,100,111,116,90,2,109, + 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 218,23,95,102,105,110,100,95,112,97,114,101,110,116,95,112, + 97,116,104,95,110,97,109,101,115,173,3,0,0,115,8,0, + 0,0,0,2,27,1,12,2,4,3,122,38,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,102,105,110,100, + 95,112,97,114,101,110,116,95,112,97,116,104,95,110,97,109, + 101,115,99,1,0,0,0,0,0,0,0,3,0,0,0,3, + 0,0,0,67,0,0,0,115,38,0,0,0,124,0,0,106, + 0,0,131,0,0,92,2,0,125,1,0,125,2,0,116,1, + 0,116,2,0,106,3,0,124,1,0,25,124,2,0,131,2, + 0,83,41,1,78,41,4,114,232,0,0,0,114,110,0,0, + 0,114,7,0,0,0,218,7,109,111,100,117,108,101,115,41, + 3,114,100,0,0,0,90,18,112,97,114,101,110,116,95,109, + 111,100,117,108,101,95,110,97,109,101,90,14,112,97,116,104, + 95,97,116,116,114,95,110,97,109,101,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,227,0,0,0,183,3, + 0,0,115,4,0,0,0,0,1,18,1,122,31,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,46,95,103,101,116, + 95,112,97,114,101,110,116,95,112,97,116,104,99,1,0,0, + 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, + 0,115,118,0,0,0,116,0,0,124,0,0,106,1,0,131, + 0,0,131,1,0,125,1,0,124,1,0,124,0,0,106,2, + 0,107,3,0,114,111,0,124,0,0,106,3,0,124,0,0, + 106,4,0,124,1,0,131,2,0,125,2,0,124,2,0,100, + 0,0,107,9,0,114,102,0,124,2,0,106,5,0,100,0, + 0,107,8,0,114,102,0,124,2,0,106,6,0,114,102,0, + 124,2,0,106,6,0,124,0,0,95,7,0,124,1,0,124, + 0,0,95,2,0,124,0,0,106,7,0,83,41,1,78,41, + 8,114,93,0,0,0,114,227,0,0,0,114,228,0,0,0, + 114,229,0,0,0,114,225,0,0,0,114,120,0,0,0,114, + 150,0,0,0,114,226,0,0,0,41,3,114,100,0,0,0, + 90,11,112,97,114,101,110,116,95,112,97,116,104,114,158,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,187, + 3,0,0,115,16,0,0,0,0,2,18,1,15,1,21,3, + 27,1,9,1,12,1,9,1,122,27,95,78,97,109,101,115, + 112,97,99,101,80,97,116,104,46,95,114,101,99,97,108,99, + 117,108,97,116,101,99,1,0,0,0,0,0,0,0,1,0, + 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,116, + 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, + 1,78,41,2,218,4,105,116,101,114,114,234,0,0,0,41, + 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,200, + 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,95,105,116,101, + 114,95,95,99,1,0,0,0,0,0,0,0,1,0,0,0, + 2,0,0,0,67,0,0,0,115,16,0,0,0,116,0,0, + 124,0,0,106,1,0,131,0,0,131,1,0,83,41,1,78, + 41,2,114,31,0,0,0,114,234,0,0,0,41,1,114,100, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,7,95,95,108,101,110,95,95,203,3,0,0,115, + 2,0,0,0,0,1,122,22,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,46,95,95,108,101,110,95,95,99,1, + 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, + 0,0,0,115,16,0,0,0,100,1,0,106,0,0,124,0, + 0,106,1,0,131,1,0,83,41,2,78,122,20,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,40,123,33,114,125, + 41,41,2,114,47,0,0,0,114,226,0,0,0,41,1,114, + 100,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,8,95,95,114,101,112,114,95,95,206,3,0, + 0,115,2,0,0,0,0,1,122,23,95,78,97,109,101,115, + 112,97,99,101,80,97,116,104,46,95,95,114,101,112,114,95, + 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, + 0,0,67,0,0,0,115,16,0,0,0,124,1,0,124,0, + 0,106,0,0,131,0,0,107,6,0,83,41,1,78,41,1, + 114,234,0,0,0,41,2,114,100,0,0,0,218,4,105,116, + 101,109,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,209, + 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,95,99,111,110, + 116,97,105,110,115,95,95,99,2,0,0,0,0,0,0,0, + 2,0,0,0,2,0,0,0,67,0,0,0,115,20,0,0, + 0,124,0,0,106,0,0,106,1,0,124,1,0,131,1,0, + 1,100,0,0,83,41,1,78,41,2,114,226,0,0,0,114, + 157,0,0,0,41,2,114,100,0,0,0,114,239,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 249,0,0,0,8,4,0,0,115,22,0,0,0,12,2,6, - 2,18,8,18,17,18,22,18,15,3,1,18,31,3,1,21, - 21,3,1,114,249,0,0,0,99,0,0,0,0,0,0,0, - 0,0,0,0,0,3,0,0,0,64,0,0,0,115,133,0, - 0,0,101,0,0,90,1,0,100,0,0,90,2,0,100,1, - 0,90,3,0,100,2,0,100,3,0,132,0,0,90,4,0, - 100,4,0,100,5,0,132,0,0,90,5,0,101,6,0,90, - 7,0,100,6,0,100,7,0,132,0,0,90,8,0,100,8, - 0,100,9,0,132,0,0,90,9,0,100,10,0,100,11,0, - 100,12,0,132,1,0,90,10,0,100,13,0,100,14,0,132, - 0,0,90,11,0,101,12,0,100,15,0,100,16,0,132,0, - 0,131,1,0,90,13,0,100,17,0,100,18,0,132,0,0, - 90,14,0,100,10,0,83,41,19,218,10,70,105,108,101,70, - 105,110,100,101,114,122,172,70,105,108,101,45,98,97,115,101, - 100,32,102,105,110,100,101,114,46,10,10,32,32,32,32,73, - 110,116,101,114,97,99,116,105,111,110,115,32,119,105,116,104, - 32,116,104,101,32,102,105,108,101,32,115,121,115,116,101,109, - 32,97,114,101,32,99,97,99,104,101,100,32,102,111,114,32, - 112,101,114,102,111,114,109,97,110,99,101,44,32,98,101,105, - 110,103,10,32,32,32,32,114,101,102,114,101,115,104,101,100, - 32,119,104,101,110,32,116,104,101,32,100,105,114,101,99,116, - 111,114,121,32,116,104,101,32,102,105,110,100,101,114,32,105, - 115,32,104,97,110,100,108,105,110,103,32,104,97,115,32,98, - 101,101,110,32,109,111,100,105,102,105,101,100,46,10,10,32, - 32,32,32,99,2,0,0,0,0,0,0,0,5,0,0,0, - 5,0,0,0,7,0,0,0,115,122,0,0,0,103,0,0, - 125,3,0,120,52,0,124,2,0,68,93,44,0,92,2,0, - 137,0,0,125,4,0,124,3,0,106,0,0,135,0,0,102, - 1,0,100,1,0,100,2,0,134,0,0,124,4,0,68,131, - 1,0,131,1,0,1,113,13,0,87,124,3,0,124,0,0, - 95,1,0,124,1,0,112,79,0,100,3,0,124,0,0,95, - 2,0,100,6,0,124,0,0,95,3,0,116,4,0,131,0, - 0,124,0,0,95,5,0,116,4,0,131,0,0,124,0,0, - 95,6,0,100,5,0,83,41,7,122,154,73,110,105,116,105, - 97,108,105,122,101,32,119,105,116,104,32,116,104,101,32,112, - 97,116,104,32,116,111,32,115,101,97,114,99,104,32,111,110, - 32,97,110,100,32,97,32,118,97,114,105,97,98,108,101,32, - 110,117,109,98,101,114,32,111,102,10,32,32,32,32,32,32, - 32,32,50,45,116,117,112,108,101,115,32,99,111,110,116,97, - 105,110,105,110,103,32,116,104,101,32,108,111,97,100,101,114, - 32,97,110,100,32,116,104,101,32,102,105,108,101,32,115,117, - 102,102,105,120,101,115,32,116,104,101,32,108,111,97,100,101, - 114,10,32,32,32,32,32,32,32,32,114,101,99,111,103,110, - 105,122,101,115,46,99,1,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,51,0,0,0,115,27,0,0,0,124, - 0,0,93,17,0,125,1,0,124,1,0,136,0,0,102,2, - 0,86,1,113,3,0,100,0,0,83,41,1,78,114,4,0, - 0,0,41,2,114,22,0,0,0,114,225,0,0,0,41,1, - 114,127,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 227,0,0,0,157,4,0,0,115,2,0,0,0,6,0,122, - 38,70,105,108,101,70,105,110,100,101,114,46,95,95,105,110, - 105,116,95,95,46,60,108,111,99,97,108,115,62,46,60,103, - 101,110,101,120,112,114,62,114,58,0,0,0,114,29,0,0, - 0,78,114,87,0,0,0,41,7,114,149,0,0,0,218,8, - 95,108,111,97,100,101,114,115,114,35,0,0,0,218,11,95, - 112,97,116,104,95,109,116,105,109,101,218,3,115,101,116,218, - 11,95,112,97,116,104,95,99,97,99,104,101,218,19,95,114, - 101,108,97,120,101,100,95,112,97,116,104,95,99,97,99,104, - 101,41,5,114,108,0,0,0,114,35,0,0,0,218,14,108, - 111,97,100,101,114,95,100,101,116,97,105,108,115,90,7,108, - 111,97,100,101,114,115,114,166,0,0,0,114,4,0,0,0, - 41,1,114,127,0,0,0,114,5,0,0,0,114,185,0,0, - 0,151,4,0,0,115,16,0,0,0,0,4,6,1,19,1, - 36,1,9,2,15,1,9,1,12,1,122,19,70,105,108,101, - 70,105,110,100,101,114,46,95,95,105,110,105,116,95,95,99, - 1,0,0,0,0,0,0,0,1,0,0,0,2,0,0,0, - 67,0,0,0,115,13,0,0,0,100,3,0,124,0,0,95, - 0,0,100,2,0,83,41,4,122,31,73,110,118,97,108,105, - 100,97,116,101,32,116,104,101,32,100,105,114,101,99,116,111, - 114,121,32,109,116,105,109,101,46,114,29,0,0,0,78,114, - 87,0,0,0,41,1,114,8,1,0,0,41,1,114,108,0, + 157,0,0,0,212,3,0,0,115,2,0,0,0,0,1,122, + 21,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, + 97,112,112,101,110,100,78,41,13,114,105,0,0,0,114,104, + 0,0,0,114,106,0,0,0,114,107,0,0,0,114,179,0, + 0,0,114,232,0,0,0,114,227,0,0,0,114,234,0,0, + 0,114,236,0,0,0,114,237,0,0,0,114,238,0,0,0, + 114,240,0,0,0,114,157,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,224, + 0,0,0,160,3,0,0,115,20,0,0,0,12,5,6,2, + 12,6,12,10,12,4,12,13,12,3,12,3,12,3,12,3, + 114,224,0,0,0,99,0,0,0,0,0,0,0,0,0,0, + 0,0,3,0,0,0,64,0,0,0,115,118,0,0,0,101, + 0,0,90,1,0,100,0,0,90,2,0,100,1,0,100,2, + 0,132,0,0,90,3,0,101,4,0,100,3,0,100,4,0, + 132,0,0,131,1,0,90,5,0,100,5,0,100,6,0,132, + 0,0,90,6,0,100,7,0,100,8,0,132,0,0,90,7, + 0,100,9,0,100,10,0,132,0,0,90,8,0,100,11,0, + 100,12,0,132,0,0,90,9,0,100,13,0,100,14,0,132, + 0,0,90,10,0,100,15,0,100,16,0,132,0,0,90,11, + 0,100,17,0,83,41,18,218,16,95,78,97,109,101,115,112, + 97,99,101,76,111,97,100,101,114,99,4,0,0,0,0,0, + 0,0,4,0,0,0,4,0,0,0,67,0,0,0,115,25, + 0,0,0,116,0,0,124,1,0,124,2,0,124,3,0,131, + 3,0,124,0,0,95,1,0,100,0,0,83,41,1,78,41, + 2,114,224,0,0,0,114,226,0,0,0,41,4,114,100,0, + 0,0,114,98,0,0,0,114,35,0,0,0,114,230,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,179,0,0,0,218,3,0,0,115,2,0,0,0,0,1, + 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, + 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, + 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, + 115,16,0,0,0,100,1,0,106,0,0,124,1,0,106,1, + 0,131,1,0,83,41,2,122,115,82,101,116,117,114,110,32, + 114,101,112,114,32,102,111,114,32,116,104,101,32,109,111,100, + 117,108,101,46,10,10,32,32,32,32,32,32,32,32,84,104, + 101,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, + 101,99,97,116,101,100,46,32,32,84,104,101,32,105,109,112, + 111,114,116,32,109,97,99,104,105,110,101,114,121,32,100,111, + 101,115,32,116,104,101,32,106,111,98,32,105,116,115,101,108, + 102,46,10,10,32,32,32,32,32,32,32,32,122,25,60,109, + 111,100,117,108,101,32,123,33,114,125,32,40,110,97,109,101, + 115,112,97,99,101,41,62,41,2,114,47,0,0,0,114,105, + 0,0,0,41,2,114,164,0,0,0,114,184,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, + 109,111,100,117,108,101,95,114,101,112,114,221,3,0,0,115, + 2,0,0,0,0,7,122,28,95,78,97,109,101,115,112,97, + 99,101,76,111,97,100,101,114,46,109,111,100,117,108,101,95, + 114,101,112,114,99,2,0,0,0,0,0,0,0,2,0,0, + 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, + 0,83,41,2,78,84,114,4,0,0,0,41,2,114,100,0, + 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,153,0,0,0,230,3,0,0,115, + 2,0,0,0,0,1,122,27,95,78,97,109,101,115,112,97, + 99,101,76,111,97,100,101,114,46,105,115,95,112,97,99,107, + 97,103,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,0, + 83,41,2,78,114,30,0,0,0,114,4,0,0,0,41,2, + 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,196,0,0,0,233,3, + 0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,101, + 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, + 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, + 0,0,0,6,0,0,0,67,0,0,0,115,22,0,0,0, + 116,0,0,100,1,0,100,2,0,100,3,0,100,4,0,100, + 5,0,131,3,1,83,41,6,78,114,30,0,0,0,122,8, + 60,115,116,114,105,110,103,62,114,183,0,0,0,114,198,0, + 0,0,84,41,1,114,199,0,0,0,41,2,114,100,0,0, + 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,181,0,0,0,236,3,0,0,115,2, + 0,0,0,0,1,122,25,95,78,97,109,101,115,112,97,99, + 101,76,111,97,100,101,114,46,103,101,116,95,99,111,100,101, + 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, + 0,67,0,0,0,115,4,0,0,0,100,1,0,83,41,2, + 122,42,85,115,101,32,100,101,102,97,117,108,116,32,115,101, + 109,97,110,116,105,99,115,32,102,111,114,32,109,111,100,117, + 108,101,32,99,114,101,97,116,105,111,110,46,78,114,4,0, + 0,0,41,2,114,100,0,0,0,114,158,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,180,0, + 0,0,239,3,0,0,115,0,0,0,0,122,30,95,78,97, + 109,101,115,112,97,99,101,76,111,97,100,101,114,46,99,114, + 101,97,116,101,95,109,111,100,117,108,101,99,2,0,0,0, + 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, + 115,4,0,0,0,100,0,0,83,41,1,78,114,4,0,0, + 0,41,2,114,100,0,0,0,114,184,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,185,0,0, + 0,242,3,0,0,115,2,0,0,0,0,1,122,28,95,78, + 97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,101, + 120,101,99,95,109,111,100,117,108,101,99,2,0,0,0,0, + 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, + 35,0,0,0,116,0,0,106,1,0,100,1,0,124,0,0, + 106,2,0,131,2,0,1,116,0,0,106,3,0,124,0,0, + 124,1,0,131,2,0,83,41,2,122,98,76,111,97,100,32, + 97,32,110,97,109,101,115,112,97,99,101,32,109,111,100,117, + 108,101,46,10,10,32,32,32,32,32,32,32,32,84,104,105, + 115,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, + 101,99,97,116,101,100,46,32,32,85,115,101,32,101,120,101, + 99,95,109,111,100,117,108,101,40,41,32,105,110,115,116,101, + 97,100,46,10,10,32,32,32,32,32,32,32,32,122,38,110, + 97,109,101,115,112,97,99,101,32,109,111,100,117,108,101,32, + 108,111,97,100,101,100,32,119,105,116,104,32,112,97,116,104, + 32,123,33,114,125,41,4,114,114,0,0,0,114,129,0,0, + 0,114,226,0,0,0,114,186,0,0,0,41,2,114,100,0, + 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,187,0,0,0,245,3,0,0,115, + 6,0,0,0,0,7,9,1,10,1,122,28,95,78,97,109, + 101,115,112,97,99,101,76,111,97,100,101,114,46,108,111,97, + 100,95,109,111,100,117,108,101,78,41,12,114,105,0,0,0, + 114,104,0,0,0,114,106,0,0,0,114,179,0,0,0,114, + 177,0,0,0,114,242,0,0,0,114,153,0,0,0,114,196, + 0,0,0,114,181,0,0,0,114,180,0,0,0,114,185,0, + 0,0,114,187,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,241,0,0,0, + 217,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, + 3,12,3,12,3,12,3,12,3,114,241,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,64, + 0,0,0,115,160,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,90,3,0,101,4,0,100,2,0, + 100,3,0,132,0,0,131,1,0,90,5,0,101,4,0,100, + 4,0,100,5,0,132,0,0,131,1,0,90,6,0,101,4, + 0,100,6,0,100,7,0,132,0,0,131,1,0,90,7,0, + 101,4,0,100,8,0,100,9,0,132,0,0,131,1,0,90, + 8,0,101,4,0,100,10,0,100,11,0,100,12,0,132,1, + 0,131,1,0,90,9,0,101,4,0,100,10,0,100,10,0, + 100,13,0,100,14,0,132,2,0,131,1,0,90,10,0,101, + 4,0,100,10,0,100,15,0,100,16,0,132,1,0,131,1, + 0,90,11,0,100,10,0,83,41,17,218,10,80,97,116,104, + 70,105,110,100,101,114,122,62,77,101,116,97,32,112,97,116, + 104,32,102,105,110,100,101,114,32,102,111,114,32,115,121,115, + 46,112,97,116,104,32,97,110,100,32,112,97,99,107,97,103, + 101,32,95,95,112,97,116,104,95,95,32,97,116,116,114,105, + 98,117,116,101,115,46,99,1,0,0,0,0,0,0,0,2, + 0,0,0,4,0,0,0,67,0,0,0,115,55,0,0,0, + 120,48,0,116,0,0,106,1,0,106,2,0,131,0,0,68, + 93,31,0,125,1,0,116,3,0,124,1,0,100,1,0,131, + 2,0,114,16,0,124,1,0,106,4,0,131,0,0,1,113, + 16,0,87,100,2,0,83,41,3,122,125,67,97,108,108,32, + 116,104,101,32,105,110,118,97,108,105,100,97,116,101,95,99, + 97,99,104,101,115,40,41,32,109,101,116,104,111,100,32,111, + 110,32,97,108,108,32,112,97,116,104,32,101,110,116,114,121, + 32,102,105,110,100,101,114,115,10,32,32,32,32,32,32,32, + 32,115,116,111,114,101,100,32,105,110,32,115,121,115,46,112, + 97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99, + 104,101,115,32,40,119,104,101,114,101,32,105,109,112,108,101, + 109,101,110,116,101,100,41,46,218,17,105,110,118,97,108,105, + 100,97,116,101,95,99,97,99,104,101,115,78,41,5,114,7, + 0,0,0,218,19,112,97,116,104,95,105,109,112,111,114,116, + 101,114,95,99,97,99,104,101,218,6,118,97,108,117,101,115, + 114,108,0,0,0,114,244,0,0,0,41,2,114,164,0,0, + 0,218,6,102,105,110,100,101,114,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,244,0,0,0,7,4,0, + 0,115,6,0,0,0,0,4,22,1,15,1,122,28,80,97, + 116,104,70,105,110,100,101,114,46,105,110,118,97,108,105,100, + 97,116,101,95,99,97,99,104,101,115,99,2,0,0,0,0, + 0,0,0,3,0,0,0,12,0,0,0,67,0,0,0,115, + 107,0,0,0,116,0,0,106,1,0,100,1,0,107,9,0, + 114,41,0,116,0,0,106,1,0,12,114,41,0,116,2,0, + 106,3,0,100,2,0,116,4,0,131,2,0,1,120,59,0, + 116,0,0,106,1,0,68,93,44,0,125,2,0,121,14,0, + 124,2,0,124,1,0,131,1,0,83,87,113,51,0,4,116, + 5,0,107,10,0,114,94,0,1,1,1,119,51,0,89,113, + 51,0,88,113,51,0,87,100,1,0,83,100,1,0,83,41, + 3,122,113,83,101,97,114,99,104,32,115,101,113,117,101,110, + 99,101,32,111,102,32,104,111,111,107,115,32,102,111,114,32, + 97,32,102,105,110,100,101,114,32,102,111,114,32,39,112,97, + 116,104,39,46,10,10,32,32,32,32,32,32,32,32,73,102, + 32,39,104,111,111,107,115,39,32,105,115,32,102,97,108,115, + 101,32,116,104,101,110,32,117,115,101,32,115,121,115,46,112, + 97,116,104,95,104,111,111,107,115,46,10,10,32,32,32,32, + 32,32,32,32,78,122,23,115,121,115,46,112,97,116,104,95, + 104,111,111,107,115,32,105,115,32,101,109,112,116,121,41,6, + 114,7,0,0,0,218,10,112,97,116,104,95,104,111,111,107, + 115,114,60,0,0,0,114,61,0,0,0,114,118,0,0,0, + 114,99,0,0,0,41,3,114,164,0,0,0,114,35,0,0, + 0,90,4,104,111,111,107,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,11,95,112,97,116,104,95,104,111, + 111,107,115,15,4,0,0,115,16,0,0,0,0,7,25,1, + 16,1,16,1,3,1,14,1,13,1,12,2,122,22,80,97, + 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,104, + 111,111,107,115,99,2,0,0,0,0,0,0,0,3,0,0, + 0,19,0,0,0,67,0,0,0,115,123,0,0,0,124,1, + 0,100,1,0,107,2,0,114,53,0,121,16,0,116,0,0, + 106,1,0,131,0,0,125,1,0,87,110,22,0,4,116,2, + 0,107,10,0,114,52,0,1,1,1,100,2,0,83,89,110, + 1,0,88,121,17,0,116,3,0,106,4,0,124,1,0,25, + 125,2,0,87,110,46,0,4,116,5,0,107,10,0,114,118, + 0,1,1,1,124,0,0,106,6,0,124,1,0,131,1,0, + 125,2,0,124,2,0,116,3,0,106,4,0,124,1,0,60, + 89,110,1,0,88,124,2,0,83,41,3,122,210,71,101,116, + 32,116,104,101,32,102,105,110,100,101,114,32,102,111,114,32, + 116,104,101,32,112,97,116,104,32,101,110,116,114,121,32,102, + 114,111,109,32,115,121,115,46,112,97,116,104,95,105,109,112, + 111,114,116,101,114,95,99,97,99,104,101,46,10,10,32,32, + 32,32,32,32,32,32,73,102,32,116,104,101,32,112,97,116, + 104,32,101,110,116,114,121,32,105,115,32,110,111,116,32,105, + 110,32,116,104,101,32,99,97,99,104,101,44,32,102,105,110, + 100,32,116,104,101,32,97,112,112,114,111,112,114,105,97,116, + 101,32,102,105,110,100,101,114,10,32,32,32,32,32,32,32, + 32,97,110,100,32,99,97,99,104,101,32,105,116,46,32,73, + 102,32,110,111,32,102,105,110,100,101,114,32,105,115,32,97, + 118,97,105,108,97,98,108,101,44,32,115,116,111,114,101,32, + 78,111,110,101,46,10,10,32,32,32,32,32,32,32,32,114, + 30,0,0,0,78,41,7,114,3,0,0,0,114,45,0,0, + 0,218,17,70,105,108,101,78,111,116,70,111,117,110,100,69, + 114,114,111,114,114,7,0,0,0,114,245,0,0,0,114,131, + 0,0,0,114,249,0,0,0,41,3,114,164,0,0,0,114, + 35,0,0,0,114,247,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,20,95,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,32,4, + 0,0,115,22,0,0,0,0,8,12,1,3,1,16,1,13, + 3,9,1,3,1,17,1,13,1,15,1,18,1,122,31,80, + 97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,99,3, + 0,0,0,0,0,0,0,6,0,0,0,3,0,0,0,67, + 0,0,0,115,119,0,0,0,116,0,0,124,2,0,100,1, + 0,131,2,0,114,39,0,124,2,0,106,1,0,124,1,0, + 131,1,0,92,2,0,125,3,0,125,4,0,110,21,0,124, + 2,0,106,2,0,124,1,0,131,1,0,125,3,0,103,0, + 0,125,4,0,124,3,0,100,0,0,107,9,0,114,88,0, + 116,3,0,106,4,0,124,1,0,124,3,0,131,2,0,83, + 116,3,0,106,5,0,124,1,0,100,0,0,131,2,0,125, + 5,0,124,4,0,124,5,0,95,6,0,124,5,0,83,41, + 2,78,114,117,0,0,0,41,7,114,108,0,0,0,114,117, + 0,0,0,114,176,0,0,0,114,114,0,0,0,114,173,0, + 0,0,114,154,0,0,0,114,150,0,0,0,41,6,114,164, + 0,0,0,114,119,0,0,0,114,247,0,0,0,114,120,0, + 0,0,114,121,0,0,0,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,16,95,108,101, + 103,97,99,121,95,103,101,116,95,115,112,101,99,54,4,0, + 0,115,18,0,0,0,0,4,15,1,24,2,15,1,6,1, + 12,1,16,1,18,1,9,1,122,27,80,97,116,104,70,105, + 110,100,101,114,46,95,108,101,103,97,99,121,95,103,101,116, + 95,115,112,101,99,78,99,4,0,0,0,0,0,0,0,9, + 0,0,0,5,0,0,0,67,0,0,0,115,243,0,0,0, + 103,0,0,125,4,0,120,230,0,124,2,0,68,93,191,0, + 125,5,0,116,0,0,124,5,0,116,1,0,116,2,0,102, + 2,0,131,2,0,115,43,0,113,13,0,124,0,0,106,3, + 0,124,5,0,131,1,0,125,6,0,124,6,0,100,1,0, + 107,9,0,114,13,0,116,4,0,124,6,0,100,2,0,131, + 2,0,114,106,0,124,6,0,106,5,0,124,1,0,124,3, + 0,131,2,0,125,7,0,110,18,0,124,0,0,106,6,0, + 124,1,0,124,6,0,131,2,0,125,7,0,124,7,0,100, + 1,0,107,8,0,114,139,0,113,13,0,124,7,0,106,7, + 0,100,1,0,107,9,0,114,158,0,124,7,0,83,124,7, + 0,106,8,0,125,8,0,124,8,0,100,1,0,107,8,0, + 114,191,0,116,9,0,100,3,0,131,1,0,130,1,0,124, + 4,0,106,10,0,124,8,0,131,1,0,1,113,13,0,87, + 116,11,0,106,12,0,124,1,0,100,1,0,131,2,0,125, + 7,0,124,4,0,124,7,0,95,8,0,124,7,0,83,100, + 1,0,83,41,4,122,63,70,105,110,100,32,116,104,101,32, + 108,111,97,100,101,114,32,111,114,32,110,97,109,101,115,112, + 97,99,101,95,112,97,116,104,32,102,111,114,32,116,104,105, + 115,32,109,111,100,117,108,101,47,112,97,99,107,97,103,101, + 32,110,97,109,101,46,78,114,175,0,0,0,122,19,115,112, + 101,99,32,109,105,115,115,105,110,103,32,108,111,97,100,101, + 114,41,13,114,137,0,0,0,114,69,0,0,0,218,5,98, + 121,116,101,115,114,251,0,0,0,114,108,0,0,0,114,175, + 0,0,0,114,252,0,0,0,114,120,0,0,0,114,150,0, + 0,0,114,99,0,0,0,114,143,0,0,0,114,114,0,0, + 0,114,154,0,0,0,41,9,114,164,0,0,0,114,119,0, + 0,0,114,35,0,0,0,114,174,0,0,0,218,14,110,97, + 109,101,115,112,97,99,101,95,112,97,116,104,90,5,101,110, + 116,114,121,114,247,0,0,0,114,158,0,0,0,114,121,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,250,0,0,0,165,4,0,0,115,2,0,0,0,0, - 2,122,28,70,105,108,101,70,105,110,100,101,114,46,105,110, - 118,97,108,105,100,97,116,101,95,99,97,99,104,101,115,99, - 2,0,0,0,0,0,0,0,3,0,0,0,2,0,0,0, - 67,0,0,0,115,59,0,0,0,124,0,0,106,0,0,124, - 1,0,131,1,0,125,2,0,124,2,0,100,1,0,107,8, - 0,114,37,0,100,1,0,103,0,0,102,2,0,83,124,2, - 0,106,1,0,124,2,0,106,2,0,112,55,0,103,0,0, - 102,2,0,83,41,2,122,197,84,114,121,32,116,111,32,102, - 105,110,100,32,97,32,108,111,97,100,101,114,32,102,111,114, - 32,116,104,101,32,115,112,101,99,105,102,105,101,100,32,109, - 111,100,117,108,101,44,32,111,114,32,116,104,101,32,110,97, - 109,101,115,112,97,99,101,10,32,32,32,32,32,32,32,32, - 112,97,99,107,97,103,101,32,112,111,114,116,105,111,110,115, - 46,32,82,101,116,117,114,110,115,32,40,108,111,97,100,101, - 114,44,32,108,105,115,116,45,111,102,45,112,111,114,116,105, - 111,110,115,41,46,10,10,32,32,32,32,32,32,32,32,84, - 104,105,115,32,109,101,116,104,111,100,32,105,115,32,100,101, - 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,102, - 105,110,100,95,115,112,101,99,40,41,32,105,110,115,116,101, - 97,100,46,10,10,32,32,32,32,32,32,32,32,78,41,3, - 114,181,0,0,0,114,127,0,0,0,114,156,0,0,0,41, - 3,114,108,0,0,0,114,126,0,0,0,114,164,0,0,0, + 0,218,9,95,103,101,116,95,115,112,101,99,69,4,0,0, + 115,40,0,0,0,0,5,6,1,13,1,21,1,3,1,15, + 1,12,1,15,1,21,2,18,1,12,1,3,1,15,1,4, + 1,9,1,12,1,12,5,17,2,18,1,9,1,122,20,80, + 97,116,104,70,105,110,100,101,114,46,95,103,101,116,95,115, + 112,101,99,99,4,0,0,0,0,0,0,0,6,0,0,0, + 4,0,0,0,67,0,0,0,115,140,0,0,0,124,2,0, + 100,1,0,107,8,0,114,21,0,116,0,0,106,1,0,125, + 2,0,124,0,0,106,2,0,124,1,0,124,2,0,124,3, + 0,131,3,0,125,4,0,124,4,0,100,1,0,107,8,0, + 114,58,0,100,1,0,83,124,4,0,106,3,0,100,1,0, + 107,8,0,114,132,0,124,4,0,106,4,0,125,5,0,124, + 5,0,114,125,0,100,2,0,124,4,0,95,5,0,116,6, + 0,124,1,0,124,5,0,124,0,0,106,2,0,131,3,0, + 124,4,0,95,4,0,124,4,0,83,100,1,0,83,110,4, + 0,124,4,0,83,100,1,0,83,41,3,122,98,102,105,110, + 100,32,116,104,101,32,109,111,100,117,108,101,32,111,110,32, + 115,121,115,46,112,97,116,104,32,111,114,32,39,112,97,116, + 104,39,32,98,97,115,101,100,32,111,110,32,115,121,115,46, + 112,97,116,104,95,104,111,111,107,115,32,97,110,100,10,32, + 32,32,32,32,32,32,32,115,121,115,46,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,46,78, + 90,9,110,97,109,101,115,112,97,99,101,41,7,114,7,0, + 0,0,114,35,0,0,0,114,255,0,0,0,114,120,0,0, + 0,114,150,0,0,0,114,152,0,0,0,114,224,0,0,0, + 41,6,114,164,0,0,0,114,119,0,0,0,114,35,0,0, + 0,114,174,0,0,0,114,158,0,0,0,114,254,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 124,0,0,0,171,4,0,0,115,8,0,0,0,0,7,15, - 1,12,1,10,1,122,22,70,105,108,101,70,105,110,100,101, - 114,46,102,105,110,100,95,108,111,97,100,101,114,99,6,0, - 0,0,0,0,0,0,7,0,0,0,7,0,0,0,67,0, - 0,0,115,40,0,0,0,124,1,0,124,2,0,124,3,0, - 131,2,0,125,6,0,116,0,0,124,2,0,124,3,0,100, - 1,0,124,6,0,100,2,0,124,4,0,131,2,2,83,41, - 3,78,114,127,0,0,0,114,156,0,0,0,41,1,114,167, - 0,0,0,41,7,114,108,0,0,0,114,165,0,0,0,114, - 126,0,0,0,114,35,0,0,0,90,4,115,109,115,108,114, - 180,0,0,0,114,127,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,5,1,0,0,183,4,0, - 0,115,6,0,0,0,0,1,15,1,18,1,122,20,70,105, - 108,101,70,105,110,100,101,114,46,95,103,101,116,95,115,112, - 101,99,78,99,3,0,0,0,0,0,0,0,14,0,0,0, - 15,0,0,0,67,0,0,0,115,234,1,0,0,100,1,0, - 125,3,0,124,1,0,106,0,0,100,2,0,131,1,0,100, - 3,0,25,125,4,0,121,34,0,116,1,0,124,0,0,106, - 2,0,112,49,0,116,3,0,106,4,0,131,0,0,131,1, - 0,106,5,0,125,5,0,87,110,24,0,4,116,6,0,107, - 10,0,114,85,0,1,1,1,100,10,0,125,5,0,89,110, - 1,0,88,124,5,0,124,0,0,106,7,0,107,3,0,114, - 120,0,124,0,0,106,8,0,131,0,0,1,124,5,0,124, - 0,0,95,7,0,116,9,0,131,0,0,114,153,0,124,0, - 0,106,10,0,125,6,0,124,4,0,106,11,0,131,0,0, - 125,7,0,110,15,0,124,0,0,106,12,0,125,6,0,124, - 4,0,125,7,0,124,7,0,124,6,0,107,6,0,114,45, - 1,116,13,0,124,0,0,106,2,0,124,4,0,131,2,0, - 125,8,0,120,100,0,124,0,0,106,14,0,68,93,77,0, - 92,2,0,125,9,0,125,10,0,100,5,0,124,9,0,23, - 125,11,0,116,13,0,124,8,0,124,11,0,131,2,0,125, - 12,0,116,15,0,124,12,0,131,1,0,114,208,0,124,0, - 0,106,16,0,124,10,0,124,1,0,124,12,0,124,8,0, - 103,1,0,124,2,0,131,5,0,83,113,208,0,87,116,17, - 0,124,8,0,131,1,0,125,3,0,120,123,0,124,0,0, - 106,14,0,68,93,112,0,92,2,0,125,9,0,125,10,0, - 116,13,0,124,0,0,106,2,0,124,4,0,124,9,0,23, - 131,2,0,125,12,0,116,18,0,100,6,0,106,19,0,124, - 12,0,131,1,0,100,7,0,100,3,0,131,1,1,1,124, - 7,0,124,9,0,23,124,6,0,107,6,0,114,55,1,116, - 15,0,124,12,0,131,1,0,114,55,1,124,0,0,106,16, - 0,124,10,0,124,1,0,124,12,0,100,8,0,124,2,0, - 131,5,0,83,113,55,1,87,124,3,0,114,230,1,116,18, - 0,100,9,0,106,19,0,124,8,0,131,1,0,131,1,0, - 1,116,20,0,106,21,0,124,1,0,100,8,0,131,2,0, - 125,13,0,124,8,0,103,1,0,124,13,0,95,22,0,124, - 13,0,83,100,8,0,83,41,11,122,125,84,114,121,32,116, - 111,32,102,105,110,100,32,97,32,108,111,97,100,101,114,32, - 102,111,114,32,116,104,101,32,115,112,101,99,105,102,105,101, - 100,32,109,111,100,117,108,101,44,32,111,114,32,116,104,101, - 32,110,97,109,101,115,112,97,99,101,10,32,32,32,32,32, - 32,32,32,112,97,99,107,97,103,101,32,112,111,114,116,105, - 111,110,115,46,32,82,101,116,117,114,110,115,32,40,108,111, - 97,100,101,114,44,32,108,105,115,116,45,111,102,45,112,111, - 114,116,105,111,110,115,41,46,70,114,58,0,0,0,114,56, - 0,0,0,114,29,0,0,0,114,185,0,0,0,122,9,116, - 114,121,105,110,103,32,123,125,114,98,0,0,0,78,122,25, - 112,111,115,115,105,98,108,101,32,110,97,109,101,115,112,97, - 99,101,32,102,111,114,32,123,125,114,87,0,0,0,41,23, - 114,32,0,0,0,114,39,0,0,0,114,35,0,0,0,114, - 3,0,0,0,114,45,0,0,0,114,219,0,0,0,114,40, - 0,0,0,114,8,1,0,0,218,11,95,102,105,108,108,95, - 99,97,99,104,101,114,6,0,0,0,114,11,1,0,0,114, - 88,0,0,0,114,10,1,0,0,114,28,0,0,0,114,7, - 1,0,0,114,44,0,0,0,114,5,1,0,0,114,46,0, - 0,0,114,105,0,0,0,114,47,0,0,0,114,121,0,0, - 0,114,160,0,0,0,114,156,0,0,0,41,14,114,108,0, - 0,0,114,126,0,0,0,114,180,0,0,0,90,12,105,115, - 95,110,97,109,101,115,112,97,99,101,90,11,116,97,105,108, - 95,109,111,100,117,108,101,114,133,0,0,0,90,5,99,97, - 99,104,101,90,12,99,97,99,104,101,95,109,111,100,117,108, - 101,90,9,98,97,115,101,95,112,97,116,104,114,225,0,0, - 0,114,165,0,0,0,90,13,105,110,105,116,95,102,105,108, - 101,110,97,109,101,90,9,102,117,108,108,95,112,97,116,104, - 114,164,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,181,0,0,0,188,4,0,0,115,68,0, - 0,0,0,3,6,1,19,1,3,1,34,1,13,1,11,1, - 15,1,10,1,9,2,9,1,9,1,15,2,9,1,6,2, - 12,1,18,1,22,1,10,1,15,1,12,1,32,4,12,2, - 22,1,22,1,25,1,16,1,12,1,29,1,6,1,19,1, - 18,1,12,1,4,1,122,20,70,105,108,101,70,105,110,100, - 101,114,46,102,105,110,100,95,115,112,101,99,99,1,0,0, - 0,0,0,0,0,9,0,0,0,13,0,0,0,67,0,0, - 0,115,11,1,0,0,124,0,0,106,0,0,125,1,0,121, - 31,0,116,1,0,106,2,0,124,1,0,112,33,0,116,1, - 0,106,3,0,131,0,0,131,1,0,125,2,0,87,110,33, - 0,4,116,4,0,116,5,0,116,6,0,102,3,0,107,10, - 0,114,75,0,1,1,1,103,0,0,125,2,0,89,110,1, - 0,88,116,7,0,106,8,0,106,9,0,100,1,0,131,1, - 0,115,112,0,116,10,0,124,2,0,131,1,0,124,0,0, - 95,11,0,110,111,0,116,10,0,131,0,0,125,3,0,120, - 90,0,124,2,0,68,93,82,0,125,4,0,124,4,0,106, - 12,0,100,2,0,131,1,0,92,3,0,125,5,0,125,6, - 0,125,7,0,124,6,0,114,191,0,100,3,0,106,13,0, - 124,5,0,124,7,0,106,14,0,131,0,0,131,2,0,125, - 8,0,110,6,0,124,5,0,125,8,0,124,3,0,106,15, - 0,124,8,0,131,1,0,1,113,128,0,87,124,3,0,124, - 0,0,95,11,0,116,7,0,106,8,0,106,9,0,116,16, - 0,131,1,0,114,7,1,100,4,0,100,5,0,132,0,0, - 124,2,0,68,131,1,0,124,0,0,95,17,0,100,6,0, - 83,41,7,122,68,70,105,108,108,32,116,104,101,32,99,97, - 99,104,101,32,111,102,32,112,111,116,101,110,116,105,97,108, - 32,109,111,100,117,108,101,115,32,97,110,100,32,112,97,99, - 107,97,103,101,115,32,102,111,114,32,116,104,105,115,32,100, - 105,114,101,99,116,111,114,121,46,114,0,0,0,0,114,58, - 0,0,0,122,5,123,125,46,123,125,99,1,0,0,0,0, - 0,0,0,2,0,0,0,3,0,0,0,83,0,0,0,115, - 28,0,0,0,104,0,0,124,0,0,93,18,0,125,1,0, - 124,1,0,106,0,0,131,0,0,146,2,0,113,6,0,83, - 114,4,0,0,0,41,1,114,88,0,0,0,41,2,114,22, - 0,0,0,90,2,102,110,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,250,9,60,115,101,116,99,111,109,112, - 62,6,5,0,0,115,2,0,0,0,9,0,122,41,70,105, - 108,101,70,105,110,100,101,114,46,95,102,105,108,108,95,99, - 97,99,104,101,46,60,108,111,99,97,108,115,62,46,60,115, - 101,116,99,111,109,112,62,78,41,18,114,35,0,0,0,114, - 3,0,0,0,90,7,108,105,115,116,100,105,114,114,45,0, - 0,0,114,0,1,0,0,218,15,80,101,114,109,105,115,115, - 105,111,110,69,114,114,111,114,218,18,78,111,116,65,68,105, - 114,101,99,116,111,114,121,69,114,114,111,114,114,7,0,0, - 0,114,8,0,0,0,114,9,0,0,0,114,9,1,0,0, - 114,10,1,0,0,114,83,0,0,0,114,47,0,0,0,114, - 88,0,0,0,218,3,97,100,100,114,10,0,0,0,114,11, - 1,0,0,41,9,114,108,0,0,0,114,35,0,0,0,90, - 8,99,111,110,116,101,110,116,115,90,21,108,111,119,101,114, - 95,115,117,102,102,105,120,95,99,111,110,116,101,110,116,115, - 114,245,0,0,0,114,106,0,0,0,114,237,0,0,0,114, - 225,0,0,0,90,8,110,101,119,95,110,97,109,101,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,13,1, - 0,0,233,4,0,0,115,34,0,0,0,0,2,9,1,3, - 1,31,1,22,3,11,3,18,1,18,7,9,1,13,1,24, - 1,6,1,27,2,6,1,17,1,9,1,18,1,122,22,70, - 105,108,101,70,105,110,100,101,114,46,95,102,105,108,108,95, - 99,97,99,104,101,99,1,0,0,0,0,0,0,0,3,0, - 0,0,3,0,0,0,7,0,0,0,115,25,0,0,0,135, - 0,0,135,1,0,102,2,0,100,1,0,100,2,0,134,0, - 0,125,2,0,124,2,0,83,41,3,97,20,1,0,0,65, - 32,99,108,97,115,115,32,109,101,116,104,111,100,32,119,104, - 105,99,104,32,114,101,116,117,114,110,115,32,97,32,99,108, - 111,115,117,114,101,32,116,111,32,117,115,101,32,111,110,32, - 115,121,115,46,112,97,116,104,95,104,111,111,107,10,32,32, - 32,32,32,32,32,32,119,104,105,99,104,32,119,105,108,108, - 32,114,101,116,117,114,110,32,97,110,32,105,110,115,116,97, - 110,99,101,32,117,115,105,110,103,32,116,104,101,32,115,112, - 101,99,105,102,105,101,100,32,108,111,97,100,101,114,115,32, - 97,110,100,32,116,104,101,32,112,97,116,104,10,32,32,32, - 32,32,32,32,32,99,97,108,108,101,100,32,111,110,32,116, - 104,101,32,99,108,111,115,117,114,101,46,10,10,32,32,32, - 32,32,32,32,32,73,102,32,116,104,101,32,112,97,116,104, - 32,99,97,108,108,101,100,32,111,110,32,116,104,101,32,99, - 108,111,115,117,114,101,32,105,115,32,110,111,116,32,97,32, - 100,105,114,101,99,116,111,114,121,44,32,73,109,112,111,114, - 116,69,114,114,111,114,32,105,115,10,32,32,32,32,32,32, - 32,32,114,97,105,115,101,100,46,10,10,32,32,32,32,32, - 32,32,32,99,1,0,0,0,0,0,0,0,1,0,0,0, - 4,0,0,0,19,0,0,0,115,43,0,0,0,116,0,0, - 124,0,0,131,1,0,115,30,0,116,1,0,100,1,0,100, - 2,0,124,0,0,131,1,1,130,1,0,136,0,0,124,0, - 0,136,1,0,140,1,0,83,41,3,122,45,80,97,116,104, - 32,104,111,111,107,32,102,111,114,32,105,109,112,111,114,116, - 108,105,98,46,109,97,99,104,105,110,101,114,121,46,70,105, - 108,101,70,105,110,100,101,114,46,122,30,111,110,108,121,32, - 100,105,114,101,99,116,111,114,105,101,115,32,97,114,101,32, - 115,117,112,112,111,114,116,101,100,114,35,0,0,0,41,2, - 114,46,0,0,0,114,107,0,0,0,41,1,114,35,0,0, - 0,41,2,114,170,0,0,0,114,12,1,0,0,114,4,0, - 0,0,114,5,0,0,0,218,24,112,97,116,104,95,104,111, - 111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101, - 114,18,5,0,0,115,6,0,0,0,0,2,12,1,18,1, - 122,54,70,105,108,101,70,105,110,100,101,114,46,112,97,116, - 104,95,104,111,111,107,46,60,108,111,99,97,108,115,62,46, - 112,97,116,104,95,104,111,111,107,95,102,111,114,95,70,105, - 108,101,70,105,110,100,101,114,114,4,0,0,0,41,3,114, - 170,0,0,0,114,12,1,0,0,114,18,1,0,0,114,4, - 0,0,0,41,2,114,170,0,0,0,114,12,1,0,0,114, - 5,0,0,0,218,9,112,97,116,104,95,104,111,111,107,8, - 5,0,0,115,4,0,0,0,0,10,21,6,122,20,70,105, - 108,101,70,105,110,100,101,114,46,112,97,116,104,95,104,111, - 111,107,99,1,0,0,0,0,0,0,0,1,0,0,0,2, - 0,0,0,67,0,0,0,115,16,0,0,0,100,1,0,106, - 0,0,124,0,0,106,1,0,131,1,0,83,41,2,78,122, - 16,70,105,108,101,70,105,110,100,101,114,40,123,33,114,125, - 41,41,2,114,47,0,0,0,114,35,0,0,0,41,1,114, - 108,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,244,0,0,0,26,5,0,0,115,2,0,0, - 0,0,1,122,19,70,105,108,101,70,105,110,100,101,114,46, - 95,95,114,101,112,114,95,95,41,15,114,112,0,0,0,114, - 111,0,0,0,114,113,0,0,0,114,114,0,0,0,114,185, - 0,0,0,114,250,0,0,0,114,130,0,0,0,114,182,0, - 0,0,114,124,0,0,0,114,5,1,0,0,114,181,0,0, - 0,114,13,1,0,0,114,183,0,0,0,114,19,1,0,0, - 114,244,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,6,1,0,0,142,4, - 0,0,115,20,0,0,0,12,7,6,2,12,14,12,4,6, - 2,12,12,12,5,15,45,12,31,18,18,114,6,1,0,0, - 99,4,0,0,0,0,0,0,0,6,0,0,0,11,0,0, - 0,67,0,0,0,115,195,0,0,0,124,0,0,106,0,0, - 100,1,0,131,1,0,125,4,0,124,0,0,106,0,0,100, - 2,0,131,1,0,125,5,0,124,4,0,115,99,0,124,5, - 0,114,54,0,124,5,0,106,1,0,125,4,0,110,45,0, - 124,2,0,124,3,0,107,2,0,114,84,0,116,2,0,124, - 1,0,124,2,0,131,2,0,125,4,0,110,15,0,116,3, - 0,124,1,0,124,2,0,131,2,0,125,4,0,124,5,0, - 115,126,0,116,4,0,124,1,0,124,2,0,100,3,0,124, - 4,0,131,2,1,125,5,0,121,44,0,124,5,0,124,0, - 0,100,2,0,60,124,4,0,124,0,0,100,1,0,60,124, - 2,0,124,0,0,100,4,0,60,124,3,0,124,0,0,100, - 5,0,60,87,110,18,0,4,116,5,0,107,10,0,114,190, - 0,1,1,1,89,110,1,0,88,100,0,0,83,41,6,78, - 218,10,95,95,108,111,97,100,101,114,95,95,218,8,95,95, - 115,112,101,99,95,95,114,127,0,0,0,90,8,95,95,102, - 105,108,101,95,95,90,10,95,95,99,97,99,104,101,100,95, - 95,41,6,218,3,103,101,116,114,127,0,0,0,114,223,0, - 0,0,114,218,0,0,0,114,167,0,0,0,218,9,69,120, - 99,101,112,116,105,111,110,41,6,90,2,110,115,114,106,0, - 0,0,90,8,112,97,116,104,110,97,109,101,90,9,99,112, - 97,116,104,110,97,109,101,114,127,0,0,0,114,164,0,0, + 175,0,0,0,101,4,0,0,115,26,0,0,0,0,4,12, + 1,9,1,21,1,12,1,4,1,15,1,9,1,6,3,9, + 1,24,1,4,2,7,2,122,20,80,97,116,104,70,105,110, + 100,101,114,46,102,105,110,100,95,115,112,101,99,99,3,0, + 0,0,0,0,0,0,4,0,0,0,3,0,0,0,67,0, + 0,0,115,41,0,0,0,124,0,0,106,0,0,124,1,0, + 124,2,0,131,2,0,125,3,0,124,3,0,100,1,0,107, + 8,0,114,34,0,100,1,0,83,124,3,0,106,1,0,83, + 41,2,122,170,102,105,110,100,32,116,104,101,32,109,111,100, + 117,108,101,32,111,110,32,115,121,115,46,112,97,116,104,32, + 111,114,32,39,112,97,116,104,39,32,98,97,115,101,100,32, + 111,110,32,115,121,115,46,112,97,116,104,95,104,111,111,107, + 115,32,97,110,100,10,32,32,32,32,32,32,32,32,115,121, + 115,46,112,97,116,104,95,105,109,112,111,114,116,101,114,95, + 99,97,99,104,101,46,10,10,32,32,32,32,32,32,32,32, + 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, + 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, + 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, + 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, + 2,114,175,0,0,0,114,120,0,0,0,41,4,114,164,0, + 0,0,114,119,0,0,0,114,35,0,0,0,114,158,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,176,0,0,0,123,4,0,0,115,8,0,0,0,0,8, + 18,1,12,1,4,1,122,22,80,97,116,104,70,105,110,100, + 101,114,46,102,105,110,100,95,109,111,100,117,108,101,41,12, + 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, + 107,0,0,0,114,177,0,0,0,114,244,0,0,0,114,249, + 0,0,0,114,251,0,0,0,114,252,0,0,0,114,255,0, + 0,0,114,175,0,0,0,114,176,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,14,95,102,105,120,95,117,112,95,109,111,100,117,108,101, - 32,5,0,0,115,34,0,0,0,0,2,15,1,15,1,6, - 1,6,1,12,1,12,1,18,2,15,1,6,1,21,1,3, - 1,10,1,10,1,10,1,14,1,13,2,114,24,1,0,0, - 99,0,0,0,0,0,0,0,0,3,0,0,0,3,0,0, - 0,67,0,0,0,115,55,0,0,0,116,0,0,116,1,0, - 106,2,0,131,0,0,102,2,0,125,0,0,116,3,0,116, - 4,0,102,2,0,125,1,0,116,5,0,116,6,0,102,2, - 0,125,2,0,124,0,0,124,1,0,124,2,0,103,3,0, - 83,41,1,122,95,82,101,116,117,114,110,115,32,97,32,108, - 105,115,116,32,111,102,32,102,105,108,101,45,98,97,115,101, - 100,32,109,111,100,117,108,101,32,108,111,97,100,101,114,115, - 46,10,10,32,32,32,32,69,97,99,104,32,105,116,101,109, - 32,105,115,32,97,32,116,117,112,108,101,32,40,108,111,97, - 100,101,114,44,32,115,117,102,102,105,120,101,115,41,46,10, - 32,32,32,32,41,7,114,224,0,0,0,114,145,0,0,0, - 218,18,101,120,116,101,110,115,105,111,110,95,115,117,102,102, - 105,120,101,115,114,218,0,0,0,114,84,0,0,0,114,223, - 0,0,0,114,74,0,0,0,41,3,90,10,101,120,116,101, - 110,115,105,111,110,115,90,6,115,111,117,114,99,101,90,8, - 98,121,116,101,99,111,100,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,161,0,0,0,55,5,0,0, - 115,8,0,0,0,0,5,18,1,12,1,12,1,114,161,0, - 0,0,99,1,0,0,0,0,0,0,0,12,0,0,0,12, - 0,0,0,67,0,0,0,115,70,2,0,0,124,0,0,97, - 0,0,116,0,0,106,1,0,97,1,0,116,0,0,106,2, - 0,97,2,0,116,1,0,106,3,0,116,4,0,25,125,1, - 0,120,76,0,100,26,0,68,93,68,0,125,2,0,124,2, - 0,116,1,0,106,3,0,107,7,0,114,83,0,116,0,0, - 106,5,0,124,2,0,131,1,0,125,3,0,110,13,0,116, - 1,0,106,3,0,124,2,0,25,125,3,0,116,6,0,124, - 1,0,124,2,0,124,3,0,131,3,0,1,113,44,0,87, - 100,5,0,100,6,0,103,1,0,102,2,0,100,7,0,100, - 8,0,100,6,0,103,2,0,102,2,0,102,2,0,125,4, - 0,120,149,0,124,4,0,68,93,129,0,92,2,0,125,5, - 0,125,6,0,116,7,0,100,9,0,100,10,0,132,0,0, - 124,6,0,68,131,1,0,131,1,0,115,199,0,116,8,0, - 130,1,0,124,6,0,100,11,0,25,125,7,0,124,5,0, - 116,1,0,106,3,0,107,6,0,114,241,0,116,1,0,106, - 3,0,124,5,0,25,125,8,0,80,113,156,0,121,20,0, - 116,0,0,106,5,0,124,5,0,131,1,0,125,8,0,80, - 87,113,156,0,4,116,9,0,107,10,0,114,28,1,1,1, - 1,119,156,0,89,113,156,0,88,113,156,0,87,116,9,0, - 100,12,0,131,1,0,130,1,0,116,6,0,124,1,0,100, - 13,0,124,8,0,131,3,0,1,116,6,0,124,1,0,100, - 14,0,124,7,0,131,3,0,1,116,6,0,124,1,0,100, - 15,0,100,16,0,106,10,0,124,6,0,131,1,0,131,3, - 0,1,121,19,0,116,0,0,106,5,0,100,17,0,131,1, - 0,125,9,0,87,110,24,0,4,116,9,0,107,10,0,114, - 147,1,1,1,1,100,18,0,125,9,0,89,110,1,0,88, - 116,6,0,124,1,0,100,17,0,124,9,0,131,3,0,1, - 116,0,0,106,5,0,100,19,0,131,1,0,125,10,0,116, - 6,0,124,1,0,100,19,0,124,10,0,131,3,0,1,124, - 5,0,100,7,0,107,2,0,114,238,1,116,0,0,106,5, - 0,100,20,0,131,1,0,125,11,0,116,6,0,124,1,0, - 100,21,0,124,11,0,131,3,0,1,116,6,0,124,1,0, - 100,22,0,116,11,0,131,0,0,131,3,0,1,116,12,0, - 106,13,0,116,2,0,106,14,0,131,0,0,131,1,0,1, - 124,5,0,100,7,0,107,2,0,114,66,2,116,15,0,106, - 16,0,100,23,0,131,1,0,1,100,24,0,116,12,0,107, - 6,0,114,66,2,100,25,0,116,17,0,95,18,0,100,18, - 0,83,41,27,122,205,83,101,116,117,112,32,116,104,101,32, - 112,97,116,104,45,98,97,115,101,100,32,105,109,112,111,114, - 116,101,114,115,32,102,111,114,32,105,109,112,111,114,116,108, - 105,98,32,98,121,32,105,109,112,111,114,116,105,110,103,32, - 110,101,101,100,101,100,10,32,32,32,32,98,117,105,108,116, - 45,105,110,32,109,111,100,117,108,101,115,32,97,110,100,32, - 105,110,106,101,99,116,105,110,103,32,116,104,101,109,32,105, - 110,116,111,32,116,104,101,32,103,108,111,98,97,108,32,110, - 97,109,101,115,112,97,99,101,46,10,10,32,32,32,32,79, - 116,104,101,114,32,99,111,109,112,111,110,101,110,116,115,32, - 97,114,101,32,101,120,116,114,97,99,116,101,100,32,102,114, - 111,109,32,116,104,101,32,99,111,114,101,32,98,111,111,116, - 115,116,114,97,112,32,109,111,100,117,108,101,46,10,10,32, - 32,32,32,114,49,0,0,0,114,60,0,0,0,218,8,98, - 117,105,108,116,105,110,115,114,142,0,0,0,90,5,112,111, - 115,105,120,250,1,47,218,2,110,116,250,1,92,99,1,0, - 0,0,0,0,0,0,2,0,0,0,3,0,0,0,115,0, - 0,0,115,33,0,0,0,124,0,0,93,23,0,125,1,0, - 116,0,0,124,1,0,131,1,0,100,0,0,107,2,0,86, - 1,113,3,0,100,1,0,83,41,2,114,29,0,0,0,78, - 41,1,114,31,0,0,0,41,2,114,22,0,0,0,114,77, + 114,243,0,0,0,3,4,0,0,115,22,0,0,0,12,2, + 6,2,18,8,18,17,18,22,18,15,3,1,18,31,3,1, + 21,21,3,1,114,243,0,0,0,99,0,0,0,0,0,0, + 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,133, + 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, + 1,0,90,3,0,100,2,0,100,3,0,132,0,0,90,4, + 0,100,4,0,100,5,0,132,0,0,90,5,0,101,6,0, + 90,7,0,100,6,0,100,7,0,132,0,0,90,8,0,100, + 8,0,100,9,0,132,0,0,90,9,0,100,10,0,100,11, + 0,100,12,0,132,1,0,90,10,0,100,13,0,100,14,0, + 132,0,0,90,11,0,101,12,0,100,15,0,100,16,0,132, + 0,0,131,1,0,90,13,0,100,17,0,100,18,0,132,0, + 0,90,14,0,100,10,0,83,41,19,218,10,70,105,108,101, + 70,105,110,100,101,114,122,172,70,105,108,101,45,98,97,115, + 101,100,32,102,105,110,100,101,114,46,10,10,32,32,32,32, + 73,110,116,101,114,97,99,116,105,111,110,115,32,119,105,116, + 104,32,116,104,101,32,102,105,108,101,32,115,121,115,116,101, + 109,32,97,114,101,32,99,97,99,104,101,100,32,102,111,114, + 32,112,101,114,102,111,114,109,97,110,99,101,44,32,98,101, + 105,110,103,10,32,32,32,32,114,101,102,114,101,115,104,101, + 100,32,119,104,101,110,32,116,104,101,32,100,105,114,101,99, + 116,111,114,121,32,116,104,101,32,102,105,110,100,101,114,32, + 105,115,32,104,97,110,100,108,105,110,103,32,104,97,115,32, + 98,101,101,110,32,109,111,100,105,102,105,101,100,46,10,10, + 32,32,32,32,99,2,0,0,0,0,0,0,0,5,0,0, + 0,5,0,0,0,7,0,0,0,115,122,0,0,0,103,0, + 0,125,3,0,120,52,0,124,2,0,68,93,44,0,92,2, + 0,137,0,0,125,4,0,124,3,0,106,0,0,135,0,0, + 102,1,0,100,1,0,100,2,0,134,0,0,124,4,0,68, + 131,1,0,131,1,0,1,113,13,0,87,124,3,0,124,0, + 0,95,1,0,124,1,0,112,79,0,100,3,0,124,0,0, + 95,2,0,100,6,0,124,0,0,95,3,0,116,4,0,131, + 0,0,124,0,0,95,5,0,116,4,0,131,0,0,124,0, + 0,95,6,0,100,5,0,83,41,7,122,154,73,110,105,116, + 105,97,108,105,122,101,32,119,105,116,104,32,116,104,101,32, + 112,97,116,104,32,116,111,32,115,101,97,114,99,104,32,111, + 110,32,97,110,100,32,97,32,118,97,114,105,97,98,108,101, + 32,110,117,109,98,101,114,32,111,102,10,32,32,32,32,32, + 32,32,32,50,45,116,117,112,108,101,115,32,99,111,110,116, + 97,105,110,105,110,103,32,116,104,101,32,108,111,97,100,101, + 114,32,97,110,100,32,116,104,101,32,102,105,108,101,32,115, + 117,102,102,105,120,101,115,32,116,104,101,32,108,111,97,100, + 101,114,10,32,32,32,32,32,32,32,32,114,101,99,111,103, + 110,105,122,101,115,46,99,1,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,51,0,0,0,115,27,0,0,0, + 124,0,0,93,17,0,125,1,0,124,1,0,136,0,0,102, + 2,0,86,1,113,3,0,100,0,0,83,41,1,78,114,4, + 0,0,0,41,2,114,22,0,0,0,114,219,0,0,0,41, + 1,114,120,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,221,0,0,0,152,4,0,0,115,2,0,0,0,6,0, + 122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105, + 110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60, + 103,101,110,101,120,112,114,62,114,58,0,0,0,114,29,0, + 0,0,78,114,87,0,0,0,41,7,114,143,0,0,0,218, + 8,95,108,111,97,100,101,114,115,114,35,0,0,0,218,11, + 95,112,97,116,104,95,109,116,105,109,101,218,3,115,101,116, + 218,11,95,112,97,116,104,95,99,97,99,104,101,218,19,95, + 114,101,108,97,120,101,100,95,112,97,116,104,95,99,97,99, + 104,101,41,5,114,100,0,0,0,114,35,0,0,0,218,14, + 108,111,97,100,101,114,95,100,101,116,97,105,108,115,90,7, + 108,111,97,100,101,114,115,114,160,0,0,0,114,4,0,0, + 0,41,1,114,120,0,0,0,114,5,0,0,0,114,179,0, + 0,0,146,4,0,0,115,16,0,0,0,0,4,6,1,19, + 1,36,1,9,2,15,1,9,1,12,1,122,19,70,105,108, + 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, + 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, + 0,67,0,0,0,115,13,0,0,0,100,3,0,124,0,0, + 95,0,0,100,2,0,83,41,4,122,31,73,110,118,97,108, + 105,100,97,116,101,32,116,104,101,32,100,105,114,101,99,116, + 111,114,121,32,109,116,105,109,101,46,114,29,0,0,0,78, + 114,87,0,0,0,41,1,114,2,1,0,0,41,1,114,100, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,227,0,0,0,91,5,0,0,115,2,0,0,0, - 6,0,122,25,95,115,101,116,117,112,46,60,108,111,99,97, - 108,115,62,46,60,103,101,110,101,120,112,114,62,114,59,0, - 0,0,122,30,105,109,112,111,114,116,108,105,98,32,114,101, - 113,117,105,114,101,115,32,112,111,115,105,120,32,111,114,32, - 110,116,114,3,0,0,0,114,25,0,0,0,114,21,0,0, - 0,114,30,0,0,0,90,7,95,116,104,114,101,97,100,78, - 90,8,95,119,101,97,107,114,101,102,90,6,119,105,110,114, - 101,103,114,169,0,0,0,114,6,0,0,0,122,4,46,112, - 121,119,122,6,95,100,46,112,121,100,84,41,4,122,3,95, - 105,111,122,9,95,119,97,114,110,105,110,103,115,122,8,98, - 117,105,108,116,105,110,115,122,7,109,97,114,115,104,97,108, - 41,19,114,121,0,0,0,114,7,0,0,0,114,145,0,0, - 0,114,239,0,0,0,114,112,0,0,0,90,18,95,98,117, - 105,108,116,105,110,95,102,114,111,109,95,110,97,109,101,114, - 116,0,0,0,218,3,97,108,108,218,14,65,115,115,101,114, - 116,105,111,110,69,114,114,111,114,114,107,0,0,0,114,26, - 0,0,0,114,11,0,0,0,114,229,0,0,0,114,149,0, - 0,0,114,25,1,0,0,114,84,0,0,0,114,163,0,0, - 0,114,168,0,0,0,114,173,0,0,0,41,12,218,17,95, - 98,111,111,116,115,116,114,97,112,95,109,111,100,117,108,101, - 90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98, - 117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105, - 108,116,105,110,95,109,111,100,117,108,101,90,10,111,115,95, - 100,101,116,97,105,108,115,90,10,98,117,105,108,116,105,110, - 95,111,115,114,21,0,0,0,114,25,0,0,0,90,9,111, - 115,95,109,111,100,117,108,101,90,13,116,104,114,101,97,100, - 95,109,111,100,117,108,101,90,14,119,101,97,107,114,101,102, - 95,109,111,100,117,108,101,90,13,119,105,110,114,101,103,95, - 109,111,100,117,108,101,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,6,95,115,101,116,117,112,66,5,0, - 0,115,82,0,0,0,0,8,6,1,9,1,9,3,13,1, - 13,1,15,1,18,2,13,1,20,3,33,1,19,2,31,1, - 10,1,15,1,13,1,4,2,3,1,15,1,5,1,13,1, - 12,2,12,1,16,1,16,1,25,3,3,1,19,1,13,2, - 11,1,16,3,15,1,16,3,12,1,15,1,16,3,19,1, - 19,1,12,1,13,1,12,1,114,33,1,0,0,99,1,0, - 0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,0, - 0,0,115,116,0,0,0,116,0,0,124,0,0,131,1,0, - 1,116,1,0,131,0,0,125,1,0,116,2,0,106,3,0, - 106,4,0,116,5,0,106,6,0,124,1,0,140,0,0,103, - 1,0,131,1,0,1,116,7,0,106,8,0,100,1,0,107, - 2,0,114,78,0,116,2,0,106,9,0,106,10,0,116,11, - 0,131,1,0,1,116,2,0,106,9,0,106,10,0,116,12, - 0,131,1,0,1,116,5,0,124,0,0,95,5,0,116,13, - 0,124,0,0,95,13,0,100,2,0,83,41,3,122,41,73, - 110,115,116,97,108,108,32,116,104,101,32,112,97,116,104,45, - 98,97,115,101,100,32,105,109,112,111,114,116,32,99,111,109, - 112,111,110,101,110,116,115,46,114,28,1,0,0,78,41,14, - 114,33,1,0,0,114,161,0,0,0,114,7,0,0,0,114, - 254,0,0,0,114,149,0,0,0,114,6,1,0,0,114,19, - 1,0,0,114,3,0,0,0,114,112,0,0,0,218,9,109, - 101,116,97,95,112,97,116,104,114,163,0,0,0,114,168,0, - 0,0,114,249,0,0,0,114,218,0,0,0,41,2,114,32, - 1,0,0,90,17,115,117,112,112,111,114,116,101,100,95,108, - 111,97,100,101,114,115,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,105,110,115,116,97,108,108,134, - 5,0,0,115,16,0,0,0,0,2,10,1,9,1,28,1, - 15,1,16,1,16,4,9,1,114,35,1,0,0,41,3,122, - 3,119,105,110,114,1,0,0,0,114,2,0,0,0,41,57, - 114,114,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 17,0,0,0,114,19,0,0,0,114,28,0,0,0,114,38, - 0,0,0,114,39,0,0,0,114,43,0,0,0,114,44,0, - 0,0,114,46,0,0,0,114,55,0,0,0,218,4,116,121, - 112,101,218,8,95,95,99,111,100,101,95,95,114,144,0,0, - 0,114,15,0,0,0,114,135,0,0,0,114,14,0,0,0, - 114,18,0,0,0,90,17,95,82,65,87,95,77,65,71,73, - 67,95,78,85,77,66,69,82,114,73,0,0,0,114,72,0, - 0,0,114,84,0,0,0,114,74,0,0,0,90,23,68,69, - 66,85,71,95,66,89,84,69,67,79,68,69,95,83,85,70, - 70,73,88,69,83,90,27,79,80,84,73,77,73,90,69,68, - 95,66,89,84,69,67,79,68,69,95,83,85,70,70,73,88, - 69,83,114,79,0,0,0,114,85,0,0,0,114,91,0,0, - 0,114,95,0,0,0,114,97,0,0,0,114,105,0,0,0, - 114,123,0,0,0,114,130,0,0,0,114,141,0,0,0,114, - 147,0,0,0,114,150,0,0,0,114,155,0,0,0,218,6, - 111,98,106,101,99,116,114,162,0,0,0,114,167,0,0,0, - 114,168,0,0,0,114,184,0,0,0,114,194,0,0,0,114, - 210,0,0,0,114,218,0,0,0,114,223,0,0,0,114,229, - 0,0,0,114,224,0,0,0,114,230,0,0,0,114,247,0, - 0,0,114,249,0,0,0,114,6,1,0,0,114,24,1,0, - 0,114,161,0,0,0,114,33,1,0,0,114,35,1,0,0, - 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,8,60,109,111,100,117,108,101,62,8,0, - 0,0,115,100,0,0,0,6,17,6,3,12,12,12,5,12, - 5,12,6,12,12,12,10,12,9,12,5,12,7,15,22,15, - 110,22,1,18,2,6,1,6,2,9,2,9,2,10,2,21, - 44,12,33,12,19,12,12,12,12,18,8,12,28,12,17,21, - 55,21,12,18,10,12,14,9,3,12,1,15,65,19,64,19, - 28,22,110,19,41,25,43,25,16,6,3,25,53,19,57,19, - 41,19,134,19,146,15,23,12,11,12,68, + 0,0,114,244,0,0,0,160,4,0,0,115,2,0,0,0, + 0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105, + 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, + 99,2,0,0,0,0,0,0,0,3,0,0,0,2,0,0, + 0,67,0,0,0,115,59,0,0,0,124,0,0,106,0,0, + 124,1,0,131,1,0,125,2,0,124,2,0,100,1,0,107, + 8,0,114,37,0,100,1,0,103,0,0,102,2,0,83,124, + 2,0,106,1,0,124,2,0,106,2,0,112,55,0,103,0, + 0,102,2,0,83,41,2,122,197,84,114,121,32,116,111,32, + 102,105,110,100,32,97,32,108,111,97,100,101,114,32,102,111, + 114,32,116,104,101,32,115,112,101,99,105,102,105,101,100,32, + 109,111,100,117,108,101,44,32,111,114,32,116,104,101,32,110, + 97,109,101,115,112,97,99,101,10,32,32,32,32,32,32,32, + 32,112,97,99,107,97,103,101,32,112,111,114,116,105,111,110, + 115,46,32,82,101,116,117,114,110,115,32,40,108,111,97,100, + 101,114,44,32,108,105,115,116,45,111,102,45,112,111,114,116, + 105,111,110,115,41,46,10,10,32,32,32,32,32,32,32,32, + 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, + 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, + 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, + 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, + 3,114,175,0,0,0,114,120,0,0,0,114,150,0,0,0, + 41,3,114,100,0,0,0,114,119,0,0,0,114,158,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,117,0,0,0,166,4,0,0,115,8,0,0,0,0,7, + 15,1,12,1,10,1,122,22,70,105,108,101,70,105,110,100, + 101,114,46,102,105,110,100,95,108,111,97,100,101,114,99,6, + 0,0,0,0,0,0,0,7,0,0,0,7,0,0,0,67, + 0,0,0,115,40,0,0,0,124,1,0,124,2,0,124,3, + 0,131,2,0,125,6,0,116,0,0,124,2,0,124,3,0, + 100,1,0,124,6,0,100,2,0,124,4,0,131,2,2,83, + 41,3,78,114,120,0,0,0,114,150,0,0,0,41,1,114, + 161,0,0,0,41,7,114,100,0,0,0,114,159,0,0,0, + 114,119,0,0,0,114,35,0,0,0,90,4,115,109,115,108, + 114,174,0,0,0,114,120,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,255,0,0,0,178,4, + 0,0,115,6,0,0,0,0,1,15,1,18,1,122,20,70, + 105,108,101,70,105,110,100,101,114,46,95,103,101,116,95,115, + 112,101,99,78,99,3,0,0,0,0,0,0,0,14,0,0, + 0,15,0,0,0,67,0,0,0,115,228,1,0,0,100,1, + 0,125,3,0,124,1,0,106,0,0,100,2,0,131,1,0, + 100,3,0,25,125,4,0,121,34,0,116,1,0,124,0,0, + 106,2,0,112,49,0,116,3,0,106,4,0,131,0,0,131, + 1,0,106,5,0,125,5,0,87,110,24,0,4,116,6,0, + 107,10,0,114,85,0,1,1,1,100,10,0,125,5,0,89, + 110,1,0,88,124,5,0,124,0,0,106,7,0,107,3,0, + 114,120,0,124,0,0,106,8,0,131,0,0,1,124,5,0, + 124,0,0,95,7,0,116,9,0,131,0,0,114,153,0,124, + 0,0,106,10,0,125,6,0,124,4,0,106,11,0,131,0, + 0,125,7,0,110,15,0,124,0,0,106,12,0,125,6,0, + 124,4,0,125,7,0,124,7,0,124,6,0,107,6,0,114, + 45,1,116,13,0,124,0,0,106,2,0,124,4,0,131,2, + 0,125,8,0,120,100,0,124,0,0,106,14,0,68,93,77, + 0,92,2,0,125,9,0,125,10,0,100,5,0,124,9,0, + 23,125,11,0,116,13,0,124,8,0,124,11,0,131,2,0, + 125,12,0,116,15,0,124,12,0,131,1,0,114,208,0,124, + 0,0,106,16,0,124,10,0,124,1,0,124,12,0,124,8, + 0,103,1,0,124,2,0,131,5,0,83,113,208,0,87,116, + 17,0,124,8,0,131,1,0,125,3,0,120,120,0,124,0, + 0,106,14,0,68,93,109,0,92,2,0,125,9,0,125,10, + 0,116,13,0,124,0,0,106,2,0,124,4,0,124,9,0, + 23,131,2,0,125,12,0,116,18,0,106,19,0,100,6,0, + 124,12,0,100,7,0,100,3,0,131,2,1,1,124,7,0, + 124,9,0,23,124,6,0,107,6,0,114,55,1,116,15,0, + 124,12,0,131,1,0,114,55,1,124,0,0,106,16,0,124, + 10,0,124,1,0,124,12,0,100,8,0,124,2,0,131,5, + 0,83,113,55,1,87,124,3,0,114,224,1,116,18,0,106, + 19,0,100,9,0,124,8,0,131,2,0,1,116,18,0,106, + 20,0,124,1,0,100,8,0,131,2,0,125,13,0,124,8, + 0,103,1,0,124,13,0,95,21,0,124,13,0,83,100,8, + 0,83,41,11,122,125,84,114,121,32,116,111,32,102,105,110, + 100,32,97,32,108,111,97,100,101,114,32,102,111,114,32,116, + 104,101,32,115,112,101,99,105,102,105,101,100,32,109,111,100, + 117,108,101,44,32,111,114,32,116,104,101,32,110,97,109,101, + 115,112,97,99,101,10,32,32,32,32,32,32,32,32,112,97, + 99,107,97,103,101,32,112,111,114,116,105,111,110,115,46,32, + 82,101,116,117,114,110,115,32,40,108,111,97,100,101,114,44, + 32,108,105,115,116,45,111,102,45,112,111,114,116,105,111,110, + 115,41,46,70,114,58,0,0,0,114,56,0,0,0,114,29, + 0,0,0,114,179,0,0,0,122,9,116,114,121,105,110,103, + 32,123,125,90,9,118,101,114,98,111,115,105,116,121,78,122, + 25,112,111,115,115,105,98,108,101,32,110,97,109,101,115,112, + 97,99,101,32,102,111,114,32,123,125,114,87,0,0,0,41, + 22,114,32,0,0,0,114,39,0,0,0,114,35,0,0,0, + 114,3,0,0,0,114,45,0,0,0,114,213,0,0,0,114, + 40,0,0,0,114,2,1,0,0,218,11,95,102,105,108,108, + 95,99,97,99,104,101,114,6,0,0,0,114,5,1,0,0, + 114,88,0,0,0,114,4,1,0,0,114,28,0,0,0,114, + 1,1,0,0,114,44,0,0,0,114,255,0,0,0,114,46, + 0,0,0,114,114,0,0,0,114,129,0,0,0,114,154,0, + 0,0,114,150,0,0,0,41,14,114,100,0,0,0,114,119, + 0,0,0,114,174,0,0,0,90,12,105,115,95,110,97,109, + 101,115,112,97,99,101,90,11,116,97,105,108,95,109,111,100, + 117,108,101,114,126,0,0,0,90,5,99,97,99,104,101,90, + 12,99,97,99,104,101,95,109,111,100,117,108,101,90,9,98, + 97,115,101,95,112,97,116,104,114,219,0,0,0,114,159,0, + 0,0,90,13,105,110,105,116,95,102,105,108,101,110,97,109, + 101,90,9,102,117,108,108,95,112,97,116,104,114,158,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,175,0,0,0,183,4,0,0,115,70,0,0,0,0,3, + 6,1,19,1,3,1,34,1,13,1,11,1,15,1,10,1, + 9,2,9,1,9,1,15,2,9,1,6,2,12,1,18,1, + 22,1,10,1,15,1,12,1,32,4,12,2,22,1,22,1, + 22,1,16,1,12,1,15,1,14,1,6,1,16,1,18,1, + 12,1,4,1,122,20,70,105,108,101,70,105,110,100,101,114, + 46,102,105,110,100,95,115,112,101,99,99,1,0,0,0,0, + 0,0,0,9,0,0,0,13,0,0,0,67,0,0,0,115, + 11,1,0,0,124,0,0,106,0,0,125,1,0,121,31,0, + 116,1,0,106,2,0,124,1,0,112,33,0,116,1,0,106, + 3,0,131,0,0,131,1,0,125,2,0,87,110,33,0,4, + 116,4,0,116,5,0,116,6,0,102,3,0,107,10,0,114, + 75,0,1,1,1,103,0,0,125,2,0,89,110,1,0,88, + 116,7,0,106,8,0,106,9,0,100,1,0,131,1,0,115, + 112,0,116,10,0,124,2,0,131,1,0,124,0,0,95,11, + 0,110,111,0,116,10,0,131,0,0,125,3,0,120,90,0, + 124,2,0,68,93,82,0,125,4,0,124,4,0,106,12,0, + 100,2,0,131,1,0,92,3,0,125,5,0,125,6,0,125, + 7,0,124,6,0,114,191,0,100,3,0,106,13,0,124,5, + 0,124,7,0,106,14,0,131,0,0,131,2,0,125,8,0, + 110,6,0,124,5,0,125,8,0,124,3,0,106,15,0,124, + 8,0,131,1,0,1,113,128,0,87,124,3,0,124,0,0, + 95,11,0,116,7,0,106,8,0,106,9,0,116,16,0,131, + 1,0,114,7,1,100,4,0,100,5,0,132,0,0,124,2, + 0,68,131,1,0,124,0,0,95,17,0,100,6,0,83,41, + 7,122,68,70,105,108,108,32,116,104,101,32,99,97,99,104, + 101,32,111,102,32,112,111,116,101,110,116,105,97,108,32,109, + 111,100,117,108,101,115,32,97,110,100,32,112,97,99,107,97, + 103,101,115,32,102,111,114,32,116,104,105,115,32,100,105,114, + 101,99,116,111,114,121,46,114,0,0,0,0,114,58,0,0, + 0,122,5,123,125,46,123,125,99,1,0,0,0,0,0,0, + 0,2,0,0,0,3,0,0,0,83,0,0,0,115,28,0, + 0,0,104,0,0,124,0,0,93,18,0,125,1,0,124,1, + 0,106,0,0,131,0,0,146,2,0,113,6,0,83,114,4, + 0,0,0,41,1,114,88,0,0,0,41,2,114,22,0,0, + 0,90,2,102,110,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,2, + 5,0,0,115,2,0,0,0,9,0,122,41,70,105,108,101, + 70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,99, + 104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,116, + 99,111,109,112,62,78,41,18,114,35,0,0,0,114,3,0, + 0,0,90,7,108,105,115,116,100,105,114,114,45,0,0,0, + 114,250,0,0,0,218,15,80,101,114,109,105,115,115,105,111, + 110,69,114,114,111,114,218,18,78,111,116,65,68,105,114,101, + 99,116,111,114,121,69,114,114,111,114,114,7,0,0,0,114, + 8,0,0,0,114,9,0,0,0,114,3,1,0,0,114,4, + 1,0,0,114,83,0,0,0,114,47,0,0,0,114,88,0, + 0,0,218,3,97,100,100,114,10,0,0,0,114,5,1,0, + 0,41,9,114,100,0,0,0,114,35,0,0,0,90,8,99, + 111,110,116,101,110,116,115,90,21,108,111,119,101,114,95,115, + 117,102,102,105,120,95,99,111,110,116,101,110,116,115,114,239, + 0,0,0,114,98,0,0,0,114,231,0,0,0,114,219,0, + 0,0,90,8,110,101,119,95,110,97,109,101,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,7,1,0,0, + 229,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, + 1,22,3,11,3,18,1,18,7,9,1,13,1,24,1,6, + 1,27,2,6,1,17,1,9,1,18,1,122,22,70,105,108, + 101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97, + 99,104,101,99,1,0,0,0,0,0,0,0,3,0,0,0, + 3,0,0,0,7,0,0,0,115,25,0,0,0,135,0,0, + 135,1,0,102,2,0,100,1,0,100,2,0,134,0,0,125, + 2,0,124,2,0,83,41,3,97,20,1,0,0,65,32,99, + 108,97,115,115,32,109,101,116,104,111,100,32,119,104,105,99, + 104,32,114,101,116,117,114,110,115,32,97,32,99,108,111,115, + 117,114,101,32,116,111,32,117,115,101,32,111,110,32,115,121, + 115,46,112,97,116,104,95,104,111,111,107,10,32,32,32,32, + 32,32,32,32,119,104,105,99,104,32,119,105,108,108,32,114, + 101,116,117,114,110,32,97,110,32,105,110,115,116,97,110,99, + 101,32,117,115,105,110,103,32,116,104,101,32,115,112,101,99, + 105,102,105,101,100,32,108,111,97,100,101,114,115,32,97,110, + 100,32,116,104,101,32,112,97,116,104,10,32,32,32,32,32, + 32,32,32,99,97,108,108,101,100,32,111,110,32,116,104,101, + 32,99,108,111,115,117,114,101,46,10,10,32,32,32,32,32, + 32,32,32,73,102,32,116,104,101,32,112,97,116,104,32,99, + 97,108,108,101,100,32,111,110,32,116,104,101,32,99,108,111, + 115,117,114,101,32,105,115,32,110,111,116,32,97,32,100,105, + 114,101,99,116,111,114,121,44,32,73,109,112,111,114,116,69, + 114,114,111,114,32,105,115,10,32,32,32,32,32,32,32,32, + 114,97,105,115,101,100,46,10,10,32,32,32,32,32,32,32, + 32,99,1,0,0,0,0,0,0,0,1,0,0,0,4,0, + 0,0,19,0,0,0,115,43,0,0,0,116,0,0,124,0, + 0,131,1,0,115,30,0,116,1,0,100,1,0,100,2,0, + 124,0,0,131,1,1,130,1,0,136,0,0,124,0,0,136, + 1,0,140,1,0,83,41,3,122,45,80,97,116,104,32,104, + 111,111,107,32,102,111,114,32,105,109,112,111,114,116,108,105, + 98,46,109,97,99,104,105,110,101,114,121,46,70,105,108,101, + 70,105,110,100,101,114,46,122,30,111,110,108,121,32,100,105, + 114,101,99,116,111,114,105,101,115,32,97,114,101,32,115,117, + 112,112,111,114,116,101,100,114,35,0,0,0,41,2,114,46, + 0,0,0,114,99,0,0,0,41,1,114,35,0,0,0,41, + 2,114,164,0,0,0,114,6,1,0,0,114,4,0,0,0, + 114,5,0,0,0,218,24,112,97,116,104,95,104,111,111,107, + 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,14, + 5,0,0,115,6,0,0,0,0,2,12,1,18,1,122,54, + 70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95, + 104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97, + 116,104,95,104,111,111,107,95,102,111,114,95,70,105,108,101, + 70,105,110,100,101,114,114,4,0,0,0,41,3,114,164,0, + 0,0,114,6,1,0,0,114,12,1,0,0,114,4,0,0, + 0,41,2,114,164,0,0,0,114,6,1,0,0,114,5,0, + 0,0,218,9,112,97,116,104,95,104,111,111,107,4,5,0, + 0,115,4,0,0,0,0,10,21,6,122,20,70,105,108,101, + 70,105,110,100,101,114,46,112,97,116,104,95,104,111,111,107, + 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, + 0,67,0,0,0,115,16,0,0,0,100,1,0,106,0,0, + 124,0,0,106,1,0,131,1,0,83,41,2,78,122,16,70, + 105,108,101,70,105,110,100,101,114,40,123,33,114,125,41,41, + 2,114,47,0,0,0,114,35,0,0,0,41,1,114,100,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,114,238,0,0,0,22,5,0,0,115,2,0,0,0,0, + 1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95, + 114,101,112,114,95,95,41,15,114,105,0,0,0,114,104,0, + 0,0,114,106,0,0,0,114,107,0,0,0,114,179,0,0, + 0,114,244,0,0,0,114,123,0,0,0,114,176,0,0,0, + 114,117,0,0,0,114,255,0,0,0,114,175,0,0,0,114, + 7,1,0,0,114,177,0,0,0,114,13,1,0,0,114,238, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,0,1,0,0,137,4,0,0, + 115,20,0,0,0,12,7,6,2,12,14,12,4,6,2,12, + 12,12,5,15,46,12,31,18,18,114,0,1,0,0,99,4, + 0,0,0,0,0,0,0,6,0,0,0,11,0,0,0,67, + 0,0,0,115,195,0,0,0,124,0,0,106,0,0,100,1, + 0,131,1,0,125,4,0,124,0,0,106,0,0,100,2,0, + 131,1,0,125,5,0,124,4,0,115,99,0,124,5,0,114, + 54,0,124,5,0,106,1,0,125,4,0,110,45,0,124,2, + 0,124,3,0,107,2,0,114,84,0,116,2,0,124,1,0, + 124,2,0,131,2,0,125,4,0,110,15,0,116,3,0,124, + 1,0,124,2,0,131,2,0,125,4,0,124,5,0,115,126, + 0,116,4,0,124,1,0,124,2,0,100,3,0,124,4,0, + 131,2,1,125,5,0,121,44,0,124,5,0,124,0,0,100, + 2,0,60,124,4,0,124,0,0,100,1,0,60,124,2,0, + 124,0,0,100,4,0,60,124,3,0,124,0,0,100,5,0, + 60,87,110,18,0,4,116,5,0,107,10,0,114,190,0,1, + 1,1,89,110,1,0,88,100,0,0,83,41,6,78,218,10, + 95,95,108,111,97,100,101,114,95,95,218,8,95,95,115,112, + 101,99,95,95,114,120,0,0,0,90,8,95,95,102,105,108, + 101,95,95,90,10,95,95,99,97,99,104,101,100,95,95,41, + 6,218,3,103,101,116,114,120,0,0,0,114,217,0,0,0, + 114,212,0,0,0,114,161,0,0,0,218,9,69,120,99,101, + 112,116,105,111,110,41,6,90,2,110,115,114,98,0,0,0, + 90,8,112,97,116,104,110,97,109,101,90,9,99,112,97,116, + 104,110,97,109,101,114,120,0,0,0,114,158,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,14, + 95,102,105,120,95,117,112,95,109,111,100,117,108,101,28,5, + 0,0,115,34,0,0,0,0,2,15,1,15,1,6,1,6, + 1,12,1,12,1,18,2,15,1,6,1,21,1,3,1,10, + 1,10,1,10,1,14,1,13,2,114,18,1,0,0,99,0, + 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,67, + 0,0,0,115,55,0,0,0,116,0,0,116,1,0,106,2, + 0,131,0,0,102,2,0,125,0,0,116,3,0,116,4,0, + 102,2,0,125,1,0,116,5,0,116,6,0,102,2,0,125, + 2,0,124,0,0,124,1,0,124,2,0,103,3,0,83,41, + 1,122,95,82,101,116,117,114,110,115,32,97,32,108,105,115, + 116,32,111,102,32,102,105,108,101,45,98,97,115,101,100,32, + 109,111,100,117,108,101,32,108,111,97,100,101,114,115,46,10, + 10,32,32,32,32,69,97,99,104,32,105,116,101,109,32,105, + 115,32,97,32,116,117,112,108,101,32,40,108,111,97,100,101, + 114,44,32,115,117,102,102,105,120,101,115,41,46,10,32,32, + 32,32,41,7,114,218,0,0,0,114,139,0,0,0,218,18, + 101,120,116,101,110,115,105,111,110,95,115,117,102,102,105,120, + 101,115,114,212,0,0,0,114,84,0,0,0,114,217,0,0, + 0,114,74,0,0,0,41,3,90,10,101,120,116,101,110,115, + 105,111,110,115,90,6,115,111,117,114,99,101,90,8,98,121, + 116,101,99,111,100,101,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,155,0,0,0,51,5,0,0,115,8, + 0,0,0,0,5,18,1,12,1,12,1,114,155,0,0,0, + 99,1,0,0,0,0,0,0,0,12,0,0,0,12,0,0, + 0,67,0,0,0,115,70,2,0,0,124,0,0,97,0,0, + 116,0,0,106,1,0,97,1,0,116,0,0,106,2,0,97, + 2,0,116,1,0,106,3,0,116,4,0,25,125,1,0,120, + 76,0,100,26,0,68,93,68,0,125,2,0,124,2,0,116, + 1,0,106,3,0,107,7,0,114,83,0,116,0,0,106,5, + 0,124,2,0,131,1,0,125,3,0,110,13,0,116,1,0, + 106,3,0,124,2,0,25,125,3,0,116,6,0,124,1,0, + 124,2,0,124,3,0,131,3,0,1,113,44,0,87,100,5, + 0,100,6,0,103,1,0,102,2,0,100,7,0,100,8,0, + 100,6,0,103,2,0,102,2,0,102,2,0,125,4,0,120, + 149,0,124,4,0,68,93,129,0,92,2,0,125,5,0,125, + 6,0,116,7,0,100,9,0,100,10,0,132,0,0,124,6, + 0,68,131,1,0,131,1,0,115,199,0,116,8,0,130,1, + 0,124,6,0,100,11,0,25,125,7,0,124,5,0,116,1, + 0,106,3,0,107,6,0,114,241,0,116,1,0,106,3,0, + 124,5,0,25,125,8,0,80,113,156,0,121,20,0,116,0, + 0,106,5,0,124,5,0,131,1,0,125,8,0,80,87,113, + 156,0,4,116,9,0,107,10,0,114,28,1,1,1,1,119, + 156,0,89,113,156,0,88,113,156,0,87,116,9,0,100,12, + 0,131,1,0,130,1,0,116,6,0,124,1,0,100,13,0, + 124,8,0,131,3,0,1,116,6,0,124,1,0,100,14,0, + 124,7,0,131,3,0,1,116,6,0,124,1,0,100,15,0, + 100,16,0,106,10,0,124,6,0,131,1,0,131,3,0,1, + 121,19,0,116,0,0,106,5,0,100,17,0,131,1,0,125, + 9,0,87,110,24,0,4,116,9,0,107,10,0,114,147,1, + 1,1,1,100,18,0,125,9,0,89,110,1,0,88,116,6, + 0,124,1,0,100,17,0,124,9,0,131,3,0,1,116,0, + 0,106,5,0,100,19,0,131,1,0,125,10,0,116,6,0, + 124,1,0,100,19,0,124,10,0,131,3,0,1,124,5,0, + 100,7,0,107,2,0,114,238,1,116,0,0,106,5,0,100, + 20,0,131,1,0,125,11,0,116,6,0,124,1,0,100,21, + 0,124,11,0,131,3,0,1,116,6,0,124,1,0,100,22, + 0,116,11,0,131,0,0,131,3,0,1,116,12,0,106,13, + 0,116,2,0,106,14,0,131,0,0,131,1,0,1,124,5, + 0,100,7,0,107,2,0,114,66,2,116,15,0,106,16,0, + 100,23,0,131,1,0,1,100,24,0,116,12,0,107,6,0, + 114,66,2,100,25,0,116,17,0,95,18,0,100,18,0,83, + 41,27,122,205,83,101,116,117,112,32,116,104,101,32,112,97, + 116,104,45,98,97,115,101,100,32,105,109,112,111,114,116,101, + 114,115,32,102,111,114,32,105,109,112,111,114,116,108,105,98, + 32,98,121,32,105,109,112,111,114,116,105,110,103,32,110,101, + 101,100,101,100,10,32,32,32,32,98,117,105,108,116,45,105, + 110,32,109,111,100,117,108,101,115,32,97,110,100,32,105,110, + 106,101,99,116,105,110,103,32,116,104,101,109,32,105,110,116, + 111,32,116,104,101,32,103,108,111,98,97,108,32,110,97,109, + 101,115,112,97,99,101,46,10,10,32,32,32,32,79,116,104, + 101,114,32,99,111,109,112,111,110,101,110,116,115,32,97,114, + 101,32,101,120,116,114,97,99,116,101,100,32,102,114,111,109, + 32,116,104,101,32,99,111,114,101,32,98,111,111,116,115,116, + 114,97,112,32,109,111,100,117,108,101,46,10,10,32,32,32, + 32,114,49,0,0,0,114,60,0,0,0,218,8,98,117,105, + 108,116,105,110,115,114,136,0,0,0,90,5,112,111,115,105, + 120,250,1,47,218,2,110,116,250,1,92,99,1,0,0,0, + 0,0,0,0,2,0,0,0,3,0,0,0,115,0,0,0, + 115,33,0,0,0,124,0,0,93,23,0,125,1,0,116,0, + 0,124,1,0,131,1,0,100,0,0,107,2,0,86,1,113, + 3,0,100,1,0,83,41,2,114,29,0,0,0,78,41,1, + 114,31,0,0,0,41,2,114,22,0,0,0,114,77,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,221,0,0,0,87,5,0,0,115,2,0,0,0,6,0, + 122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115, + 62,46,60,103,101,110,101,120,112,114,62,114,59,0,0,0, + 122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,117, + 105,114,101,115,32,112,111,115,105,120,32,111,114,32,110,116, + 114,3,0,0,0,114,25,0,0,0,114,21,0,0,0,114, + 30,0,0,0,90,7,95,116,104,114,101,97,100,78,90,8, + 95,119,101,97,107,114,101,102,90,6,119,105,110,114,101,103, + 114,163,0,0,0,114,6,0,0,0,122,4,46,112,121,119, + 122,6,95,100,46,112,121,100,84,41,4,122,3,95,105,111, + 122,9,95,119,97,114,110,105,110,103,115,122,8,98,117,105, + 108,116,105,110,115,122,7,109,97,114,115,104,97,108,41,19, + 114,114,0,0,0,114,7,0,0,0,114,139,0,0,0,114, + 233,0,0,0,114,105,0,0,0,90,18,95,98,117,105,108, + 116,105,110,95,102,114,111,109,95,110,97,109,101,114,109,0, + 0,0,218,3,97,108,108,218,14,65,115,115,101,114,116,105, + 111,110,69,114,114,111,114,114,99,0,0,0,114,26,0,0, + 0,114,11,0,0,0,114,223,0,0,0,114,143,0,0,0, + 114,19,1,0,0,114,84,0,0,0,114,157,0,0,0,114, + 162,0,0,0,114,167,0,0,0,41,12,218,17,95,98,111, + 111,116,115,116,114,97,112,95,109,111,100,117,108,101,90,11, + 115,101,108,102,95,109,111,100,117,108,101,90,12,98,117,105, + 108,116,105,110,95,110,97,109,101,90,14,98,117,105,108,116, + 105,110,95,109,111,100,117,108,101,90,10,111,115,95,100,101, + 116,97,105,108,115,90,10,98,117,105,108,116,105,110,95,111, + 115,114,21,0,0,0,114,25,0,0,0,90,9,111,115,95, + 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, + 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, + 111,100,117,108,101,90,13,119,105,110,114,101,103,95,109,111, + 100,117,108,101,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,6,95,115,101,116,117,112,62,5,0,0,115, + 82,0,0,0,0,8,6,1,9,1,9,3,13,1,13,1, + 15,1,18,2,13,1,20,3,33,1,19,2,31,1,10,1, + 15,1,13,1,4,2,3,1,15,1,5,1,13,1,12,2, + 12,1,16,1,16,1,25,3,3,1,19,1,13,2,11,1, + 16,3,15,1,16,3,12,1,15,1,16,3,19,1,19,1, + 12,1,13,1,12,1,114,27,1,0,0,99,1,0,0,0, + 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, + 115,116,0,0,0,116,0,0,124,0,0,131,1,0,1,116, + 1,0,131,0,0,125,1,0,116,2,0,106,3,0,106,4, + 0,116,5,0,106,6,0,124,1,0,140,0,0,103,1,0, + 131,1,0,1,116,7,0,106,8,0,100,1,0,107,2,0, + 114,78,0,116,2,0,106,9,0,106,10,0,116,11,0,131, + 1,0,1,116,2,0,106,9,0,106,10,0,116,12,0,131, + 1,0,1,116,5,0,124,0,0,95,5,0,116,13,0,124, + 0,0,95,13,0,100,2,0,83,41,3,122,41,73,110,115, + 116,97,108,108,32,116,104,101,32,112,97,116,104,45,98,97, + 115,101,100,32,105,109,112,111,114,116,32,99,111,109,112,111, + 110,101,110,116,115,46,114,22,1,0,0,78,41,14,114,27, + 1,0,0,114,155,0,0,0,114,7,0,0,0,114,248,0, + 0,0,114,143,0,0,0,114,0,1,0,0,114,13,1,0, + 0,114,3,0,0,0,114,105,0,0,0,218,9,109,101,116, + 97,95,112,97,116,104,114,157,0,0,0,114,162,0,0,0, + 114,243,0,0,0,114,212,0,0,0,41,2,114,26,1,0, + 0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,97, + 100,101,114,115,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,8,95,105,110,115,116,97,108,108,130,5,0, + 0,115,16,0,0,0,0,2,10,1,9,1,28,1,15,1, + 16,1,16,4,9,1,114,29,1,0,0,41,3,122,3,119, + 105,110,114,1,0,0,0,114,2,0,0,0,41,56,114,107, + 0,0,0,114,10,0,0,0,114,11,0,0,0,114,17,0, + 0,0,114,19,0,0,0,114,28,0,0,0,114,38,0,0, + 0,114,39,0,0,0,114,43,0,0,0,114,44,0,0,0, + 114,46,0,0,0,114,55,0,0,0,218,4,116,121,112,101, + 218,8,95,95,99,111,100,101,95,95,114,138,0,0,0,114, + 15,0,0,0,114,128,0,0,0,114,14,0,0,0,114,18, + 0,0,0,90,17,95,82,65,87,95,77,65,71,73,67,95, + 78,85,77,66,69,82,114,73,0,0,0,114,72,0,0,0, + 114,84,0,0,0,114,74,0,0,0,90,23,68,69,66,85, + 71,95,66,89,84,69,67,79,68,69,95,83,85,70,70,73, + 88,69,83,90,27,79,80,84,73,77,73,90,69,68,95,66, + 89,84,69,67,79,68,69,95,83,85,70,70,73,88,69,83, + 114,79,0,0,0,114,85,0,0,0,114,91,0,0,0,114, + 95,0,0,0,114,97,0,0,0,114,116,0,0,0,114,123, + 0,0,0,114,135,0,0,0,114,141,0,0,0,114,144,0, + 0,0,114,149,0,0,0,218,6,111,98,106,101,99,116,114, + 156,0,0,0,114,161,0,0,0,114,162,0,0,0,114,178, + 0,0,0,114,188,0,0,0,114,204,0,0,0,114,212,0, + 0,0,114,217,0,0,0,114,223,0,0,0,114,218,0,0, + 0,114,224,0,0,0,114,241,0,0,0,114,243,0,0,0, + 114,0,1,0,0,114,18,1,0,0,114,155,0,0,0,114, + 27,1,0,0,114,29,1,0,0,114,4,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,218,8,60, + 109,111,100,117,108,101,62,8,0,0,0,115,98,0,0,0, + 6,17,6,3,12,12,12,5,12,5,12,6,12,12,12,10, + 12,9,12,5,12,7,15,22,15,110,22,1,18,2,6,1, + 6,2,9,2,9,2,10,2,21,44,12,33,12,19,12,12, + 12,12,12,28,12,17,21,55,21,12,18,10,12,14,9,3, + 12,1,15,65,19,64,19,28,22,110,19,41,25,45,25,16, + 6,3,25,53,19,57,19,42,19,134,19,147,15,23,12,11, + 12,68, }; -- cgit v0.12 From 98de5340d4d782b2a0595c9fa729460a2c1b4b91 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Sep 2015 09:43:45 +0200 Subject: Issue #25220: Create Lib/test/libregrtest/ Start to split regrtest.py into smaller parts with the creation of Lib/test/libregrtest/cmdline.py: code to handle the command line, especially parsing command line arguments. This part of the code is tested by test_regrtest. --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 340 +++++++++++++++++++++++++++++++++++++++ Lib/test/regrtest.py | 332 +------------------------------------- Lib/test/test_regrtest.py | 4 +- 4 files changed, 345 insertions(+), 332 deletions(-) create mode 100644 Lib/test/libregrtest/__init__.py create mode 100644 Lib/test/libregrtest/cmdline.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py new file mode 100644 index 0000000..554aeed --- /dev/null +++ b/Lib/test/libregrtest/__init__.py @@ -0,0 +1 @@ +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py new file mode 100644 index 0000000..9464996 --- /dev/null +++ b/Lib/test/libregrtest/cmdline.py @@ -0,0 +1,340 @@ +import argparse +import faulthandler +import os + +from test import support + +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b49e66b..51e9e18 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,119 +6,6 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - # We import importlib *ASAP* in order to test #15386 import importlib @@ -153,6 +40,8 @@ try: except ImportError: multiprocessing = None +from test.libregrtest import _parse_args + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -198,9 +87,6 @@ CHILD_ERROR = -5 # error in a child process from test import support -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -210,220 +96,6 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns - - def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a398a4f..cf4b84f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support +from test import regrtest, support, libregrtest class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(regrtest.RESOURCE_NAMES) + expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v0.12 From 7a84552c84867b548a72b20c177542e892c528f6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 01:30:51 -0700 Subject: Hoist constant expression out of an inner loop. --- Modules/_collectionsmodule.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index d50bc42..2dcdf8b 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -371,6 +371,7 @@ static PyObject * deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + PyObject *(*iternext)(PyObject *); int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ @@ -399,7 +400,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (deque->maxlen == 0) return consume_iterator(it); - while ((item = PyIter_Next(it)) != NULL) { + iternext = *Py_TYPE(it)->tp_iternext; + while ((item = iternext(it)) != NULL) { deque->state++; if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque)); @@ -422,8 +424,12 @@ deque_extend(dequeobject *deque, PyObject *iterable) deque_trim_left(deque); } if (PyErr_Occurred()) { - Py_DECREF(it); - return NULL; + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else { + Py_DECREF(it); + return NULL; + } } Py_DECREF(it); Py_RETURN_NONE; @@ -436,6 +442,7 @@ static PyObject * deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + PyObject *(*iternext)(PyObject *); int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ @@ -464,7 +471,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (deque->maxlen == 0) return consume_iterator(it); - while ((item = PyIter_Next(it)) != NULL) { + iternext = *Py_TYPE(it)->tp_iternext; + while ((item = iternext(it)) != NULL) { deque->state++; if (deque->leftindex == 0) { block *b = newblock(Py_SIZE(deque)); @@ -487,8 +495,12 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) deque_trim_right(deque); } if (PyErr_Occurred()) { - Py_DECREF(it); - return NULL; + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else { + Py_DECREF(it); + return NULL; + } } Py_DECREF(it); Py_RETURN_NONE; -- cgit v0.12 From c22eee6b5917568933bbfec2be1069a4340efd84 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 02:14:50 -0700 Subject: Precomputing the number iterations allows the inner-loop to be vectorizable. --- Modules/_collectionsmodule.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 2dcdf8b..487c765 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -557,7 +557,7 @@ static void deque_clear(dequeobject *deque); static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) { - Py_ssize_t i, size; + Py_ssize_t i, m, size; PyObject *seq; PyObject *rv; @@ -598,7 +598,11 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) MARK_END(b->rightlink); deque->rightindex = -1; } - for ( ; i < n-1 && deque->rightindex != BLOCKLEN - 1 ; i++) { + m = n - 1 - i; + if (m > BLOCKLEN - 1 - deque->rightindex) + m = BLOCKLEN - 1 - deque->rightindex; + i += m; + while (m--) { deque->rightindex++; Py_INCREF(item); deque->rightblock->data[deque->rightindex] = item; -- cgit v0.12 From 3844fe5ed8da2755c1036deda8d7a8387170b72b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Sep 2015 10:38:01 +0200 Subject: Issue #25220: Move most regrtest.py code to libregrtest --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 2 +- Lib/test/libregrtest/main.py | 547 +++++++++++++++++ Lib/test/libregrtest/refleak.py | 165 +++++ Lib/test/libregrtest/runtest.py | 271 +++++++++ Lib/test/libregrtest/save_env.py | 284 +++++++++ Lib/test/regrtest.py | 1225 +------------------------------------- Lib/test/test_regrtest.py | 95 +-- 8 files changed, 1318 insertions(+), 1272 deletions(-) create mode 100644 Lib/test/libregrtest/main.py create mode 100644 Lib/test/libregrtest/refleak.py create mode 100644 Lib/test/libregrtest/runtest.py create mode 100644 Lib/test/libregrtest/save_env.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 554aeed..a882ed2 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1 +1,2 @@ from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES +from test.libregrtest.main import main_in_temp_cwd diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 9464996..19c52a2 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,9 +1,9 @@ import argparse import faulthandler import os - from test import support + USAGE = """\ python -m test [options] [test_name1 [test_name2 ...]] python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py new file mode 100644 index 0000000..5d34cff --- /dev/null +++ b/Lib/test/libregrtest/main.py @@ -0,0 +1,547 @@ +import faulthandler +import json +import os +import re +import sys +import tempfile +import sysconfig +import signal +import random +import platform +import traceback +import unittest +from test.libregrtest.runtest import ( + findtests, runtest, run_test_in_subprocess, + STDTESTS, NOTTESTS, + PASSED, FAILED, ENV_CHANGED, SKIPPED, + RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR) +from test.libregrtest.refleak import warm_caches +from test.libregrtest.cmdline import _parse_args +from test import support +try: + import threading +except ImportError: + threading = None + + +# Some times __path__ and __file__ are not absolute (e.g. while running from +# Lib/) and, if we change the CWD to run the tests in a temporary dir, some +# imports might fail. This affects only the modules imported before os.chdir(). +# These modules are searched first in sys.path[0] (so '' -- the CWD) and if +# they are found in the CWD their __file__ and __path__ will be relative (this +# happens before the chdir). All the modules imported after the chdir, are +# not found in the CWD, and since the other paths in sys.path[1:] are absolute +# (site.py absolutize them), the __file__ and __path__ will be absolute too. +# Therefore it is necessary to absolutize manually the __file__ and __path__ of +# the packages to prevent later imports to fail when the CWD is different. +for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + +# MacOSX (a.k.a. Darwin) has a default stack size that is too small +# for deeply recursive regular expressions. We see this as crashes in +# the Python test suite when running test_re.py and test_sre.py. The +# fix is to set the stack limit to 2048. +# This approach may also be useful for other Unixy platforms that +# suffer from small default stack limits. +if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + +# When tests are run from the Python build directory, it is best practice +# to keep the test files in a subfolder. This eases the cleanup of leftover +# files using the "make distclean" command. +if sysconfig.is_python_build(): + TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') +else: + TEMPDIR = tempfile.gettempdir() +TEMPDIR = os.path.abspath(TEMPDIR) + + +def main(tests=None, **kwargs): + """Execute a test suite. + + This also parses command-line options and modifies its behavior + accordingly. + + tests -- a list of strings containing test names (optional) + testdir -- the directory in which to look for tests (optional) + + Users other than the Python test suite will certainly want to + specify testdir; if it's omitted, the directory containing the + Python test suite is searched for. + + If the tests argument is omitted, the tests listed on the + command-line will be used. If that's empty, too, then all *.py + files beginning with test_ will be used. + + The other default arguments (verbose, quiet, exclude, + single, randomize, findleaks, use_resources, trace, coverdir, + print_slow, and random_seed) allow programmers calling main() + directly to set the values that would normally be set by flags + on the command line. + """ + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + + support.record_original_stdout(sys.stdout) + + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.huntrleaks: + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + if ns.threshold is not None: + import gc + gc.set_threshold(ns.threshold) + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + if ns.wait: + input("Press any key to continue...") + + if ns.slaveargs is not None: + args, kwargs = json.loads(ns.slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + good = [] + bad = [] + skipped = [] + resource_denieds = [] + environment_changed = [] + interrupted = False + + if ns.findleaks: + try: + import gc + except ImportError: + print('No GC available, disabling findleaks.') + ns.findleaks = False + else: + # Uncomment the line below to report garbage that is not + # freeable by reference counting alone. By default only + # garbage that is not collectable by the GC is reported. + #gc.set_debug(gc.DEBUG_SAVEALL) + found_garbage = [] + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + if ns.single: + filename = os.path.join(TEMPDIR, 'pynexttest') + try: + with open(filename, 'r') as fp: + next_test = fp.read().strip() + tests = [next_test] + except OSError: + pass + + if ns.fromfile: + tests = [] + with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') + for line in fp: + line = count_pat.sub('', line) + guts = line.split() # assuming no test has whitespace in its name + if guts and not guts[0].startswith('#'): + tests.extend(guts) + + # Strip .py extensions. + removepy(ns.args) + removepy(tests) + + stdtests = STDTESTS[:] + nottests = NOTTESTS.copy() + if ns.exclude: + for arg in ns.args: + if arg in stdtests: + stdtests.remove(arg) + nottests.add(arg) + ns.args = [] + + # For a partial run, we do not need to clutter the output. + if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if ns.testdir: + alltests = findtests(ns.testdir, list(), set()) + else: + alltests = findtests(ns.testdir, stdtests, nottests) + + selected = tests or ns.args or alltests + if ns.single: + selected = selected[:1] + try: + next_single_test = alltests[alltests.index(selected[0])+1] + except IndexError: + next_single_test = None + # Remove all the selected tests that precede start if it's set. + if ns.start: + try: + del selected[:selected.index(ns.start)] + except ValueError: + print("Couldn't find starting test (%s), using all tests" % ns.start) + if ns.randomize: + if ns.random_seed is None: + ns.random_seed = random.randrange(10000000) + random.seed(ns.random_seed) + print("Using random seed", ns.random_seed) + random.shuffle(selected) + if ns.trace: + import trace, tempfile + tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + + test_times = [] + support.verbose = ns.verbose # Tell tests to be moderately quiet + support.use_resources = ns.use_resources + save_modules = sys.modules.keys() + + def accumulate_result(test, result): + ok, test_time = result + test_times.append((test_time, test)) + if ok == PASSED: + good.append(test) + elif ok == FAILED: + bad.append(test) + elif ok == ENV_CHANGED: + environment_changed.append(test) + elif ok == SKIPPED: + skipped.append(test) + elif ok == RESOURCE_DENIED: + skipped.append(test) + resource_denieds.append(test) + + if ns.forever: + def test_forever(tests=list(selected)): + while True: + for test in tests: + yield test + if bad: + return + tests = test_forever() + test_count = '' + test_count_width = 3 + else: + tests = iter(selected) + test_count = '/{}'.format(len(selected)) + test_count_width = len(test_count) - 1 + + if ns.use_mp: + try: + from threading import Thread + except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + from queue import Queue + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") + output = Queue() + pending = MultiprocessTests(tests) + def work(): + # A worker thread. + try: + while True: + try: + test = next(pending) + except StopIteration: + output.put((None, None, None, None)) + return + retcode, stdout, stderr = run_test_in_subprocess(test, ns) + # Strip last refcount output line if it exists, since it + # comes from the shutdown of the interpreter in the subcommand. + stderr = debug_output_pat.sub("", stderr) + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return + if not result: + output.put((None, None, None, None)) + return + result = json.loads(result) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + except BaseException: + output.put((None, None, None, None)) + raise + workers = [Thread(target=work) for i in range(ns.use_mp)] + for worker in workers: + worker.start() + finished = 0 + test_index = 1 + try: + while finished < ns.use_mp: + test, stdout, stderr, result = output.get() + if test is None: + finished += 1 + continue + accumulate_result(test, result) + if not ns.quiet: + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, + len(bad), test)) + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + sys.stdout.flush() + sys.stderr.flush() + if result[0] == INTERRUPTED: + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) + test_index += 1 + except KeyboardInterrupt: + interrupted = True + pending.interrupted = True + for worker in workers: + worker.join() + else: + for test_index, test in enumerate(tests, 1): + if not ns.quiet: + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, len(bad), test)) + sys.stdout.flush() + if ns.trace: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', + globals=globals(), locals=vars()) + else: + try: + result = runtest(test, ns.verbose, ns.quiet, + ns.huntrleaks, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests) + accumulate_result(test, result) + except KeyboardInterrupt: + interrupted = True + break + if ns.findleaks: + gc.collect() + if gc.garbage: + print("Warning: test created", len(gc.garbage), end=' ') + print("uncollectable object(s).") + # move the uncollectable objects somewhere so we don't see + # them again + found_garbage.extend(gc.garbage) + del gc.garbage[:] + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) + + if interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + omitted = set(selected) - set(good) - set(bad) - set(skipped) + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + if good and not ns.quiet: + if not bad and not skipped and not interrupted and len(good) > 1: + print("All", end=' ') + print(count(len(good), "test"), "OK.") + if ns.print_slow: + test_times.sort(reverse=True) + print("10 slowest tests:") + for time, test in test_times[:10]: + print("%s: %.1fs" % (test, time)) + if bad: + print(count(len(bad), "test"), "failed:") + printlist(bad) + if environment_changed: + print("{} altered the execution environment:".format( + count(len(environment_changed), "test"))) + printlist(environment_changed) + if skipped and not ns.quiet: + print(count(len(skipped), "test"), "skipped:") + printlist(skipped) + + if ns.verbose2 and bad: + print("Re-running failed tests in verbose mode") + for test in bad[:]: + print("Re-running test %r in verbose mode" % test) + sys.stdout.flush() + try: + ns.verbose = True + ok = runtest(test, True, ns.quiet, ns.huntrleaks, + timeout=ns.timeout) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + bad.remove(test) + else: + if bad: + print(count(len(bad), 'test'), "failed again:") + printlist(bad) + + if ns.single: + if next_single_test: + with open(filename, 'w') as fp: + fp.write(next_single_test + '\n') + else: + os.unlink(filename) + + if ns.trace: + r = tracer.results() + r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) + + if ns.runleaks: + os.system("leaks %d" % os.getpid()) + + sys.exit(len(bad) > 0 or interrupted) + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessTests(object): + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests): + self.interrupted = False + self.lock = threading.Lock() + self.tests = tests + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.interrupted: + raise StopIteration('tests interrupted') + return next(self.tests) + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + import atexit + + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) + + +def removepy(names): + if not names: + return + for idx, name in enumerate(names): + basename, ext = os.path.splitext(name) + if ext == '.py': + names[idx] = basename + + +def count(n, word): + if n == 1: + return "%d %s" % (n, word) + else: + return "%d %ss" % (n, word) + + +def printlist(x, width=70, indent=4): + """Print the elements of iterable x to stdout. + + Optional arg width (default 70) is the maximum line length. + Optional arg indent (default 4) is the number of blanks with which to + begin each line. + """ + + from textwrap import fill + blanks = ' ' * indent + # Print the sorted list: 'x' may be a '--random' list or a set() + print(fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks)) + + +def main_in_temp_cwd(): + """Run main() in a temporary working directory.""" + if sysconfig.is_python_build(): + try: + os.mkdir(TEMPDIR) + except FileExistsError: + pass + + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + test_cwd = 'test_python_{}'.format(os.getpid()) + test_cwd = os.path.join(TEMPDIR, test_cwd) + + # Run the tests in a context manager that temporarily changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + main() diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py new file mode 100644 index 0000000..dd0d05d --- /dev/null +++ b/Lib/test/libregrtest/refleak.py @@ -0,0 +1,165 @@ +import os +import re +import sys +import warnings +from inspect import isabstract +from test import support + + +def dash_R(the_module, test, indirect_test, huntrleaks): + """Run a test multiple times, looking for reference leaks. + + Returns: + False if the test didn't leak references; True if we detected refleaks. + """ + # This code is hackish and inelegant, but it seems to do the job. + import copyreg + import collections.abc + + if not hasattr(sys, 'gettotalrefcount'): + raise Exception("Tracking reference leaks requires a debug build " + "of Python") + + # Save current values for dash_R_cleanup() to restore. + fs = warnings.filters[:] + ps = copyreg.dispatch_table.copy() + pic = sys.path_importer_cache.copy() + try: + import zipimport + except ImportError: + zdc = None # Run unmodified on platforms without zipimport support + else: + zdc = zipimport._zip_directory_cache.copy() + abcs = {} + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = obj._abc_registry.copy() + + nwarmup, ntracked, fname = huntrleaks + fname = os.path.join(support.SAVEDCWD, fname) + repcount = nwarmup + ntracked + rc_deltas = [0] * repcount + alloc_deltas = [0] * repcount + + print("beginning", repcount, "repetitions", file=sys.stderr) + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) + sys.stderr.flush() + for i in range(repcount): + indirect_test() + alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) + sys.stderr.write('.') + sys.stderr.flush() + if i >= nwarmup: + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + alloc_before, rc_before = alloc_after, rc_after + print(file=sys.stderr) + # These checkers return False on success, True on failure + def check_rc_deltas(deltas): + return any(deltas) + def check_alloc_deltas(deltas): + # At least 1/3rd of 0s + if 3 * deltas.count(0) < len(deltas): + return True + # Nothing else than 1s, 0s and -1s + if not set(deltas) <= {1,0,-1}: + return True + return False + failed = False + for deltas, item_name, checker in [ + (rc_deltas, 'references', check_rc_deltas), + (alloc_deltas, 'memory blocks', check_alloc_deltas)]: + if checker(deltas): + msg = '%s leaked %s %s, sum=%s' % ( + test, deltas[nwarmup:], item_name, sum(deltas)) + print(msg, file=sys.stderr) + sys.stderr.flush() + with open(fname, "a") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + return failed + + +def dash_R_cleanup(fs, ps, pic, zdc, abcs): + import gc, copyreg + import _strptime, linecache + import urllib.parse, urllib.request, mimetypes, doctest + import struct, filecmp, collections.abc + from distutils.dir_util import _path_created + from weakref import WeakSet + + # Clear the warnings registry, so they can be displayed again + for mod in sys.modules.values(): + if hasattr(mod, '__warningregistry__'): + del mod.__warningregistry__ + + # Restore some original values. + warnings.filters[:] = fs + copyreg.dispatch_table.clear() + copyreg.dispatch_table.update(ps) + sys.path_importer_cache.clear() + sys.path_importer_cache.update(pic) + try: + import zipimport + except ImportError: + pass # Run unmodified on platforms without zipimport support + else: + zipimport._zip_directory_cache.clear() + zipimport._zip_directory_cache.update(zdc) + + # clear type cache + sys._clear_type_cache() + + # Clear ABC registries, restoring previously saved ABC registries. + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + obj._abc_registry = abcs.get(obj, WeakSet()).copy() + obj._abc_cache.clear() + obj._abc_negative_cache.clear() + + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + + # Clear assorted module caches. + _path_created.clear() + re.purge() + _strptime._regex_cache.clear() + urllib.parse.clear_cache() + urllib.request.urlcleanup() + linecache.clearcache() + mimetypes._default_mime_types() + filecmp._cache.clear() + struct._clearcache() + doctest.master = None + try: + import ctypes + except ImportError: + # Don't worry about resetting the cache if ctypes is not supported + pass + else: + ctypes._reset_cache() + + # Collect cyclic trash and read memory statistics immediately after. + func1 = sys.getallocatedblocks + func2 = sys.gettotalrefcount + gc.collect() + return func1(), func2() + + +def warm_caches(): + # char cache + s = bytes(range(256)) + for i in range(256): + s[i:i+1] + # unicode cache + x = [chr(i) for i in range(256)] + # int cache + x = list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py new file mode 100644 index 0000000..d8c0eb2 --- /dev/null +++ b/Lib/test/libregrtest/runtest.py @@ -0,0 +1,271 @@ +import faulthandler +import importlib +import io +import json +import os +import sys +import time +import traceback +import unittest +from test import support +from test.libregrtest.refleak import dash_R +from test.libregrtest.save_env import saved_test_environment + + +# Test result constants. +PASSED = 1 +FAILED = 0 +ENV_CHANGED = -1 +SKIPPED = -2 +RESOURCE_DENIED = -3 +INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process + + +def run_test_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) +STDTESTS = [ + 'test_grammar', + 'test_opcodes', + 'test_dict', + 'test_builtin', + 'test_exceptions', + 'test_types', + 'test_unittest', + 'test_doctest', + 'test_doctest2', + 'test_support' +] + +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() + + +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + names = os.listdir(testdir) + tests = [] + others = set(stdtests) | nottests + for name in names: + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) + return stdtests + sorted(tests) + + +def runtest(test, verbose, quiet, + huntrleaks=False, use_resources=None, + output_on_failure=False, failfast=False, match_tests=None, + timeout=None): + """Run a single test. + + test -- the name of the test + verbose -- if true, print more messages + quiet -- if true, don't print 'skipped' messages (probably redundant) + huntrleaks -- run multiple times to test for leaks; requires a debug + build; a triple corresponding to -R's three arguments + use_resources -- list of extra resources to use + output_on_failure -- if true, display test output on failure + timeout -- dump the traceback and exit if a test takes more than + timeout seconds + failfast, match_tests -- See regrtest command-line flags for these. + + Returns the tuple result, test_time, where result is one of the constants: + INTERRUPTED KeyboardInterrupt when run under -j + RESOURCE_DENIED test skipped because resource denied + SKIPPED test skipped for some other reason + ENV_CHANGED test failed because it changed the execution environment + FAILED test failed + PASSED test passed + """ + + if use_resources is not None: + support.use_resources = use_resources + use_timeout = (timeout is not None) + if use_timeout: + faulthandler.dump_traceback_later(timeout, exit=True) + try: + support.match_tests = match_tests + if failfast: + support.failfast = True + if output_on_failure: + support.verbose = True + + # Reuse the same instance to all calls to runtest(). Some + # tests keep a reference to sys.stdout or sys.stderr + # (eg. test_argparse). + if runtest.stringio is None: + stream = io.StringIO() + runtest.stringio = stream + else: + stream = runtest.stringio + stream.seek(0) + stream.truncate() + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + try: + sys.stdout = stream + sys.stderr = stream + result = runtest_inner(test, verbose, quiet, huntrleaks, + display_failure=False) + if result[0] == FAILED: + output = stream.getvalue() + orig_stderr.write(output) + orig_stderr.flush() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + else: + support.verbose = verbose # Tell tests to be moderately quiet + result = runtest_inner(test, verbose, quiet, huntrleaks, + display_failure=not verbose) + return result + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + cleanup_test_droppings(test, verbose) +runtest.stringio = None + + +def runtest_inner(test, verbose, quiet, + huntrleaks=False, display_failure=True): + support.unload(test) + + test_time = 0.0 + refleak = False # True if the test leaked references. + try: + if test.startswith('test.'): + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test + with saved_test_environment(test, verbose, quiet) as environment: + start_time = time.time() + the_module = importlib.import_module(abstest) + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + def test_runner(): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(the_module) + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + support.run_unittest(tests) + test_runner() + if huntrleaks: + refleak = dash_R(the_module, test, test_runner, huntrleaks) + test_time = time.time() - start_time + except support.ResourceDenied as msg: + if not quiet: + print(test, "skipped --", msg) + sys.stdout.flush() + return RESOURCE_DENIED, test_time + except unittest.SkipTest as msg: + if not quiet: + print(test, "skipped --", msg) + sys.stdout.flush() + return SKIPPED, test_time + except KeyboardInterrupt: + raise + except support.TestFailed as msg: + if display_failure: + print("test", test, "failed --", msg, file=sys.stderr) + else: + print("test", test, "failed", file=sys.stderr) + sys.stderr.flush() + return FAILED, test_time + except: + msg = traceback.format_exc() + print("test", test, "crashed --", msg, file=sys.stderr) + sys.stderr.flush() + return FAILED, test_time + else: + if refleak: + return FAILED, test_time + if environment.changed: + return ENV_CHANGED, test_time + return PASSED, test_time + + +def cleanup_test_droppings(testname, verbose): + import shutil + import stat + import gc + + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce failures. + gc.collect() + + # Try to clean up junk commonly left behind. While tests shouldn't leave + # any files or directories behind, when a test fails that can be tedious + # for it to arrange. The consequences can be especially nasty on Windows, + # since if a test leaves a file open, it cannot be deleted by name (while + # there's nothing we can do about that here either, we can display the + # name of the offending test, which is a real help). + for name in (support.TESTFN, + "db_home", + ): + if not os.path.exists(name): + continue + + if os.path.isdir(name): + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise SystemError("os.path says %r exists but is neither " + "directory nor file" % name) + + if verbose: + print("%r left behind %s %r" % (testname, kind, name)) + try: + # if we have chmod, fix possible permissions problems + # that might prevent cleanup + if (hasattr(os, 'chmod')): + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as msg: + print(("%r left behind %s %r and it couldn't be " + "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) + + +def findtestdir(path=None): + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py new file mode 100644 index 0000000..4f6a1aa --- /dev/null +++ b/Lib/test/libregrtest/save_env.py @@ -0,0 +1,284 @@ +import builtins +import locale +import logging +import os +import shutil +import sys +import sysconfig +import warnings +from test import support +try: + import threading +except ImportError: + threading = None +try: + import _multiprocessing, multiprocessing.process +except ImportError: + multiprocessing = None + + +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. + +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. + + with saved_test_environment(testname, verbose, quiet): + #stuff + + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + + changed = False + + def __init__(self, testname, verbose=0, quiet=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + + # To add things to save and restore, add a name XXX to the resources list + # and add corresponding get_XXX/restore_XXX functions. get_XXX should + # return the value to be saved and compared against a second call to the + # get function when test execution completes. restore_XXX should accept + # the saved value and restore the resource using it. It will be called if + # and only if a change in the value is detected. + # + # Note: XXX will have any '.' replaced with '_' characters when determining + # the corresponding method names. + + resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', + 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', + 'warnings.filters', 'asyncore.socket_map', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', + 'sys.warnoptions', + # multiprocessing.process._cleanup() may release ref + # to a thread, so check processes first. + 'multiprocessing.process._dangling', 'threading._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', + 'files', 'locale', 'warnings.showwarning', + ) + + def get_sys_argv(self): + return id(sys.argv), sys.argv, sys.argv[:] + def restore_sys_argv(self, saved_argv): + sys.argv = saved_argv[1] + sys.argv[:] = saved_argv[2] + + def get_cwd(self): + return os.getcwd() + def restore_cwd(self, saved_cwd): + os.chdir(saved_cwd) + + def get_sys_stdout(self): + return sys.stdout + def restore_sys_stdout(self, saved_stdout): + sys.stdout = saved_stdout + + def get_sys_stderr(self): + return sys.stderr + def restore_sys_stderr(self, saved_stderr): + sys.stderr = saved_stderr + + def get_sys_stdin(self): + return sys.stdin + def restore_sys_stdin(self, saved_stdin): + sys.stdin = saved_stdin + + def get_os_environ(self): + return id(os.environ), os.environ, dict(os.environ) + def restore_os_environ(self, saved_environ): + os.environ = saved_environ[1] + os.environ.clear() + os.environ.update(saved_environ[2]) + + def get_sys_path(self): + return id(sys.path), sys.path, sys.path[:] + def restore_sys_path(self, saved_path): + sys.path = saved_path[1] + sys.path[:] = saved_path[2] + + def get_sys_path_hooks(self): + return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] + def restore_sys_path_hooks(self, saved_hooks): + sys.path_hooks = saved_hooks[1] + sys.path_hooks[:] = saved_hooks[2] + + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + + def get___import__(self): + return builtins.__import__ + def restore___import__(self, import_): + builtins.__import__ = import_ + + def get_warnings_filters(self): + return id(warnings.filters), warnings.filters, warnings.filters[:] + def restore_warnings_filters(self, saved_filters): + warnings.filters = saved_filters[1] + warnings.filters[:] = saved_filters[2] + + def get_asyncore_socket_map(self): + asyncore = sys.modules.get('asyncore') + # XXX Making a copy keeps objects alive until __exit__ gets called. + return asyncore and asyncore.socket_map.copy() or {} + def restore_asyncore_socket_map(self, saved_map): + asyncore = sys.modules.get('asyncore') + if asyncore is not None: + asyncore.close_all(ignore_all=True) + asyncore.socket_map.update(saved_map) + + def get_shutil_archive_formats(self): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() + def restore_shutil_archive_formats(self, saved): + shutil._ARCHIVE_FORMATS = saved[0] + shutil._ARCHIVE_FORMATS.clear() + shutil._ARCHIVE_FORMATS.update(saved[1]) + + def get_shutil_unpack_formats(self): + return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() + def restore_shutil_unpack_formats(self, saved): + shutil._UNPACK_FORMATS = saved[0] + shutil._UNPACK_FORMATS.clear() + shutil._UNPACK_FORMATS.update(saved[1]) + + def get_logging__handlers(self): + # _handlers is a WeakValueDictionary + return id(logging._handlers), logging._handlers, logging._handlers.copy() + def restore_logging__handlers(self, saved_handlers): + # Can't easily revert the logging state + pass + + def get_logging__handlerList(self): + # _handlerList is a list of weakrefs to handlers + return id(logging._handlerList), logging._handlerList, logging._handlerList[:] + def restore_logging__handlerList(self, saved_handlerList): + # Can't easily revert the logging state + pass + + def get_sys_warnoptions(self): + return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] + def restore_sys_warnoptions(self, saved_options): + sys.warnoptions = saved_options[1] + sys.warnoptions[:] = saved_options[2] + + # Controlling dangling references to Thread objects can make it easier + # to track reference leaks. + def get_threading__dangling(self): + if not threading: + return None + # This copies the weakrefs without making any strong reference + return threading._dangling.copy() + def restore_threading__dangling(self, saved): + if not threading: + return + threading._dangling.clear() + threading._dangling.update(saved) + + # Same for Process objects + def get_multiprocessing_process__dangling(self): + if not multiprocessing: + return None + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + def restore_multiprocessing_process__dangling(self, saved): + if not multiprocessing: + return + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(saved) + + def get_sysconfig__CONFIG_VARS(self): + # make sure the dict is initialized + sysconfig.get_config_var('prefix') + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__INSTALL_SCHEMES(self): + return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, + sysconfig._INSTALL_SCHEMES.copy()) + def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig._INSTALL_SCHEMES = saved[1] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(saved[2]) + + def get_files(self): + return sorted(fn + ('/' if os.path.isdir(fn) else '') + for fn in os.listdir()) + def restore_files(self, saved_value): + fn = support.TESTFN + if fn not in saved_value and (fn + '/') not in saved_value: + if os.path.isfile(fn): + support.unlink(fn) + elif os.path.isdir(fn): + support.rmtree(fn) + + _lc = [getattr(locale, lc) for lc in dir(locale) + if lc.startswith('LC_')] + def get_locale(self): + pairings = [] + for lc in self._lc: + try: + pairings.append((lc, locale.setlocale(lc, None))) + except (TypeError, ValueError): + continue + return pairings + def restore_locale(self, saved): + for lc, setting in saved: + locale.setlocale(lc, setting) + + def get_warnings_showwarning(self): + return warnings.showwarning + def restore_warnings_showwarning(self, fxn): + warnings.showwarning = fxn + + def resource_info(self): + for name in self.resources: + method_suffix = name.replace('.', '_') + get_name = 'get_' + method_suffix + restore_name = 'restore_' + method_suffix + yield name, getattr(self, get_name), getattr(self, restore_name) + + def __enter__(self): + self.saved_values = dict((name, get()) for name, get, restore + in self.resource_info()) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + saved_values = self.saved_values + del self.saved_values + for name, get, restore in self.resource_info(): + current = get() + original = saved_values.pop(name) + # Check for changes to the resource's value + if current != original: + self.changed = True + restore(original) + if not self.quiet: + print("Warning -- {} was modified by {}".format( + name, self.testname), + file=sys.stderr) + if self.verbose > 1: + print(" Before: {}\n After: {} ".format( + original, current), + file=sys.stderr) + return False diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 51e9e18..f01cad4 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -9,1232 +9,9 @@ Run this script with -h or --help for documentation. # We import importlib *ASAP* in order to test #15386 import importlib -import argparse -import builtins -import faulthandler -import io -import json -import locale -import logging import os -import platform -import random -import re -import shutil -import signal import sys -import sysconfig -import tempfile -import time -import traceback -import unittest -import warnings -from inspect import isabstract - -try: - import threading -except ImportError: - threading = None -try: - import _multiprocessing, multiprocessing.process -except ImportError: - multiprocessing = None - -from test.libregrtest import _parse_args - - -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process - -from test import support - -# When tests are run from the Python build directory, it is best practice -# to keep the test files in a subfolder. This eases the cleanup of leftover -# files using the "make distclean" command. -if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') -else: - TEMPDIR = tempfile.gettempdir() -TEMPDIR = os.path.abspath(TEMPDIR) - -def run_test_in_subprocess(testname, ns): - """Run the given test in a subprocess with --slaveargs. - - ns is the option Namespace parsed from command-line arguments. regrtest - is invoked in a subprocess with the --slaveargs argument; when the - subprocess exits, its return code, stdout and stderr are returned as a - 3-tuple. - """ - from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() - return retcode, stdout, stderr - - -def main(tests=None, **kwargs): - """Execute a test suite. - - This also parses command-line options and modifies its behavior - accordingly. - - tests -- a list of strings containing test names (optional) - testdir -- the directory in which to look for tests (optional) - - Users other than the Python test suite will certainly want to - specify testdir; if it's omitted, the directory containing the - Python test suite is searched for. - - If the tests argument is omitted, the tests listed on the - command-line will be used. If that's empty, too, then all *.py - files beginning with test_ will be used. - - The other default arguments (verbose, quiet, exclude, - single, randomize, findleaks, use_resources, trace, coverdir, - print_slow, and random_seed) allow programmers calling main() - directly to set the values that would normally be set by flags - on the command line. - """ - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - - support.record_original_stdout(sys.stdout) - - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - if ns.threshold is not None: - import gc - gc.set_threshold(ns.threshold) - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if ns.wait: - input("Press any key to continue...") - - if ns.slaveargs is not None: - args, kwargs = json.loads(ns.slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - good = [] - bad = [] - skipped = [] - resource_denieds = [] - environment_changed = [] - interrupted = False - - if ns.findleaks: - try: - import gc - except ImportError: - print('No GC available, disabling findleaks.') - ns.findleaks = False - else: - # Uncomment the line below to report garbage that is not - # freeable by reference counting alone. By default only - # garbage that is not collectable by the GC is reported. - #gc.set_debug(gc.DEBUG_SAVEALL) - found_garbage = [] - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - if ns.single: - filename = os.path.join(TEMPDIR, 'pynexttest') - try: - with open(filename, 'r') as fp: - next_test = fp.read().strip() - tests = [next_test] - except OSError: - pass - - if ns.fromfile: - tests = [] - with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - - # Strip .py extensions. - removepy(ns.args) - removepy(tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if ns.exclude: - for arg in ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - ns.args = [] - - # For a partial run, we do not need to clutter the output. - if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if ns.testdir: - alltests = findtests(ns.testdir, list(), set()) - else: - alltests = findtests(ns.testdir, stdtests, nottests) - - selected = tests or ns.args or alltests - if ns.single: - selected = selected[:1] - try: - next_single_test = alltests[alltests.index(selected[0])+1] - except IndexError: - next_single_test = None - # Remove all the selected tests that precede start if it's set. - if ns.start: - try: - del selected[:selected.index(ns.start)] - except ValueError: - print("Couldn't find starting test (%s), using all tests" % ns.start) - if ns.randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) - print("Using random seed", ns.random_seed) - random.shuffle(selected) - if ns.trace: - import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - - test_times = [] - support.verbose = ns.verbose # Tell tests to be moderately quiet - support.use_resources = ns.use_resources - save_modules = sys.modules.keys() - - def accumulate_result(test, result): - ok, test_time = result - test_times.append((test_time, test)) - if ok == PASSED: - good.append(test) - elif ok == FAILED: - bad.append(test) - elif ok == ENV_CHANGED: - environment_changed.append(test) - elif ok == SKIPPED: - skipped.append(test) - elif ok == RESOURCE_DENIED: - skipped.append(test) - resource_denieds.append(test) - - if ns.forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() - test_count = '' - test_count_width = 3 - else: - tests = iter(selected) - test_count = '/{}'.format(len(selected)) - test_count_width = len(test_count) - 1 - - if ns.use_mp: - try: - from threading import Thread - except ImportError: - print("Multiprocess option requires thread support") - sys.exit(2) - from queue import Queue - debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - output = Queue() - pending = MultiprocessTests(tests) - def work(): - # A worker thread. - try: - while True: - try: - test = next(pending) - except StopIteration: - output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_test_in_subprocess(test, ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return - if not result: - output.put((None, None, None, None)) - return - result = json.loads(result) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - except BaseException: - output.put((None, None, None, None)) - raise - workers = [Thread(target=work) for i in range(ns.use_mp)] - for worker in workers: - worker.start() - finished = 0 - test_index = 1 - try: - while finished < ns.use_mp: - test, stdout, stderr, result = output.get() - if test is None: - finished += 1 - continue - accumulate_result(test, result) - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, - len(bad), test)) - if stdout: - print(stdout) - if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() - if result[0] == INTERRUPTED: - raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) - test_index += 1 - except KeyboardInterrupt: - interrupted = True - pending.interrupted = True - for worker in workers: - worker.join() - else: - for test_index, test in enumerate(tests, 1): - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, len(bad), test)) - sys.stdout.flush() - if ns.trace: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', - globals=globals(), locals=vars()) - else: - try: - result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests) - accumulate_result(test, result) - except KeyboardInterrupt: - interrupted = True - break - if ns.findleaks: - gc.collect() - if gc.garbage: - print("Warning: test created", len(gc.garbage), end=' ') - print("uncollectable object(s).") - # move the uncollectable objects somewhere so we don't see - # them again - found_garbage.extend(gc.garbage) - del gc.garbage[:] - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) - - if interrupted: - # print a newline after ^C - print() - print("Test suite interrupted by signal SIGINT.") - omitted = set(selected) - set(good) - set(bad) - set(skipped) - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if good and not ns.quiet: - if not bad and not skipped and not interrupted and len(good) > 1: - print("All", end=' ') - print(count(len(good), "test"), "OK.") - if ns.print_slow: - test_times.sort(reverse=True) - print("10 slowest tests:") - for time, test in test_times[:10]: - print("%s: %.1fs" % (test, time)) - if bad: - print(count(len(bad), "test"), "failed:") - printlist(bad) - if environment_changed: - print("{} altered the execution environment:".format( - count(len(environment_changed), "test"))) - printlist(environment_changed) - if skipped and not ns.quiet: - print(count(len(skipped), "test"), "skipped:") - printlist(skipped) - - if ns.verbose2 and bad: - print("Re-running failed tests in verbose mode") - for test in bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() - try: - ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, - timeout=ns.timeout) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - bad.remove(test) - else: - if bad: - print(count(len(bad), 'test'), "failed again:") - printlist(bad) - - if ns.single: - if next_single_test: - with open(filename, 'w') as fp: - fp.write(next_single_test + '\n') - else: - os.unlink(filename) - - if ns.trace: - r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) - - if ns.runleaks: - os.system("leaks %d" % os.getpid()) - - sys.exit(len(bad) > 0 or interrupted) - - -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): - """Return a list of all applicable test modules.""" - testdir = findtestdir(testdir) - names = os.listdir(testdir) - tests = [] - others = set(stdtests) | nottests - for name in names: - mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) - return stdtests + sorted(tests) - -# We do not use a generator so multiple threads can call next(). -class MultiprocessTests(object): - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.interrupted = False - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - if self.interrupted: - raise StopIteration('tests interrupted') - return next(self.tests) - -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - -def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None, - timeout=None): - """Run a single test. - - test -- the name of the test - verbose -- if true, print more messages - quiet -- if true, don't print 'skipped' messages (probably redundant) - huntrleaks -- run multiple times to test for leaks; requires a debug - build; a triple corresponding to -R's three arguments - use_resources -- list of extra resources to use - output_on_failure -- if true, display test output on failure - timeout -- dump the traceback and exit if a test takes more than - timeout seconds - failfast, match_tests -- See regrtest command-line flags for these. - - Returns the tuple result, test_time, where result is one of the constants: - INTERRUPTED KeyboardInterrupt when run under -j - RESOURCE_DENIED test skipped because resource denied - SKIPPED test skipped for some other reason - ENV_CHANGED test failed because it changed the execution environment - FAILED test failed - PASSED test passed - """ - - if use_resources is not None: - support.use_resources = use_resources - use_timeout = (timeout is not None) - if use_timeout: - faulthandler.dump_traceback_later(timeout, exit=True) - try: - support.match_tests = match_tests - if failfast: - support.failfast = True - if output_on_failure: - support.verbose = True - - # Reuse the same instance to all calls to runtest(). Some - # tests keep a reference to sys.stdout or sys.stderr - # (eg. test_argparse). - if runtest.stringio is None: - stream = io.StringIO() - runtest.stringio = stream - else: - stream = runtest.stringio - stream.seek(0) - stream.truncate() - - orig_stdout = sys.stdout - orig_stderr = sys.stderr - try: - sys.stdout = stream - sys.stderr = stream - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=False) - if result[0] == FAILED: - output = stream.getvalue() - orig_stderr.write(output) - orig_stderr.flush() - finally: - sys.stdout = orig_stdout - sys.stderr = orig_stderr - else: - support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=not verbose) - return result - finally: - if use_timeout: - faulthandler.cancel_dump_traceback_later() - cleanup_test_droppings(test, verbose) -runtest.stringio = None - -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. - -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. - - with saved_test_environment(testname, verbose, quiet): - #stuff - - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. - - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - - changed = False - - def __init__(self, testname, verbose=0, quiet=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - - # To add things to save and restore, add a name XXX to the resources list - # and add corresponding get_XXX/restore_XXX functions. get_XXX should - # return the value to be saved and compared against a second call to the - # get function when test execution completes. restore_XXX should accept - # the saved value and restore the resource using it. It will be called if - # and only if a change in the value is detected. - # - # Note: XXX will have any '.' replaced with '_' characters when determining - # the corresponding method names. - - resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', - 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', - 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', 'sys.gettrace', - 'sys.warnoptions', - # multiprocessing.process._cleanup() may release ref - # to a thread, so check processes first. - 'multiprocessing.process._dangling', 'threading._dangling', - 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'files', 'locale', 'warnings.showwarning', - ) - - def get_sys_argv(self): - return id(sys.argv), sys.argv, sys.argv[:] - def restore_sys_argv(self, saved_argv): - sys.argv = saved_argv[1] - sys.argv[:] = saved_argv[2] - - def get_cwd(self): - return os.getcwd() - def restore_cwd(self, saved_cwd): - os.chdir(saved_cwd) - - def get_sys_stdout(self): - return sys.stdout - def restore_sys_stdout(self, saved_stdout): - sys.stdout = saved_stdout - - def get_sys_stderr(self): - return sys.stderr - def restore_sys_stderr(self, saved_stderr): - sys.stderr = saved_stderr - - def get_sys_stdin(self): - return sys.stdin - def restore_sys_stdin(self, saved_stdin): - sys.stdin = saved_stdin - - def get_os_environ(self): - return id(os.environ), os.environ, dict(os.environ) - def restore_os_environ(self, saved_environ): - os.environ = saved_environ[1] - os.environ.clear() - os.environ.update(saved_environ[2]) - - def get_sys_path(self): - return id(sys.path), sys.path, sys.path[:] - def restore_sys_path(self, saved_path): - sys.path = saved_path[1] - sys.path[:] = saved_path[2] - - def get_sys_path_hooks(self): - return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] - def restore_sys_path_hooks(self, saved_hooks): - sys.path_hooks = saved_hooks[1] - sys.path_hooks[:] = saved_hooks[2] - - def get_sys_gettrace(self): - return sys.gettrace() - def restore_sys_gettrace(self, trace_fxn): - sys.settrace(trace_fxn) - - def get___import__(self): - return builtins.__import__ - def restore___import__(self, import_): - builtins.__import__ = import_ - - def get_warnings_filters(self): - return id(warnings.filters), warnings.filters, warnings.filters[:] - def restore_warnings_filters(self, saved_filters): - warnings.filters = saved_filters[1] - warnings.filters[:] = saved_filters[2] - - def get_asyncore_socket_map(self): - asyncore = sys.modules.get('asyncore') - # XXX Making a copy keeps objects alive until __exit__ gets called. - return asyncore and asyncore.socket_map.copy() or {} - def restore_asyncore_socket_map(self, saved_map): - asyncore = sys.modules.get('asyncore') - if asyncore is not None: - asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(saved_map) - - def get_shutil_archive_formats(self): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() - def restore_shutil_archive_formats(self, saved): - shutil._ARCHIVE_FORMATS = saved[0] - shutil._ARCHIVE_FORMATS.clear() - shutil._ARCHIVE_FORMATS.update(saved[1]) - - def get_shutil_unpack_formats(self): - return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() - def restore_shutil_unpack_formats(self, saved): - shutil._UNPACK_FORMATS = saved[0] - shutil._UNPACK_FORMATS.clear() - shutil._UNPACK_FORMATS.update(saved[1]) - - def get_logging__handlers(self): - # _handlers is a WeakValueDictionary - return id(logging._handlers), logging._handlers, logging._handlers.copy() - def restore_logging__handlers(self, saved_handlers): - # Can't easily revert the logging state - pass - - def get_logging__handlerList(self): - # _handlerList is a list of weakrefs to handlers - return id(logging._handlerList), logging._handlerList, logging._handlerList[:] - def restore_logging__handlerList(self, saved_handlerList): - # Can't easily revert the logging state - pass - - def get_sys_warnoptions(self): - return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] - def restore_sys_warnoptions(self, saved_options): - sys.warnoptions = saved_options[1] - sys.warnoptions[:] = saved_options[2] - - # Controlling dangling references to Thread objects can make it easier - # to track reference leaks. - def get_threading__dangling(self): - if not threading: - return None - # This copies the weakrefs without making any strong reference - return threading._dangling.copy() - def restore_threading__dangling(self, saved): - if not threading: - return - threading._dangling.clear() - threading._dangling.update(saved) - - # Same for Process objects - def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) - - def get_sysconfig__CONFIG_VARS(self): - # make sure the dict is initialized - sysconfig.get_config_var('prefix') - return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, - dict(sysconfig._CONFIG_VARS)) - def restore_sysconfig__CONFIG_VARS(self, saved): - sysconfig._CONFIG_VARS = saved[1] - sysconfig._CONFIG_VARS.clear() - sysconfig._CONFIG_VARS.update(saved[2]) - - def get_sysconfig__INSTALL_SCHEMES(self): - return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, - sysconfig._INSTALL_SCHEMES.copy()) - def restore_sysconfig__INSTALL_SCHEMES(self, saved): - sysconfig._INSTALL_SCHEMES = saved[1] - sysconfig._INSTALL_SCHEMES.clear() - sysconfig._INSTALL_SCHEMES.update(saved[2]) - - def get_files(self): - return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) - def restore_files(self, saved_value): - fn = support.TESTFN - if fn not in saved_value and (fn + '/') not in saved_value: - if os.path.isfile(fn): - support.unlink(fn) - elif os.path.isdir(fn): - support.rmtree(fn) - - _lc = [getattr(locale, lc) for lc in dir(locale) - if lc.startswith('LC_')] - def get_locale(self): - pairings = [] - for lc in self._lc: - try: - pairings.append((lc, locale.setlocale(lc, None))) - except (TypeError, ValueError): - continue - return pairings - def restore_locale(self, saved): - for lc, setting in saved: - locale.setlocale(lc, setting) - - def get_warnings_showwarning(self): - return warnings.showwarning - def restore_warnings_showwarning(self, fxn): - warnings.showwarning = fxn - - def resource_info(self): - for name in self.resources: - method_suffix = name.replace('.', '_') - get_name = 'get_' + method_suffix - restore_name = 'restore_' + method_suffix - yield name, getattr(self, get_name), getattr(self, restore_name) - - def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - saved_values = self.saved_values - del self.saved_values - for name, get, restore in self.resource_info(): - current = get() - original = saved_values.pop(name) - # Check for changes to the resource's value - if current != original: - self.changed = True - restore(original) - if not self.quiet: - print("Warning -- {} was modified by {}".format( - name, self.testname), - file=sys.stderr) - if self.verbose > 1: - print(" Before: {}\n After: {} ".format( - original, current), - file=sys.stderr) - return False - - -def runtest_inner(test, verbose, quiet, - huntrleaks=False, display_failure=True): - support.unload(test) - - test_time = 0.0 - refleak = False # True if the test leaked references. - try: - if test.startswith('test.'): - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test - with saved_test_environment(test, verbose, quiet) as environment: - start_time = time.time() - the_module = importlib.import_module(abstest) - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - def test_runner(): - loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(the_module) - for error in loader.errors: - print(error, file=sys.stderr) - if loader.errors: - raise Exception("errors while loading tests") - support.run_unittest(tests) - test_runner() - if huntrleaks: - refleak = dash_R(the_module, test, test_runner, huntrleaks) - test_time = time.time() - start_time - except support.ResourceDenied as msg: - if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() - return RESOURCE_DENIED, test_time - except unittest.SkipTest as msg: - if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() - return SKIPPED, test_time - except KeyboardInterrupt: - raise - except support.TestFailed as msg: - if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) - else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - except: - msg = traceback.format_exc() - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - else: - if refleak: - return FAILED, test_time - if environment.changed: - return ENV_CHANGED, test_time - return PASSED, test_time - -def cleanup_test_droppings(testname, verbose): - import shutil - import stat - import gc - - # First kill any dangling references to open files etc. - # This can also issue some ResourceWarnings which would otherwise get - # triggered during the following test run, and possibly produce failures. - gc.collect() - - # Try to clean up junk commonly left behind. While tests shouldn't leave - # any files or directories behind, when a test fails that can be tedious - # for it to arrange. The consequences can be especially nasty on Windows, - # since if a test leaves a file open, it cannot be deleted by name (while - # there's nothing we can do about that here either, we can display the - # name of the offending test, which is a real help). - for name in (support.TESTFN, - "db_home", - ): - if not os.path.exists(name): - continue - - if os.path.isdir(name): - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise SystemError("os.path says %r exists but is neither " - "directory nor file" % name) - - if verbose: - print("%r left behind %s %r" % (testname, kind, name)) - try: - # if we have chmod, fix possible permissions problems - # that might prevent cleanup - if (hasattr(os, 'chmod')): - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as msg: - print(("%r left behind %s %r and it couldn't be " - "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) - -def dash_R(the_module, test, indirect_test, huntrleaks): - """Run a test multiple times, looking for reference leaks. - - Returns: - False if the test didn't leak references; True if we detected refleaks. - """ - # This code is hackish and inelegant, but it seems to do the job. - import copyreg - import collections.abc - - if not hasattr(sys, 'gettotalrefcount'): - raise Exception("Tracking reference leaks requires a debug build " - "of Python") - - # Save current values for dash_R_cleanup() to restore. - fs = warnings.filters[:] - ps = copyreg.dispatch_table.copy() - pic = sys.path_importer_cache.copy() - try: - import zipimport - except ImportError: - zdc = None # Run unmodified on platforms without zipimport support - else: - zdc = zipimport._zip_directory_cache.copy() - abcs = {} - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = obj._abc_registry.copy() - - nwarmup, ntracked, fname = huntrleaks - fname = os.path.join(support.SAVEDCWD, fname) - repcount = nwarmup + ntracked - rc_deltas = [0] * repcount - alloc_deltas = [0] * repcount - - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() - for i in range(repcount): - indirect_test() - alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() - if i >= nwarmup: - rc_deltas[i] = rc_after - rc_before - alloc_deltas[i] = alloc_after - alloc_before - alloc_before, rc_before = alloc_after, rc_after - print(file=sys.stderr) - # These checkers return False on success, True on failure - def check_rc_deltas(deltas): - return any(deltas) - def check_alloc_deltas(deltas): - # At least 1/3rd of 0s - if 3 * deltas.count(0) < len(deltas): - return True - # Nothing else than 1s, 0s and -1s - if not set(deltas) <= {1,0,-1}: - return True - return False - failed = False - for deltas, item_name, checker in [ - (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas)]: - if checker(deltas): - msg = '%s leaked %s %s, sum=%s' % ( - test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() - with open(fname, "a") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return failed - -def dash_R_cleanup(fs, ps, pic, zdc, abcs): - import gc, copyreg - import _strptime, linecache - import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, collections.abc - from distutils.dir_util import _path_created - from weakref import WeakSet - - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - - # Restore some original values. - warnings.filters[:] = fs - copyreg.dispatch_table.clear() - copyreg.dispatch_table.update(ps) - sys.path_importer_cache.clear() - sys.path_importer_cache.update(pic) - try: - import zipimport - except ImportError: - pass # Run unmodified on platforms without zipimport support - else: - zipimport._zip_directory_cache.clear() - zipimport._zip_directory_cache.update(zdc) - - # clear type cache - sys._clear_type_cache() - - # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - obj._abc_registry = abcs.get(obj, WeakSet()).copy() - obj._abc_cache.clear() - obj._abc_negative_cache.clear() - - # Flush standard output, so that buffered data is sent to the OS and - # associated Python objects are reclaimed. - for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): - if stream is not None: - stream.flush() - - # Clear assorted module caches. - _path_created.clear() - re.purge() - _strptime._regex_cache.clear() - urllib.parse.clear_cache() - urllib.request.urlcleanup() - linecache.clearcache() - mimetypes._default_mime_types() - filecmp._cache.clear() - struct._clearcache() - doctest.master = None - try: - import ctypes - except ImportError: - # Don't worry about resetting the cache if ctypes is not supported - pass - else: - ctypes._reset_cache() - - # Collect cyclic trash and read memory statistics immediately after. - func1 = sys.getallocatedblocks - func2 = sys.gettotalrefcount - gc.collect() - return func1(), func2() - -def warm_caches(): - # char cache - s = bytes(range(256)) - for i in range(256): - s[i:i+1] - # unicode cache - x = [chr(i) for i in range(256)] - # int cache - x = list(range(-5, 257)) - -def findtestdir(path=None): - return path or os.path.dirname(__file__) or os.curdir - -def removepy(names): - if not names: - return - for idx, name in enumerate(names): - basename, ext = os.path.splitext(name) - if ext == '.py': - names[idx] = basename - -def count(n, word): - if n == 1: - return "%d %s" % (n, word) - else: - return "%d %ss" % (n, word) - -def printlist(x, width=70, indent=4): - """Print the elements of iterable x to stdout. - - Optional arg width (default 70) is the maximum line length. - Optional arg indent (default 4) is the number of blanks with which to - begin each line. - """ - - from textwrap import fill - blanks = ' ' * indent - # Print the sorted list: 'x' may be a '--random' list or a set() - print(fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) - - -def main_in_temp_cwd(): - """Run main() in a temporary working directory.""" - if sysconfig.is_python_build(): - try: - os.mkdir(TEMPDIR) - except FileExistsError: - pass - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from support.SAVEDCWD. - with support.temp_cwd(test_cwd, quiet=True): - main() +from test.libregrtest import main_in_temp_cwd if __name__ == '__main__': diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index cf4b84f..f744127 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,8 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support, libregrtest +from test import libregrtest +from test import support class ParseArgsTestCase(unittest.TestCase): @@ -15,7 +16,7 @@ class ParseArgsTestCase(unittest.TestCase): def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): - regrtest._parse_args(args) + libregrtest._parse_args(args) self.assertIn(msg, err.getvalue()) def test_help(self): @@ -23,82 +24,82 @@ class ParseArgsTestCase(unittest.TestCase): with self.subTest(opt=opt): with support.captured_stdout() as out, \ self.assertRaises(SystemExit): - regrtest._parse_args([opt]) + libregrtest._parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), "faulthandler.dump_traceback_later() required") def test_timeout(self): - ns = regrtest._parse_args(['--timeout', '4.2']) + ns = libregrtest._parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) self.checkError(['--timeout'], 'expected one argument') self.checkError(['--timeout', 'foo'], 'invalid float value') def test_wait(self): - ns = regrtest._parse_args(['--wait']) + ns = libregrtest._parse_args(['--wait']) self.assertTrue(ns.wait) def test_slaveargs(self): - ns = regrtest._parse_args(['--slaveargs', '[[], {}]']) + ns = libregrtest._parse_args(['--slaveargs', '[[], {}]']) self.assertEqual(ns.slaveargs, '[[], {}]') self.checkError(['--slaveargs'], 'expected one argument') def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.start, 'foo') self.checkError([opt], 'expected one argument') def test_verbose(self): - ns = regrtest._parse_args(['-v']) + ns = libregrtest._parse_args(['-v']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['-vvv']) + ns = libregrtest._parse_args(['-vvv']) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args(['--verbose']) + ns = libregrtest._parse_args(['--verbose']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['--verbose'] * 3) + ns = libregrtest._parse_args(['--verbose'] * 3) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args([]) + ns = libregrtest._parse_args([]) self.assertEqual(ns.verbose, 0) def test_verbose2(self): for opt in '-w', '--verbose2': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose2) def test_verbose3(self): for opt in '-W', '--verbose3': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose3) def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_slow(self): for opt in '-o', '--slow': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.print_slow) def test_header(self): - ns = regrtest._parse_args(['--header']) + ns = libregrtest._parse_args(['--header']) self.assertTrue(ns.header) def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.randomize) def test_randseed(self): - ns = regrtest._parse_args(['--randseed', '12345']) + ns = libregrtest._parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) self.assertTrue(ns.randomize) self.checkError(['--randseed'], 'expected one argument') @@ -107,7 +108,7 @@ class ParseArgsTestCase(unittest.TestCase): def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.fromfile, 'foo') self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo', '-s'], "don't go together") @@ -115,42 +116,42 @@ class ParseArgsTestCase(unittest.TestCase): def test_exclude(self): for opt in '-x', '--exclude': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.exclude) def test_single(self): for opt in '-s', '--single': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'pattern']) + ns = libregrtest._parse_args([opt, 'pattern']) self.assertEqual(ns.match_tests, 'pattern') self.checkError([opt], 'expected one argument') def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '-v']) + ns = libregrtest._parse_args([opt, '-v']) self.assertTrue(ns.failfast) - ns = regrtest._parse_args([opt, '-W']) + ns = libregrtest._parse_args([opt, '-W']) self.assertTrue(ns.failfast) self.checkError([opt], '-G/--failfast needs either -v or -W') def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'gui,network']) + ns = libregrtest._parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) - ns = regrtest._parse_args([opt, 'gui,none,network']) + ns = libregrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') - ns = regrtest._parse_args([opt, 'all,-gui']) + ns = libregrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') @@ -158,31 +159,31 @@ class ParseArgsTestCase(unittest.TestCase): def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '4G']) + ns = libregrtest._parse_args([opt, '4G']) self.assertEqual(ns.memlimit, '4G') self.checkError([opt], 'expected one argument') def test_testdir(self): - ns = regrtest._parse_args(['--testdir', 'foo']) + ns = libregrtest._parse_args(['--testdir', 'foo']) self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError(['--testdir'], 'expected one argument') def test_runleaks(self): for opt in '-L', '--runleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.runleaks) def test_huntrleaks(self): for opt in '-R', '--huntrleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, ':']) + ns = libregrtest._parse_args([opt, ':']) self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:']) + ns = libregrtest._parse_args([opt, '6:']) self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, ':3']) + ns = libregrtest._parse_args([opt, ':3']) self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:3:leaks.log']) + ns = libregrtest._parse_args([opt, '6:3:leaks.log']) self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) self.checkError([opt], 'expected one argument') self.checkError([opt, '6'], @@ -193,7 +194,7 @@ class ParseArgsTestCase(unittest.TestCase): def test_multiprocess(self): for opt in '-j', '--multiprocess': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '2']) + ns = libregrtest._parse_args([opt, '2']) self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -204,13 +205,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_coverage(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.coverdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError([opt], 'expected one argument') @@ -218,13 +219,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_nocoverdir(self): for opt in '-N', '--nocoverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertIsNone(ns.coverdir) def test_threshold(self): for opt in '-t', '--threshold': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '1000']) + ns = libregrtest._parse_args([opt, '1000']) self.assertEqual(ns.threshold, 1000) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -232,13 +233,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.nowindows) def test_forever(self): for opt in '-F', '--forever': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.forever) @@ -246,26 +247,26 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError(['--xxx'], 'usage:') def test_long_option__partial(self): - ns = regrtest._parse_args(['--qui']) + ns = libregrtest._parse_args(['--qui']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_two_options(self): - ns = regrtest._parse_args(['--quiet', '--exclude']) + ns = libregrtest._parse_args(['--quiet', '--exclude']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertTrue(ns.exclude) def test_option_with_empty_string_value(self): - ns = regrtest._parse_args(['--start', '']) + ns = libregrtest._parse_args(['--start', '']) self.assertEqual(ns.start, '') def test_arg(self): - ns = regrtest._parse_args(['foo']) + ns = libregrtest._parse_args(['foo']) self.assertEqual(ns.args, ['foo']) def test_option_and_arg(self): - ns = regrtest._parse_args(['--quiet', 'foo']) + ns = libregrtest._parse_args(['--quiet', 'foo']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertEqual(ns.args, ['foo']) -- cgit v0.12 From f2b02ced7edfa30e3ee5f15abcdd8f45b36476f5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 17:47:02 -0700 Subject: Bump up the maximum number of freeblocks --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 487c765..9210e24 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -121,7 +121,7 @@ static PyTypeObject deque_type; added at about the same rate as old blocks are being freed. */ -#define MAXFREEBLOCKS 10 +#define MAXFREEBLOCKS 16 static Py_ssize_t numfreeblocks = 0; static block *freeblocks[MAXFREEBLOCKS]; -- cgit v0.12 From 7c0b70f41910fb70432f32f8ed263a56110a5525 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 21:11:05 -0700 Subject: Minor tweak to the order of variable updates. --- Modules/_collectionsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 9210e24..66c2a74 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -315,8 +315,8 @@ deque_append(dequeobject *deque, PyObject *item) MARK_END(b->rightlink); deque->rightindex = -1; } - Py_INCREF(item); Py_SIZE(deque)++; + Py_INCREF(item); deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; deque_trim_left(deque); @@ -340,8 +340,8 @@ deque_appendleft(dequeobject *deque, PyObject *item) MARK_END(b->leftlink); deque->leftindex = BLOCKLEN; } - Py_INCREF(item); Py_SIZE(deque)++; + Py_INCREF(item); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; deque_trim_right(deque); -- cgit v0.12 From 8299e9b59ecea5cd7236949bf17392e2fd56aec8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 21:31:23 -0700 Subject: Move the copy and clear functions upwards to eliminate unnecessary forward references. --- Modules/_collectionsmodule.c | 230 +++++++++++++++++++++---------------------- 1 file changed, 113 insertions(+), 117 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 66c2a74..5db7aed 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -522,7 +522,40 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) return (PyObject *)deque; } -static PyObject *deque_copy(PyObject *deque); +static PyObject * +deque_copy(PyObject *deque) +{ + dequeobject *old_deque = (dequeobject *)deque; + if (Py_TYPE(deque) == &deque_type) { + dequeobject *new_deque; + PyObject *rv; + + new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); + if (new_deque == NULL) + return NULL; + new_deque->maxlen = old_deque->maxlen; + /* Fast path for the deque_repeat() common case where len(deque) == 1 */ + if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { + PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; + rv = deque_append(new_deque, item); + } else { + rv = deque_extend(new_deque, deque); + } + if (rv != NULL) { + Py_DECREF(rv); + return (PyObject *)new_deque; + } + Py_DECREF(new_deque); + return NULL; + } + if (old_deque->maxlen < 0) + return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); + else + return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", + deque, old_deque->maxlen, NULL); +} + +PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); static PyObject * deque_concat(dequeobject *deque, PyObject *other) @@ -552,7 +585,85 @@ deque_concat(dequeobject *deque, PyObject *other) return new_deque; } -static void deque_clear(dequeobject *deque); +static void +deque_clear(dequeobject *deque) +{ + block *b; + block *prevblock; + block *leftblock; + Py_ssize_t leftindex; + Py_ssize_t n; + PyObject *item; + + /* During the process of clearing a deque, decrefs can cause the + deque to mutate. To avoid fatal confusion, we have to make the + deque empty before clearing the blocks and never refer to + anything via deque->ref while clearing. (This is the same + technique used for clearing lists, sets, and dicts.) + + Making the deque empty requires allocating a new empty block. In + the unlikely event that memory is full, we fall back to an + alternate method that doesn't require a new block. Repeating + pops in a while-loop is slower, possibly re-entrant (and a clever + adversary could cause it to never terminate). + */ + + b = newblock(0); + if (b == NULL) { + PyErr_Clear(); + goto alternate_method; + } + + /* Remember the old size, leftblock, and leftindex */ + leftblock = deque->leftblock; + leftindex = deque->leftindex; + n = Py_SIZE(deque); + + /* Set the deque to be empty using the newly allocated block */ + MARK_END(b->leftlink); + MARK_END(b->rightlink); + Py_SIZE(deque) = 0; + deque->leftblock = b; + deque->rightblock = b; + deque->leftindex = CENTER + 1; + deque->rightindex = CENTER; + deque->state++; + + /* Now the old size, leftblock, and leftindex are disconnected from + the empty deque and we can use them to decref the pointers. + */ + while (n--) { + item = leftblock->data[leftindex]; + Py_DECREF(item); + leftindex++; + if (leftindex == BLOCKLEN && n) { + CHECK_NOT_END(leftblock->rightlink); + prevblock = leftblock; + leftblock = leftblock->rightlink; + leftindex = 0; + freeblock(prevblock); + } + } + CHECK_END(leftblock->rightlink); + freeblock(leftblock); + return; + + alternate_method: + while (Py_SIZE(deque)) { + item = deque_pop(deque, NULL); + assert (item != NULL); + Py_DECREF(item); + } +} + +static PyObject * +deque_clearmethod(dequeobject *deque) +{ + deque_clear(deque); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) @@ -1058,77 +1169,6 @@ deque_remove(dequeobject *deque, PyObject *value) PyDoc_STRVAR(remove_doc, "D.remove(value) -- remove first occurrence of value."); -static void -deque_clear(dequeobject *deque) -{ - block *b; - block *prevblock; - block *leftblock; - Py_ssize_t leftindex; - Py_ssize_t n; - PyObject *item; - - /* During the process of clearing a deque, decrefs can cause the - deque to mutate. To avoid fatal confusion, we have to make the - deque empty before clearing the blocks and never refer to - anything via deque->ref while clearing. (This is the same - technique used for clearing lists, sets, and dicts.) - - Making the deque empty requires allocating a new empty block. In - the unlikely event that memory is full, we fall back to an - alternate method that doesn't require a new block. Repeating - pops in a while-loop is slower, possibly re-entrant (and a clever - adversary could cause it to never terminate). - */ - - b = newblock(0); - if (b == NULL) { - PyErr_Clear(); - goto alternate_method; - } - - /* Remember the old size, leftblock, and leftindex */ - leftblock = deque->leftblock; - leftindex = deque->leftindex; - n = Py_SIZE(deque); - - /* Set the deque to be empty using the newly allocated block */ - MARK_END(b->leftlink); - MARK_END(b->rightlink); - Py_SIZE(deque) = 0; - deque->leftblock = b; - deque->rightblock = b; - deque->leftindex = CENTER + 1; - deque->rightindex = CENTER; - deque->state++; - - /* Now the old size, leftblock, and leftindex are disconnected from - the empty deque and we can use them to decref the pointers. - */ - while (n--) { - item = leftblock->data[leftindex]; - Py_DECREF(item); - leftindex++; - if (leftindex == BLOCKLEN && n) { - CHECK_NOT_END(leftblock->rightlink); - prevblock = leftblock; - leftblock = leftblock->rightlink; - leftindex = 0; - freeblock(prevblock); - } - } - CHECK_END(leftblock->rightlink); - freeblock(leftblock); - return; - - alternate_method: - while (Py_SIZE(deque)) { - item = deque_pop(deque, NULL); - assert (item != NULL); - Py_DECREF(item); - } -} - static int valid_index(Py_ssize_t i, Py_ssize_t limit) { @@ -1229,15 +1269,6 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) return 0; } -static PyObject * -deque_clearmethod(dequeobject *deque) -{ - deque_clear(deque); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); - static void deque_dealloc(dequeobject *deque) { @@ -1277,41 +1308,6 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) } static PyObject * -deque_copy(PyObject *deque) -{ - dequeobject *old_deque = (dequeobject *)deque; - if (Py_TYPE(deque) == &deque_type) { - dequeobject *new_deque; - PyObject *rv; - - new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); - if (new_deque == NULL) - return NULL; - new_deque->maxlen = old_deque->maxlen; - /* Fast path for the deque_repeat() common case where len(deque) == 1 */ - if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { - PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; - rv = deque_append(new_deque, item); - } else { - rv = deque_extend(new_deque, deque); - } - if (rv != NULL) { - Py_DECREF(rv); - return (PyObject *)new_deque; - } - Py_DECREF(new_deque); - return NULL; - } - if (old_deque->maxlen < 0) - return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); - else - return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", - deque, old_deque->maxlen, NULL); -} - -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); - -static PyObject * deque_reduce(dequeobject *deque) { PyObject *dict, *result, *aslist; -- cgit v0.12 From 36b3fbb0ee6e266381cec91a0a3c15fd403c3cfd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 27 Sep 2015 11:19:08 +0200 Subject: Issue #25220: Fix Lib/test/autotest.py --- Lib/test/libregrtest/__init__.py | 2 +- Lib/test/regrtest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index a882ed2..9f7b1c1 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,2 +1,2 @@ from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES -from test.libregrtest.main import main_in_temp_cwd +from test.libregrtest.main import main, main_in_temp_cwd diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index f01cad4..fcc3937 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -11,7 +11,7 @@ import importlib import os import sys -from test.libregrtest import main_in_temp_cwd +from test.libregrtest import main, main_in_temp_cwd if __name__ == '__main__': -- cgit v0.12 From 8ace8e99b375f978111b62f2cc976ce4f55ecd6b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Sep 2015 13:26:03 +0300 Subject: Issue #25209: rlcomplete now can add a space or a colon after completed keyword. --- Lib/rlcompleter.py | 6 ++++++ Lib/test/test_rlcompleter.py | 13 +++++++++---- Misc/NEWS | 2 ++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index d517c0e..568cf36 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -106,6 +106,12 @@ class Completer: n = len(text) for word in keyword.kwlist: if word[:n] == text: + if word in {'finally', 'try'}: + word = word + ':' + elif word not in {'False', 'None', 'True', + 'break', 'continue', 'pass', + 'else'}: + word = word + ' ' matches.append(word) for nspace in [builtins.__dict__, self.namespace]: for word, val in nspace.items(): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d37b620..11a8305 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -67,10 +67,15 @@ class TestRlcompleter(unittest.TestCase): def test_complete(self): completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') - self.assertEqual(completer.complete('a', 0), 'and') - self.assertEqual(completer.complete('a', 1), 'as') - self.assertEqual(completer.complete('as', 2), 'assert') - self.assertEqual(completer.complete('an', 0), 'and') + self.assertEqual(completer.complete('a', 0), 'and ') + self.assertEqual(completer.complete('a', 1), 'as ') + self.assertEqual(completer.complete('as', 2), 'assert ') + self.assertEqual(completer.complete('an', 0), 'and ') + self.assertEqual(completer.complete('pa', 0), 'pass') + self.assertEqual(completer.complete('Fa', 0), 'False') + self.assertEqual(completer.complete('el', 0), 'elif ') + self.assertEqual(completer.complete('el', 1), 'else') + self.assertEqual(completer.complete('tr', 0), 'try:') if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 38eb843..8d4549a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,8 @@ Core and Builtins Library ------- +- Issue #25209: rlcomplete now can add a space or a colon after completed keyword. + - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of -- cgit v0.12 From ab824222d1356ce731f18e9d585d58d84a64637d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Sep 2015 13:43:50 +0300 Subject: Issue #25011: rlcomplete now omits private and special attribute names unless the prefix starts with underscores. --- Doc/whatsnew/3.6.rst | 8 ++++++++ Lib/rlcompleter.py | 35 +++++++++++++++++++++++++---------- Lib/test/test_rlcompleter.py | 15 +++++++++++++++ Misc/NEWS | 3 +++ 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8a2b5d3..0ea3f3b 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -103,6 +103,14 @@ operator (Contributed by Joe Jevnik in :issue:`24379`.) +rlcomplete +---------- + +Private and special attribute names now are omitted unless the prefix starts +with underscores. A space or a colon can be added after completed keyword. +(Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) + + Optimizations ============= diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 568cf36..613848f 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -142,20 +142,35 @@ class Completer: return [] # get the content of the object, except __builtins__ - words = dir(thisobject) - if "__builtins__" in words: - words.remove("__builtins__") + words = set(dir(thisobject)) + words.discard("__builtins__") if hasattr(thisobject, '__class__'): - words.append('__class__') - words.extend(get_class_members(thisobject.__class__)) + words.add('__class__') + words.update(get_class_members(thisobject.__class__)) matches = [] n = len(attr) - for word in words: - if word[:n] == attr and hasattr(thisobject, word): - val = getattr(thisobject, word) - word = self._callable_postfix(val, "%s.%s" % (expr, word)) - matches.append(word) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + while True: + for word in words: + if (word[:n] == attr and + not (noprefix and word[:n+1] == noprefix) and + hasattr(thisobject, word)): + val = getattr(thisobject, word) + word = self._callable_postfix(val, "%s.%s" % (expr, word)) + matches.append(word) + if matches or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + matches.sort() return matches def get_class_members(klass): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 11a8305..2ff0788 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -5,6 +5,7 @@ import rlcompleter class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ spam = 1 + _ham = 2 class TestRlcompleter(unittest.TestCase): @@ -51,11 +52,25 @@ class TestRlcompleter(unittest.TestCase): ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) + expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') + for x in dir(None)}) + self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None.__'), expected) # test with a customized namespace self.assertEqual(self.completer.attr_matches('CompleteMe.sp'), ['CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('Completeme.egg'), []) + self.assertEqual(self.completer.attr_matches('CompleteMe.'), + ['CompleteMe.mro(', 'CompleteMe.spam']) + self.assertEqual(self.completer.attr_matches('CompleteMe._'), + ['CompleteMe._ham']) + matches = self.completer.attr_matches('CompleteMe.__') + for x in matches: + self.assertTrue(x.startswith('CompleteMe.__'), x) + self.assertIn('CompleteMe.__name__', matches) + self.assertIn('CompleteMe.__new__(', matches) CompleteMe.me = CompleteMe self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), diff --git a/Misc/NEWS b/Misc/NEWS index 8d4549a..a44d5c9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- Issue #25011: rlcomplete now omits private and special attribute names unless + the prefix starts with underscores. + - Issue #25209: rlcomplete now can add a space or a colon after completed keyword. - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. -- cgit v0.12 From ff6ee25102acbfa61e30a410c3fc75311a3413f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Sep 2015 15:04:11 +0200 Subject: Issue #25122: Remove verbose mode of test_eintr "./python -m test -W test_eintr" wrote Lib/test/eintrdata/eintr_tester.py output to stdout which was not expected. Since test_eintr doesn't hang anymore, remove the verbose mode instead. --- Lib/test/test_eintr.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index aabad83..d3cdda0 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -16,14 +16,7 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - - if support.verbose: - args = [sys.executable, tester] - with subprocess.Popen(args) as proc: - exitcode = proc.wait() - self.assertEqual(exitcode, 0) - else: - script_helper.assert_python_ok(tester) + script_helper.assert_python_ok(tester) if __name__ == "__main__": -- cgit v0.12 From 6b415101a24c26accd409c9e72840953f340a6e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Sep 2015 23:16:17 +0200 Subject: Issue #25220: Add functional tests to test_regrtest * test all available ways to run the Python test suite * test many regrtest options: --slow, --coverage, -r, -u, etc. Note: python -m test --coverage doesn't work on Windows. --- Lib/test/test_regrtest.py | 293 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 291 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f744127..c0f1e7b 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1,18 +1,33 @@ """ Tests of regrtest.py. + +Note: test_regrtest cannot be run twice in parallel. """ import argparse import faulthandler import getopt import os.path +import platform +import re +import subprocess +import sys +import textwrap import unittest from test import libregrtest from test import support +from test.support import script_helper -class ParseArgsTestCase(unittest.TestCase): - """Test regrtest's argument parsing.""" +Py_DEBUG = hasattr(sys, 'getobjects') +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') +ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) + + +class ParseArgsTestCase(unittest.TestCase): + """ + Test regrtest's argument parsing, function _parse_args(). + """ def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): @@ -272,5 +287,279 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.args, ['foo']) +class BaseTestCase(unittest.TestCase): + TEST_UNIQUE_ID = 1 + TESTNAME_PREFIX = 'test_regrtest_' + TESTNAME_REGEX = r'test_[a-z0-9_]+' + + def setUp(self): + self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test') + + # When test_regrtest is interrupted by CTRL+c, it can leave + # temporary test files + remove = [entry.path + for entry in os.scandir(self.testdir) + if (entry.name.startswith(self.TESTNAME_PREFIX) + and entry.name.endswith(".py"))] + for path in remove: + print("WARNING: test_regrtest: remove %s" % path) + support.unlink(path) + + def create_test(self, name=None, code=''): + if not name: + name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID + BaseTestCase.TEST_UNIQUE_ID += 1 + + # test_regrtest cannot be run twice in parallel because + # of setUp() and create_test() + name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name) + path = os.path.join(self.testdir, name + '.py') + + self.addCleanup(support.unlink, path) + # Use 'x' mode to ensure that we do not override existing tests + with open(path, 'x', encoding='utf-8') as fp: + fp.write(code) + return name + + def regex_search(self, regex, output): + match = re.search(regex, output, re.MULTILINE) + if not match: + self.fail("%r not found in %r" % (regex, output)) + return match + + def check_line(self, output, regex): + regex = re.compile(r'^' + regex, re.MULTILINE) + self.assertRegex(output, regex) + + def parse_executed_tests(self, output): + parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX, + output, + re.MULTILINE) + return set(match.group(1) for match in parser) + + def check_executed_tests(self, output, tests, skipped=None): + if isinstance(tests, str): + tests = [tests] + executed = self.parse_executed_tests(output) + self.assertEqual(executed, set(tests), output) + ntest = len(tests) + if skipped: + if isinstance(skipped, str): + skipped = [skipped] + nskipped = len(skipped) + + plural = 's' if nskipped != 1 else '' + names = ' '.join(sorted(skipped)) + expected = (r'%s test%s skipped:\n %s$' + % (nskipped, plural, names)) + self.check_line(output, expected) + + ok = ntest - nskipped + if ok: + self.check_line(output, r'%s test OK\.$' % ok) + else: + self.check_line(output, r'All %s tests OK\.$' % ntest) + + def parse_random_seed(self, output): + match = self.regex_search(r'Using random seed ([0-9]+)', output) + randseed = int(match.group(1)) + self.assertTrue(0 <= randseed <= 10000000, randseed) + return randseed + + +class ProgramsTestCase(BaseTestCase): + """ + Test various ways to run the Python test suite. Use options close + to options used on the buildbot. + """ + + NTEST = 4 + + def setUp(self): + super().setUp() + + # Create NTEST tests doing nothing + self.tests = [self.create_test() for index in range(self.NTEST)] + + self.python_args = ['-Wd', '-E', '-bb'] + self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4'] + if sys.platform == 'win32': + self.regrtest_args.append('-n') + + def check_output(self, output): + self.parse_random_seed(output) + self.check_executed_tests(output, self.tests) + + def run_tests(self, args): + res = script_helper.assert_python_ok(*args) + output = os.fsdecode(res.out) + self.check_output(output) + + def test_script_regrtest(self): + # Lib/test/regrtest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py') + + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_test(self): + # -m test + args = [*self.python_args, '-m', 'test', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_regrtest(self): + # -m test.regrtest + args = [*self.python_args, '-m', 'test.regrtest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_autotest(self): + # -m test.autotest + args = [*self.python_args, '-m', 'test.autotest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_from_test_autotest(self): + # from test import autotest + code = 'from test import autotest' + args = [*self.python_args, '-c', code, + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_script_autotest(self): + # Lib/test/autotest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py') + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_tools_script_run_tests(self): + # Tools/scripts/run_tests.py + script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') + self.run_tests([script, *self.tests]) + + def run_rt_bat(self, script, *args): + rt_args = [] + if platform.architecture()[0] == '64bit': + rt_args.append('-x64') # 64-bit build + if Py_DEBUG: + rt_args.append('-d') # Debug build + + args = [script, *rt_args, *args] + proc = subprocess.run(args, + check=True, universal_newlines=True, + input='', + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.check_output(proc.stdout) + + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_tools_buildbot_test(self): + # Tools\buildbot\test.bat + script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') + self.run_rt_bat(script, *self.tests) + + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_pcbuild_rt(self): + # PCbuild\rt.bat + script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') + # -q: quick, don't run tests twice + rt_args = ["-q"] + self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests) + + +class ArgsTestCase(BaseTestCase): + """ + Test arguments of the Python test suite. + """ + + def run_tests(self, *args): + args = ['-m', 'test', *args] + res = script_helper.assert_python_ok(*args) + return os.fsdecode(res.out) + + def test_resources(self): + # test -u command line option + tests = {} + for resource in ('audio', 'network'): + code = 'from test import support\nsupport.requires(%r)' % resource + tests[resource] = self.create_test(resource, code) + test_names = sorted(tests.values()) + + # -u all: 2 resources enabled + output = self.run_tests('-u', 'all', *test_names) + self.check_executed_tests(output, test_names) + + # -u audio: 1 resource enabled + output = self.run_tests('-uaudio', *test_names) + self.check_executed_tests(output, test_names, + skipped=tests['network']) + + # no option: 0 resources enabled + output = self.run_tests(*test_names) + self.check_executed_tests(output, test_names, + skipped=test_names) + + def test_random(self): + # test -r and --randseed command line option + code = textwrap.dedent(""" + import random + print("TESTRANDOM: %s" % random.randint(1, 1000)) + """) + test = self.create_test('random', code) + + # first run to get the output with the random seed + output = self.run_tests('-r', test) + randseed = self.parse_random_seed(output) + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random = int(match.group(1)) + + # try to reproduce with the random seed + output = self.run_tests('-r', '--randseed=%s' % randseed, test) + randseed2 = self.parse_random_seed(output) + self.assertEqual(randseed2, randseed) + + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random2 = int(match.group(1)) + self.assertEqual(test_random2, test_random) + + def test_fromfile(self): + # test --fromfile + tests = [self.create_test() for index in range(5)] + + # Write the list of files using a format similar to regrtest output: + # [1/2] test_1 + # [2/2] test_2 + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w") as fp: + for index, name in enumerate(tests, 1): + print("[%s/%s] %s" % (index, len(tests), name), file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + def test_slow(self): + # test --slow + tests = [self.create_test() for index in range(3)] + output = self.run_tests("--slow", *tests) + self.check_executed_tests(output, tests) + regex = ('10 slowest tests:\n' + '(?:%s: [0-9]+\.[0-9]+s\n){%s}' + % (self.TESTNAME_REGEX, len(tests))) + self.check_line(output, regex) + + @unittest.skipIf(sys.platform == 'win32', + "FIXME: coverage doesn't work on Windows") + def test_coverage(self): + # test --coverage + test = self.create_test() + output = self.run_tests("--coverage", test) + executed = self.parse_executed_tests(output) + self.assertEqual(executed, {test}, output) + regex = ('lines +cov% +module +$path$\n' + '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +$[^)]+$+)+') + self.check_line(output, regex) + + if __name__ == '__main__': unittest.main() -- cgit v0.12 From e143c1f94f923db17a0f6d17d05e4af440178c1b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 01:02:37 +0200 Subject: Fix test_regrtest.test_tools_buildbot_test() Issue #25220: Fix test_regrtest.test_tools_buildbot_test() on release build (on Windows), pass "+d" option to test.bat. --- Lib/test/test_regrtest.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c0f1e7b..ca4b356 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -438,14 +438,7 @@ class ProgramsTestCase(BaseTestCase): script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') self.run_tests([script, *self.tests]) - def run_rt_bat(self, script, *args): - rt_args = [] - if platform.architecture()[0] == '64bit': - rt_args.append('-x64') # 64-bit build - if Py_DEBUG: - rt_args.append('-d') # Debug build - - args = [script, *rt_args, *args] + def run_batch(self, *args): proc = subprocess.run(args, check=True, universal_newlines=True, input='', @@ -456,15 +449,23 @@ class ProgramsTestCase(BaseTestCase): def test_tools_buildbot_test(self): # Tools\buildbot\test.bat script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') - self.run_rt_bat(script, *self.tests) + test_args = [] + if platform.architecture()[0] == '64bit': + test_args.append('-x64') # 64-bit build + if not Py_DEBUG: + test_args.append('+d') # Release build, use python.exe + self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') def test_pcbuild_rt(self): # PCbuild\rt.bat script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') - # -q: quick, don't run tests twice - rt_args = ["-q"] - self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests) + rt_args = ["-q"] # Quick, don't run tests twice + if platform.architecture()[0] == '64bit': + rt_args.append('-x64') # 64-bit build + if Py_DEBUG: + rt_args.append('-d') # Debug build, use python_d.exe + self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) class ArgsTestCase(BaseTestCase): -- cgit v0.12 From c3713e9706e51bbd30958c27d35e7fda764b0c4a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 12:32:13 +0200 Subject: Optimize ascii/latin1+surrogateescape encoders Issue #25227: Optimize ASCII and latin1 encoders with the ``surrogateescape`` error handler: the encoders are now up to 3 times as fast. Initial patch written by Serhiy Storchaka. --- Doc/whatsnew/3.6.rst | 3 +++ Lib/test/test_codecs.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ Objects/unicodeobject.c | 16 +++++++++++++ 4 files changed, 83 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 0ea3f3b..16cdca0 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -117,6 +117,9 @@ Optimizations * The ASCII decoder is now up to 60 times as fast for error handlers: ``surrogateescape``, ``ignore`` and ``replace``. +* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error + error ``surrogateescape``. + Build and C API Changes ======================= diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index e0e3119..254c0c1 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3060,7 +3060,31 @@ class CodePageTest(unittest.TestCase): class ASCIITest(unittest.TestCase): + def test_encode(self): + self.assertEqual('abc123'.encode('ascii'), b'abc123') + + def test_encode_error(self): + for data, error_handler, expected in ( + ('[\x80\xff\u20ac]', 'ignore', b'[]'), + ('[\x80\xff\u20ac]', 'replace', b'[???]'), + ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[€ÿ€]'), + ('[\x80\xff\u20ac]', 'backslashreplace', b'[\\x80\\xff\\u20ac]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('ascii', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\xff'.encode('ascii', 'surrogateescape') + def test_decode(self): + self.assertEqual(b'abc'.decode('ascii'), 'abc') + + def test_decode_error(self): for data, error_handler, expected in ( (b'[\x80\xff]', 'ignore', '[]'), (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), @@ -3073,5 +3097,41 @@ class ASCIITest(unittest.TestCase): expected) +class Latin1Test(unittest.TestCase): + def test_encode(self): + for data, expected in ( + ('abc', b'abc'), + ('\x80\xe9\xff', b'\x80\xe9\xff'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.encode('latin1'), expected) + + def test_encode_errors(self): + for data, error_handler, expected in ( + ('[\u20ac\udc80]', 'ignore', b'[]'), + ('[\u20ac\udc80]', 'replace', b'[??]'), + ('[\u20ac\udc80]', 'backslashreplace', b'[\\u20ac\\udc80]'), + ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€�]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('latin1', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\u20ac'.encode('latin1', 'surrogateescape') + + def test_decode(self): + for data, expected in ( + (b'abc', 'abc'), + (b'[\x80\xff]', '[\x80\xff]'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.decode('latin1'), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 72e521d..ceac0a3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25227: Optimize ASCII and latin1 encoders with the ``surrogateescape`` + error handler: the encoders are now up to 3 times as fast. Initial patch + written by Serhiy Storchaka. + - Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the getrandom() function instead of the getentropy() function. The getentropy() function is blocking to generate very good quality entropy, os.urandom() diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index da2aac7..6657cd4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6532,6 +6532,22 @@ unicode_encode_ucs1(PyObject *unicode, pos = collend; break; + case _Py_ERROR_SURROGATEESCAPE: + for (i = collstart; i < collend; ++i) { + ch = PyUnicode_READ(kind, data, i); + if (ch < 0xdc80 || 0xdcff < ch) { + /* Not a UTF-8b surrogate */ + break; + } + *str++ = (char)(ch - 0xdc00); + ++pos; + } + if (i >= collend) + break; + collstart = pos; + assert(collstart != collend); + /* fallback to general error handling */ + default: repunicode = unicode_encode_call_errorhandler(errors, &error_handler_obj, encoding, reason, unicode, &exc, -- cgit v0.12 From 2bfed53b88cea7a4ec4b45cac33c62d9342796bb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 13:41:46 +0200 Subject: Try to fix _PyTime_AsTimevalStruct_impl() on OpenBSD It looks like the check for integer overflow doesn't work on x86 OpenBSD 5.8. --- Python/pytime.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 9889a3b..53611b1 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -454,7 +454,7 @@ static int _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - _PyTime_t secs; + _PyTime_t secs, secs2; int us; int res; @@ -467,7 +467,8 @@ _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, #endif tv->tv_usec = us; - if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { + secs2 = (_PyTime_t)tv->tv_sec; + if (res < 0 || secs2 != secs) { if (raise) error_time_t_overflow(); return -1; -- cgit v0.12 From 3f7468507ae83d66cee059fb8e51692d8adb797b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 13:47:15 +0200 Subject: test --- Lib/test/libregrtest/main.py | 4 +--- Lib/test/test_regrtest.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 5d34cff..3724f27 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -246,9 +246,7 @@ def main(tests=None, **kwargs): random.shuffle(selected) if ns.trace: import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) + tracer = trace.Trace(trace=False, count=True) test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ca4b356..5906c17 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -549,8 +549,6 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - @unittest.skipIf(sys.platform == 'win32', - "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() -- cgit v0.12 From 449b27179931aa570597556ed4865ac2f1f981c9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 13:59:50 +0200 Subject: Issue #18174: Explain why is_valid_fd() uses dup() instead of fstat() --- Python/pylifecycle.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4f5efc9..857a543 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -972,6 +972,9 @@ is_valid_fd(int fd) if (fd < 0 || !_PyVerify_fd(fd)) return 0; _Py_BEGIN_SUPPRESS_IPH + /* Prefer dup() over fstat(). fstat() can require input/output whereas + dup() doesn't, there is a low risk of EMFILE/ENFILE at Python + startup. */ fd2 = dup(fd); if (fd2 >= 0) close(fd2); -- cgit v0.12 From feabaed054ae94cf1104d21f2c13568c5c8b6622 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 14:02:35 +0200 Subject: Oops, revert unwanted change, sorry --- Lib/test/libregrtest/main.py | 4 +++- Lib/test/test_regrtest.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3724f27..5d34cff 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -246,7 +246,9 @@ def main(tests=None, **kwargs): random.shuffle(selected) if ns.trace: import trace, tempfile - tracer = trace.Trace(trace=False, count=True) + tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 5906c17..ca4b356 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -549,6 +549,8 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) + @unittest.skipIf(sys.platform == 'win32', + "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() -- cgit v0.12 From 427713403546696d1171905095089dccef39c54c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 14:17:09 +0200 Subject: Issue #25220: Add test for --wait in test_regrtest Replace script_helper.assert_python_ok() with subprocess.run(). --- Lib/test/test_regrtest.py | 48 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ca4b356..54bbfe4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -16,7 +16,6 @@ import textwrap import unittest from test import libregrtest from test import support -from test.support import script_helper Py_DEBUG = hasattr(sys, 'getobjects') @@ -366,6 +365,31 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed + def run_command(self, args, input=None): + if not input: + input = '' + try: + return subprocess.run(args, + check=True, universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except subprocess.CalledProcessError as exc: + self.fail("%s\n" + "\n" + "stdout:\n" + "%s\n" + "\n" + "stderr:\n" + "%s" + % (str(exc), exc.stdout, exc.stderr)) + + + def run_python(self, args, **kw): + args = [sys.executable, '-X', 'faulthandler', '-I', *args] + proc = self.run_command(args, **kw) + return proc.stdout + class ProgramsTestCase(BaseTestCase): """ @@ -391,9 +415,8 @@ class ProgramsTestCase(BaseTestCase): self.check_executed_tests(output, self.tests) def run_tests(self, args): - res = script_helper.assert_python_ok(*args) - output = os.fsdecode(res.out) - self.check_output(output) + stdout = self.run_python(args) + self.check_output(stdout) def test_script_regrtest(self): # Lib/test/regrtest.py @@ -439,10 +462,7 @@ class ProgramsTestCase(BaseTestCase): self.run_tests([script, *self.tests]) def run_batch(self, *args): - proc = subprocess.run(args, - check=True, universal_newlines=True, - input='', - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = self.run_command(args) self.check_output(proc.stdout) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -473,10 +493,8 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args): - args = ['-m', 'test', *args] - res = script_helper.assert_python_ok(*args) - return os.fsdecode(res.out) + def run_tests(self, *args, input=None): + return self.run_python(['-m', 'test', *args], input=input) def test_resources(self): # test -u command line option @@ -561,6 +579,12 @@ class ArgsTestCase(BaseTestCase): '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +$[^)]+$+)+') self.check_line(output, regex) + def test_wait(self): + # test --wait + test = self.create_test() + output = self.run_tests("--wait", test, input='key') + self.check_line(output, 'Press any key to continue') + if __name__ == '__main__': unittest.main() -- cgit v0.12 From dad20e4876b339a559d61810299f96712719687e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 22:48:52 +0200 Subject: Issue #25220: Split the huge main() function of libregrtest.main into a class with attributes and methods. The --threshold command line option is now ignored if the gc module is missing. * Convert main() variables to Regrtest attributes, document some attributes * Convert accumulate_result() function to a method * Create setup_python() function and setup_regrtest() method. * Import gc at top level * Move resource.setrlimit() and the code to make the module paths absolute into the new setup_python() function. So this code is no more executed when the module is imported, only when main() is executed. We have a better control on when the setup is done. * Move textwrap import from printlist() to the top level. * Some other minor cleanup. --- Lib/test/libregrtest/main.py | 701 ++++++++++++++++++++++++------------------- 1 file changed, 387 insertions(+), 314 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 5d34cff..63388c9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,13 +1,14 @@ import faulthandler import json import os +import platform +import random import re +import signal import sys -import tempfile import sysconfig -import signal -import random -import platform +import tempfile +import textwrap import traceback import unittest from test.libregrtest.runtest import ( @@ -19,45 +20,15 @@ from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args from test import support try: + import gc +except ImportError: + gc = None +try: import threading except ImportError: threading = None -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -68,7 +39,73 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def main(tests=None, **kwargs): +def slave_runner(slaveargs): + args, kwargs = json.loads(slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + +def setup_python(): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + +class Regrtest: """Execute a test suite. This also parses command-line options and modifies its behavior @@ -91,210 +128,257 @@ def main(tests=None, **kwargs): directly to set the values that would normally be set by flags on the command line. """ - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) + def __init__(self): + # Namespace of command line options + self.ns = None + + # tests + self.tests = [] + self.selected = [] + + # test results + self.good = [] + self.bad = [] + self.skipped = [] + self.resource_denieds = [] + self.environment_changed = [] + self.interrupted = False - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) + # used by --slow + self.test_times = [] - replace_stdout() + # used by --coverage, trace.Trace instance + self.tracer = None - support.record_original_stdout(sys.stdout) + # used by --findleaks, store for gc.garbage + self.found_garbage = [] - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - if ns.threshold is not None: - import gc - gc.set_threshold(ns.threshold) - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if ns.wait: - input("Press any key to continue...") - - if ns.slaveargs is not None: - args, kwargs = json.loads(ns.slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - good = [] - bad = [] - skipped = [] - resource_denieds = [] - environment_changed = [] - interrupted = False - - if ns.findleaks: - try: - import gc - except ImportError: - print('No GC available, disabling findleaks.') - ns.findleaks = False - else: - # Uncomment the line below to report garbage that is not - # freeable by reference counting alone. By default only - # garbage that is not collectable by the GC is reported. - #gc.set_debug(gc.DEBUG_SAVEALL) - found_garbage = [] - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + # used to display the progress bar "[ 3/100]" + self.test_count = '' + self.test_count_width = 1 - if ns.single: - filename = os.path.join(TEMPDIR, 'pynexttest') - try: - with open(filename, 'r') as fp: - next_test = fp.read().strip() - tests = [next_test] - except OSError: - pass + # used by --single + self.next_single_test = None + self.next_single_filename = None - if ns.fromfile: - tests = [] - with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - - # Strip .py extensions. - removepy(ns.args) - removepy(tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if ns.exclude: - for arg in ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - ns.args = [] - - # For a partial run, we do not need to clutter the output. - if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if ns.testdir: - alltests = findtests(ns.testdir, list(), set()) - else: - alltests = findtests(ns.testdir, stdtests, nottests) - - selected = tests or ns.args or alltests - if ns.single: - selected = selected[:1] - try: - next_single_test = alltests[alltests.index(selected[0])+1] - except IndexError: - next_single_test = None - # Remove all the selected tests that precede start if it's set. - if ns.start: - try: - del selected[:selected.index(ns.start)] - except ValueError: - print("Couldn't find starting test (%s), using all tests" % ns.start) - if ns.randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) - print("Using random seed", ns.random_seed) - random.shuffle(selected) - if ns.trace: - import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - - test_times = [] - support.verbose = ns.verbose # Tell tests to be moderately quiet - support.use_resources = ns.use_resources - save_modules = sys.modules.keys() - - def accumulate_result(test, result): + def accumulate_result(self, test, result): ok, test_time = result - test_times.append((test_time, test)) + self.test_times.append((test_time, test)) if ok == PASSED: - good.append(test) + self.good.append(test) elif ok == FAILED: - bad.append(test) + self.bad.append(test) elif ok == ENV_CHANGED: - environment_changed.append(test) + self.environment_changed.append(test) elif ok == SKIPPED: - skipped.append(test) + self.skipped.append(test) elif ok == RESOURCE_DENIED: - skipped.append(test) - resource_denieds.append(test) - - if ns.forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() - test_count = '' - test_count_width = 3 - else: - tests = iter(selected) - test_count = '/{}'.format(len(selected)) - test_count_width = len(test_count) - 1 + self.skipped.append(test) + self.resource_denieds.append(test) + + def display_progress(self, test_index, test): + if self.ns.quiet: + return + fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + self.test_count_width, test_index, self.test_count, len(self.bad), test)) + sys.stdout.flush() + + def setup_regrtest(self): + if self.ns.huntrleaks: + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() - if ns.use_mp: + if self.ns.memlimit is not None: + support.set_memlimit(self.ns.memlimit) + + if self.ns.threshold is not None: + if gc is not None: + gc.set_threshold(self.ns.threshold) + else: + print('No GC available, ignore --threshold.') + + if self.ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + + if self.ns.findleaks: + if gc is not None: + # Uncomment the line below to report garbage that is not + # freeable by reference counting alone. By default only + # garbage that is not collectable by the GC is reported. + pass + #gc.set_debug(gc.DEBUG_SAVEALL) + else: + print('No GC available, disabling --findleaks') + self.ns.findleaks = False + + if self.ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Strip .py extensions. + removepy(self.ns.args) + + if self.ns.trace: + import trace + self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, + sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + + def find_tests(self, tests): + self.tests = tests + + if self.ns.single: + self.next_single_filename = os.path.join(TEMPDIR, 'pynexttest') + try: + with open(self.next_single_filename, 'r') as fp: + next_test = fp.read().strip() + self.tests = [next_test] + except OSError: + pass + + if self.ns.fromfile: + self.tests = [] + with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') + for line in fp: + line = count_pat.sub('', line) + guts = line.split() # assuming no test has whitespace in its name + if guts and not guts[0].startswith('#'): + self.tests.extend(guts) + + removepy(self.tests) + + stdtests = STDTESTS[:] + nottests = NOTTESTS.copy() + if self.ns.exclude: + for arg in self.ns.args: + if arg in stdtests: + stdtests.remove(arg) + nottests.add(arg) + self.ns.args = [] + + # For a partial run, we do not need to clutter the output. + if self.ns.verbose or self.ns.header or not (self.ns.quiet or self.ns.single or self.tests or self.ns.args): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if self.ns.testdir: + alltests = findtests(self.ns.testdir, list(), set()) + else: + alltests = findtests(self.ns.testdir, stdtests, nottests) + + self.selected = self.tests or self.ns.args or alltests + if self.ns.single: + self.selected = self.selected[:1] + try: + pos = alltests.index(self.selected[0]) + self.next_single_test = alltests[pos + 1] + except IndexError: + pass + + # Remove all the self.selected tests that precede start if it's set. + if self.ns.start: + try: + del self.selected[:self.selected.index(self.ns.start)] + except ValueError: + print("Couldn't find starting test (%s), using all tests" % self.ns.start) + + if self.ns.randomize: + if self.ns.random_seed is None: + self.ns.random_seed = random.randrange(10000000) + random.seed(self.ns.random_seed) + print("Using random seed", self.ns.random_seed) + random.shuffle(self.selected) + + def display_result(self): + if self.interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + omitted = set(self.selected) - set(self.good) - set(self.bad) - set(self.skipped) + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + + if self.good and not self.ns.quiet: + if not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1: + print("All", end=' ') + print(count(len(self.good), "test"), "OK.") + + if self.ns.print_slow: + self.test_times.sort(reverse=True) + print("10 slowest tests:") + for time, test in self.test_times[:10]: + print("%s: %.1fs" % (test, time)) + + if self.bad: + print(count(len(self.bad), "test"), "failed:") + printlist(self.bad) + + if self.environment_changed: + print("{} altered the execution environment:".format( + count(len(self.environment_changed), "test"))) + printlist(self.environment_changed) + + if self.skipped and not self.ns.quiet: + print(count(len(self.skipped), "test"), "skipped:") + printlist(self.skipped) + + if self.ns.verbose2 and self.bad: + print("Re-running failed tests in verbose mode") + for test in self.bad[:]: + print("Re-running test %r in verbose mode" % test) + sys.stdout.flush() + try: + self.ns.verbose = True + ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, + timeout=self.ns.timeout) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + self.bad.remove(test) + else: + if self.bad: + print(count(len(self.bad), 'test'), "failed again:") + printlist(self.bad) + + def _run_tests_mp(self): try: from threading import Thread except ImportError: print("Multiprocess option requires thread support") sys.exit(2) from queue import Queue + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") output = Queue() - pending = MultiprocessTests(tests) + pending = MultiprocessTests(self.tests) + def work(): # A worker thread. try: @@ -304,7 +388,7 @@ def main(tests=None, **kwargs): except StopIteration: output.put((None, None, None, None)) return - retcode, stdout, stderr = run_test_in_subprocess(test, ns) + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) @@ -321,23 +405,20 @@ def main(tests=None, **kwargs): except BaseException: output.put((None, None, None, None)) raise - workers = [Thread(target=work) for i in range(ns.use_mp)] + + workers = [Thread(target=work) for i in range(self.ns.use_mp)] for worker in workers: worker.start() finished = 0 test_index = 1 try: - while finished < ns.use_mp: + while finished < self.ns.use_mp: test, stdout, stderr, result = output.get() if test is None: finished += 1 continue - accumulate_result(test, result) - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, - len(bad), test)) + self.accumulate_result(test, result) + self.display_progress(test_index, test) if stdout: print(stdout) if stderr: @@ -350,110 +431,99 @@ def main(tests=None, **kwargs): raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: - interrupted = True + self.interrupted = True pending.interrupted = True for worker in workers: worker.join() - else: - for test_index, test in enumerate(tests, 1): - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, len(bad), test)) - sys.stdout.flush() - if ns.trace: + + def _run_tests_sequential(self): + save_modules = sys.modules.keys() + + for test_index, test in enumerate(self.tests, 1): + self.display_progress(test_index, test) + if self.ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', - globals=globals(), locals=vars()) + cmd = 'runtest(test, self.ns.verbose, self.ns.quiet, timeout=self.ns.timeout)' + self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests) - accumulate_result(test, result) + result = runtest(test, self.ns.verbose, self.ns.quiet, + self.ns.huntrleaks, + output_on_failure=self.ns.verbose3, + timeout=self.ns.timeout, failfast=self.ns.failfast, + match_tests=self.ns.match_tests) + self.accumulate_result(test, result) except KeyboardInterrupt: - interrupted = True + self.interrupted = True break - if ns.findleaks: + if self.ns.findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again - found_garbage.extend(gc.garbage) + self.found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) - if interrupted: - # print a newline after ^C - print() - print("Test suite interrupted by signal SIGINT.") - omitted = set(selected) - set(good) - set(bad) - set(skipped) - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if good and not ns.quiet: - if not bad and not skipped and not interrupted and len(good) > 1: - print("All", end=' ') - print(count(len(good), "test"), "OK.") - if ns.print_slow: - test_times.sort(reverse=True) - print("10 slowest tests:") - for time, test in test_times[:10]: - print("%s: %.1fs" % (test, time)) - if bad: - print(count(len(bad), "test"), "failed:") - printlist(bad) - if environment_changed: - print("{} altered the execution environment:".format( - count(len(environment_changed), "test"))) - printlist(environment_changed) - if skipped and not ns.quiet: - print(count(len(skipped), "test"), "skipped:") - printlist(skipped) - - if ns.verbose2 and bad: - print("Re-running failed tests in verbose mode") - for test in bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() - try: - ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, - timeout=ns.timeout) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - bad.remove(test) - else: - if bad: - print(count(len(bad), 'test'), "failed again:") - printlist(bad) - - if ns.single: - if next_single_test: - with open(filename, 'w') as fp: - fp.write(next_single_test + '\n') - else: - os.unlink(filename) + def run_tests(self): + support.verbose = self.ns.verbose # Tell tests to be moderately quiet + support.use_resources = self.ns.use_resources - if ns.trace: - r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) + if self.ns.forever: + def test_forever(tests): + while True: + for test in tests: + yield test + if self.bad: + return + self.tests = test_forever(list(self.selected)) + self.test_count = '' + self.test_count_width = 3 + else: + self.tests = iter(self.selected) + self.test_count = '/{}'.format(len(self.selected)) + self.test_count_width = len(self.test_count) - 1 - if ns.runleaks: - os.system("leaks %d" % os.getpid()) + if self.ns.use_mp: + self._run_tests_mp() + else: + self._run_tests_sequential() - sys.exit(len(bad) > 0 or interrupted) + def finalize(self): + if self.next_single_filename: + if self.next_single_test: + with open(self.next_single_filename, 'w') as fp: + fp.write(self.next_single_test + '\n') + else: + os.unlink(self.next_single_filename) + + if self.ns.trace: + r = self.tracer.results() + r.write_results(show_missing=True, summary=True, + coverdir=self.ns.coverdir) + + if self.ns.runleaks: + os.system("leaks %d" % os.getpid()) + + def main(self, tests=None, **kwargs): + setup_python() + self.ns = _parse_args(sys.argv[1:], **kwargs) + self.setup_regrtest() + if self.ns.wait: + input("Press any key to continue...") + if self.ns.slaveargs is not None: + slave_runner(self.ns.slaveargs) + self.find_tests(tests) + self.run_tests() + self.display_result() + self.finalize() + sys.exit(len(self.bad) > 0 or self.interrupted) # We do not use a generator so multiple threads can call next(). @@ -518,11 +588,14 @@ def printlist(x, width=70, indent=4): begin each line. """ - from textwrap import fill blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() - print(fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) + print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks)) + + +def main(tests=None, **kwargs): + Regrtest().main(tests=tests, **kwargs) def main_in_temp_cwd(): -- cgit v0.12 From 56e05dd0b03386b2da879283f6ab165f857417be Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:15:38 +0200 Subject: Issue #25220: Create libregrtest/runtest_mp.py Move the code to run tests in multiple processes using threading and subprocess to a new submodule. Move also slave_runner() (renamed to run_tests_slave()) and run_test_in_subprocess() (renamed to run_tests_in_subprocess()) there. --- Lib/test/libregrtest/main.py | 120 ++-------------------------- Lib/test/libregrtest/runtest.py | 33 -------- Lib/test/libregrtest/runtest_mp.py | 158 +++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 147 deletions(-) create mode 100644 Lib/test/libregrtest/runtest_mp.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 63388c9..b66f045 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,4 @@ import faulthandler -import json import os import platform import random @@ -9,13 +8,10 @@ import sys import sysconfig import tempfile import textwrap -import traceback import unittest from test.libregrtest.runtest import ( - findtests, runtest, run_test_in_subprocess, - STDTESTS, NOTTESTS, - PASSED, FAILED, ENV_CHANGED, SKIPPED, - RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR) + findtests, runtest, + STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args from test import support @@ -39,23 +35,6 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def slave_runner(slaveargs): - args, kwargs = json.loads(slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - def setup_python(): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -367,75 +346,6 @@ class Regrtest: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - def _run_tests_mp(self): - try: - from threading import Thread - except ImportError: - print("Multiprocess option requires thread support") - sys.exit(2) - from queue import Queue - - debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - output = Queue() - pending = MultiprocessTests(self.tests) - - def work(): - # A worker thread. - try: - while True: - try: - test = next(pending) - except StopIteration: - output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return - if not result: - output.put((None, None, None, None)) - return - result = json.loads(result) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - except BaseException: - output.put((None, None, None, None)) - raise - - workers = [Thread(target=work) for i in range(self.ns.use_mp)] - for worker in workers: - worker.start() - finished = 0 - test_index = 1 - try: - while finished < self.ns.use_mp: - test, stdout, stderr, result = output.get() - if test is None: - finished += 1 - continue - self.accumulate_result(test, result) - self.display_progress(test_index, test) - if stdout: - print(stdout) - if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() - if result[0] == INTERRUPTED: - raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) - test_index += 1 - except KeyboardInterrupt: - self.interrupted = True - pending.interrupted = True - for worker in workers: - worker.join() - def _run_tests_sequential(self): save_modules = sys.modules.keys() @@ -491,7 +401,8 @@ class Regrtest: self.test_count_width = len(self.test_count) - 1 if self.ns.use_mp: - self._run_tests_mp() + from test.libregrtest.runtest_mp import run_tests_multiprocess + run_tests_multiprocess(self) else: self._run_tests_sequential() @@ -518,7 +429,8 @@ class Regrtest: if self.ns.wait: input("Press any key to continue...") if self.ns.slaveargs is not None: - slave_runner(self.ns.slaveargs) + from test.libregrtest.runtest_mp import run_tests_slave + run_tests_slave(self.ns.slaveargs) self.find_tests(tests) self.run_tests() self.display_result() @@ -526,26 +438,6 @@ class Regrtest: sys.exit(len(self.bad) > 0 or self.interrupted) -# We do not use a generator so multiple threads can call next(). -class MultiprocessTests(object): - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.interrupted = False - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - if self.interrupted: - raise StopIteration('tests interrupted') - return next(self.tests) - - def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index d8c0eb2..60b8f5f 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -1,7 +1,6 @@ import faulthandler import importlib import io -import json import os import sys import time @@ -22,38 +21,6 @@ INTERRUPTED = -4 CHILD_ERROR = -5 # error in a child process -def run_test_in_subprocess(testname, ns): - """Run the given test in a subprocess with --slaveargs. - - ns is the option Namespace parsed from command-line arguments. regrtest - is invoked in a subprocess with the --slaveargs argument; when the - subprocess exits, its return code, stdout and stderr are returned as a - 3-tuple. - """ - from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() - return retcode, stdout, stderr - - # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py new file mode 100644 index 0000000..b55c11a --- /dev/null +++ b/Lib/test/libregrtest/runtest_mp.py @@ -0,0 +1,158 @@ +import json +import os +import re +import sys +import traceback +import unittest +from queue import Queue +from test import support +try: + import threading +except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + +from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR + + +debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") + + +def run_tests_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +def run_tests_slave(slaveargs): + args, kwargs = json.loads(slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessIterator: + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests): + self.interrupted = False + self.lock = threading.Lock() + self.tests = tests + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.interrupted: + raise StopIteration('tests interrupted') + return next(self.tests) + + +class MultiprocessThread(threading.Thread): + def __init__(self, pending, output, ns): + super().__init__() + self.pending = pending + self.output = output + self.ns = ns + + def run(self): + # A worker thread. + try: + while True: + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return + retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) + # Strip last refcount output line if it exists, since it + # comes from the shutdown of the interpreter in the subcommand. + stderr = debug_output_pat.sub("", stderr) + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return + if not result: + self.output.put((None, None, None, None)) + return + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + except BaseException: + self.output.put((None, None, None, None)) + raise + + +def run_tests_multiprocess(regrtest): + output = Queue() + pending = MultiprocessIterator(regrtest.tests) + + workers = [MultiprocessThread(pending, output, regrtest.ns) + for i in range(regrtest.ns.use_mp)] + for worker in workers: + worker.start() + finished = 0 + test_index = 1 + try: + while finished < regrtest.ns.use_mp: + test, stdout, stderr, result = output.get() + if test is None: + finished += 1 + continue + regrtest.accumulate_result(test, result) + regrtest.display_progress(test_index, test) + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + sys.stdout.flush() + sys.stderr.flush() + if result[0] == INTERRUPTED: + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) + test_index += 1 + except KeyboardInterrupt: + regrtest.interrupted = True + pending.interrupted = True + for worker in workers: + worker.join() -- cgit v0.12 From bd1a72c455a18e73e59c502cb28cb0d0667e1087 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:36:27 +0200 Subject: Issue #25220: Enhance regrtest --coverage Add a new Regrtest.run_test() method to ensure that --coverage pass the same options to the runtest() function. --- Lib/test/libregrtest/main.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b66f045..41c32d6 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -346,7 +346,18 @@ class Regrtest: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - def _run_tests_sequential(self): + def run_test(self, test): + result = runtest(test, + self.ns.verbose, + self.ns.quiet, + self.ns.huntrleaks, + output_on_failure=self.ns.verbose3, + timeout=self.ns.timeout, + failfast=self.ns.failfast, + match_tests=self.ns.match_tests) + self.accumulate_result(test, result) + + def run_tests_sequential(self): save_modules = sys.modules.keys() for test_index, test in enumerate(self.tests, 1): @@ -354,19 +365,15 @@ class Regrtest: if self.ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = 'runtest(test, self.ns.verbose, self.ns.quiet, timeout=self.ns.timeout)' + cmd = 'self.run_test(test)' self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - result = runtest(test, self.ns.verbose, self.ns.quiet, - self.ns.huntrleaks, - output_on_failure=self.ns.verbose3, - timeout=self.ns.timeout, failfast=self.ns.failfast, - match_tests=self.ns.match_tests) - self.accumulate_result(test, result) + self.run_test(test) except KeyboardInterrupt: self.interrupted = True break + if self.ns.findleaks: gc.collect() if gc.garbage: @@ -376,13 +383,14 @@ class Regrtest: # them again self.found_garbage.extend(gc.garbage) del gc.garbage[:] + # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) def run_tests(self): - support.verbose = self.ns.verbose # Tell tests to be moderately quiet + support.verbose = self.ns.verbose # Tell tests to be moderately quiet support.use_resources = self.ns.use_resources if self.ns.forever: @@ -404,7 +412,7 @@ class Regrtest: from test.libregrtest.runtest_mp import run_tests_multiprocess run_tests_multiprocess(self) else: - self._run_tests_sequential() + self.run_tests_sequential() def finalize(self): if self.next_single_filename: -- cgit v0.12 From 37554525aab167c88e61cac0d78dfdefc31fdee9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:37:14 +0200 Subject: Issue #25220: regrtest setups Python after parsing command line options --- Lib/test/libregrtest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 41c32d6..2716536 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -431,8 +431,8 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): - setup_python() self.ns = _parse_args(sys.argv[1:], **kwargs) + setup_python() self.setup_regrtest() if self.ns.wait: input("Press any key to continue...") -- cgit v0.12 From 6448b8041a80d29de47799cb8eccd94fd1dde3be Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:43:33 +0200 Subject: Issue #25220: truncate some long lines in libregrtest/*.py --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++------- Lib/test/libregrtest/runtest_mp.py | 12 ++++++++---- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2716536..ee1591d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -63,7 +63,8 @@ def setup_python(): # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] + module.__path__ = [os.path.abspath(path) + for path in module.__path__] if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) @@ -159,8 +160,8 @@ class Regrtest: if self.ns.quiet: return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - self.test_count_width, test_index, self.test_count, len(self.bad), test)) + print(fmt.format(self.test_count_width, test_index, + self.test_count, len(self.bad), test)) sys.stdout.flush() def setup_regrtest(self): @@ -252,7 +253,10 @@ class Regrtest: self.ns.args = [] # For a partial run, we do not need to clutter the output. - if self.ns.verbose or self.ns.header or not (self.ns.quiet or self.ns.single or self.tests or self.ns.args): + if (self.ns.verbose + or self.ns.header + or not (self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("== ", platform.platform(aliased=True), @@ -283,7 +287,8 @@ class Regrtest: try: del self.selected[:self.selected.index(self.ns.start)] except ValueError: - print("Couldn't find starting test (%s), using all tests" % self.ns.start) + print("Couldn't find starting test (%s), using all tests" + % self.ns.start) if self.ns.randomize: if self.ns.random_seed is None: @@ -297,12 +302,16 @@ class Regrtest: # print a newline after ^C print() print("Test suite interrupted by signal SIGINT.") - omitted = set(self.selected) - set(self.good) - set(self.bad) - set(self.skipped) + executed = set(self.good) | set(self.bad) | set(self.skipped) + omitted = set(self.selected) - executed print(count(len(omitted), "test"), "omitted:") printlist(omitted) if self.good and not self.ns.quiet: - if not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1: + if (not self.bad + and not self.skipped + and not self.interrupted + and len(self.good) > 1): print("All", end=' ') print(count(len(self.good), "test"), "OK.") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index b55c11a..c5a6b3d 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -103,20 +103,23 @@ class MultiprocessThread(threading.Thread): except StopIteration: self.output.put((None, None, None, None)) return - retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) + retcode, stdout, stderr = run_tests_in_subprocess(test, + self.ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) return if not result: self.output.put((None, None, None, None)) return result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) except BaseException: self.output.put((None, None, None, None)) raise @@ -149,7 +152,8 @@ def run_tests_multiprocess(regrtest): if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) + msg = "Child error on {}: {}".format(test, result[1]) + raise Exception(msg) test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True -- cgit v0.12 From 86e8c31b8da163eb1f412c68ef96c2a4fda7adcf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:50:19 +0200 Subject: Issue #25220, libregrtest: Remove unused import --- Lib/test/libregrtest/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ee1591d..306beb8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -19,10 +19,6 @@ try: import gc except ImportError: gc = None -try: - import threading -except ImportError: - threading = None # When tests are run from the Python build directory, it is best practice -- cgit v0.12 From 02319804ea4b92e11fef6adce63698da8c44052a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:52:33 +0200 Subject: Don't strip refcount in libregrtest/runtest_mp.py Python doesn't display the refcount anymore by default. It only displays it when -X showrefcount command line option is used, which is not the case here. regrtest can be run with -X showrefcount, the option is not inherited by child processes. --- Lib/test/libregrtest/runtest_mp.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index c5a6b3d..8311ecf 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -15,9 +15,6 @@ except ImportError: from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR -debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - - def run_tests_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -105,9 +102,6 @@ class MultiprocessThread(threading.Thread): return retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) -- cgit v0.12 From 76f756d9344157d893ce49503327af315c7c3b9c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:33:29 +0200 Subject: Issue #25220: Enhance regrtest -jN Running the Python test suite with -jN now: - Display the duration of tests which took longer than 30 seconds - Display the tests currently running since at least 30 seconds - Display the tests we are waiting for when the test suite is interrupted Clenaup also run_test_in_subprocess() code. --- Lib/test/libregrtest/runtest_mp.py | 127 ++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 37 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 8311ecf..71096d3 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,7 +1,7 @@ import json import os -import re import sys +import time import traceback import unittest from queue import Queue @@ -15,7 +15,12 @@ except ImportError: from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR -def run_tests_in_subprocess(testname, ns): +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + +def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. ns is the option Namespace parsed from command-line arguments. regrtest @@ -24,26 +29,33 @@ def run_tests_in_subprocess(testname, ns): 3-tuple. """ from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) + + args = (testname, ns.verbose, ns.quiet) + kwargs = dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, + failfast=ns.failfast, + match_tests=ns.match_tests) + slaveargs = (args, kwargs) + slaveargs = json.dumps(slaveargs) + + cmd = [sys.executable, *support.args_from_interpreter_flags(), + '-X', 'faulthandler', + '-m', 'test.regrtest', + '--slaveargs', slaveargs] + # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + popen = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, close_fds=(os.name != 'nt'), cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() + with popen: + stdout, stderr = popen.communicate() + retcode = popen.wait() return retcode, stdout, stderr @@ -90,30 +102,45 @@ class MultiprocessThread(threading.Thread): self.pending = pending self.output = output self.ns = ns + self.current_test = None + self.start_time = None + + def _runtest(self): + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return True + + try: + self.start_time = time.monotonic() + self.current_test = test + + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) + finally: + self.current_test = None + + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return True + + if not result: + self.output.put((None, None, None, None)) + return True + + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return False def run(self): - # A worker thread. try: - while True: - try: - test = next(self.pending) - except StopIteration: - self.output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_tests_in_subprocess(test, - self.ns) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) - return - if not result: - self.output.put((None, None, None, None)) - return - result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) + stop = False + while not stop: + stop = self._runtest() except BaseException: self.output.put((None, None, None, None)) raise @@ -136,13 +163,33 @@ def run_tests_multiprocess(regrtest): finished += 1 continue regrtest.accumulate_result(test, result) - regrtest.display_progress(test_index, test) + + # Display progress + text = test + ok, test_time = result + if (ok not in (CHILD_ERROR, INTERRUPTED) + and test_time >= PROGRESS_MIN_TIME): + text += ' (%.0f sec)' % test_time + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + if running: + text += ' -- running: %s' % ', '.join(running) + regrtest.display_progress(test_index, text) + + # Copy stdout and stderr from the child process if stdout: print(stdout) if stderr: print(stderr, file=sys.stderr) sys.stdout.flush() sys.stderr.flush() + if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: @@ -152,5 +199,11 @@ def run_tests_multiprocess(regrtest): except KeyboardInterrupt: regrtest.interrupted = True pending.interrupted = True + print() + + running = [worker.current_test for worker in workers] + running = list(filter(bool, running)) + if running: + print("Waiting for %s" % ', '.join(running)) for worker in workers: worker.join() -- cgit v0.12 From f33536c4308ef1b56a062391ac955c746423e1ee Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:48:27 +0200 Subject: Issue #25220: Use print(flush=True) in libregrtest --- Lib/test/libregrtest/main.py | 7 +++---- Lib/test/libregrtest/refleak.py | 10 ++++------ Lib/test/libregrtest/runtest.py | 14 +++++--------- Lib/test/libregrtest/runtest_mp.py | 9 +++------ 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 306beb8..7440e9d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -157,8 +157,8 @@ class Regrtest: return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" print(fmt.format(self.test_count_width, test_index, - self.test_count, len(self.bad), test)) - sys.stdout.flush() + self.test_count, len(self.bad), test), + flush=True) def setup_regrtest(self): if self.ns.huntrleaks: @@ -333,8 +333,7 @@ class Regrtest: if self.ns.verbose2 and self.bad: print("Re-running failed tests in verbose mode") for test in self.bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() + print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index dd0d05d..db8a445 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -44,13 +44,12 @@ def dash_R(the_module, test, indirect_test, huntrleaks): alloc_deltas = [0] * repcount print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, + flush=True) for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() + print('.', end='', flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before @@ -74,8 +73,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() + print(msg, file=sys.stderr, flush=True) with open(fname, "a") as refrep: print(msg, file=refrep) refrep.flush() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 60b8f5f..f57784d 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -161,27 +161,23 @@ def runtest_inner(test, verbose, quiet, test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return RESOURCE_DENIED, test_time except unittest.SkipTest as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return SKIPPED, test_time except KeyboardInterrupt: raise except support.TestFailed as msg: if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) + print("test", test, "failed --", msg, file=sys.stderr, flush=True) else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() + print("test", test, "failed", file=sys.stderr, flush=True) return FAILED, test_time except: msg = traceback.format_exc() - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() + print("test", test, "crashed --", msg, file=sys.stderr, flush=True) return FAILED, test_time else: if refleak: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 71096d3..1a82b3d 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -70,9 +70,8 @@ def run_tests_slave(slaveargs): except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) - sys.stdout.flush() print() # Force a newline (just in case) - print(json.dumps(result)) + print(json.dumps(result), flush=True) sys.exit(0) @@ -184,11 +183,9 @@ def run_tests_multiprocess(regrtest): # Copy stdout and stderr from the child process if stdout: - print(stdout) + print(stdout, flush=True) if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() + print(stderr, file=sys.stderr, flush=True) if result[0] == INTERRUPTED: raise KeyboardInterrupt -- cgit v0.12 From c7eab0528c00791da8368abe950a7b6f97aa747d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:59:35 +0200 Subject: Issue #25220, libregrtest: Cleanup setup code --- Lib/test/libregrtest/main.py | 96 ++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 7440e9d..f410bfd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -31,7 +31,7 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(): +def setup_python(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -80,6 +80,38 @@ def setup_python(): newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + if gc is not None: + gc.set_threshold(ns.threshold) + else: + print('No GC available, ignore --threshold.') + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + class Regrtest: """Execute a test suite. @@ -161,36 +193,6 @@ class Regrtest: flush=True) def setup_regrtest(self): - if self.ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if self.ns.memlimit is not None: - support.set_memlimit(self.ns.memlimit) - - if self.ns.threshold is not None: - if gc is not None: - gc.set_threshold(self.ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if self.ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if self.ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not @@ -202,19 +204,9 @@ class Regrtest: print('No GC available, disabling --findleaks') self.ns.findleaks = False - if self.ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - # Strip .py extensions. removepy(self.ns.args) - if self.ns.trace: - import trace - self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, - sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - def find_tests(self, tests): self.tests = tests @@ -278,7 +270,7 @@ class Regrtest: except IndexError: pass - # Remove all the self.selected tests that precede start if it's set. + # Remove all the selected tests that precede start if it's set. if self.ns.start: try: del self.selected[:self.selected.index(self.ns.start)] @@ -362,11 +354,18 @@ class Regrtest: self.accumulate_result(test, result) def run_tests_sequential(self): + if self.ns.trace: + import trace + self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, + sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + save_modules = sys.modules.keys() for test_index, test in enumerate(self.tests, 1): self.display_progress(test_index, test) - if self.ns.trace: + if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = 'self.run_test(test)' @@ -426,7 +425,7 @@ class Regrtest: else: os.unlink(self.next_single_filename) - if self.ns.trace: + if self.tracer: r = self.tracer.results() r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) @@ -436,15 +435,18 @@ class Regrtest: def main(self, tests=None, **kwargs): self.ns = _parse_args(sys.argv[1:], **kwargs) - setup_python() + setup_python(self.ns) self.setup_regrtest() - if self.ns.wait: - input("Press any key to continue...") + if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: + input("Press any key to continue...") + self.find_tests(tests) self.run_tests() + self.display_result() self.finalize() sys.exit(len(self.bad) > 0 or self.interrupted) -- cgit v0.12 From 234cbef39fe7f154d46afcc6fe865db2e120bba2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:13:53 +0200 Subject: Issue #25220, libregrtest: Move setup_python() to a new submodule --- Lib/test/libregrtest/main.py | 125 +++++------------------------------------- Lib/test/libregrtest/setup.py | 108 ++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 110 deletions(-) create mode 100644 Lib/test/libregrtest/setup.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f410bfd..fbbfa73 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,19 +1,16 @@ -import faulthandler import os import platform import random import re -import signal import sys import sysconfig import tempfile import textwrap -import unittest from test.libregrtest.runtest import ( findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) -from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args +from test.libregrtest.setup import setup_python from test import support try: import gc @@ -31,88 +28,6 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(ns): - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - support.record_original_stdout(sys.stdout) - - # Some times __path__ and __file__ are not absolute (e.g. while running from - # Lib/) and, if we change the CWD to run the tests in a temporary dir, some - # imports might fail. This affects only the modules imported before os.chdir(). - # These modules are searched first in sys.path[0] (so '' -- the CWD) and if - # they are found in the CWD their __file__ and __path__ will be relative (this - # happens before the chdir). All the modules imported after the chdir, are - # not found in the CWD, and since the other paths in sys.path[1:] are absolute - # (site.py absolutize them), the __file__ and __path__ will be absolute too. - # Therefore it is necessary to absolutize manually the __file__ and __path__ of - # the packages to prevent later imports to fail when the CWD is different. - for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) - for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - # MacOSX (a.k.a. Darwin) has a default stack size that is too small - # for deeply recursive regular expressions. We see this as crashes in - # the Python test suite when running test_re.py and test_sre.py. The - # fix is to set the stack limit to 2048. - # This approach may also be useful for other Unixy platforms that - # suffer from small default stack limits. - if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - - if ns.threshold is not None: - if gc is not None: - gc.set_threshold(ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - - class Regrtest: """Execute a test suite. @@ -192,8 +107,14 @@ class Regrtest: self.test_count, len(self.bad), test), flush=True) - def setup_regrtest(self): - if self.ns.findleaks: + def parse_args(self, kwargs): + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.threshold is not None and gc is None: + print('No GC available, ignore --threshold.') + ns.threshold = None + + if ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only @@ -202,10 +123,12 @@ class Regrtest: #gc.set_debug(gc.DEBUG_SAVEALL) else: print('No GC available, disabling --findleaks') - self.ns.findleaks = False + ns.findleaks = False # Strip .py extensions. - removepy(self.ns.args) + removepy(ns.args) + + return ns def find_tests(self, tests): self.tests = tests @@ -434,9 +357,9 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): - self.ns = _parse_args(sys.argv[1:], **kwargs) + self.ns = self.parse_args(kwargs) + setup_python(self.ns) - self.setup_regrtest() if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave @@ -452,24 +375,6 @@ class Regrtest: sys.exit(len(self.bad) > 0 or self.interrupted) -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - - def removepy(names): if not names: return diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py new file mode 100644 index 0000000..a7dfa79 --- /dev/null +++ b/Lib/test/libregrtest/setup.py @@ -0,0 +1,108 @@ +import atexit +import faulthandler +import os +import signal +import sys +import unittest +from test import support +try: + import gc +except ImportError: + gc = None + +from test.libregrtest.refleak import warm_caches + + +def setup_python(ns): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) + for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + gc.set_threshold(ns.threshold) + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) -- cgit v0.12 From 8bb19f094ba6ba12f70a6458729c911d93c85209 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:32:39 +0200 Subject: Issue #25220, libregrtest: Add runtest_ns() function * Factorize code to run tests. * run_test_in_subprocess() now pass the whole "ns" namespace to the child process. --- Lib/test/libregrtest/main.py | 17 ++++++----------- Lib/test/libregrtest/runtest.py | 7 +++++++ Lib/test/libregrtest/runtest_mp.py | 24 +++++++++++++----------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fbbfa73..df2329f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import sysconfig import tempfile import textwrap from test.libregrtest.runtest import ( - findtests, runtest, + findtests, runtest_ns, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args from test.libregrtest.setup import setup_python @@ -251,8 +251,7 @@ class Regrtest: print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True - ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, - timeout=self.ns.timeout) + ok = runtest_ns(test, True, self.ns) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -266,14 +265,10 @@ class Regrtest: printlist(self.bad) def run_test(self, test): - result = runtest(test, - self.ns.verbose, - self.ns.quiet, - self.ns.huntrleaks, - output_on_failure=self.ns.verbose3, - timeout=self.ns.timeout, - failfast=self.ns.failfast, - match_tests=self.ns.match_tests) + result = runtest_ns(test, self.ns.verbose, self.ns, + output_on_failure=self.ns.verbose3, + failfast=self.ns.failfast, + match_tests=self.ns.match_tests) self.accumulate_result(test, result) def run_tests_sequential(self): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index f57784d..fb7f821 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -53,6 +53,13 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) +def runtest_ns(test, verbose, ns, **kw): + return runtest(test, verbose, ns.quiet, + huntrleaks=ns.huntrleaks, + timeout=ns.timeout, + **kw) + + def runtest(test, verbose, quiet, huntrleaks=False, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 1a82b3d..74424c1 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -3,6 +3,7 @@ import os import sys import time import traceback +import types import unittest from queue import Queue from test import support @@ -30,14 +31,8 @@ def run_test_in_subprocess(testname, ns): """ from subprocess import Popen, PIPE - args = (testname, ns.verbose, ns.quiet) - kwargs = dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, - failfast=ns.failfast, - match_tests=ns.match_tests) - slaveargs = (args, kwargs) + ns_dict = vars(ns) + slaveargs = (ns_dict, testname) slaveargs = json.dumps(slaveargs) cmd = [sys.executable, *support.args_from_interpreter_flags(), @@ -60,11 +55,18 @@ def run_test_in_subprocess(testname, ns): def run_tests_slave(slaveargs): - args, kwargs = json.loads(slaveargs) - if kwargs.get('huntrleaks'): + ns_dict, testname = json.loads(slaveargs) + ns = types.SimpleNamespace(**ns_dict) + + if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False + try: - result = runtest(*args, **kwargs) + result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + failfast=ns.failfast, + match_tests=ns.match_tests) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: -- cgit v0.12 From ecef622fec197f4377b5b5ec58f80f5fd000210f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:39:28 +0200 Subject: Issue #25220, libregrtest: Call setup_python(ns) in the slaves Slaves (child processes running tests for regrtest -jN) now inherit --memlimit/-M, --threshold/-t and --nowindows/-n options. * -M, -t and -n are now supported with -jN * Factorize code to run tests. * run_test_in_subprocess() now pass the whole "ns" namespace to the child process. --- Lib/test/libregrtest/cmdline.py | 2 -- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/libregrtest/runtest_mp.py | 8 ++++---- Lib/test/test_regrtest.py | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 19c52a2..e55e53f 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -295,8 +295,6 @@ def _parse_args(args, **kwargs): parser.error("-T and -j don't go together!") if ns.use_mp and ns.findleaks: parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") if ns.failfast and not (ns.verbose or ns.verbose3): parser.error("-G/--failfast needs either -v or -W") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index df2329f..e685970 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -354,14 +354,15 @@ class Regrtest: def main(self, tests=None, **kwargs): self.ns = self.parse_args(kwargs) - setup_python(self.ns) - if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: input("Press any key to continue...") + setup_python(self.ns) + self.find_tests(tests) self.run_tests() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 74424c1..47393aa 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,7 +13,8 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR +from test.libregrtest.setup import setup_python # Minimum duration of a test to display its duration or to mention that @@ -58,11 +59,10 @@ def run_tests_slave(slaveargs): ns_dict, testname = json.loads(slaveargs) ns = types.SimpleNamespace(**ns_dict) - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + setup_python(ns) try: - result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + result = runtest_ns(testname, ns.verbose, ns, use_resources=ns.use_resources, output_on_failure=ns.verbose3, failfast=ns.failfast, diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 54bbfe4..c277e10 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -214,7 +214,6 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError([opt, 'foo'], 'invalid int value') self.checkError([opt, '2', '-T'], "don't go together") self.checkError([opt, '2', '-l'], "don't go together") - self.checkError([opt, '2', '-M', '4G'], "don't go together") def test_coverage(self): for opt in '-T', '--coverage': -- cgit v0.12 From 00b8f9bb9ea1a246acc27f371c123016d50842aa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:02:49 +0200 Subject: Issue #25274: Workaround test_sys crash just to keep buildbots usable --- Lib/test/test_sys.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 83549bc..ccbf262 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -208,7 +208,10 @@ class SysModuleTest(unittest.TestCase): def f(): f() try: - for i in (50, 1000): + # FIXME: workaround crash for the issue #25274 + # FIXME: until the crash is fixed + #for i in (50, 1000): + for i in (150, 1000): # Issue #5392: stack overflow after hitting recursion limit twice sys.setrecursionlimit(i) self.assertRaises(RecursionError, f) -- cgit v0.12 From a204502dbf38318ea926eeccab886dcb52305a16 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:17:28 +0200 Subject: Issue #25220, libregrtest: Set support.use_resources in setup_tests() * Rename setup_python() to setup_tests() * Remove use_resources parameter of runtest() --- Lib/test/libregrtest/main.py | 5 ++--- Lib/test/libregrtest/runtest.py | 5 +---- Lib/test/libregrtest/runtest_mp.py | 5 ++--- Lib/test/libregrtest/setup.py | 4 +++- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e685970..9ca407a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,7 +10,7 @@ from test.libregrtest.runtest import ( findtests, runtest_ns, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args -from test.libregrtest.setup import setup_python +from test.libregrtest.setup import setup_tests from test import support try: import gc @@ -312,7 +312,6 @@ class Regrtest: def run_tests(self): support.verbose = self.ns.verbose # Tell tests to be moderately quiet - support.use_resources = self.ns.use_resources if self.ns.forever: def test_forever(tests): @@ -361,7 +360,7 @@ class Regrtest: if self.ns.wait: input("Press any key to continue...") - setup_python(self.ns) + setup_tests(self.ns) self.find_tests(tests) self.run_tests() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index fb7f821..a3d4e79 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -61,7 +61,7 @@ def runtest_ns(test, verbose, ns, **kw): def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, + huntrleaks=False, output_on_failure=False, failfast=False, match_tests=None, timeout=None): """Run a single test. @@ -71,7 +71,6 @@ def runtest(test, verbose, quiet, quiet -- if true, don't print 'skipped' messages (probably redundant) huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments - use_resources -- list of extra resources to use output_on_failure -- if true, display test output on failure timeout -- dump the traceback and exit if a test takes more than timeout seconds @@ -86,8 +85,6 @@ def runtest(test, verbose, quiet, PASSED test passed """ - if use_resources is not None: - support.use_resources = use_resources use_timeout = (timeout is not None) if use_timeout: faulthandler.dump_traceback_later(timeout, exit=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 47393aa..8332a0b 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -14,7 +14,7 @@ except ImportError: sys.exit(2) from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR -from test.libregrtest.setup import setup_python +from test.libregrtest.setup import setup_tests # Minimum duration of a test to display its duration or to mention that @@ -59,11 +59,10 @@ def run_tests_slave(slaveargs): ns_dict, testname = json.loads(slaveargs) ns = types.SimpleNamespace(**ns_dict) - setup_python(ns) + setup_tests(ns) try: result = runtest_ns(testname, ns.verbose, ns, - use_resources=ns.use_resources, output_on_failure=ns.verbose3, failfast=ns.failfast, match_tests=ns.match_tests) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index a7dfa79..6a1c308 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -13,7 +13,7 @@ except ImportError: from test.libregrtest.refleak import warm_caches -def setup_python(ns): +def setup_tests(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -91,6 +91,8 @@ def setup_python(ns): msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + support.use_resources = ns.use_resources + def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error -- cgit v0.12 From 6f20a2e01ffe897c20d0085f63371c467d118dac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:32:11 +0200 Subject: Issue #25220, libregrtest: Pass directly ns to runtest() * Remove runtest_ns(): pass directly ns to runtest(). * Create also Regrtest.rerun_failed_tests() method. * Inline again Regrtest.run_test(): it's no more justified to have a method --- Lib/test/libregrtest/main.py | 62 ++++++++++++++++++++------------------ Lib/test/libregrtest/runtest.py | 20 ++++++------ Lib/test/libregrtest/runtest_mp.py | 7 ++--- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9ca407a..c102ee0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import sysconfig import tempfile import textwrap from test.libregrtest.runtest import ( - findtests, runtest_ns, + findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args from test.libregrtest.setup import setup_tests @@ -208,6 +208,30 @@ class Regrtest: print("Using random seed", self.ns.random_seed) random.shuffle(self.selected) + def rerun_failed_tests(self): + self.ns.verbose = True + self.ns.failfast = False + self.ns.verbose3 = False + self.ns.match_tests = None + + print("Re-running failed tests in verbose mode") + for test in self.bad[:]: + print("Re-running test %r in verbose mode" % test, flush=True) + try: + self.ns.verbose = True + ok = runtest(self.ns, test) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + self.bad.remove(test) + else: + if self.bad: + print(count(len(self.bad), 'test'), "failed again:") + printlist(self.bad) + def display_result(self): if self.interrupted: # print a newline after ^C @@ -245,32 +269,6 @@ class Regrtest: print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) - if self.ns.verbose2 and self.bad: - print("Re-running failed tests in verbose mode") - for test in self.bad[:]: - print("Re-running test %r in verbose mode" % test, flush=True) - try: - self.ns.verbose = True - ok = runtest_ns(test, True, self.ns) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - self.bad.remove(test) - else: - if self.bad: - print(count(len(self.bad), 'test'), "failed again:") - printlist(self.bad) - - def run_test(self, test): - result = runtest_ns(test, self.ns.verbose, self.ns, - output_on_failure=self.ns.verbose3, - failfast=self.ns.failfast, - match_tests=self.ns.match_tests) - self.accumulate_result(test, result) - def run_tests_sequential(self): if self.ns.trace: import trace @@ -286,11 +284,13 @@ class Regrtest: if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = 'self.run_test(test)' + cmd = ('result = runtest(self.ns, test); ' + 'self.accumulate_result(test, result)') self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - self.run_test(test) + result = runtest(self.ns, test) + self.accumulate_result(test, result) except KeyboardInterrupt: self.interrupted = True break @@ -366,6 +366,10 @@ class Regrtest: self.run_tests() self.display_result() + + if self.ns.verbose2 and self.bad: + self.rerun_failed_tests() + self.finalize() sys.exit(len(self.bad) > 0 or self.interrupted) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index a3d4e79..4cc2588 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -53,17 +53,7 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) -def runtest_ns(test, verbose, ns, **kw): - return runtest(test, verbose, ns.quiet, - huntrleaks=ns.huntrleaks, - timeout=ns.timeout, - **kw) - - -def runtest(test, verbose, quiet, - huntrleaks=False, - output_on_failure=False, failfast=False, match_tests=None, - timeout=None): +def runtest(ns, test): """Run a single test. test -- the name of the test @@ -85,6 +75,14 @@ def runtest(test, verbose, quiet, PASSED test passed """ + verbose = ns.verbose + quiet = ns.quiet + huntrleaks = ns.huntrleaks + output_on_failure = ns.verbose3 + failfast = ns.failfast + match_tests = ns.match_tests + timeout = ns.timeout + use_timeout = (timeout is not None) if use_timeout: faulthandler.dump_traceback_later(timeout, exit=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 8332a0b..df075c1 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,7 +13,7 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR from test.libregrtest.setup import setup_tests @@ -62,10 +62,7 @@ def run_tests_slave(slaveargs): setup_tests(ns) try: - result = runtest_ns(testname, ns.verbose, ns, - output_on_failure=ns.verbose3, - failfast=ns.failfast, - match_tests=ns.match_tests) + result = runtest(ns, testname) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: -- cgit v0.12 From b40843546b1dc558da7217f8c0fe72fbca38e797 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:39:22 +0200 Subject: Issue #25220, libregrtest: Cleanup No need to support.verbose in Regrtest.run_tests(), it's always set in runtest(). --- Lib/test/libregrtest/main.py | 17 ++++++++--------- Lib/test/libregrtest/runtest_mp.py | 1 + 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c102ee0..fdb925d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -310,17 +310,16 @@ class Regrtest: if module not in save_modules and module.startswith("test."): support.unload(module) - def run_tests(self): - support.verbose = self.ns.verbose # Tell tests to be moderately quiet + def _test_forever(self, tests): + while True: + for test in tests: + yield test + if self.bad: + return + def run_tests(self): if self.ns.forever: - def test_forever(tests): - while True: - for test in tests: - yield test - if self.bad: - return - self.tests = test_forever(list(self.selected)) + self.tests = _test_forever(list(self.selected)) self.test_count = '' self.test_count_width = 3 else: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index df075c1..6ed8dc2 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -68,6 +68,7 @@ def run_tests_slave(slaveargs): except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) + print() # Force a newline (just in case) print(json.dumps(result), flush=True) sys.exit(0) -- cgit v0.12 From 17f97166766fd2ebe46df0f9fe9308182d141529 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 03:05:43 +0200 Subject: Issue #25220, libregrtest: more verbose output for -jN When the -jN command line option is used, display tests running since at least 30 seconds every minute. --- Lib/test/libregrtest/runtest_mp.py | 39 +++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 6ed8dc2..b31b51e 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,11 +1,11 @@ import json import os +import queue import sys import time import traceback import types import unittest -from queue import Queue from test import support try: import threading @@ -21,6 +21,9 @@ from test.libregrtest.setup import setup_tests # the test is running in background PROGRESS_MIN_TIME = 30.0 # seconds +# Display the running tests if nothing happened last N seconds +PROGRESS_UPDATE = 60.0 # seconds + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -145,18 +148,39 @@ class MultiprocessThread(threading.Thread): def run_tests_multiprocess(regrtest): - output = Queue() + output = queue.Queue() pending = MultiprocessIterator(regrtest.tests) workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] for worker in workers: worker.start() + + def get_running(workers): + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + return running + finished = 0 test_index = 1 + timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: while finished < regrtest.ns.use_mp: - test, stdout, stderr, result = output.get() + try: + item = output.get(timeout=PROGRESS_UPDATE) + except queue.Empty: + running = get_running(workers) + if running: + print('running: %s' % ', '.join(running)) + continue + + test, stdout, stderr, result = item if test is None: finished += 1 continue @@ -168,14 +192,7 @@ def run_tests_multiprocess(regrtest): if (ok not in (CHILD_ERROR, INTERRUPTED) and test_time >= PROGRESS_MIN_TIME): text += ' (%.0f sec)' % test_time - running = [] - for worker in workers: - current_test = worker.current_test - if not current_test: - continue - dt = time.monotonic() - worker.start_time - if dt >= PROGRESS_MIN_TIME: - running.append('%s (%.0f sec)' % (current_test, dt)) + running = get_running(workers) if running: text += ' -- running: %s' % ', '.join(running) regrtest.display_progress(test_index, text) -- cgit v0.12 From 38031143fbab6267fe464a1cb39eeb5c68de38f4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 29 Sep 2015 22:45:05 -0700 Subject: Add an early-out for deque_clear() --- Modules/_collectionsmodule.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 5db7aed..d9df574 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -595,6 +595,9 @@ deque_clear(dequeobject *deque) Py_ssize_t n; PyObject *item; + if (Py_SIZE(deque) == 0) + return; + /* During the process of clearing a deque, decrefs can cause the deque to mutate. To avoid fatal confusion, we have to make the deque empty before clearing the blocks and never refer to -- cgit v0.12 From 9a14214aee22512f833a8d5cf8627ca45627d330 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 13:51:17 +0200 Subject: Issue #25220: Fix "-m test --forever" * Fix "-m test --forever": replace _test_forever() with self._test_forever() * Add unit test for --forever * Add unit test for a failing test * Fix also some pyflakes warnings in libregrtest --- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/refleak.py | 6 +- Lib/test/libregrtest/runtest_mp.py | 3 +- Lib/test/test_regrtest.py | 133 ++++++++++++++++++++++++++----------- 4 files changed, 100 insertions(+), 44 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fdb925d..e1a99fb 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -319,7 +319,7 @@ class Regrtest: def run_tests(self): if self.ns.forever: - self.tests = _test_forever(list(self.selected)) + self.tests = self._test_forever(list(self.selected)) self.test_count = '' self.test_count_width = 3 else: diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index db8a445..9be0dec 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -46,6 +46,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) + # initialize variables to make pyflakes quiet + rc_before = alloc_before = 0 for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) @@ -158,6 +160,6 @@ def warm_caches(): for i in range(256): s[i:i+1] # unicode cache - x = [chr(i) for i in range(256)] + [chr(i) for i in range(256)] # int cache - x = list(range(-5, 257)) + list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index b31b51e..a732d70 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -5,7 +5,6 @@ import sys import time import traceback import types -import unittest from test import support try: import threading @@ -173,7 +172,7 @@ def run_tests_multiprocess(regrtest): try: while finished < regrtest.ns.use_mp: try: - item = output.get(timeout=PROGRESS_UPDATE) + item = output.get(timeout=timeout) except queue.Empty: running = get_running(workers) if running: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c277e10..897598d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -330,33 +330,52 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX, - output, - re.MULTILINE) - return set(match.group(1) for match in parser) + regex = r'^\[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + parser = re.finditer(regex, output, re.MULTILINE) + return list(match.group(1) for match in parser) - def check_executed_tests(self, output, tests, skipped=None): + def check_executed_tests(self, output, tests, skipped=(), failed=(), + randomize=False): if isinstance(tests, str): tests = [tests] - executed = self.parse_executed_tests(output) - self.assertEqual(executed, set(tests), output) + if isinstance(skipped, str): + skipped = [skipped] + if isinstance(failed, str): + failed = [failed] ntest = len(tests) - if skipped: - if isinstance(skipped, str): - skipped = [skipped] - nskipped = len(skipped) - - plural = 's' if nskipped != 1 else '' - names = ' '.join(sorted(skipped)) - expected = (r'%s test%s skipped:\n %s$' - % (nskipped, plural, names)) - self.check_line(output, expected) - - ok = ntest - nskipped - if ok: - self.check_line(output, r'%s test OK\.$' % ok) + nskipped = len(skipped) + nfailed = len(failed) + + executed = self.parse_executed_tests(output) + if randomize: + self.assertEqual(set(executed), set(tests), output) else: - self.check_line(output, r'All %s tests OK\.$' % ntest) + self.assertEqual(executed, tests, output) + + def plural(count): + return 's' if count != 1 else '' + + def list_regex(line_format, tests): + count = len(tests) + names = ' '.join(sorted(tests)) + regex = line_format % (count, plural(count)) + regex = r'%s:\n %s$' % (regex, names) + return regex + + if skipped: + regex = list_regex('%s test%s skipped', skipped) + self.check_line(output, regex) + + if failed: + regex = list_regex('%s test%s failed', failed) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed + if good: + regex = r'%s test%s OK\.$' % (good, plural(good)) + if not skipped and not failed and good > 1: + regex = 'All %s' % regex + self.check_line(output, regex) def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) @@ -364,24 +383,28 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed - def run_command(self, args, input=None): + def run_command(self, args, input=None, exitcode=0): if not input: input = '' - try: - return subprocess.run(args, - check=True, universal_newlines=True, - input=input, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except subprocess.CalledProcessError as exc: - self.fail("%s\n" + proc = subprocess.run(args, + universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if proc.returncode != exitcode: + self.fail("Command %s failed with exit code %s\n" "\n" "stdout:\n" + "---\n" "%s\n" + "---\n" "\n" "stderr:\n" + "---\n" "%s" - % (str(exc), exc.stdout, exc.stderr)) + "---\n" + % (str(args), proc.returncode, proc.stdout, proc.stderr)) + return proc def run_python(self, args, **kw): @@ -411,11 +434,11 @@ class ProgramsTestCase(BaseTestCase): def check_output(self, output): self.parse_random_seed(output) - self.check_executed_tests(output, self.tests) + self.check_executed_tests(output, self.tests, randomize=True) def run_tests(self, args): - stdout = self.run_python(args) - self.check_output(stdout) + output = self.run_python(args) + self.check_output(output) def test_script_regrtest(self): # Lib/test/regrtest.py @@ -492,8 +515,24 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args, input=None): - return self.run_python(['-m', 'test', *args], input=input) + def run_tests(self, *args, **kw): + return self.run_python(['-m', 'test', *args], **kw) + + def test_failing_test(self): + # test a failing test + code = textwrap.dedent(""" + import unittest + + class FailingTest(unittest.TestCase): + def test_failing(self): + self.fail("bug") + """) + test_ok = self.create_test() + test_failing = self.create_test(code=code) + tests = [test_ok, test_failing] + + output = self.run_tests(*tests, exitcode=1) + self.check_executed_tests(output, tests, failed=test_failing) def test_resources(self): # test -u command line option @@ -572,8 +611,7 @@ class ArgsTestCase(BaseTestCase): # test --coverage test = self.create_test() output = self.run_tests("--coverage", test) - executed = self.parse_executed_tests(output) - self.assertEqual(executed, {test}, output) + self.check_executed_tests(output, [test]) regex = ('lines +cov% +module +$path$\n' '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +$[^)]+$+)+') self.check_line(output, regex) @@ -584,6 +622,23 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests("--wait", test, input='key') self.check_line(output, 'Press any key to continue') + def test_forever(self): + # test --forever + code = textwrap.dedent(""" + import unittest + + class ForeverTester(unittest.TestCase): + RUN = 1 + + def test_run(self): + ForeverTester.RUN += 1 + if ForeverTester.RUN > 3: + self.fail("fail at the 3rd runs") + """) + test = self.create_test(code=code) + output = self.run_tests('--forever', test, exitcode=1) + self.check_executed_tests(output, [test]*3, failed=test) + if __name__ == '__main__': unittest.main() -- cgit v0.12 From c51d244fc922610ee58e30dbd99d8d42f32d061b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 22:06:51 +0200 Subject: Issue #25171: Fix compilation issue on OpenBSD in random.c Patch written by Remi Pointel. --- Python/random.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/random.c b/Python/random.c index 8f3e6d6..8caaa95 100644 --- a/Python/random.c +++ b/Python/random.c @@ -364,7 +364,7 @@ _PyOS_URandom(void *buffer, Py_ssize_t size) #ifdef MS_WINDOWS return win32_urandom((unsigned char *)buffer, size, 1); -#elif PY_GETENTROPY +#elif defined(PY_GETENTROPY) return py_getentropy(buffer, size, 0); #else return dev_urandom_python((char*)buffer, size); @@ -411,7 +411,7 @@ _PyRandom_Init(void) else { #ifdef MS_WINDOWS (void)win32_urandom(secret, secret_size, 0); -#elif PY_GETENTROPY +#elif defined(PY_GETENTROPY) (void)py_getentropy(secret, secret_size, 1); #else dev_urandom_noraise(secret, secret_size); @@ -427,7 +427,7 @@ _PyRandom_Fini(void) CryptReleaseContext(hCryptProv, 0); hCryptProv = 0; } -#elif PY_GETENTROPY +#elif defined(PY_GETENTROPY) /* nothing to clean */ #else dev_urandom_close(); -- cgit v0.12 From c29f399e7e59c6e3d369568af784958321c151ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 22:50:12 +0200 Subject: Backout change 28d3bcb1bad6: "Try to fix _PyTime_AsTimevalStruct_impl() on OpenBSD", I'm not sure that the change was really needed. I read the test result of an old build because the OpenBSD was 100 builds late. --- Python/pytime.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 53611b1..9889a3b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -454,7 +454,7 @@ static int _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - _PyTime_t secs, secs2; + _PyTime_t secs; int us; int res; @@ -467,8 +467,7 @@ _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, #endif tv->tv_usec = us; - secs2 = (_PyTime_t)tv->tv_sec; - if (res < 0 || secs2 != secs) { + if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { if (raise) error_time_t_overflow(); return -1; -- cgit v0.12 From a53a818c3c67c927bcf8ee118fd29e21045dd9ab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 00:53:09 +0200 Subject: Fix regrtest --coverage on Windows Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the list of ignored directories. --- Lib/test/libregrtest/main.py | 5 +---- Lib/test/test_regrtest.py | 2 -- Misc/NEWS | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e1a99fb..c1ce3b1 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -272,10 +272,7 @@ class Regrtest: def run_tests_sequential(self): if self.ns.trace: import trace - self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, - sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) + self.tracer = trace.Trace(trace=False, count=True) save_modules = sys.modules.keys() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 897598d..0f5f22c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -605,8 +605,6 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - @unittest.skipIf(sys.platform == 'win32', - "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() diff --git a/Misc/NEWS b/Misc/NEWS index 72d6fef..a1aa269 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -157,6 +157,9 @@ Documentation Tests ----- +- Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the + list of ignored directories. + - PCbuild\rt.bat now accepts an unlimited number of arguments to pass along to regrtest.py. Previously there was a limit of 9. -- cgit v0.12 From 0d30940dd256da65c91c3bb2d1e135f90eb8c223 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 30 Sep 2015 23:15:02 -0700 Subject: Add fast paths to deque_init() for the common cases --- Modules/_collectionsmodule.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index d9df574..0c64713 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1456,8 +1456,14 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) Py_ssize_t maxlen = -1; char *kwlist[] = {"iterable", "maxlen", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, &iterable, &maxlenobj)) - return -1; + if (kwdargs == NULL) { + if (!PyArg_UnpackTuple(args, "deque()", 0, 2, &iterable, &maxlenobj)) + return -1; + } else { + if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, + &iterable, &maxlenobj)) + return -1; + } if (maxlenobj != NULL && maxlenobj != Py_None) { maxlen = PyLong_AsSsize_t(maxlenobj); if (maxlen == -1 && PyErr_Occurred()) @@ -1468,7 +1474,8 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) } } deque->maxlen = maxlen; - deque_clear(deque); + if (Py_SIZE(deque) > 0) + deque_clear(deque); if (iterable != NULL) { PyObject *rv = deque_extend(deque, iterable); if (rv == NULL) -- cgit v0.12 From b7a8af20ff09efdfa6896ecaff7d036562c17897 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 08:44:03 +0200 Subject: Fix _PyTime_AsTimevalStruct_impl() on OpenBSD On the x86 OpenBSD 5.8 buildbot, the integer overflow check is ignored. Copy the tv_sec variable into a Py_time_t variable instead of "simply" casting it to Py_time_t, to fix the integer overflow check. --- Python/pytime.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 9889a3b..53611b1 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -454,7 +454,7 @@ static int _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - _PyTime_t secs; + _PyTime_t secs, secs2; int us; int res; @@ -467,7 +467,8 @@ _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, #endif tv->tv_usec = us; - if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { + secs2 = (_PyTime_t)tv->tv_sec; + if (res < 0 || secs2 != secs) { if (raise) error_time_t_overflow(); return -1; -- cgit v0.12 From 98f223dfa0440529261e921c8b9ec9a955c6d27f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 13:16:43 +0200 Subject: Issue #25277: Set a timeout of 10 minutes in test_eintr using faulthandler to try to debug a hang on the FreeBSD 9 buildbot. Run also eintr_tester.py with python "-u" command line option to try to get the full output on hang/crash. --- Lib/test/eintrdata/eintr_tester.py | 5 +++++ Lib/test/test_eintr.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1e0d91..443ccd5 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -9,6 +9,7 @@ sub-second periodicity (contrarily to signal()). """ import contextlib +import faulthandler import io import os import select @@ -50,6 +51,9 @@ class EINTRBaseTest(unittest.TestCase): signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) + # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD + faulthandler.dump_traceback_later(10 * 60, exit=True) + @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) @@ -58,6 +62,7 @@ class EINTRBaseTest(unittest.TestCase): def tearDownClass(cls): cls.stop_alarm() signal.signal(signal.SIGALRM, cls.orig_handler) + faulthandler.cancel_dump_traceback_later() @classmethod def _sleep(cls): diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index d3cdda0..75452f2 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -16,7 +16,8 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - script_helper.assert_python_ok(tester) + # use -u to try to get the full output if the test hangs or crash + script_helper.assert_python_ok("-u", tester) if __name__ == "__main__": -- cgit v0.12 From 29a1445136c7353543b516a085c38b8be9ce5109 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 1 Oct 2015 20:54:41 +0100 Subject: Closes #24884: refactored WatchedFileHandler file reopening into a separate method, based on a suggestion and patch by Marian Horban. --- Doc/library/logging.handlers.rst | 12 +++++++++--- Lib/logging/handlers.py | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 67403a9..0d3928c 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -162,11 +162,17 @@ for this value. first call to :meth:`emit`. By default, the file grows indefinitely. + .. method:: reopenIfNeeded() + + Checks to see if the file has changed. If it has, the existing stream is + flushed and closed and the file opened again, typically as a precursor to + outputting the record to the file. + + .. method:: emit(record) - Outputs the record to the file, but first checks to see if the file has - changed. If it has, the existing stream is flushed and closed and the - file opened again, before outputting the record to the file. + Outputs the record to the file, but first calls :meth:`reopenIfNeeded` to + reopen the file if it has changed. .. _base-rotating-handler: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 02a5fc1..54bee89 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -440,11 +440,11 @@ class WatchedFileHandler(logging.FileHandler): sres = os.fstat(self.stream.fileno()) self.dev, self.ino = sres[ST_DEV], sres[ST_INO] - def emit(self, record): + def reopenIfNeeded(self): """ - Emit a record. + Reopen log file if needed. - First check if the underlying file has changed, and if it + Checks if the underlying file has changed, and if it has, close the old stream and reopen the file to get the current stream. """ @@ -467,6 +467,15 @@ class WatchedFileHandler(logging.FileHandler): # open a new file handle and get new stat info from that fd self.stream = self._open() self._statstream() + + def emit(self, record): + """ + Emit a record. + + If underlying file has changed, reopen the file before emitting the + record to it. + """ + self.reopenIfNeeded() logging.FileHandler.emit(self, record) -- cgit v0.12 From 01ada3996bded57d1baf9c54b050fc55907d9b13 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 21:54:51 +0200 Subject: Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. Patch co-written with Serhiy Storchaka. --- Doc/whatsnew/3.6.rst | 3 + Lib/test/test_codecs.py | 37 +++++++++--- Misc/NEWS | 4 ++ Objects/stringlib/codecs.h | 147 +++++++++++++++++++++++++++++---------------- Objects/unicodeobject.c | 7 ++- 5 files changed, 135 insertions(+), 63 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 16cdca0..ca83ef9 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -120,6 +120,9 @@ Optimizations * The ASCII and the Latin1 encoders are now up to 3 times as fast for the error error ``surrogateescape``. +* The UTF-8 encoder is now up to 75 times as fast for error handlers: + ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. + Build and C API Changes ======================= diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 254c0c1..4658497 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -361,6 +361,12 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual("[\uDC80]".encode(self.encoding, "replace"), "[?]".encode(self.encoding)) + # sequential surrogate characters + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"), + "[]".encode(self.encoding)) + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"), + "[??]".encode(self.encoding)) + bom = "".encode(self.encoding) for before, after in [("\U00010fff", "A"), ("[", "]"), ("A", "\U00010fff")]: @@ -753,6 +759,7 @@ class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" ill_formed_sequence_replace = "\ufffd" * 3 + BOM = b'' def test_partial(self): self.check_partial( @@ -785,23 +792,32 @@ class UTF8Test(ReadTest, unittest.TestCase): super().test_lone_surrogates() # not sure if this is making sense for # UTF-16 and UTF-32 - self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"), - b'[\x80]') + self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"), + self.BOM + b'[\x80]') + + with self.assertRaises(UnicodeEncodeError) as cm: + "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape") + exc = cm.exception + self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF') def test_surrogatepass_handler(self): - self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"), - b"abc\xed\xa0\x80def") - self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"), + self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"), + self.BOM + b"abc\xed\xa0\x80def") + self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"), + self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"), + self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]') + + self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"), "abc\ud800def") - self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"), - b"\xf0\x90\xbf\xbf\xed\xa0\x80") - self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"), + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"), "\U00010fff\uD800") + self.assertTrue(codecs.lookup_error("surrogatepass")) with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0".decode("utf-8", "surrogatepass") + b"abc\xed\xa0".decode(self.encoding, "surrogatepass") with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + b"abc\xed\xa0z".decode(self.encoding, "surrogatepass") @unittest.skipUnless(sys.platform == 'win32', @@ -1008,6 +1024,7 @@ class ReadBufferTest(unittest.TestCase): class UTF8SigTest(UTF8Test, unittest.TestCase): encoding = "utf-8-sig" + BOM = codecs.BOM_UTF8 def test_partial(self): self.check_partial( diff --git a/Misc/NEWS b/Misc/NEWS index 68c0be1..1fae34a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error + handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. + Patch co-written with Serhiy Storchaka. + - Issue #25280: Import trace messages emitted in verbose (-v) mode are no longer formatted twice. diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 0fc6b58..562191c 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -268,9 +268,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, Py_ssize_t nallocated; /* number of result bytes allocated */ Py_ssize_t nneeded; /* number of result bytes needed */ #if STRINGLIB_SIZEOF_CHAR > 1 - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; PyObject *rep = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; #endif #if STRINGLIB_SIZEOF_CHAR == 1 const Py_ssize_t max_char_size = 2; @@ -326,72 +327,116 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } #if STRINGLIB_SIZEOF_CHAR > 1 else if (Py_UNICODE_IS_SURROGATE(ch)) { - Py_ssize_t newpos; - Py_ssize_t repsize, k, startpos; + Py_ssize_t startpos, endpos, newpos; + Py_ssize_t repsize, k; + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + startpos = i-1; - rep = unicode_encode_call_errorhandler( - errors, &errorHandler, "utf-8", "surrogates not allowed", - unicode, &exc, startpos, startpos+1, &newpos); - if (!rep) - goto error; - - if (PyBytes_Check(rep)) - repsize = PyBytes_GET_SIZE(rep); - else - repsize = PyUnicode_GET_LENGTH(rep); + endpos = startpos+1; + + while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos])) + endpos++; + + switch (error_handler) + { + case _Py_ERROR_REPLACE: + memset(p, '?', endpos - startpos); + p += (endpos - startpos); + /* fall through the ignore handler */ + case _Py_ERROR_IGNORE: + i += (endpos - startpos - 1); + break; - if (repsize > max_char_size) { - Py_ssize_t offset; - if (result == NULL) - offset = p - stackbuf; - else - offset = p - PyBytes_AS_STRING(result); + case _Py_ERROR_SURROGATEPASS: + for (k=startpos; k> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + i += (endpos - startpos - 1); + break; - if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { - /* integer overflow */ - PyErr_NoMemory(); - goto error; + case _Py_ERROR_SURROGATEESCAPE: + for (k=startpos; k= endpos) { + i += (endpos - startpos - 1); + break; + } + startpos = k; + assert(startpos < endpos); + /* fall through the default handler */ + + default: + rep = unicode_encode_call_errorhandler( + errors, &error_handler_obj, "utf-8", "surrogates not allowed", + unicode, &exc, startpos, endpos, &newpos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) + repsize = PyBytes_GET_SIZE(rep); + else + repsize = PyUnicode_GET_LENGTH(rep); + + if (repsize > max_char_size) { + Py_ssize_t offset; + if (result == NULL) + offset = p - stackbuf; + else + offset = p - PyBytes_AS_STRING(result); + + if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { + /* integer overflow */ + PyErr_NoMemory(); goto error; - Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + nallocated += repsize - max_char_size; + if (result != NULL) { + if (_PyBytes_Resize(&result, nallocated) < 0) + goto error; + } else { + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + goto error; + Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + p = PyBytes_AS_STRING(result) + offset; } - p = PyBytes_AS_STRING(result) + offset; - } - - if (PyBytes_Check(rep)) { - char *prep = PyBytes_AS_STRING(rep); - for(k = repsize; k > 0; k--) - *p++ = *prep++; - } else /* rep is unicode */ { - enum PyUnicode_Kind repkind; - void *repdata; - if (PyUnicode_READY(rep) < 0) - goto error; - repkind = PyUnicode_KIND(rep); - repdata = PyUnicode_DATA(rep); + if (PyBytes_Check(rep)) { + memcpy(p, PyBytes_AS_STRING(rep), repsize); + p += repsize; + } + else { + /* rep is unicode */ + if (PyUnicode_READY(rep) < 0) + goto error; - for(k=0; k 2 @@ -430,7 +475,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } #if STRINGLIB_SIZEOF_CHAR > 1 - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); #endif return result; @@ -438,7 +483,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #if STRINGLIB_SIZEOF_CHAR > 1 error: Py_XDECREF(rep); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); Py_XDECREF(result); return NULL; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6657cd4..df3a1b5 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -297,6 +297,7 @@ typedef enum { _Py_ERROR_UNKNOWN=0, _Py_ERROR_STRICT, _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_SURROGATEPASS, _Py_ERROR_REPLACE, _Py_ERROR_IGNORE, _Py_ERROR_XMLCHARREFREPLACE, @@ -312,6 +313,8 @@ get_error_handler(const char *errors) return _Py_ERROR_STRICT; if (strcmp(errors, "surrogateescape") == 0) return _Py_ERROR_SURROGATEESCAPE; + if (strcmp(errors, "surrogatepass") == 0) + return _Py_ERROR_SURROGATEPASS; if (strcmp(errors, "ignore") == 0) return _Py_ERROR_IGNORE; if (strcmp(errors, "replace") == 0) @@ -6479,8 +6482,8 @@ unicode_encode_ucs1(PyObject *unicode, goto onError; case _Py_ERROR_REPLACE: - while (collstart++ < collend) - *str++ = '?'; + memset(str, '?', collend - collstart); + str += (collend - collstart); /* fall through ignore error handler */ case _Py_ERROR_IGNORE: pos = collend; -- cgit v0.12 From 13f7fc5771f07d401002dbe7e67ad1a62d0e8ce5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 22:06:54 +0200 Subject: Update importlib_external.h --- Python/importlib_external.h | 113 ++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/Python/importlib_external.h b/Python/importlib_external.h index f4f5ac4..c4b6dc9 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -613,7 +613,7 @@ const unsigned char _Py_M__importlib_external[] = { 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, 109,135,1,0,0,115,10,0,0,0,0,10,21,1,24,1, 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, - 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,240, + 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,252, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, 9,0,114,31,0,124,2,0,124,4,0,100,2,0,60,110, 6,0,100,3,0,125,2,0,124,3,0,100,1,0,107,9, @@ -621,59 +621,60 @@ const unsigned char _Py_M__importlib_external[] = { 0,100,1,0,100,5,0,133,2,0,25,125,5,0,124,0, 0,100,5,0,100,6,0,133,2,0,25,125,6,0,124,0, 0,100,6,0,100,7,0,133,2,0,25,125,7,0,124,5, - 0,116,0,0,107,3,0,114,168,0,100,8,0,106,1,0, + 0,116,0,0,107,3,0,114,171,0,100,8,0,106,1,0, 124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,106, - 3,0,124,8,0,131,1,0,1,116,4,0,124,8,0,124, - 4,0,141,1,0,130,1,0,110,119,0,116,5,0,124,6, - 0,131,1,0,100,5,0,107,3,0,114,229,0,100,9,0, - 106,1,0,124,2,0,131,1,0,125,8,0,116,2,0,106, - 3,0,124,8,0,131,1,0,1,116,6,0,124,8,0,131, - 1,0,130,1,0,110,58,0,116,5,0,124,7,0,131,1, - 0,100,5,0,107,3,0,114,31,1,100,10,0,106,1,0, - 124,2,0,131,1,0,125,8,0,116,2,0,106,3,0,124, - 8,0,131,1,0,1,116,6,0,124,8,0,131,1,0,130, - 1,0,124,1,0,100,1,0,107,9,0,114,226,1,121,20, - 0,116,7,0,124,1,0,100,11,0,25,131,1,0,125,9, - 0,87,110,18,0,4,116,8,0,107,10,0,114,83,1,1, - 1,1,89,110,62,0,88,116,9,0,124,6,0,131,1,0, - 124,9,0,107,3,0,114,145,1,100,12,0,106,1,0,124, - 2,0,131,1,0,125,8,0,116,2,0,106,3,0,124,8, - 0,131,1,0,1,116,4,0,124,8,0,124,4,0,141,1, - 0,130,1,0,121,18,0,124,1,0,100,13,0,25,100,14, - 0,64,125,10,0,87,110,18,0,4,116,8,0,107,10,0, - 114,183,1,1,1,1,89,110,43,0,88,116,9,0,124,7, - 0,131,1,0,124,10,0,107,3,0,114,226,1,116,4,0, - 100,12,0,106,1,0,124,2,0,131,1,0,124,4,0,141, - 1,0,130,1,0,124,0,0,100,7,0,100,1,0,133,2, - 0,25,83,41,15,97,122,1,0,0,86,97,108,105,100,97, - 116,101,32,116,104,101,32,104,101,97,100,101,114,32,111,102, - 32,116,104,101,32,112,97,115,115,101,100,45,105,110,32,98, - 121,116,101,99,111,100,101,32,97,103,97,105,110,115,116,32, - 115,111,117,114,99,101,95,115,116,97,116,115,32,40,105,102, - 10,32,32,32,32,103,105,118,101,110,41,32,97,110,100,32, - 114,101,116,117,114,110,105,110,103,32,116,104,101,32,98,121, - 116,101,99,111,100,101,32,116,104,97,116,32,99,97,110,32, - 98,101,32,99,111,109,112,105,108,101,100,32,98,121,32,99, - 111,109,112,105,108,101,40,41,46,10,10,32,32,32,32,65, - 108,108,32,111,116,104,101,114,32,97,114,103,117,109,101,110, - 116,115,32,97,114,101,32,117,115,101,100,32,116,111,32,101, - 110,104,97,110,99,101,32,101,114,114,111,114,32,114,101,112, - 111,114,116,105,110,103,46,10,10,32,32,32,32,73,109,112, - 111,114,116,69,114,114,111,114,32,105,115,32,114,97,105,115, - 101,100,32,119,104,101,110,32,116,104,101,32,109,97,103,105, - 99,32,110,117,109,98,101,114,32,105,115,32,105,110,99,111, - 114,114,101,99,116,32,111,114,32,116,104,101,32,98,121,116, - 101,99,111,100,101,32,105,115,10,32,32,32,32,102,111,117, - 110,100,32,116,111,32,98,101,32,115,116,97,108,101,46,32, - 69,79,70,69,114,114,111,114,32,105,115,32,114,97,105,115, - 101,100,32,119,104,101,110,32,116,104,101,32,100,97,116,97, - 32,105,115,32,102,111,117,110,100,32,116,111,32,98,101,10, - 32,32,32,32,116,114,117,110,99,97,116,101,100,46,10,10, - 32,32,32,32,78,114,98,0,0,0,122,10,60,98,121,116, - 101,99,111,100,101,62,114,35,0,0,0,114,12,0,0,0, - 233,8,0,0,0,233,12,0,0,0,122,30,98,97,100,32, - 109,97,103,105,99,32,110,117,109,98,101,114,32,105,110,32, - 123,33,114,125,58,32,123,33,114,125,122,43,114,101,97,99, + 3,0,100,9,0,124,8,0,131,2,0,1,116,4,0,124, + 8,0,124,4,0,141,1,0,130,1,0,110,125,0,116,5, + 0,124,6,0,131,1,0,100,5,0,107,3,0,114,235,0, + 100,10,0,106,1,0,124,2,0,131,1,0,125,8,0,116, + 2,0,106,3,0,100,9,0,124,8,0,131,2,0,1,116, + 6,0,124,8,0,131,1,0,130,1,0,110,61,0,116,5, + 0,124,7,0,131,1,0,100,5,0,107,3,0,114,40,1, + 100,11,0,106,1,0,124,2,0,131,1,0,125,8,0,116, + 2,0,106,3,0,100,9,0,124,8,0,131,2,0,1,116, + 6,0,124,8,0,131,1,0,130,1,0,124,1,0,100,1, + 0,107,9,0,114,238,1,121,20,0,116,7,0,124,1,0, + 100,12,0,25,131,1,0,125,9,0,87,110,18,0,4,116, + 8,0,107,10,0,114,92,1,1,1,1,89,110,65,0,88, + 116,9,0,124,6,0,131,1,0,124,9,0,107,3,0,114, + 157,1,100,13,0,106,1,0,124,2,0,131,1,0,125,8, + 0,116,2,0,106,3,0,100,9,0,124,8,0,131,2,0, + 1,116,4,0,124,8,0,124,4,0,141,1,0,130,1,0, + 121,18,0,124,1,0,100,14,0,25,100,15,0,64,125,10, + 0,87,110,18,0,4,116,8,0,107,10,0,114,195,1,1, + 1,1,89,110,43,0,88,116,9,0,124,7,0,131,1,0, + 124,10,0,107,3,0,114,238,1,116,4,0,100,13,0,106, + 1,0,124,2,0,131,1,0,124,4,0,141,1,0,130,1, + 0,124,0,0,100,7,0,100,1,0,133,2,0,25,83,41, + 16,97,122,1,0,0,86,97,108,105,100,97,116,101,32,116, + 104,101,32,104,101,97,100,101,114,32,111,102,32,116,104,101, + 32,112,97,115,115,101,100,45,105,110,32,98,121,116,101,99, + 111,100,101,32,97,103,97,105,110,115,116,32,115,111,117,114, + 99,101,95,115,116,97,116,115,32,40,105,102,10,32,32,32, + 32,103,105,118,101,110,41,32,97,110,100,32,114,101,116,117, + 114,110,105,110,103,32,116,104,101,32,98,121,116,101,99,111, + 100,101,32,116,104,97,116,32,99,97,110,32,98,101,32,99, + 111,109,112,105,108,101,100,32,98,121,32,99,111,109,112,105, + 108,101,40,41,46,10,10,32,32,32,32,65,108,108,32,111, + 116,104,101,114,32,97,114,103,117,109,101,110,116,115,32,97, + 114,101,32,117,115,101,100,32,116,111,32,101,110,104,97,110, + 99,101,32,101,114,114,111,114,32,114,101,112,111,114,116,105, + 110,103,46,10,10,32,32,32,32,73,109,112,111,114,116,69, + 114,114,111,114,32,105,115,32,114,97,105,115,101,100,32,119, + 104,101,110,32,116,104,101,32,109,97,103,105,99,32,110,117, + 109,98,101,114,32,105,115,32,105,110,99,111,114,114,101,99, + 116,32,111,114,32,116,104,101,32,98,121,116,101,99,111,100, + 101,32,105,115,10,32,32,32,32,102,111,117,110,100,32,116, + 111,32,98,101,32,115,116,97,108,101,46,32,69,79,70,69, + 114,114,111,114,32,105,115,32,114,97,105,115,101,100,32,119, + 104,101,110,32,116,104,101,32,100,97,116,97,32,105,115,32, + 102,111,117,110,100,32,116,111,32,98,101,10,32,32,32,32, + 116,114,117,110,99,97,116,101,100,46,10,10,32,32,32,32, + 78,114,98,0,0,0,122,10,60,98,121,116,101,99,111,100, + 101,62,114,35,0,0,0,114,12,0,0,0,233,8,0,0, + 0,233,12,0,0,0,122,30,98,97,100,32,109,97,103,105, + 99,32,110,117,109,98,101,114,32,105,110,32,123,33,114,125, + 58,32,123,33,114,125,122,2,123,125,122,43,114,101,97,99, 104,101,100,32,69,79,70,32,119,104,105,108,101,32,114,101, 97,100,105,110,103,32,116,105,109,101,115,116,97,109,112,32, 105,110,32,123,33,114,125,122,48,114,101,97,99,104,101,100, @@ -699,9 +700,9 @@ const unsigned char _Py_M__importlib_external[] = { 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, 95,104,101,97,100,101,114,152,1,0,0,115,76,0,0,0, 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, - 16,1,16,1,12,1,18,1,13,1,18,1,18,1,15,1, - 13,1,15,1,18,1,15,1,13,1,12,1,12,1,3,1, - 20,1,13,1,5,2,18,1,15,1,13,1,15,1,3,1, + 16,1,16,1,12,1,18,1,16,1,18,1,18,1,15,1, + 16,1,15,1,18,1,15,1,16,1,12,1,12,1,3,1, + 20,1,13,1,5,2,18,1,15,1,16,1,15,1,3,1, 18,1,13,1,5,2,18,1,15,1,9,1,114,135,0,0, 0,99,4,0,0,0,0,0,0,0,5,0,0,0,6,0, 0,0,67,0,0,0,115,115,0,0,0,116,0,0,106,1, -- cgit v0.12 From 3222da26fe873410cbdbc18744e19a7d0f6a7523 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 22:07:32 +0200 Subject: Make _PyUnicode_TranslateCharmap() symbol private unicodeobject.h exposes PyUnicode_TranslateCharmap() and PyUnicode_Translate(). --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index df3a1b5..93c4ad9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8683,7 +8683,7 @@ exit: return res; } -PyObject * +static PyObject * _PyUnicode_TranslateCharmap(PyObject *input, PyObject *mapping, const char *errors) -- cgit v0.12 From 6661d885a34b355200b8ee6bd559af0a5cfc9c8b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2015 23:00:39 +0200 Subject: Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not supported. Check if it is supported, it may not be supported on OpenBSD for example. --- Doc/library/crypt.rst | 2 +- Lib/crypt.py | 3 +-- Misc/NEWS | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index b4c90cd..04ffdb2 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -64,7 +64,7 @@ Module Attributes A list of available password hashing algorithms, as ``crypt.METHOD_*`` objects. This list is sorted from strongest to - weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``. + weakest. Module Functions diff --git a/Lib/crypt.py b/Lib/crypt.py index 49ab96e..fbc5f4c 100644 --- a/Lib/crypt.py +++ b/Lib/crypt.py @@ -54,9 +54,8 @@ METHOD_SHA256 = _Method('SHA256', '5', 16, 63) METHOD_SHA512 = _Method('SHA512', '6', 16, 106) methods = [] -for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5): +for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5, METHOD_CRYPT): _result = crypt('', _method) if _result and len(_result) == _method.total_size: methods.append(_method) -methods.append(METHOD_CRYPT) del _result, _method diff --git a/Misc/NEWS b/Misc/NEWS index b43073f..8aad689 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -40,6 +40,10 @@ Core and Builtins Library ------- +- Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not + supported. Check if it is supported, it may not be supported on OpenBSD for + example. + - Issue #23600: Default implementation of tzinfo.fromutc() was returning wrong results in some cases. -- cgit v0.12 From 076fc872bca0f3aa4e6140021e5c05fc4360e82a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 00:20:56 +0200 Subject: Issue #18174: "python -m test --huntrleaks ..." now also checks for leak of file descriptors. Patch written by Richard Oudkerk. --- Lib/test/libregrtest/refleak.py | 47 ++++++++++++++++++++++++++---- Lib/test/test_regrtest.py | 63 ++++++++++++++++++++++++++++++++--------- Misc/NEWS | 3 ++ 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 9be0dec..59dc49f 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,3 +1,4 @@ +import errno import os import re import sys @@ -6,6 +7,36 @@ from inspect import isabstract from test import support +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except Exception: + MAXFD = 256 + + +def fd_count(): + """Count the number of open file descriptors""" + if sys.platform.startswith(('linux', 'freebsd')): + try: + names = os.listdir("/proc/self/fd") + return len(names) + except FileNotFoundError: + pass + + count = 0 + for fd in range(MAXFD): + try: + # Prefer dup() over fstat(). fstat() can require input/output + # whereas dup() doesn't. + fd2 = os.dup(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + os.close(fd2) + count += 1 + return count + + def dash_R(the_module, test, indirect_test, huntrleaks): """Run a test multiple times, looking for reference leaks. @@ -42,20 +73,25 @@ def dash_R(the_module, test, indirect_test, huntrleaks): repcount = nwarmup + ntracked rc_deltas = [0] * repcount alloc_deltas = [0] * repcount + fd_deltas = [0] * repcount print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) # initialize variables to make pyflakes quiet - rc_before = alloc_before = 0 + rc_before = alloc_before = fd_before = 0 for i in range(repcount): indirect_test() - alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) + alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, + abcs) print('.', end='', flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before - alloc_before, rc_before = alloc_after, rc_after + fd_deltas[i] = fd_after - fd_before + alloc_before = alloc_after + rc_before = rc_after + fd_before = fd_after print(file=sys.stderr) # These checkers return False on success, True on failure def check_rc_deltas(deltas): @@ -71,7 +107,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): failed = False for deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas)]: + (alloc_deltas, 'memory blocks', check_alloc_deltas), + (fd_deltas, 'file descriptors', check_rc_deltas)]: if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test, deltas[nwarmup:], item_name, sum(deltas)) @@ -151,7 +188,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): func1 = sys.getallocatedblocks func2 = sys.gettotalrefcount gc.collect() - return func1(), func2() + return func1(), func2(), fd_count() def warm_caches(): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0f5f22c..8b2449a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -383,27 +383,32 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed - def run_command(self, args, input=None, exitcode=0): + def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' + if 'stderr' not in kw: + kw['stderr'] = subprocess.PIPE proc = subprocess.run(args, universal_newlines=True, input=input, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + **kw) if proc.returncode != exitcode: - self.fail("Command %s failed with exit code %s\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s" - "---\n" - % (str(args), proc.returncode, proc.stdout, proc.stderr)) + msg = ("Command %s failed with exit code %s\n" + "\n" + "stdout:\n" + "---\n" + "%s\n" + "---\n" + % (str(args), proc.returncode, proc.stdout)) + if proc.stderr: + msg += ("\n" + "stderr:\n" + "---\n" + "%s" + "---\n" + % proc.stderr) + self.fail(msg) return proc @@ -637,6 +642,36 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) + def test_huntrleaks_fd_leak(self): + # test --huntrleaks for file descriptor leak + code = textwrap.dedent(""" + import os + import unittest + + class FDLeakTest(unittest.TestCase): + def test_leak(self): + fd = os.open(__file__, os.O_RDONLY) + # bug: never cloes the file descriptor + """) + test = self.create_test(code=code) + + filename = 'reflog.txt' + self.addCleanup(support.unlink, filename) + output = self.run_tests('--huntrleaks', '3:3:', test, + exitcode=1, + stderr=subprocess.STDOUT) + self.check_executed_tests(output, [test], failed=test) + + line = 'beginning 6 repetitions\n123456\n......\n' + self.check_line(output, re.escape(line)) + + line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test + self.check_line(output, re.escape(line2)) + + with open(filename) as fp: + reflog = fp.read() + self.assertEqual(reflog, line2) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 8aad689..5b708ea 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,9 @@ Documentation Tests ----- +- Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of + file descriptors. Patch written by Richard Oudkerk. + - Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the list of ignored directories. -- cgit v0.12 From 5f9d3acc5e8050a43c79cf4d2a373be67f3c27fc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 00:21:12 +0200 Subject: Issue #22806: Add ``python -m test --list-tests`` command to list tests. --- Lib/test/libregrtest/cmdline.py | 11 +++------ Lib/test/libregrtest/main.py | 55 +++++++++++++++++++++++++++-------------- Lib/test/test_regrtest.py | 7 ++++++ Misc/NEWS | 2 ++ 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index e55e53f..ae1aeb9 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,4 @@ import argparse -import faulthandler import os from test import support @@ -234,6 +233,9 @@ def _create_parser(): group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' 'error happens') + group.add_argument('--list-tests', action='store_true', + help="only write the name of tests that will be run, " + "don't execute them") parser.add_argument('args', nargs=argparse.REMAINDER, help=argparse.SUPPRESS) @@ -301,12 +303,7 @@ def _parse_args(args, **kwargs): if ns.quiet: ns.verbose = 0 if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") + if ns.timeout <= 0: ns.timeout = None if ns.use_mp is not None: if ns.use_mp <= 0: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c1ce3b1..30d8a59 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import faulthandler import os import platform import random @@ -110,8 +111,13 @@ class Regrtest: def parse_args(self, kwargs): ns = _parse_args(sys.argv[1:], **kwargs) + if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later", file=sys.stderr) + ns.timeout = None + if ns.threshold is not None and gc is None: - print('No GC available, ignore --threshold.') + print('No GC available, ignore --threshold.', file=sys.stderr) ns.threshold = None if ns.findleaks: @@ -122,7 +128,8 @@ class Regrtest: pass #gc.set_debug(gc.DEBUG_SAVEALL) else: - print('No GC available, disabling --findleaks') + print('No GC available, disabling --findleaks', + file=sys.stderr) ns.findleaks = False # Strip .py extensions. @@ -163,20 +170,6 @@ class Regrtest: nottests.add(arg) self.ns.args = [] - # For a partial run, we do not need to clutter the output. - if (self.ns.verbose - or self.ns.header - or not (self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - # if testdir is set, then we are not running the python tests suite, so # don't add default tests to be executed or skipped (pass empty values) if self.ns.testdir: @@ -199,15 +192,18 @@ class Regrtest: del self.selected[:self.selected.index(self.ns.start)] except ValueError: print("Couldn't find starting test (%s), using all tests" - % self.ns.start) + % self.ns.start, file=sys.stderr) if self.ns.randomize: if self.ns.random_seed is None: self.ns.random_seed = random.randrange(10000000) random.seed(self.ns.random_seed) - print("Using random seed", self.ns.random_seed) random.shuffle(self.selected) + def list_tests(self): + for name in self.selected: + print(name) + def rerun_failed_tests(self): self.ns.verbose = True self.ns.failfast = False @@ -315,6 +311,23 @@ class Regrtest: return def run_tests(self): + # For a partial run, we do not need to clutter the output. + if (self.ns.verbose + or self.ns.header + or not (self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + if self.ns.randomize: + print("Using random seed", self.ns.random_seed) + if self.ns.forever: self.tests = self._test_forever(list(self.selected)) self.test_count = '' @@ -359,8 +372,12 @@ class Regrtest: setup_tests(self.ns) self.find_tests(tests) - self.run_tests() + if self.ns.list_tests: + self.list_tests() + sys.exit(0) + + self.run_tests() self.display_result() if self.ns.verbose2 and self.bad: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 8b2449a..e15e724 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -672,6 +672,13 @@ class ArgsTestCase(BaseTestCase): reflog = fp.read() self.assertEqual(reflog, line2) + def test_list_tests(self): + # test --list-tests + tests = [self.create_test() for i in range(5)] + output = self.run_tests('--list-tests', *tests) + self.assertEqual(output.rstrip().splitlines(), + tests) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 5b708ea..6fb0be7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,8 @@ Documentation Tests ----- +- Issue #22806: Add ``python -m test --list-tests`` command to list tests. + - Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of file descriptors. Patch written by Richard Oudkerk. -- cgit v0.12 From eb36fdaad8bd38c60973eb69b1307315fa950372 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 01:55:51 +0200 Subject: Fix _PyUnicodeWriter_PrepareKind() Initialize kind to 0 (PyUnicode_WCHAR_KIND) to ensure that _PyUnicodeWriter_PrepareKind() handles correctly read-only buffer: copy the buffer. --- Objects/unicodeobject.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4fd0430..bc98287 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13294,27 +13294,38 @@ unicode_endswith(PyObject *self, Py_LOCAL_INLINE(void) _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { - if (!writer->readonly) + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); + writer->data = PyUnicode_DATA(writer->buffer); + + if (!writer->readonly) { + writer->kind = PyUnicode_KIND(writer->buffer); writer->size = PyUnicode_GET_LENGTH(writer->buffer); + } else { + /* use a value smaller than PyUnicode_1BYTE_KIND() so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + writer->kind = PyUnicode_WCHAR_KIND; + assert(writer->kind <= PyUnicode_1BYTE_KIND); + /* Copy-on-write mode: set buffer size to 0 so * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on * next write. */ writer->size = 0; } - writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); - writer->data = PyUnicode_DATA(writer->buffer); - writer->kind = PyUnicode_KIND(writer->buffer); } void _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) { memset(writer, 0, sizeof(*writer)); -#ifdef Py_DEBUG - writer->kind = 5; /* invalid kind */ -#endif + + /* ASCII is the bare minimum */ writer->min_char = 127; + + /* use a value smaller than PyUnicode_1BYTE_KIND() so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + writer->kind = PyUnicode_WCHAR_KIND; + assert(writer->kind <= PyUnicode_1BYTE_KIND); } int -- cgit v0.12 From 46d75353c5688e0273ca90efcbdb61687c085a80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 02:21:35 +0200 Subject: Issue #18174: Fix test_regrtest when Python is compiled in release mode --- Lib/test/test_regrtest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e15e724..e0307a8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -642,6 +642,7 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) + @unittest.skipUnless(Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" -- cgit v0.12 From fd265f4a1892f7be64ceccadb276e37b3fb61f61 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 2 Oct 2015 23:17:33 -0700 Subject: Factor out common iterator finalization code --- Modules/_collectionsmodule.c | 45 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 0c64713..9312636 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -350,21 +350,34 @@ deque_appendleft(dequeobject *deque, PyObject *item) PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque."); +static PyObject* +finalize_iterator(PyObject *it) +{ + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else { + Py_DECREF(it); + return NULL; + } + } + Py_DECREF(it); + Py_RETURN_NONE; +} /* Run an iterator to exhaustion. Shortcut for the extend/extendleft methods when maxlen == 0. */ static PyObject* consume_iterator(PyObject *it) { + PyObject *(*iternext)(PyObject *); PyObject *item; - while ((item = PyIter_Next(it)) != NULL) { + iternext = *Py_TYPE(it)->tp_iternext; + while ((item = iternext(it)) != NULL) { Py_DECREF(item); } - Py_DECREF(it); - if (PyErr_Occurred()) - return NULL; - Py_RETURN_NONE; + return finalize_iterator(it); } static PyObject * @@ -423,16 +436,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (trim) deque_trim_left(deque); } - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else { - Py_DECREF(it); - return NULL; - } - } - Py_DECREF(it); - Py_RETURN_NONE; + return finalize_iterator(it); } PyDoc_STRVAR(extend_doc, @@ -494,16 +498,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (trim) deque_trim_right(deque); } - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else { - Py_DECREF(it); - return NULL; - } - } - Py_DECREF(it); - Py_RETURN_NONE; + return finalize_iterator(it); } PyDoc_STRVAR(extendleft_doc, -- cgit v0.12 From 909a950ffc279d463d916ac03c10fb4168ccc007 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 3 Oct 2015 06:25:43 +0000 Subject: Issues #25232, #24657: Add NEWS to 3.6.0a1 section --- Misc/NEWS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 754041f..42d7a86 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -40,6 +40,12 @@ Core and Builtins Library ------- +- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the + first question mark (?) rather than the last. Patch from Xiang Zhang. + +- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the + query part of the URL as if it were a path. Patch from Xiang Zhang. + - Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not supported. Check if it is supported, it may not be supported on OpenBSD for example. -- cgit v0.12 From 33c30131db0249dec166fcbe22cd9152b405e170 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 21:20:41 +0200 Subject: Issue #25306: Skip test_huntrleaks_fd_leak() of test_regrtest until the bug is fixed. --- Lib/test/test_regrtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e0307a8..edbb4b0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -643,6 +643,8 @@ class ArgsTestCase(BaseTestCase): self.check_executed_tests(output, [test]*3, failed=test) @unittest.skipUnless(Py_DEBUG, 'need a debug build') + # Issue #25306: the test hangs sometimes on Windows + @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" -- cgit v0.12 From 20d2118248f9313a9b77bb4d299252da7fc7fa2f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 21:40:21 +0200 Subject: Issue #25306: Try to fix test_huntrleaks_fd_leak() on Windows Issue #25306: Disable popup and logs to stderr on assertion failures in MSCRT. --- Lib/test/test_regrtest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index edbb4b0..2e33555 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -643,14 +643,24 @@ class ArgsTestCase(BaseTestCase): self.check_executed_tests(output, [test]*3, failed=test) @unittest.skipUnless(Py_DEBUG, 'need a debug build') - # Issue #25306: the test hangs sometimes on Windows - @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" import os import unittest + # Issue #25306: Disable popups and logs to stderr on assertion + # failures in MSCRT + try: + import msvcrt + msvcrt.CrtSetReportMode + except (ImportError, AttributeError): + # no Windows, o release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, 0) + class FDLeakTest(unittest.TestCase): def test_leak(self): fd = os.open(__file__, os.O_RDONLY) -- cgit v0.12 From 1d65d9192dac57776693c55a9ccefbde2ca74c23 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2015 13:43:50 +0200 Subject: Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error handlers: ``ignore``, ``replace`` and ``surrogateescape``. --- Doc/whatsnew/3.6.rst | 3 +++ Lib/test/test_codecs.py | 12 ++++++++++++ Misc/NEWS | 3 +++ Objects/unicodeobject.c | 48 +++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index ca83ef9..24fd822 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -123,6 +123,9 @@ Optimizations * The UTF-8 encoder is now up to 75 times as fast for error handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. +* The UTF-8 decoder is now up to 15 times as fast for error handlers: + ``ignore``, ``replace`` and ``surrogateescape``. + Build and C API Changes ======================= diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index bdc331e..7b6883f 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -788,6 +788,18 @@ class UTF8Test(ReadTest, unittest.TestCase): self.check_state_handling_decode(self.encoding, u, u.encode(self.encoding)) + def test_decode_error(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode(self.encoding, error_handler), + expected) + def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for diff --git a/Misc/NEWS b/Misc/NEWS index d809377..3991d6b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +* Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error + handlers: ``ignore``, ``replace`` and ``surrogateescape``. + - Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data. - Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index bc98287..56614e6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4714,8 +4714,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; const char *errmsg = ""; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) { if (consumed) @@ -4740,6 +4741,7 @@ PyUnicode_DecodeUTF8Stateful(const char *s, while (s < end) { Py_UCS4 ch; int kind = writer.kind; + if (kind == PyUnicode_1BYTE_KIND) { if (PyUnicode_IS_ASCII(writer.buffer)) ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); @@ -4778,24 +4780,52 @@ PyUnicode_DecodeUTF8Stateful(const char *s, continue; } - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) { + case _Py_ERROR_IGNORE: + s += (endinpos - startinpos); + break; + + case _Py_ERROR_REPLACE: + if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) + goto onError; + s += (endinpos - startinpos); + break; + + case _Py_ERROR_SURROGATEESCAPE: + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + for (Py_ssize_t i=startinpos; i Date: Mon, 5 Oct 2015 13:49:26 +0200 Subject: Issue #25301: Fix compatibility with ISO C90 --- Objects/unicodeobject.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 56614e6..3d78404 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4795,9 +4795,12 @@ PyUnicode_DecodeUTF8Stateful(const char *s, break; case _Py_ERROR_SURROGATEESCAPE: + { + Py_ssize_t i; + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) goto onError; - for (Py_ssize_t i=startinpos; i Date: Mon, 5 Oct 2015 22:52:37 -0400 Subject: Eliminate unnecessary test --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 9312636..4ea9140 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -530,7 +530,7 @@ deque_copy(PyObject *deque) return NULL; new_deque->maxlen = old_deque->maxlen; /* Fast path for the deque_repeat() common case where len(deque) == 1 */ - if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { + if (Py_SIZE(deque) == 1) { PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; rv = deque_append(new_deque, item); } else { -- cgit v0.12 From 68713e41a5df7228e6971e894d8fc2fda720b5ce Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Tue, 6 Oct 2015 13:29:56 -0400 Subject: Closes issue #12006: Add ISO 8601 year, week, and day directives to strptime. This commit adds %G, %V, and %u directives to strptime. Thanks Ashley Anderson for the implementation. --- Doc/library/datetime.rst | 37 +++++++++++++++++++++- Doc/whatsnew/3.6.rst | 8 +++++ Lib/_strptime.py | 81 +++++++++++++++++++++++++++++++++++++++-------- Lib/test/test_strptime.py | 57 +++++++++++++++++++++++++-------- Misc/NEWS | 3 ++ 5 files changed, 159 insertions(+), 27 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 5c659e7..cf5d5b8 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1909,6 +1909,34 @@ format codes. | ``%%`` | A literal ``'%'`` character. | % | | +-----------+--------------------------------+------------------------+-------+ +Several additional directives not required by the C89 standard are included for +convenience. These parameters all correspond to ISO 8601 date values. These +may not be available on all platforms when used with the :meth:`strftime` +method. The ISO 8601 year and ISO 8601 week directives are not interchangeable +with the year and week number directives above. Calling :meth:`strptime` with +incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. + ++-----------+--------------------------------+------------------------+-------+ +| Directive | Meaning | Example | Notes | ++===========+================================+========================+=======+ +| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | +| | representing the year that | 2014, ..., 9998, 9999 | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | +| | number where 1 is Monday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | +| | number with Monday as | | | +| | the first day of the week. | | | +| | Week 01 is the week containing | | | +| | Jan 4. | | | ++-----------+--------------------------------+------------------------+-------+ + +.. versionadded:: 3.6 + ``%G``, ``%u`` and ``%V`` were added. + Notes: (1) @@ -1973,7 +2001,14 @@ Notes: (7) When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used - in calculations when the day of the week and the year are specified. + in calculations when the day of the week and the calendar year (``%Y``) + are specified. + +(8) + Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the + day of the week and the ISO year (``%G``) are specified in a + :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not + interchangable. .. rubric:: Footnotes diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 24fd822..dd35c9a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -110,6 +110,14 @@ Private and special attribute names now are omitted unless the prefix starts with underscores. A space or a colon can be added after completed keyword. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) +datetime +-------- + +* :meth:`datetime.stftime ` and + :meth:`date.stftime ` methods now support ISO 8601 + date directives ``%G``, ``%u`` and ``%V``. + (Contributed by Ashley Anderson in :issue:`12006`.) + Optimizations ============= diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 374923d..fe5b046 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -195,12 +195,15 @@ class TimeRE(dict): 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", 'M': r"(?P[0-5]\d|\d)", 'S': r"(?P6[0-1]|[0-5]\d|\d)", 'U': r"(?P5[0-3]|[0-4]\d|\d)", 'w': r"(?P[0-6])", + 'u': r"(?P[1-7])", + 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", #XXX: Does 'Y' need to worry about having less or more than @@ -295,6 +298,22 @@ def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): return 1 + days_to_week + day_of_week +def _calc_julian_from_V(iso_year, iso_week, iso_weekday): + """Calculate the Julian day based on the ISO 8601 year, week, and weekday. + ISO weeks start on Mondays, with week 01 being the week containing 4 Jan. + ISO week days range from 1 (Monday) to 7 (Sunday). + """ + correction = datetime_date(iso_year, 1, 4).isoweekday() + 3 + ordinal = (iso_week * 7) + iso_weekday - correction + # ordinal may be negative or 0 now, which means the date is in the previous + # calendar year + if ordinal < 1: + ordinal += datetime_date(iso_year, 1, 1).toordinal() + iso_year -= 1 + ordinal -= datetime_date(iso_year, 1, 1).toordinal() + return iso_year, ordinal + + def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a 2-tuple consisting of a time struct and an int containing the number of microseconds based on the input string and the @@ -339,15 +358,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - year = None + iso_year = year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 tzoffset = None # Default to -1 to signify that values not known; not critical to have, # though - week_of_year = -1 - week_of_year_start = -1 + iso_week = week_of_year = None + week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate # values weekday = julian = None @@ -369,6 +388,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) + elif group_key == 'G': + iso_year = int(found_dict['G']) elif group_key == 'm': month = int(found_dict['m']) elif group_key == 'B': @@ -414,6 +435,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): weekday = 6 else: weekday -= 1 + elif group_key == 'u': + weekday = int(found_dict['u']) + weekday -= 1 elif group_key == 'j': julian = int(found_dict['j']) elif group_key in ('U', 'W'): @@ -424,6 +448,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): else: # W starts week on Monday. week_of_year_start = 0 + elif group_key == 'V': + iso_week = int(found_dict['V']) elif group_key == 'z': z = found_dict['z'] tzoffset = int(z[1:3]) * 60 + int(z[3:5]) @@ -444,28 +470,57 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): else: tz = value break + # Deal with the cases where ambiguities arize + # don't assume default values for ISO week/year + if year is None and iso_year is not None: + if iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + if julian is not None: + raise ValueError("Day of the year directive '%j' is not " + "compatible with ISO year directive '%G'. " + "Use '%Y' instead.") + elif week_of_year is None and iso_week is not None: + if weekday is None: + raise ValueError("ISO week directive '%V' must be used with " + "the ISO year directive '%G' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + else: + raise ValueError("ISO week directive '%V' is incompatible with " + "the year directive '%Y'. Use the ISO year '%G' " + "instead.") + leap_year_fix = False if year is None and month == 2 and day == 29: year = 1904 # 1904 is first leap year of 20th century leap_year_fix = True elif year is None: year = 1900 + + # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. - if julian is None and week_of_year != -1 and weekday is not None: - week_starts_Mon = True if week_of_year_start == 0 else False - julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, - week_starts_Mon) - # Cannot pre-calculate datetime_date() since can change in Julian - # calculation and thus could have different value for the day of the week - # calculation. + if julian is None and weekday is not None: + if week_of_year is not None: + week_starts_Mon = True if week_of_year_start == 0 else False + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + week_starts_Mon) + elif iso_year is not None and iso_week is not None: + year, julian = _calc_julian_from_V(iso_year, iso_week, weekday + 1) + if julian is None: + # Cannot pre-calculate datetime_date() since can change in Julian + # calculation and thus could have different value for the day of + # the week calculation. # Need to add 1 to result since first day of the year is 1, not 0. julian = datetime_date(year, month, day).toordinal() - \ datetime_date(year, 1, 1).toordinal() + 1 - else: # Assume that if they bothered to include Julian day it will - # be accurate. - datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal()) + else: # Assume that if they bothered to include Julian day (or if it was + # calculated above with year/week/weekday) it will be accurate. + datetime_result = datetime_date.fromordinal( + (julian - 1) + + datetime_date(year, 1, 1).toordinal()) year = datetime_result.year month = datetime_result.month day = datetime_result.day diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 346e2c6..6b26c8a 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -152,8 +152,8 @@ class TimeRETests(unittest.TestCase): "'%s' using '%s'; group 'a' = '%s', group 'b' = %s'" % (found.string, found.re.pattern, found.group('a'), found.group('b'))) - for directive in ('a','A','b','B','c','d','H','I','j','m','M','p','S', - 'U','w','W','x','X','y','Y','Z','%'): + for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', + 'S','u','U','V','w','W','x','X','y','Y','Z','%'): compiled = self.time_re.compile("%" + directive) found = compiled.match(time.strftime("%" + directive)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % @@ -218,6 +218,26 @@ class StrptimeTests(unittest.TestCase): else: self.fail("'%s' did not raise ValueError" % bad_format) + # Ambiguous or incomplete cases using ISO year/week/weekday directives + # 1. ISO week (%V) is specified, but the year is specified with %Y + # instead of %G + with self.assertRaises(ValueError): + _strptime._strptime("1999 50", "%Y %V") + # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not + with self.assertRaises(ValueError): + _strptime._strptime("1999 51", "%G %V") + # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not + for w in ('A', 'a', 'w', 'u'): + with self.assertRaises(ValueError): + _strptime._strptime("1999 51","%G %{}".format(w)) + # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) + with self.assertRaises(ValueError): + _strptime._strptime("2015", "%G") + # 5. Julian/ordinal day (%j) is specified with %G, but not %Y + with self.assertRaises(ValueError): + _strptime._strptime("1999 256", "%G %j") + + def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) with self.assertRaises(ValueError) as e: @@ -289,7 +309,7 @@ class StrptimeTests(unittest.TestCase): def test_weekday(self): # Test weekday directives - for directive in ('A', 'a', 'w'): + for directive in ('A', 'a', 'w', 'u'): self.helper(directive,6) def test_julian(self): @@ -458,16 +478,20 @@ class CalculationTests(unittest.TestCase): # Should be able to infer date if given year, week of year (%U or %W) # and day of the week def test_helper(ymd_tuple, test_reason): - for directive in ('W', 'U'): - format_string = "%%Y %%%s %%w" % directive - dt_date = datetime_date(*ymd_tuple) - strp_input = dt_date.strftime(format_string) - strp_output = _strptime._strptime_time(strp_input, format_string) - self.assertTrue(strp_output[:3] == ymd_tuple, - "%s(%s) test failed w/ '%s': %s != %s (%s != %s)" % - (test_reason, directive, strp_input, - strp_output[:3], ymd_tuple, - strp_output[7], dt_date.timetuple()[7])) + for year_week_format in ('%Y %W', '%Y %U', '%G %V'): + for weekday_format in ('%w', '%u', '%a', '%A'): + format_string = year_week_format + ' ' + weekday_format + with self.subTest(test_reason, + date=ymd_tuple, + format=format_string): + dt_date = datetime_date(*ymd_tuple) + strp_input = dt_date.strftime(format_string) + strp_output = _strptime._strptime_time(strp_input, + format_string) + msg = "%r: %s != %s" % (strp_input, + strp_output[7], + dt_date.timetuple()[7]) + self.assertEqual(strp_output[:3], ymd_tuple, msg) test_helper((1901, 1, 3), "week 0") test_helper((1901, 1, 8), "common case") test_helper((1901, 1, 13), "day on Sunday") @@ -499,18 +523,25 @@ class CalculationTests(unittest.TestCase): self.assertEqual(_strptime._strptime_time(value, format)[:-1], expected) check('2015 0 0', '%Y %U %w', 2014, 12, 28, 0, 0, 0, 6, -3) check('2015 0 0', '%Y %W %w', 2015, 1, 4, 0, 0, 0, 6, 4) + check('2015 1 1', '%G %V %u', 2014, 12, 29, 0, 0, 0, 0, 363) check('2015 0 1', '%Y %U %w', 2014, 12, 29, 0, 0, 0, 0, -2) check('2015 0 1', '%Y %W %w', 2014, 12, 29, 0, 0, 0, 0, -2) + check('2015 1 2', '%G %V %u', 2014, 12, 30, 0, 0, 0, 1, 364) check('2015 0 2', '%Y %U %w', 2014, 12, 30, 0, 0, 0, 1, -1) check('2015 0 2', '%Y %W %w', 2014, 12, 30, 0, 0, 0, 1, -1) + check('2015 1 3', '%G %V %u', 2014, 12, 31, 0, 0, 0, 2, 365) check('2015 0 3', '%Y %U %w', 2014, 12, 31, 0, 0, 0, 2, 0) check('2015 0 3', '%Y %W %w', 2014, 12, 31, 0, 0, 0, 2, 0) + check('2015 1 4', '%G %V %u', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %U %w', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %W %w', 2015, 1, 1, 0, 0, 0, 3, 1) + check('2015 1 5', '%G %V %u', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %U %w', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %W %w', 2015, 1, 2, 0, 0, 0, 4, 2) + check('2015 1 6', '%G %V %u', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %U %w', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %W %w', 2015, 1, 3, 0, 0, 0, 5, 3) + check('2015 1 7', '%G %V %u', 2015, 1, 4, 0, 0, 0, 6, 4) class CacheTests(unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS index dec38ba..074577f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -383,6 +383,9 @@ Library - Issue #23572: Fixed functools.singledispatch on classes with falsy metaclasses. Patch by Ethan Furman. +- Issue #12006: Add ISO 8601 year, week, and day directives (%G, %V, %u) to + strptime. + Documentation ------------- -- cgit v0.12 From 3c5f0030238125a5a2e87d713e6d6c4a22b945ae Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 6 Oct 2015 21:17:02 -0700 Subject: make configure executable --- configure | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 configure diff --git a/configure b/configure old mode 100644 new mode 100755 -- cgit v0.12 From 6f038ada5bd1bc2838e20f0b22de1780e7ce709d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 7 Oct 2015 07:54:23 +0300 Subject: Add a versionadded directive for reopenIfNeeded() --- Doc/library/logging.handlers.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 0d3928c..c830efd 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -168,6 +168,8 @@ for this value. flushed and closed and the file opened again, typically as a precursor to outputting the record to the file. + .. versionadded:: 3.6 + .. method:: emit(record) -- cgit v0.12 From c2432f6edbec7a8cc1021d2f4fe625ba92dc8ac4 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 7 Oct 2015 11:15:15 +0000 Subject: One more typo in a comment --- Python/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index 88fcd37..5a7a745 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4035,7 +4035,7 @@ fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, assert(expr_end >= 0 && expr_end < PyUnicode_GET_LENGTH(str)); assert(expr_end >= expr_start); - /* There has to be at least on character on each side of the + /* There has to be at least one character on each side of the expression inside this str. This will have been caught before we're called. */ assert(expr_start >= 1); -- cgit v0.12 From 960e848f0d32399824d61dce2ff5736bb596ed44 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 8 Oct 2015 12:27:06 +0300 Subject: Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate extensions. Patch by Nikolay Bogoychev. --- Doc/library/urllib.robotparser.rst | 30 ++++++++++++- Doc/whatsnew/3.6.rst | 8 ++++ Lib/test/test_robotparser.py | 90 +++++++++++++++++++++++++++++--------- Lib/urllib/robotparser.py | 39 ++++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 5 ++- 6 files changed, 147 insertions(+), 26 deletions(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index f179de2..c2e1bef 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -53,15 +53,41 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. Sets the time the ``robots.txt`` file was last fetched to the current time. + .. method:: crawl_delay(useragent) -The following example demonstrates basic use of the RobotFileParser class. + Returns the value of the ``Crawl-delay`` parameter from ``robots.txt`` + for the *useragent* in question. If there is no such parameter or it + doesn't apply to the *useragent* specified or the ``robots.txt`` entry + for this parameter has invalid syntax, return ``None``. + + .. versionadded:: 3.6 + + .. method:: request_rate(useragent) + + Returns the contents of the ``Request-rate`` parameter from + ``robots.txt`` in the form of a :func:`~collections.namedtuple` + ``(requests, seconds)``. If there is no such parameter or it doesn't + apply to the *useragent* specified or the ``robots.txt`` entry for this + parameter has invalid syntax, return ``None``. + + .. versionadded:: 3.6 + + +The following example demonstrates basic use of the :class:`RobotFileParser` +class:: >>> import urllib.robotparser >>> rp = urllib.robotparser.RobotFileParser() >>> rp.set_url("http://www.musi-cal.com/robots.txt") >>> rp.read() + >>> rrate = rp.request_rate("*") + >>> rrate.requests + 3 + >>> rrate.seconds + 20 + >>> rp.crawl_delay("*") + 6 >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") False >>> rp.can_fetch("*", "http://www.musi-cal.com/") True - diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index dd35c9a..3080820 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -119,6 +119,14 @@ datetime (Contributed by Ashley Anderson in :issue:`12006`.) +urllib.robotparser +------------------ + +:class:`~urllib.robotparser.RobotFileParser` now supports ``Crawl-delay`` and +``Request-rate`` extensions. +(Contributed by Nikolay Bogoychev in :issue:`16099`.) + + Optimizations ============= diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index d01266f..90b3072 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -1,6 +1,7 @@ import io import unittest import urllib.robotparser +from collections import namedtuple from urllib.error import URLError, HTTPError from urllib.request import urlopen from test import support @@ -12,7 +13,8 @@ except ImportError: class RobotTestCase(unittest.TestCase): - def __init__(self, index=None, parser=None, url=None, good=None, agent=None): + def __init__(self, index=None, parser=None, url=None, good=None, + agent=None, request_rate=None, crawl_delay=None): # workaround to make unittest discovery work (see #17066) if not isinstance(index, int): return @@ -25,6 +27,8 @@ class RobotTestCase(unittest.TestCase): self.url = url self.good = good self.agent = agent + self.request_rate = request_rate + self.crawl_delay = crawl_delay def runTest(self): if isinstance(self.url, tuple): @@ -34,6 +38,18 @@ class RobotTestCase(unittest.TestCase): agent = self.agent if self.good: self.assertTrue(self.parser.can_fetch(agent, url)) + self.assertEqual(self.parser.crawl_delay(agent), self.crawl_delay) + # if we have actual values for request rate + if self.request_rate and self.parser.request_rate(agent): + self.assertEqual( + self.parser.request_rate(agent).requests, + self.request_rate.requests + ) + self.assertEqual( + self.parser.request_rate(agent).seconds, + self.request_rate.seconds + ) + self.assertEqual(self.parser.request_rate(agent), self.request_rate) else: self.assertFalse(self.parser.can_fetch(agent, url)) @@ -43,15 +59,17 @@ class RobotTestCase(unittest.TestCase): tests = unittest.TestSuite() def RobotTest(index, robots_txt, good_urls, bad_urls, - agent="test_robotparser"): + request_rate, crawl_delay, agent="test_robotparser"): lines = io.StringIO(robots_txt).readlines() parser = urllib.robotparser.RobotFileParser() parser.parse(lines) for url in good_urls: - tests.addTest(RobotTestCase(index, parser, url, 1, agent)) + tests.addTest(RobotTestCase(index, parser, url, 1, agent, + request_rate, crawl_delay)) for url in bad_urls: - tests.addTest(RobotTestCase(index, parser, url, 0, agent)) + tests.addTest(RobotTestCase(index, parser, url, 0, agent, + request_rate, crawl_delay)) # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002) @@ -65,14 +83,18 @@ Disallow: /foo.html good = ['/','/test.html'] bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html'] +request_rate = None +crawl_delay = None -RobotTest(1, doc, good, bad) +RobotTest(1, doc, good, bad, request_rate, crawl_delay) # 2. doc = """ # robots.txt for http://www.example.com/ User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. @@ -83,8 +105,10 @@ Disallow: good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')] bad = ['/cyberworld/map/index.html'] +request_rate = None # The parameters should be equal to None since they +crawl_delay = None # don't apply to the cybermapper user agent -RobotTest(2, doc, good, bad) +RobotTest(2, doc, good, bad, request_rate, crawl_delay) # 3. doc = """ @@ -95,14 +119,18 @@ Disallow: / good = [] bad = ['/cyberworld/map/index.html','/','/tmp/'] +request_rate = None +crawl_delay = None -RobotTest(3, doc, good, bad) +RobotTest(3, doc, good, bad, request_rate, crawl_delay) # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002) # 4. doc = """ User-agent: figtree +Crawl-delay: 3 +Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html @@ -115,8 +143,17 @@ bad = ['/tmp','/tmp.html','/tmp/a.html', '/~joe/index.html' ] -RobotTest(4, doc, good, bad, 'figtree') -RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04') +request_rate = namedtuple('req_rate', 'requests seconds') +request_rate.requests = 9 +request_rate.seconds = 30 +crawl_delay = 3 +request_rate_bad = None # not actually tested, but we still need to parse it +crawl_delay_bad = None # in order to accommodate the input parameters + + +RobotTest(4, doc, good, bad, request_rate, crawl_delay, 'figtree' ) +RobotTest(5, doc, good, bad, request_rate_bad, crawl_delay_bad, + 'FigTree Robot libwww-perl/5.04') # 6. doc = """ @@ -125,14 +162,18 @@ Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html +Crawl-delay: 3 +Request-rate: 9/banana """ good = ['/tmp',] # XFAIL: '/a%2fb.html' bad = ['/tmp/','/tmp/a.html', '/a%3cd.html','/a%3Cd.html',"/a/b.html", '/%7Ejoe/index.html'] +crawl_delay = 3 +request_rate = None # since request rate has invalid syntax, return None -RobotTest(6, doc, good, bad) +RobotTest(6, doc, good, bad, None, None) # From bug report #523041 @@ -140,12 +181,16 @@ RobotTest(6, doc, good, bad) doc = """ User-Agent: * Disallow: /. +Crawl-delay: pears """ good = ['/foo.html'] -bad = [] # Bug report says "/" should be denied, but that is not in the RFC +bad = [] # bug report says "/" should be denied, but that is not in the RFC + +crawl_delay = None # since crawl delay has invalid syntax, return None +request_rate = None -RobotTest(7, doc, good, bad) +RobotTest(7, doc, good, bad, crawl_delay, request_rate) # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364 @@ -154,12 +199,15 @@ doc = """ User-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ +Request-rate: whale/banana """ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] +crawl_delay = None +request_rate = None # invalid syntax, return none -RobotTest(8, doc, good, bad, agent="Googlebot") +RobotTest(8, doc, good, bad, crawl_delay, request_rate, agent="Googlebot") # 9. This file is incorrect because "Googlebot" is a substring of # "Googlebot-Mobile", so test 10 works just like test 9. @@ -174,12 +222,12 @@ Allow: / good = [] bad = ['/something.jpg'] -RobotTest(9, doc, good, bad, agent="Googlebot") +RobotTest(9, doc, good, bad, None, None, agent="Googlebot") good = [] bad = ['/something.jpg'] -RobotTest(10, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(10, doc, good, bad, None, None, agent="Googlebot-Mobile") # 11. Get the order correct. doc = """ @@ -193,12 +241,12 @@ Disallow: / good = [] bad = ['/something.jpg'] -RobotTest(11, doc, good, bad, agent="Googlebot") +RobotTest(11, doc, good, bad, None, None, agent="Googlebot") good = ['/something.jpg'] bad = [] -RobotTest(12, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(12, doc, good, bad, None, None, agent="Googlebot-Mobile") # 13. Google also got the order wrong in #8. You need to specify the @@ -212,7 +260,7 @@ Disallow: /folder1/ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] -RobotTest(13, doc, good, bad, agent="googlebot") +RobotTest(13, doc, good, bad, None, None, agent="googlebot") # 14. For issue #6325 (query string support) @@ -224,7 +272,7 @@ Disallow: /some/path?name=value good = ['/some/path'] bad = ['/some/path?name=value'] -RobotTest(14, doc, good, bad) +RobotTest(14, doc, good, bad, None, None) # 15. For issue #4108 (obey first * entry) doc = """ @@ -238,7 +286,7 @@ Disallow: /another/path good = ['/another/path'] bad = ['/some/path'] -RobotTest(15, doc, good, bad) +RobotTest(15, doc, good, bad, None, None) # 16. Empty query (issue #17403). Normalizing the url first. doc = """ @@ -250,7 +298,7 @@ Disallow: /another/path? good = ['/some/path?'] bad = ['/another/path?'] -RobotTest(16, doc, good, bad) +RobotTest(16, doc, good, bad, None, None) class RobotHandler(BaseHTTPRequestHandler): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 4fbb0cb..4ac553a 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -10,7 +10,9 @@ http://www.robotstxt.org/norobots-rfc.txt """ -import urllib.parse, urllib.request +import collections +import urllib.parse +import urllib.request __all__ = ["RobotFileParser"] @@ -120,10 +122,29 @@ class RobotFileParser: if state != 0: entry.rulelines.append(RuleLine(line[1], True)) state = 2 + elif line[0] == "crawl-delay": + if state != 0: + # before trying to convert to int we need to make + # sure that robots.txt has valid syntax otherwise + # it will crash + if line[1].strip().isdigit(): + entry.delay = int(line[1]) + state = 2 + elif line[0] == "request-rate": + if state != 0: + numbers = line[1].split('/') + # check if all values are sane + if (len(numbers) == 2 and numbers[0].strip().isdigit() + and numbers[1].strip().isdigit()): + req_rate = collections.namedtuple('req_rate', + 'requests seconds') + entry.req_rate = req_rate + entry.req_rate.requests = int(numbers[0]) + entry.req_rate.seconds = int(numbers[1]) + state = 2 if state == 2: self._add_entry(entry) - def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: @@ -153,6 +174,18 @@ class RobotFileParser: # agent not found ==> access granted return True + def crawl_delay(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.delay + return None + + def request_rate(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.req_rate + return None + def __str__(self): return ''.join([str(entry) + "\n" for entry in self.entries]) @@ -180,6 +213,8 @@ class Entry: def __init__(self): self.useragents = [] self.rulelines = [] + self.delay = None + self.req_rate = None def __str__(self): ret = [] diff --git a/Misc/ACKS b/Misc/ACKS index 9678bc3..221d6e2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -151,6 +151,7 @@ Finn Bock Paul Boddie Matthew Boedicker Robin Boerdijk +Nikolay Bogoychev David Bolen Wouter Bolsterlee Gawain Bolton diff --git a/Misc/NEWS b/Misc/NEWS index daa40fc..b543088 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,4 +1,4 @@ -+++++++++++ ++++++++++++ Python News +++++++++++ @@ -46,6 +46,9 @@ Core and Builtins Library ------- +- Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate + extensions. Patch by Nikolay Bogoychev. + - Issue #25316: distutils raises OSError instead of DistutilsPlatformError when MSVC is not installed. -- cgit v0.12 From b6c9572fa9e88029beda2d9cc6b387020f0a7d69 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 8 Oct 2015 13:58:49 +0300 Subject: Sort module names in whatsnew/3.6.rst --- Doc/whatsnew/3.6.rst | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3080820..50d2a6a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -95,12 +95,21 @@ New Modules Improved Modules ================ +datetime +-------- + +:meth:`datetime.stftime ` and +:meth:`date.stftime ` methods now support ISO 8601 date +directives ``%G``, ``%u`` and ``%V``. +(Contributed by Ashley Anderson in :issue:`12006`.) + + operator -------- -* New object :data:`operator.subscript` makes it easier to create complex - indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` - (Contributed by Joe Jevnik in :issue:`24379`.) +New object :data:`operator.subscript` makes it easier to create complex +indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` +(Contributed by Joe Jevnik in :issue:`24379`.) rlcomplete @@ -110,14 +119,6 @@ Private and special attribute names now are omitted unless the prefix starts with underscores. A space or a colon can be added after completed keyword. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) -datetime --------- - -* :meth:`datetime.stftime ` and - :meth:`date.stftime ` methods now support ISO 8601 - date directives ``%G``, ``%u`` and ``%V``. - (Contributed by Ashley Anderson in :issue:`12006`.) - urllib.robotparser ------------------ -- cgit v0.12 From 12c2945ccf8624e3978c5d2069dd18194f88becf Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 8 Oct 2015 09:05:36 -0700 Subject: Issue #23919: Prevents assert dialogs appearing in the test suite. --- Lib/test/libregrtest/cmdline.py | 4 ++++ Lib/test/libregrtest/setup.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 6b18b3a..21c3e66 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -304,6 +304,10 @@ def _parse_args(args, **kwargs): if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): parser.error("--pgo/-v don't go together!") + if ns.nowindows: + print("Warning: the --nowindows (-n) option is deprecated. " + "Use -vv to display assertions in stderr.", file=sys.stderr) + if ns.quiet: ns.verbose = 0 if ns.timeout is not None: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 6a1c308..6e05c7e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -75,8 +75,11 @@ def setup_tests(ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) - if ns.nowindows: + try: import msvcrt + except ImportError: + pass + else: msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| @@ -88,8 +91,11 @@ def setup_tests(ns): pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + if ns.verbose and ns.verbose >= 2: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + else: + msvcrt.CrtSetReportMode(m, 0) support.use_resources = ns.use_resources -- cgit v0.12 From 08ec6d9611b612182808bdf1d30cdd786cde9c3e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 8 Oct 2015 11:34:07 -0700 Subject: Fix missing import in libregrtest. --- Lib/test/libregrtest/cmdline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 21c3e66..a9cee6e 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,6 @@ import argparse import os +import sys from test import support -- cgit v0.12 From fdfbf781140f22619b0ef6bfeac792496774bb69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 00:33:49 +0200 Subject: Issue #25318: Add _PyBytesWriter API Add a new private API to optimize Unicode encoders. It uses a small buffer allocated on the stack and supports overallocation. Use _PyBytesWriter API for UCS1 (ASCII and Latin1) and UTF-8 encoders. Enable overallocation for the UTF-8 encoder with error handlers. unicode_encode_ucs1(): initialize collend to collstart+1 to not check the current character twice, we already know that it is not ASCII. --- Include/unicodeobject.h | 2 +- Objects/stringlib/codecs.h | 84 +++--------- Objects/unicodeobject.c | 316 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 269 insertions(+), 133 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index d0e0142..adcb64c 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -908,7 +908,7 @@ typedef struct { /* minimum character (default: 127, ASCII) */ Py_UCS4 min_char; - /* If non-zero, overallocate the buffer by 25% (default: 0). */ + /* If non-zero, overallocate the buffer (default: 0). */ unsigned char overallocate; /* If readonly is 1, buffer is a shared string (cannot be modified) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 562191c..d7a9918 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -263,10 +263,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */ Py_ssize_t i; /* index into s of next input byte */ - PyObject *result; /* result string object */ char *p; /* next free byte in output buffer */ - Py_ssize_t nallocated; /* number of result bytes allocated */ - Py_ssize_t nneeded; /* number of result bytes needed */ #if STRINGLIB_SIZEOF_CHAR > 1 PyObject *error_handler_obj = NULL; PyObject *exc = NULL; @@ -275,39 +272,25 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #endif #if STRINGLIB_SIZEOF_CHAR == 1 const Py_ssize_t max_char_size = 2; - char stackbuf[MAX_SHORT_UNICHARS * 2]; #elif STRINGLIB_SIZEOF_CHAR == 2 const Py_ssize_t max_char_size = 3; - char stackbuf[MAX_SHORT_UNICHARS * 3]; #else /* STRINGLIB_SIZEOF_CHAR == 4 */ const Py_ssize_t max_char_size = 4; - char stackbuf[MAX_SHORT_UNICHARS * 4]; #endif + _PyBytesWriter writer; assert(size >= 0); + _PyBytesWriter_Init(&writer); - if (size <= MAX_SHORT_UNICHARS) { - /* Write into the stack buffer; nallocated can't overflow. - * At the end, we'll allocate exactly as much heap space as it - * turns out we need. - */ - nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int); - result = NULL; /* will allocate after we're done */ - p = stackbuf; - } - else { - if (size > PY_SSIZE_T_MAX / max_char_size) { - /* integer overflow */ - return PyErr_NoMemory(); - } - /* Overallocate on the heap, and give the excess back at the end. */ - nallocated = size * max_char_size; - result = PyBytes_FromStringAndSize(NULL, nallocated); - if (result == NULL) - return NULL; - p = PyBytes_AS_STRING(result); + if (size > PY_SSIZE_T_MAX / max_char_size) { + /* integer overflow */ + return PyErr_NoMemory(); } + p = _PyBytesWriter_Alloc(&writer, size * max_char_size); + if (p == NULL) + return NULL; + for (i = 0; i < size;) { Py_UCS4 ch = data[i++]; @@ -338,6 +321,9 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos])) endpos++; + /* Only overallocate the buffer if it's not the last write */ + writer.overallocate = (endpos < size); + switch (error_handler) { case _Py_ERROR_REPLACE: @@ -387,29 +373,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, repsize = PyUnicode_GET_LENGTH(rep); if (repsize > max_char_size) { - Py_ssize_t offset; - - if (result == NULL) - offset = p - stackbuf; - else - offset = p - PyBytes_AS_STRING(result); - - if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { - /* integer overflow */ - PyErr_NoMemory(); + p = _PyBytesWriter_Prepare(&writer, p, + repsize - max_char_size); + if (p == NULL) goto error; - } - nallocated += repsize - max_char_size; - if (result != NULL) { - if (_PyBytes_Resize(&result, nallocated) < 0) - goto error; - } else { - result = PyBytes_FromStringAndSize(NULL, nallocated); - if (result == NULL) - goto error; - Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); - } - p = PyBytes_AS_STRING(result) + offset; } if (PyBytes_Check(rep)) { @@ -437,6 +404,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, i = newpos; } + + /* If overallocation was disabled, ensure that it was the last + write. Otherwise, we missed an optimization */ + assert(writer.overallocate || i == size); } else #if STRINGLIB_SIZEOF_CHAR > 2 @@ -461,31 +432,18 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #endif /* STRINGLIB_SIZEOF_CHAR > 1 */ } - if (result == NULL) { - /* This was stack allocated. */ - nneeded = p - stackbuf; - assert(nneeded <= nallocated); - result = PyBytes_FromStringAndSize(stackbuf, nneeded); - } - else { - /* Cut back to size actually needed. */ - nneeded = p - PyBytes_AS_STRING(result); - assert(nneeded <= nallocated); - _PyBytes_Resize(&result, nneeded); - } - #if STRINGLIB_SIZEOF_CHAR > 1 Py_XDECREF(error_handler_obj); Py_XDECREF(exc); #endif - return result; + return _PyBytesWriter_Finish(&writer, p); #if STRINGLIB_SIZEOF_CHAR > 1 error: Py_XDECREF(rep); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - Py_XDECREF(result); + _PyBytesWriter_Dealloc(&writer); return NULL; #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 3d78404..010a610 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -163,6 +163,14 @@ extern "C" { *_to++ = (to_type) *_iter++; \ } while (0) +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ +# define OVERALLOCATE_FACTOR 2 +#else + /* On Linux, overallocate by 25% is the best factor */ +# define OVERALLOCATE_FACTOR 4 +#endif + /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -338,6 +346,216 @@ PyUnicode_GetMax(void) #endif } +/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer". + A _PyBytesWriter variable must be declared at the end of variables in a + function to optimize the memory allocation on the stack. */ +typedef struct { + /* bytes object */ + PyObject *buffer; + + /* Number of allocated size */ + Py_ssize_t allocated; + + /* Current size of the buffer (can be smaller than the allocated size) */ + Py_ssize_t size; + + /* If non-zero, overallocate the buffer (default: 0). */ + int overallocate; + + /* Stack buffer */ + int use_stack_buffer; + char stack_buffer[512]; +} _PyBytesWriter; + +static void +_PyBytesWriter_Init(_PyBytesWriter *writer) +{ + writer->buffer = NULL; + writer->allocated = 0; + writer->size = 0; + writer->overallocate = 0; + writer->use_stack_buffer = 0; +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); +#endif +} + +static void +_PyBytesWriter_Dealloc(_PyBytesWriter *writer) +{ + Py_CLEAR(writer->buffer); +} + +static char* +_PyBytesWriter_AsString(_PyBytesWriter *writer) +{ + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + return PyBytes_AS_STRING(writer->buffer); + } + else { + assert(writer->buffer == NULL); + return writer->stack_buffer; + } +} + +Py_LOCAL_INLINE(Py_ssize_t) +_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) +{ + char *start = _PyBytesWriter_AsString(writer); + assert(str != NULL); + assert(str >= start); + return str - start; +} + +Py_LOCAL_INLINE(void) +_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) +{ +#ifdef Py_DEBUG + char *start, *end; + + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + assert(PyBytes_CheckExact(writer->buffer)); + assert(Py_REFCNT(writer->buffer) == 1); + } + else { + assert(writer->buffer == NULL); + } + + start = _PyBytesWriter_AsString(writer); + assert(0 <= writer->size && writer->size <= writer->allocated); + /* the last byte must always be null */ + assert(start[writer->allocated] == 0); + + end = start + writer->allocated; + assert(str != NULL); + assert(start <= str && str <= end); +#endif +} + +/* Add *size* bytes to the buffer. + str is the current pointer inside the buffer. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. */ +static char* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) +{ + Py_ssize_t allocated, pos; + + _PyBytesWriter_CheckConsistency(writer, str); + assert(size >= 0); + + if (size == 0) { + /* nothing to do */ + return str; + } + + if (writer->size > PY_SSIZE_T_MAX - size) { + PyErr_NoMemory(); + _PyBytesWriter_Dealloc(writer); + return NULL; + } + writer->size += size; + + allocated = writer->allocated; + if (writer->size <= allocated) + return str; + + allocated = writer->size; + if (writer->overallocate + && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { + /* overallocate to limit the number of realloc() */ + allocated += allocated / OVERALLOCATE_FACTOR; + } + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + /* Note: Don't use a bytearray object because the conversion from + byterray to bytes requires to copy all bytes. */ + if (_PyBytes_Resize(&writer->buffer, allocated)) { + assert(writer->buffer == NULL); + return NULL; + } + } + else { + /* convert from stack buffer to bytes object buffer */ + assert(writer->buffer == NULL); + + writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); + if (writer->buffer == NULL) + return NULL; + + if (pos != 0) { + Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), + writer->stack_buffer, + pos); + } + +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); +#endif + + writer->use_stack_buffer = 0; + } + writer->allocated = allocated; + + str = _PyBytesWriter_AsString(writer) + pos; + _PyBytesWriter_CheckConsistency(writer, str); + return str; +} + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +static char* +_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) +{ + /* ensure that _PyBytesWriter_Alloc() is only called once */ + assert(writer->size == 0 && writer->buffer == NULL); + assert(size >= 0); + + writer->use_stack_buffer = 1; +#if Py_DEBUG + /* the last byte is reserved, it must be '\0' */ + writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; + writer->allocated = sizeof(writer->stack_buffer) - 1; +#else + writer->allocated = sizeof(writer->stack_buffer); +#endif + return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); +} + +/* Get the buffer content and reset the writer. + Return a bytes object. + Raise an exception and return NULL on error. */ +static PyObject * +_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) +{ + Py_ssize_t pos; + PyObject *result; + + _PyBytesWriter_CheckConsistency(writer, str); + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + if (pos != writer->allocated) { + if (_PyBytes_Resize(&writer->buffer, pos)) { + assert(writer->buffer == NULL); + return NULL; + } + } + + result = writer->buffer; + writer->buffer = NULL; + } + else { + result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); + } + + return result; +} + #ifdef Py_DEBUG int _PyUnicode_CheckConsistency(PyObject *op, int check_content) @@ -6460,17 +6678,15 @@ unicode_encode_ucs1(PyObject *unicode, Py_ssize_t pos=0, size; int kind; void *data; - /* output object */ - PyObject *res; /* pointer into the output */ char *str; - /* current output position */ - Py_ssize_t ressize; const char *encoding = (limit == 256) ? "latin-1" : "ascii"; const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)"; PyObject *error_handler_obj = NULL; PyObject *exc = NULL; _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; + /* output object */ + _PyBytesWriter writer; if (PyUnicode_READY(unicode) == -1) return NULL; @@ -6481,11 +6697,11 @@ unicode_encode_ucs1(PyObject *unicode, replacements, if we need more, we'll resize */ if (size == 0) return PyBytes_FromStringAndSize(NULL, 0); - res = PyBytes_FromStringAndSize(NULL, size); - if (res == NULL) + + _PyBytesWriter_Init(&writer); + str = _PyBytesWriter_Alloc(&writer, size); + if (str == NULL) return NULL; - str = PyBytes_AS_STRING(res); - ressize = size; while (pos < size) { Py_UCS4 ch = PyUnicode_READ(kind, data, pos); @@ -6499,15 +6715,18 @@ unicode_encode_ucs1(PyObject *unicode, else { Py_ssize_t requiredsize; PyObject *repunicode; - Py_ssize_t repsize, newpos, respos, i; + Py_ssize_t repsize, newpos, i; /* startpos for collecting unencodable chars */ Py_ssize_t collstart = pos; - Py_ssize_t collend = pos; + Py_ssize_t collend = collstart + 1; /* find all unecodable characters */ while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit)) ++collend; + /* Only overallocate the buffer if it's not the last write */ + writer.overallocate = (collend < size); + /* cache callback name lookup (if not done yet, i.e. it's the first error) */ if (error_handler == _Py_ERROR_UNKNOWN) error_handler = get_error_handler(errors); @@ -6526,8 +6745,7 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - respos = str - PyBytes_AS_STRING(res); - requiredsize = respos; + requiredsize = 0; /* determine replacement size */ for (i = collstart; i < collend; ++i) { Py_ssize_t incr; @@ -6553,17 +6771,11 @@ unicode_encode_ucs1(PyObject *unicode, goto overflow; requiredsize += incr; } - if (requiredsize > PY_SSIZE_T_MAX - (size - collend)) - goto overflow; - requiredsize += size - collend; - if (requiredsize > ressize) { - if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) - requiredsize = 2*ressize; - if (_PyBytes_Resize(&res, requiredsize)) - goto onError; - str = PyBytes_AS_STRING(res) + respos; - ressize = requiredsize; - } + + str = _PyBytesWriter_Prepare(&writer, str, requiredsize-1); + if (str == NULL) + goto onError; + /* generate replacement */ for (i = collstart; i < collend; ++i) { str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); @@ -6598,20 +6810,9 @@ unicode_encode_ucs1(PyObject *unicode, if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ repsize = PyBytes_Size(repunicode); - if (repsize > 1) { - /* Make room for all additional bytes. */ - respos = str - PyBytes_AS_STRING(res); - if (ressize > PY_SSIZE_T_MAX - repsize - 1) { - Py_DECREF(repunicode); - goto overflow; - } - if (_PyBytes_Resize(&res, ressize+repsize-1)) { - Py_DECREF(repunicode); - goto onError; - } - str = PyBytes_AS_STRING(res) + respos; - ressize += repsize-1; - } + str = _PyBytesWriter_Prepare(&writer, str, repsize-1); + if (str == NULL) + goto onError; memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; pos = newpos; @@ -6622,24 +6823,11 @@ unicode_encode_ucs1(PyObject *unicode, /* need more space? (at least enough for what we have+the replacement+the rest of the string, so we won't have to check space for encodable characters) */ - respos = str - PyBytes_AS_STRING(res); repsize = PyUnicode_GET_LENGTH(repunicode); - requiredsize = respos; - if (requiredsize > PY_SSIZE_T_MAX - repsize) - goto overflow; - requiredsize += repsize; - if (requiredsize > PY_SSIZE_T_MAX - (size - collend)) - goto overflow; - requiredsize += size - collend; - if (requiredsize > ressize) { - if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) - requiredsize = 2*ressize; - if (_PyBytes_Resize(&res, requiredsize)) { - Py_DECREF(repunicode); + if (repsize > 1) { + str = _PyBytesWriter_Prepare(&writer, str, repsize-1); + if (str == NULL) goto onError; - } - str = PyBytes_AS_STRING(res) + respos; - ressize = requiredsize; } /* check if there is anything unencodable in the replacement @@ -6657,26 +6845,23 @@ unicode_encode_ucs1(PyObject *unicode, pos = newpos; Py_DECREF(repunicode); } + + /* If overallocation was disabled, ensure that it was the last + write. Otherwise, we missed an optimization */ + assert(writer.overallocate || pos == size); } } - /* Resize if we allocated to much */ - size = str - PyBytes_AS_STRING(res); - if (size < ressize) { /* If this falls res will be NULL */ - assert(size >= 0); - if (_PyBytes_Resize(&res, size) < 0) - goto onError; - } Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return res; + return _PyBytesWriter_Finish(&writer, str); overflow: PyErr_SetString(PyExc_OverflowError, "encoded result is too long for a Python string"); onError: - Py_XDECREF(res); + _PyBytesWriter_Dealloc(&writer); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; @@ -13366,13 +13551,6 @@ int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) { -#ifdef MS_WINDOWS - /* On Windows, overallocate by 50% is the best factor */ -# define OVERALLOCATE_FACTOR 2 -#else - /* On Linux, overallocate by 25% is the best factor */ -# define OVERALLOCATE_FACTOR 4 -#endif Py_ssize_t newlen; PyObject *newbuffer; -- cgit v0.12 From e7bf86cd7d7c9a3924501875a08c4ef4a0063103 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 01:39:28 +0200 Subject: Optimize backslashreplace error handler Issue #25318: Optimize backslashreplace and xmlcharrefreplace error handlers in UTF-8 encoder. Optimize also backslashreplace error handler for ASCII and Latin1 encoders. Use the new _PyBytesWriter API to optimize these error handlers for the encoders. It avoids to create an exception and call the slow implementation of the error handler. --- Objects/stringlib/codecs.h | 18 ++++- Objects/unicodeobject.c | 193 +++++++++++++++++++++++++++++++++------------ 2 files changed, 160 insertions(+), 51 deletions(-) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index d7a9918..ae99d1a 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -334,7 +334,6 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, i += (endpos - startpos - 1); break; - case _Py_ERROR_SURROGATEPASS: for (k=startpos; k PY_SSIZE_T_MAX - incr) { + PyErr_SetString(PyExc_OverflowError, + "encoded result is too long for a Python string"); + return NULL; + } + size += incr; + } + + prealloc = prealloc_per_char * (collend - collstart); + if (size > prealloc) { + str = _PyBytesWriter_Prepare(writer, str, size - prealloc); + if (str == NULL) + return NULL; + } + + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + ch = PyUnicode_READ(kind, data, i); + if (ch < 0x100) + str += sprintf(str, "\\x%02x", ch); + else if (ch < 0x10000) + str += sprintf(str, "\\u%04x", ch); + else { + assert(ch <= MAX_UNICODE); + str += sprintf(str, "\\U%08x", ch); + } + } + return str; +} + +/* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings: + ASCII, Latin1, UTF-8, etc. */ +static char* +xmlcharrefreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, + char *str, + PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) +{ + Py_ssize_t size, i, prealloc; + Py_UCS4 ch; + enum PyUnicode_Kind kind; + void *data; + + assert(PyUnicode_IS_READY(unicode)); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + + size = 0; + /* determine replacement size */ + for (i = collstart; i < collend; ++i) { + Py_ssize_t incr; + + ch = PyUnicode_READ(kind, data, i); + if (ch < 10) + incr = 2+1+1; + else if (ch < 100) + incr = 2+2+1; + else if (ch < 1000) + incr = 2+3+1; + else if (ch < 10000) + incr = 2+4+1; + else if (ch < 100000) + incr = 2+5+1; + else if (ch < 1000000) + incr = 2+6+1; + else { + assert(ch <= MAX_UNICODE); + incr = 2+7+1; + } + if (size > PY_SSIZE_T_MAX - incr) { + PyErr_SetString(PyExc_OverflowError, + "encoded result is too long for a Python string"); + return NULL; + } + size += incr; + } + + prealloc = prealloc_per_char * (collend - collstart); + if (size > prealloc) { + str = _PyBytesWriter_Prepare(writer, str, size - prealloc); + if (str == NULL) + return NULL; + } + + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); + } + return str; +} + /* --- Bloom Filters ----------------------------------------------------- */ /* stuff to implement simple "bloom filters" for Unicode characters. @@ -6713,7 +6834,6 @@ unicode_encode_ucs1(PyObject *unicode, ++pos; } else { - Py_ssize_t requiredsize; PyObject *repunicode; Py_ssize_t repsize, newpos, i; /* startpos for collecting unencodable chars */ @@ -6744,42 +6864,19 @@ unicode_encode_ucs1(PyObject *unicode, pos = collend; break; - case _Py_ERROR_XMLCHARREFREPLACE: - requiredsize = 0; - /* determine replacement size */ - for (i = collstart; i < collend; ++i) { - Py_ssize_t incr; - - ch = PyUnicode_READ(kind, data, i); - if (ch < 10) - incr = 2+1+1; - else if (ch < 100) - incr = 2+2+1; - else if (ch < 1000) - incr = 2+3+1; - else if (ch < 10000) - incr = 2+4+1; - else if (ch < 100000) - incr = 2+5+1; - else if (ch < 1000000) - incr = 2+6+1; - else { - assert(ch <= MAX_UNICODE); - incr = 2+7+1; - } - if (requiredsize > PY_SSIZE_T_MAX - incr) - goto overflow; - requiredsize += incr; - } - - str = _PyBytesWriter_Prepare(&writer, str, requiredsize-1); + case _Py_ERROR_BACKSLASHREPLACE: + str = backslashreplace(&writer, 1, str, + unicode, collstart, collend); if (str == NULL) goto onError; + pos = collend; + break; - /* generate replacement */ - for (i = collstart; i < collend; ++i) { - str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); - } + case _Py_ERROR_XMLCHARREFREPLACE: + str = xmlcharrefreplace(&writer, 1, str, + unicode, collstart, collend); + if (str == NULL) + goto onError; pos = collend; break; @@ -6810,9 +6907,11 @@ unicode_encode_ucs1(PyObject *unicode, if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ repsize = PyBytes_Size(repunicode); - str = _PyBytesWriter_Prepare(&writer, str, repsize-1); - if (str == NULL) - goto onError; + if (repsize > 1) { + str = _PyBytesWriter_Prepare(&writer, str, repsize-1); + if (str == NULL) + goto onError; + } memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; pos = newpos; @@ -6856,10 +6955,6 @@ unicode_encode_ucs1(PyObject *unicode, Py_XDECREF(exc); return _PyBytesWriter_Finish(&writer, str); - overflow: - PyErr_SetString(PyExc_OverflowError, - "encoded result is too long for a Python string"); - onError: _PyBytesWriter_Dealloc(&writer); Py_XDECREF(error_handler_obj); -- cgit v0.12 From 0016507c168fa942d7856bdef371cd8d494b140b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 01:53:21 +0200 Subject: Issue #25318: Move _PyBytesWriter to bytesobject.c Declare also the private API in bytesobject.h. --- Include/bytesobject.h | 52 ++++++++++++ Objects/bytesobject.c | 193 ++++++++++++++++++++++++++++++++++++++++++++ Objects/unicodeobject.c | 210 ------------------------------------------------ 3 files changed, 245 insertions(+), 210 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index e379bac..eafcdea 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -123,6 +123,58 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer, #define F_ALT (1<<3) #define F_ZERO (1<<4) +#ifndef Py_LIMITED_API +/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer". + A _PyBytesWriter variable must be declared at the end of variables in a + function to optimize the memory allocation on the stack. */ +typedef struct { + /* bytes object */ + PyObject *buffer; + + /* Number of allocated size */ + Py_ssize_t allocated; + + /* Current size of the buffer (can be smaller than the allocated size) */ + Py_ssize_t size; + + /* If non-zero, overallocate the buffer (default: 0). */ + int overallocate; + + /* Stack buffer */ + int use_stack_buffer; + char stack_buffer[512]; +} _PyBytesWriter; + +/* Initialize a bytes writer + + By default, the overallocation is disabled. Set the overallocate attribute + to control the allocation of the buffer. */ +PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); + +/* Get the buffer content and reset the writer. + Return a bytes object. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, + char *str); + +/* Deallocate memory of a writer (clear its internal buffer). */ +PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, + Py_ssize_t size); + +/* Add *size* bytes to the buffer. + str is the current pointer inside the buffer. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, + char *str, + Py_ssize_t size); +#endif /* Py_LIMITED_API */ + #ifdef __cplusplus } #endif diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 08275ad..46322aa 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3735,3 +3735,196 @@ bytes_iter(PyObject *seq) _PyObject_GC_TRACK(it); return (PyObject *)it; } + + +/* _PyBytesWriter API */ + +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ +# define OVERALLOCATE_FACTOR 2 +#else + /* On Linux, overallocate by 25% is the best factor */ +# define OVERALLOCATE_FACTOR 4 +#endif + +void +_PyBytesWriter_Init(_PyBytesWriter *writer) +{ + writer->buffer = NULL; + writer->allocated = 0; + writer->size = 0; + writer->overallocate = 0; + writer->use_stack_buffer = 0; +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); +#endif +} + +void +_PyBytesWriter_Dealloc(_PyBytesWriter *writer) +{ + Py_CLEAR(writer->buffer); +} + +Py_LOCAL_INLINE(char*) +_PyBytesWriter_AsString(_PyBytesWriter *writer) +{ + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + return PyBytes_AS_STRING(writer->buffer); + } + else { + assert(writer->buffer == NULL); + return writer->stack_buffer; + } +} + +Py_LOCAL_INLINE(Py_ssize_t) +_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) +{ + char *start = _PyBytesWriter_AsString(writer); + assert(str != NULL); + assert(str >= start); + return str - start; +} + +Py_LOCAL_INLINE(void) +_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) +{ +#ifdef Py_DEBUG + char *start, *end; + + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + assert(PyBytes_CheckExact(writer->buffer)); + assert(Py_REFCNT(writer->buffer) == 1); + } + else { + assert(writer->buffer == NULL); + } + + start = _PyBytesWriter_AsString(writer); + assert(0 <= writer->size && writer->size <= writer->allocated); + /* the last byte must always be null */ + assert(start[writer->allocated] == 0); + + end = start + writer->allocated; + assert(str != NULL); + assert(start <= str && str <= end); +#endif +} + +char* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) +{ + Py_ssize_t allocated, pos; + + _PyBytesWriter_CheckConsistency(writer, str); + assert(size >= 0); + + if (size == 0) { + /* nothing to do */ + return str; + } + + if (writer->size > PY_SSIZE_T_MAX - size) { + PyErr_NoMemory(); + _PyBytesWriter_Dealloc(writer); + return NULL; + } + writer->size += size; + + allocated = writer->allocated; + if (writer->size <= allocated) + return str; + + allocated = writer->size; + if (writer->overallocate + && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { + /* overallocate to limit the number of realloc() */ + allocated += allocated / OVERALLOCATE_FACTOR; + } + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + /* Note: Don't use a bytearray object because the conversion from + byterray to bytes requires to copy all bytes. */ + if (_PyBytes_Resize(&writer->buffer, allocated)) { + assert(writer->buffer == NULL); + return NULL; + } + } + else { + /* convert from stack buffer to bytes object buffer */ + assert(writer->buffer == NULL); + + writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); + if (writer->buffer == NULL) + return NULL; + + if (pos != 0) { + Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), + writer->stack_buffer, + pos); + } + +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); +#endif + + writer->use_stack_buffer = 0; + } + writer->allocated = allocated; + + str = _PyBytesWriter_AsString(writer) + pos; + _PyBytesWriter_CheckConsistency(writer, str); + return str; +} + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +char* +_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) +{ + /* ensure that _PyBytesWriter_Alloc() is only called once */ + assert(writer->size == 0 && writer->buffer == NULL); + assert(size >= 0); + + writer->use_stack_buffer = 1; +#if Py_DEBUG + /* the last byte is reserved, it must be '\0' */ + writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; + writer->allocated = sizeof(writer->stack_buffer) - 1; +#else + writer->allocated = sizeof(writer->stack_buffer); +#endif + return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); +} + +PyObject * +_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) +{ + Py_ssize_t pos; + PyObject *result; + + _PyBytesWriter_CheckConsistency(writer, str); + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + if (pos != writer->allocated) { + if (_PyBytes_Resize(&writer->buffer, pos)) { + assert(writer->buffer == NULL); + return NULL; + } + } + + result = writer->buffer; + writer->buffer = NULL; + } + else { + result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); + } + + return result; +} diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e0b3c68..614d214 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -347,216 +347,6 @@ PyUnicode_GetMax(void) #endif } -/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes object */ - PyObject *buffer; - - /* Number of allocated size */ - Py_ssize_t allocated; - - /* Current size of the buffer (can be smaller than the allocated size) */ - Py_ssize_t size; - - /* If non-zero, overallocate the buffer (default: 0). */ - int overallocate; - - /* Stack buffer */ - int use_stack_buffer; - char stack_buffer[512]; -} _PyBytesWriter; - -static void -_PyBytesWriter_Init(_PyBytesWriter *writer) -{ - writer->buffer = NULL; - writer->allocated = 0; - writer->size = 0; - writer->overallocate = 0; - writer->use_stack_buffer = 0; -#ifdef Py_DEBUG - memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); -#endif -} - -static void -_PyBytesWriter_Dealloc(_PyBytesWriter *writer) -{ - Py_CLEAR(writer->buffer); -} - -static char* -_PyBytesWriter_AsString(_PyBytesWriter *writer) -{ - if (!writer->use_stack_buffer) { - assert(writer->buffer != NULL); - return PyBytes_AS_STRING(writer->buffer); - } - else { - assert(writer->buffer == NULL); - return writer->stack_buffer; - } -} - -Py_LOCAL_INLINE(Py_ssize_t) -_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) -{ - char *start = _PyBytesWriter_AsString(writer); - assert(str != NULL); - assert(str >= start); - return str - start; -} - -Py_LOCAL_INLINE(void) -_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) -{ -#ifdef Py_DEBUG - char *start, *end; - - if (!writer->use_stack_buffer) { - assert(writer->buffer != NULL); - assert(PyBytes_CheckExact(writer->buffer)); - assert(Py_REFCNT(writer->buffer) == 1); - } - else { - assert(writer->buffer == NULL); - } - - start = _PyBytesWriter_AsString(writer); - assert(0 <= writer->size && writer->size <= writer->allocated); - /* the last byte must always be null */ - assert(start[writer->allocated] == 0); - - end = start + writer->allocated; - assert(str != NULL); - assert(start <= str && str <= end); -#endif -} - -/* Add *size* bytes to the buffer. - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -static char* -_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) -{ - Py_ssize_t allocated, pos; - - _PyBytesWriter_CheckConsistency(writer, str); - assert(size >= 0); - - if (size == 0) { - /* nothing to do */ - return str; - } - - if (writer->size > PY_SSIZE_T_MAX - size) { - PyErr_NoMemory(); - _PyBytesWriter_Dealloc(writer); - return NULL; - } - writer->size += size; - - allocated = writer->allocated; - if (writer->size <= allocated) - return str; - - allocated = writer->size; - if (writer->overallocate - && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - allocated += allocated / OVERALLOCATE_FACTOR; - } - - pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { - /* Note: Don't use a bytearray object because the conversion from - byterray to bytes requires to copy all bytes. */ - if (_PyBytes_Resize(&writer->buffer, allocated)) { - assert(writer->buffer == NULL); - return NULL; - } - } - else { - /* convert from stack buffer to bytes object buffer */ - assert(writer->buffer == NULL); - - writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); - if (writer->buffer == NULL) - return NULL; - - if (pos != 0) { - Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), - writer->stack_buffer, - pos); - } - -#ifdef Py_DEBUG - memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); -#endif - - writer->use_stack_buffer = 0; - } - writer->allocated = allocated; - - str = _PyBytesWriter_AsString(writer) + pos; - _PyBytesWriter_CheckConsistency(writer, str); - return str; -} - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -static char* -_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) -{ - /* ensure that _PyBytesWriter_Alloc() is only called once */ - assert(writer->size == 0 && writer->buffer == NULL); - assert(size >= 0); - - writer->use_stack_buffer = 1; -#if Py_DEBUG - /* the last byte is reserved, it must be '\0' */ - writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; - writer->allocated = sizeof(writer->stack_buffer) - 1; -#else - writer->allocated = sizeof(writer->stack_buffer); -#endif - return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); -} - -/* Get the buffer content and reset the writer. - Return a bytes object. - Raise an exception and return NULL on error. */ -static PyObject * -_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) -{ - Py_ssize_t pos; - PyObject *result; - - _PyBytesWriter_CheckConsistency(writer, str); - - pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { - if (pos != writer->allocated) { - if (_PyBytes_Resize(&writer->buffer, pos)) { - assert(writer->buffer == NULL); - return NULL; - } - } - - result = writer->buffer; - writer->buffer = NULL; - } - else { - result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); - } - - return result; -} - #ifdef Py_DEBUG int _PyUnicode_CheckConsistency(PyObject *op, int check_content) -- cgit v0.12 From b13b97d3b881daaab8f3bf32cc851eac9b895c52 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 02:52:16 +0200 Subject: Issue #25318: Fix compilation error Replace "#if Py_DEBUG" with "#ifdef Py_DEBUG". --- Objects/bytesobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 46322aa..56231a2 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3892,7 +3892,7 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) assert(size >= 0); writer->use_stack_buffer = 1; -#if Py_DEBUG +#ifdef Py_DEBUG /* the last byte is reserved, it must be '\0' */ writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; writer->allocated = sizeof(writer->stack_buffer) - 1; -- cgit v0.12 From 797485e10135ca323565b22b4fabf1e161a5ec7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 03:17:30 +0200 Subject: Issue #25318: Avoid sprintf() in backslashreplace() Rewrite backslashreplace() to be closer to PyCodec_BackslashReplaceErrors(). Add also unit tests for non-BMP characters. --- Lib/test/test_codecs.py | 6 ++++-- Objects/unicodeobject.c | 25 ++++++++++++++++++------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 7b6883f..ff314b1 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3155,7 +3155,8 @@ class ASCIITest(unittest.TestCase): ('[\x80\xff\u20ac]', 'ignore', b'[]'), ('[\x80\xff\u20ac]', 'replace', b'[???]'), ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[€ÿ€]'), - ('[\x80\xff\u20ac]', 'backslashreplace', b'[\\x80\\xff\\u20ac]'), + ('[\x80\xff\u20ac\U000abcde]', 'backslashreplace', + b'[\\x80\\xff\\u20ac\\U000abcde]'), ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), ): with self.subTest(data=data, error_handler=error_handler, @@ -3197,7 +3198,8 @@ class Latin1Test(unittest.TestCase): for data, error_handler, expected in ( ('[\u20ac\udc80]', 'ignore', b'[]'), ('[\u20ac\udc80]', 'replace', b'[??]'), - ('[\u20ac\udc80]', 'backslashreplace', b'[\\u20ac\\udc80]'), + ('[\u20ac\U000abcde]', 'backslashreplace', + b'[\\u20ac\\U000abcde]'), ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€�]'), ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), ): diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 614d214..10cdcc0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -610,14 +610,25 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, /* generate replacement */ for (i = collstart; i < collend; ++i) { ch = PyUnicode_READ(kind, data, i); - if (ch < 0x100) - str += sprintf(str, "\\x%02x", ch); - else if (ch < 0x10000) - str += sprintf(str, "\\u%04x", ch); - else { - assert(ch <= MAX_UNICODE); - str += sprintf(str, "\\U%08x", ch); + *str++ = '\\'; + if (ch >= 0x00010000) { + *str++ = 'U'; + *str++ = Py_hexdigits[(ch>>28)&0xf]; + *str++ = Py_hexdigits[(ch>>24)&0xf]; + *str++ = Py_hexdigits[(ch>>20)&0xf]; + *str++ = Py_hexdigits[(ch>>16)&0xf]; + *str++ = Py_hexdigits[(ch>>12)&0xf]; + *str++ = Py_hexdigits[(ch>>8)&0xf]; + } + else if (ch >= 0x100) { + *str++ = 'u'; + *str++ = Py_hexdigits[(ch>>12)&0xf]; + *str++ = Py_hexdigits[(ch>>8)&0xf]; } + else + *str++ = 'x'; + *str++ = Py_hexdigits[(ch>>4)&0xf]; + *str++ = Py_hexdigits[ch&0xf]; } return str; } -- cgit v0.12 From 3fa36ff5e4991550e31cc7ab55dc3a2165c2ffa3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 03:37:11 +0200 Subject: Issue #25318: Fix backslashreplace() Fix code to estimate the needed space. --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 10cdcc0..a3bbf92 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -590,7 +590,7 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, incr = 2+4; else { assert(ch <= MAX_UNICODE); - incr = 2+6; + incr = 2+8; } if (size > PY_SSIZE_T_MAX - incr) { PyErr_SetString(PyExc_OverflowError, -- cgit v0.12 From b3653a34580684d210daed5cbb8a676c4d7a33fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 03:38:24 +0200 Subject: Issue #25318: cleanup code _PyBytesWriter Rename "stack buffer" to "small buffer". Add also an assertion in _PyBytesWriter_GetPos(). --- Include/bytesobject.h | 4 ++-- Objects/bytesobject.c | 34 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index eafcdea..ffa529b 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -141,8 +141,8 @@ typedef struct { int overallocate; /* Stack buffer */ - int use_stack_buffer; - char stack_buffer[512]; + int use_small_buffer; + char small_buffer[512]; } _PyBytesWriter; /* Initialize a bytes writer diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 56231a2..722978d 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3754,9 +3754,9 @@ _PyBytesWriter_Init(_PyBytesWriter *writer) writer->allocated = 0; writer->size = 0; writer->overallocate = 0; - writer->use_stack_buffer = 0; + writer->use_small_buffer = 0; #ifdef Py_DEBUG - memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); + memset(writer->small_buffer, 0xCB, sizeof(writer->small_buffer)); #endif } @@ -3769,13 +3769,13 @@ _PyBytesWriter_Dealloc(_PyBytesWriter *writer) Py_LOCAL_INLINE(char*) _PyBytesWriter_AsString(_PyBytesWriter *writer) { - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { assert(writer->buffer != NULL); return PyBytes_AS_STRING(writer->buffer); } else { assert(writer->buffer == NULL); - return writer->stack_buffer; + return writer->small_buffer; } } @@ -3785,6 +3785,7 @@ _PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) char *start = _PyBytesWriter_AsString(writer); assert(str != NULL); assert(str >= start); + assert(str - start <= writer->allocated); return str - start; } @@ -3794,7 +3795,7 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) #ifdef Py_DEBUG char *start, *end; - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { assert(writer->buffer != NULL); assert(PyBytes_CheckExact(writer->buffer)); assert(Py_REFCNT(writer->buffer) == 1); @@ -3846,7 +3847,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) } pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { /* Note: Don't use a bytearray object because the conversion from byterray to bytes requires to copy all bytes. */ if (_PyBytes_Resize(&writer->buffer, allocated)) { @@ -3864,15 +3865,14 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) if (pos != 0) { Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), - writer->stack_buffer, + writer->small_buffer, pos); } + writer->use_small_buffer = 0; #ifdef Py_DEBUG - memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); + memset(writer->small_buffer, 0xDB, sizeof(writer->small_buffer)); #endif - - writer->use_stack_buffer = 0; } writer->allocated = allocated; @@ -3891,15 +3891,15 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) assert(writer->size == 0 && writer->buffer == NULL); assert(size >= 0); - writer->use_stack_buffer = 1; + writer->use_small_buffer = 1; #ifdef Py_DEBUG /* the last byte is reserved, it must be '\0' */ - writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; - writer->allocated = sizeof(writer->stack_buffer) - 1; + writer->allocated = sizeof(writer->small_buffer) - 1; + writer->small_buffer[writer->allocated] = 0; #else - writer->allocated = sizeof(writer->stack_buffer); + writer->allocated = sizeof(writer->small_buffer); #endif - return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); + return _PyBytesWriter_Prepare(writer, writer->small_buffer, size); } PyObject * @@ -3911,7 +3911,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) _PyBytesWriter_CheckConsistency(writer, str); pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { if (pos != writer->allocated) { if (_PyBytes_Resize(&writer->buffer, pos)) { assert(writer->buffer == NULL); @@ -3923,7 +3923,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) writer->buffer = NULL; } else { - result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); + result = PyBytes_FromStringAndSize(writer->small_buffer, pos); } return result; -- cgit v0.12 From 7836a27ceba2395572f1b3b385c11e6a43910185 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Oct 2015 00:03:51 -0400 Subject: Issue #25298: Add lock and rlock weakref tests (Contributed by Nir Soffer). --- Lib/test/lock_tests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index afd6873..055bf28 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -7,6 +7,7 @@ import time from _thread import start_new_thread, TIMEOUT_MAX import threading import unittest +import weakref from test import support @@ -198,6 +199,17 @@ class BaseLockTests(BaseTestCase): self.assertFalse(results[0]) self.assertTimeout(results[1], 0.5) + def test_weakref_exists(self): + lock = self.locktype() + ref = weakref.ref(lock) + self.assertIsNotNone(ref()) + + def test_weakref_deleted(self): + lock = self.locktype() + ref = weakref.ref(lock) + del lock + self.assertIsNone(ref()) + class LockTests(BaseLockTests): """ -- cgit v0.12 From 5098b58381e572ccd55982004da903a59ffec95c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Oct 2015 00:42:47 -0400 Subject: Make comparison more consistent --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2038d2b..3f2e2c0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -331,7 +331,7 @@ builtin_any(PyModuleDef *module, PyObject *iterable) Py_DECREF(it); return NULL; } - if (cmp == 1) { + if (cmp > 0) { Py_DECREF(it); Py_RETURN_TRUE; } -- cgit v0.12 From bd5f0e8c1cf633a238c10e1d3199c0d572a4df1d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Oct 2015 01:34:08 -0400 Subject: Hoist constant expression out of the inner loop. --- Python/bltinmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3f2e2c0..3c8f1e9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -469,6 +469,7 @@ filter_next(filterobject *lz) PyObject *it = lz->it; long ok; PyObject *(*iternext)(PyObject *); + int checktrue = lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type; iternext = *Py_TYPE(it)->tp_iternext; for (;;) { @@ -476,12 +477,11 @@ filter_next(filterobject *lz) if (item == NULL) return NULL; - if (lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type) { + if (checktrue) { ok = PyObject_IsTrue(item); } else { PyObject *good; - good = PyObject_CallFunctionObjArgs(lz->func, - item, NULL); + good = PyObject_CallFunctionObjArgs(lz->func, item, NULL); if (good == NULL) { Py_DECREF(item); return NULL; -- cgit v0.12 From fa7762ec066aa3632a25b6a52bb7597b8f17c2f3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 11:48:06 +0200 Subject: Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API * Thanks to the _PyBytesWriter API, output smaller than 512 bytes are allocated on the stack and so avoid calling _PyBytes_Resize(). Because of that, change the default buffer size to fmtcnt instead of fmtcnt+100. * Rely on _PyBytesWriter algorithm to overallocate the buffer instead of using a custom code. For example, _PyBytesWriter uses a different overallocation factor (25% or 50%) depending on the platform to get best performances. * Disable overallocation for the last write. * Replace C loops to fill characters with memset() * Add also many comments to _PyBytes_Format() * Remove unused FORMATBUFLEN constant * Avoid the creation of a temporary bytes object when formatting a floating point number (when no custom formatting option is used) * Fix also reference leaks on error handling * Use Py_MEMCPY() to copy bytes between two formatters (%) --- Misc/NEWS | 2 + Objects/bytesobject.c | 187 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 130 insertions(+), 59 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 2dc4d9b..9a7a85a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API. + - Issue #24806: Prevent builtin types that are not allowed to be subclassed from being subclassed through multiple inheritance. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 722978d..ef53288 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -409,12 +409,15 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) /* Returns a new reference to a PyBytes object, or NULL on failure. */ -static PyObject * -formatfloat(PyObject *v, int flags, int prec, int type) +static char* +formatfloat(PyObject *v, int flags, int prec, int type, + PyObject **p_result, _PyBytesWriter *writer, char *str, + Py_ssize_t prealloc) { char *p; PyObject *result; double x; + size_t len; x = PyFloat_AsDouble(v); if (x == -1.0 && PyErr_Occurred()) { @@ -431,9 +434,23 @@ formatfloat(PyObject *v, int flags, int prec, int type) if (p == NULL) return NULL; - result = PyBytes_FromStringAndSize(p, strlen(p)); + + len = strlen(p); + if (writer != NULL) { + if ((Py_ssize_t)len > prealloc) { + str = _PyBytesWriter_Prepare(writer, str, len - prealloc); + if (str == NULL) + return NULL; + } + Py_MEMCPY(str, p, len); + str += len; + return str; + } + + result = PyBytes_FromStringAndSize(p, len); PyMem_Free(p); - return result; + *p_result = result; + return str; } static PyObject * @@ -557,36 +574,32 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) return NULL; } -/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) - - FORMATBUFLEN is the length of the buffer in which the ints & - chars are formatted. XXX This is a magic number. Each formatting - routine does bounds checking to ensure no overflow, but a better - solution may be to malloc a buffer of appropriate size for each - format. For now, the current solution is sufficient. -*/ -#define FORMATBUFLEN (size_t)120 +/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */ PyObject * _PyBytes_Format(PyObject *format, PyObject *args) { char *fmt, *res; Py_ssize_t arglen, argidx; - Py_ssize_t reslen, rescnt, fmtcnt; + Py_ssize_t fmtcnt; int args_owned = 0; - PyObject *result; PyObject *dict = NULL; + _PyBytesWriter writer; + if (format == NULL || !PyBytes_Check(format) || args == NULL) { PyErr_BadInternalCall(); return NULL; } fmt = PyBytes_AS_STRING(format); fmtcnt = PyBytes_GET_SIZE(format); - reslen = rescnt = fmtcnt + 100; - result = PyBytes_FromStringAndSize((char *)NULL, reslen); - if (result == NULL) + + _PyBytesWriter_Init(&writer); + + res = _PyBytesWriter_Alloc(&writer, fmtcnt); + if (res == NULL) return NULL; - res = PyBytes_AsString(result); + writer.overallocate = 1; + if (PyTuple_Check(args)) { arglen = PyTuple_GET_SIZE(args); argidx = 0; @@ -600,18 +613,25 @@ _PyBytes_Format(PyObject *format, PyObject *args) !PyByteArray_Check(args)) { dict = args; } + while (--fmtcnt >= 0) { if (*fmt != '%') { - if (--rescnt < 0) { - rescnt = fmtcnt + 100; - reslen += rescnt; - if (_PyBytes_Resize(&result, reslen)) - return NULL; - res = PyBytes_AS_STRING(result) - + reslen - rescnt; - --rescnt; + Py_ssize_t len; + char *pos; + + pos = strchr(fmt + 1, '%'); + if (pos != NULL) + len = pos - fmt; + else { + len = PyBytes_GET_SIZE(format); + len -= (fmt - PyBytes_AS_STRING(format)); } - *res++ = *fmt++; + assert(len != 0); + + Py_MEMCPY(res, fmt, len); + res += len; + fmt += len; + fmtcnt -= (len - 1); } else { /* Got a format specifier */ @@ -626,6 +646,10 @@ _PyBytes_Format(PyObject *format, PyObject *args) int sign; Py_ssize_t len = 0; char onechar; /* For byte_converter() */ + Py_ssize_t alloc; +#ifdef Py_DEBUG + char *before; +#endif fmt++; if (*fmt == '(') { @@ -673,6 +697,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) arglen = -1; argidx = -2; } + + /* Parse flags. Example: "%+i" => flags=F_SIGN. */ while (--fmtcnt >= 0) { switch (c = *fmt++) { case '-': flags |= F_LJUST; continue; @@ -683,6 +709,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) } break; } + + /* Parse width. Example: "%10s" => width=10 */ if (c == '*') { v = getnextarg(args, arglen, &argidx); if (v == NULL) @@ -717,6 +745,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) width = width*10 + (c - '0'); } } + + /* Parse precision. Example: "%.3f" => prec=3 */ if (c == '.') { prec = 0; if (--fmtcnt >= 0) @@ -771,6 +801,12 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (v == NULL) goto error; } + + if (fmtcnt < 0) { + /* last writer: disable writer overallocation */ + writer.overallocate = 0; + } + sign = 0; fill = ' '; switch (c) { @@ -778,6 +814,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) pbuf = "%"; len = 1; break; + case 'r': // %r is only for 2/3 code; 3 only code should use %a case 'a': @@ -790,6 +827,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (prec >= 0 && len > prec) len = prec; break; + case 's': // %s is only for 2/3 code; 3 only code should use %b case 'b': @@ -799,6 +837,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (prec >= 0 && len > prec) len = prec; break; + case 'i': case 'd': case 'u': @@ -815,14 +854,24 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (flags & F_ZERO) fill = '0'; break; + case 'e': case 'E': case 'f': case 'F': case 'g': case 'G': - temp = formatfloat(v, flags, prec, c); - if (temp == NULL) + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + res = formatfloat(v, flags, prec, c, NULL, &writer, res, 1); + if (res == NULL) + goto error; + continue; + } + + if (!formatfloat(v, flags, prec, c, &temp, NULL, res, 1)) goto error; pbuf = PyBytes_AS_STRING(temp); len = PyBytes_GET_SIZE(temp); @@ -830,12 +879,14 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (flags & F_ZERO) fill = '0'; break; + case 'c': pbuf = &onechar; len = byte_converter(v, &onechar); if (!len) goto error; break; + default: PyErr_Format(PyExc_ValueError, "unsupported format character '%c' (0x%x) " @@ -845,6 +896,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) PyBytes_AsString(format))); goto error; } + if (sign) { if (*pbuf == '-' || *pbuf == '+') { sign = *pbuf++; @@ -859,29 +911,30 @@ _PyBytes_Format(PyObject *format, PyObject *args) } if (width < len) width = len; - if (rescnt - (sign != 0) < width) { - reslen -= rescnt; - rescnt = width + fmtcnt + 100; - reslen += rescnt; - if (reslen < 0) { - Py_DECREF(result); - Py_XDECREF(temp); - return PyErr_NoMemory(); - } - if (_PyBytes_Resize(&result, reslen)) { - Py_XDECREF(temp); - return NULL; - } - res = PyBytes_AS_STRING(result) - + reslen - rescnt; + + alloc = width; + if (sign != 0 && len == width) + alloc++; + if (alloc > 1) { + res = _PyBytesWriter_Prepare(&writer, res, alloc - 1); + if (res == NULL) + goto error; } +#ifdef Py_DEBUG + before = res; +#endif + + /* Write the sign if needed */ if (sign) { if (fill != ' ') *res++ = sign; - rescnt--; if (width > len) width--; } + + /* Write the numeric prefix for "x", "X" and "o" formats + if the alternate form is used. + For example, write "0x" for the "%#x" format. */ if ((flags & F_ALT) && (c == 'x' || c == 'X')) { assert(pbuf[0] == '0'); assert(pbuf[1] == c); @@ -889,18 +942,21 @@ _PyBytes_Format(PyObject *format, PyObject *args) *res++ = *pbuf++; *res++ = *pbuf++; } - rescnt -= 2; width -= 2; if (width < 0) width = 0; len -= 2; } + + /* Pad left with the fill character if needed */ if (width > len && !(flags & F_LJUST)) { - do { - --rescnt; - *res++ = fill; - } while (--width > len); + memset(res, fill, width - len); + res += (width - len); + width = len; } + + /* If padding with spaces: write sign if needed and/or numeric + prefix if the alternate form is used */ if (fill == ' ') { if (sign) *res++ = sign; @@ -912,13 +968,17 @@ _PyBytes_Format(PyObject *format, PyObject *args) *res++ = *pbuf++; } } + + /* Copy bytes */ Py_MEMCPY(res, pbuf, len); res += len; - rescnt -= len; - while (--width >= len) { - --rescnt; - *res++ = ' '; + + /* Pad right with the fill character if needed */ + if (width > len) { + memset(res, ' ', width - len); + res += (width - len); } + if (dict && (argidx < arglen) && c != '%') { PyErr_SetString(PyExc_TypeError, "not all arguments converted during bytes formatting"); @@ -926,22 +986,31 @@ _PyBytes_Format(PyObject *format, PyObject *args) goto error; } Py_XDECREF(temp); + +#ifdef Py_DEBUG + /* check that we computed the exact size for this write */ + assert((res - before) == alloc); +#endif } /* '%' */ + + /* If overallocation was disabled, ensure that it was the last + write. Otherwise, we missed an optimization */ + assert(writer.overallocate || fmtcnt < 0); } /* until end */ + if (argidx < arglen && !dict) { PyErr_SetString(PyExc_TypeError, "not all arguments converted during bytes formatting"); goto error; } + if (args_owned) { Py_DECREF(args); } - if (_PyBytes_Resize(&result, reslen - rescnt)) - return NULL; - return result; + return _PyBytesWriter_Finish(&writer, res); error: - Py_DECREF(result); + _PyBytesWriter_Dealloc(&writer); if (args_owned) { Py_DECREF(args); } -- cgit v0.12 From 53926a1ce2772ea5dc1c4fb8c19f5f9ae461750d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 12:37:03 +0200 Subject: _PyBytesWriter: rename size attribute to min_size --- Include/bytesobject.h | 5 +++-- Objects/bytesobject.c | 14 +++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index ffa529b..6cd5a34 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -134,8 +134,9 @@ typedef struct { /* Number of allocated size */ Py_ssize_t allocated; - /* Current size of the buffer (can be smaller than the allocated size) */ - Py_ssize_t size; + /* Minimum number of allocated bytes, + incremented by _PyBytesWriter_Prepare() */ + Py_ssize_t min_size; /* If non-zero, overallocate the buffer (default: 0). */ int overallocate; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ef53288..4a0735f 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3821,7 +3821,7 @@ _PyBytesWriter_Init(_PyBytesWriter *writer) { writer->buffer = NULL; writer->allocated = 0; - writer->size = 0; + writer->min_size = 0; writer->overallocate = 0; writer->use_small_buffer = 0; #ifdef Py_DEBUG @@ -3874,7 +3874,7 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) } start = _PyBytesWriter_AsString(writer); - assert(0 <= writer->size && writer->size <= writer->allocated); + assert(0 <= writer->min_size && writer->min_size <= writer->allocated); /* the last byte must always be null */ assert(start[writer->allocated] == 0); @@ -3897,18 +3897,18 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) return str; } - if (writer->size > PY_SSIZE_T_MAX - size) { + if (writer->min_size > PY_SSIZE_T_MAX - size) { PyErr_NoMemory(); _PyBytesWriter_Dealloc(writer); return NULL; } - writer->size += size; + writer->min_size += size; allocated = writer->allocated; - if (writer->size <= allocated) + if (writer->min_size <= allocated) return str; - allocated = writer->size; + allocated = writer->min_size; if (writer->overallocate && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { /* overallocate to limit the number of realloc() */ @@ -3957,7 +3957,7 @@ char* _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) { /* ensure that _PyBytesWriter_Alloc() is only called once */ - assert(writer->size == 0 && writer->buffer == NULL); + assert(writer->min_size == 0 && writer->buffer == NULL); assert(size >= 0); writer->use_small_buffer = 1; -- cgit v0.12 From ad7715891e3c6c51b4860e0d496843ee5417206b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 12:38:53 +0200 Subject: _PyBytesWriter: simplify code to avoid "prealloc" parameters Substract preallocate bytes from min_size before calling _PyBytesWriter_Prepare(). --- Objects/bytesobject.c | 16 ++++++------- Objects/stringlib/codecs.h | 20 +++++++++------- Objects/unicodeobject.c | 58 ++++++++++++++++++++++------------------------ 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4a0735f..075edf8 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -411,8 +411,7 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) static char* formatfloat(PyObject *v, int flags, int prec, int type, - PyObject **p_result, _PyBytesWriter *writer, char *str, - Py_ssize_t prealloc) + PyObject **p_result, _PyBytesWriter *writer, char *str) { char *p; PyObject *result; @@ -437,11 +436,9 @@ formatfloat(PyObject *v, int flags, int prec, int type, len = strlen(p); if (writer != NULL) { - if ((Py_ssize_t)len > prealloc) { - str = _PyBytesWriter_Prepare(writer, str, len - prealloc); - if (str == NULL) - return NULL; - } + str = _PyBytesWriter_Prepare(writer, str, len); + if (str == NULL) + return NULL; Py_MEMCPY(str, p, len); str += len; return str; @@ -865,13 +862,14 @@ _PyBytes_Format(PyObject *format, PyObject *args) && !(flags & (F_SIGN | F_BLANK))) { /* Fast path */ - res = formatfloat(v, flags, prec, c, NULL, &writer, res, 1); + writer.min_size -= 2; /* size preallocated by "%f" */ + res = formatfloat(v, flags, prec, c, NULL, &writer, res); if (res == NULL) goto error; continue; } - if (!formatfloat(v, flags, prec, c, &temp, NULL, res, 1)) + if (!formatfloat(v, flags, prec, c, &temp, NULL, res)) goto error; pbuf = PyBytes_AS_STRING(temp); len = PyBytes_GET_SIZE(temp); diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index ae99d1a..6842f67 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -345,7 +345,9 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, break; case _Py_ERROR_BACKSLASHREPLACE: - p = backslashreplace(&writer, max_char_size, p, + /* substract preallocated bytes */ + writer.min_size -= max_char_size * (endpos - startpos); + p = backslashreplace(&writer, p, unicode, startpos, endpos); if (p == NULL) goto error; @@ -353,7 +355,9 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - p = xmlcharrefreplace(&writer, max_char_size, p, + /* substract preallocated bytes */ + writer.min_size -= max_char_size * (endpos - startpos); + p = xmlcharrefreplace(&writer, p, unicode, startpos, endpos); if (p == NULL) goto error; @@ -381,17 +385,17 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, if (!rep) goto error; + /* substract preallocated bytes */ + writer.min_size -= max_char_size; + if (PyBytes_Check(rep)) repsize = PyBytes_GET_SIZE(rep); else repsize = PyUnicode_GET_LENGTH(rep); - if (repsize > max_char_size) { - p = _PyBytesWriter_Prepare(&writer, p, - repsize - max_char_size); - if (p == NULL) - goto error; - } + p = _PyBytesWriter_Prepare(&writer, p, repsize); + if (p == NULL) + goto error; if (PyBytes_Check(rep)) { memcpy(p, PyBytes_AS_STRING(rep), repsize); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a3bbf92..0bcacd8 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -565,11 +565,10 @@ unicode_result_unchanged(PyObject *unicode) /* Implementation of the "backslashreplace" error handler for 8-bit encodings: ASCII, Latin1, UTF-8, etc. */ static char* -backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, - char *str, +backslashreplace(_PyBytesWriter *writer, char *str, PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) { - Py_ssize_t size, i, prealloc; + Py_ssize_t size, i; Py_UCS4 ch; enum PyUnicode_Kind kind; void *data; @@ -600,12 +599,9 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, size += incr; } - prealloc = prealloc_per_char * (collend - collstart); - if (size > prealloc) { - str = _PyBytesWriter_Prepare(writer, str, size - prealloc); - if (str == NULL) - return NULL; - } + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; /* generate replacement */ for (i = collstart; i < collend; ++i) { @@ -636,11 +632,10 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, /* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings: ASCII, Latin1, UTF-8, etc. */ static char* -xmlcharrefreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, - char *str, +xmlcharrefreplace(_PyBytesWriter *writer, char *str, PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) { - Py_ssize_t size, i, prealloc; + Py_ssize_t size, i; Py_UCS4 ch; enum PyUnicode_Kind kind; void *data; @@ -679,12 +674,9 @@ xmlcharrefreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, size += incr; } - prealloc = prealloc_per_char * (collend - collstart); - if (size > prealloc) { - str = _PyBytesWriter_Prepare(writer, str, size - prealloc); - if (str == NULL) - return NULL; - } + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; /* generate replacement */ for (i = collstart; i < collend; ++i) { @@ -6666,7 +6658,9 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_BACKSLASHREPLACE: - str = backslashreplace(&writer, 1, str, + /* substract preallocated bytes */ + writer.min_size -= (collend - collstart); + str = backslashreplace(&writer, str, unicode, collstart, collend); if (str == NULL) goto onError; @@ -6674,7 +6668,9 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - str = xmlcharrefreplace(&writer, 1, str, + /* substract preallocated bytes */ + writer.min_size -= (collend - collstart); + str = xmlcharrefreplace(&writer, str, unicode, collstart, collend); if (str == NULL) goto onError; @@ -6705,14 +6701,17 @@ unicode_encode_ucs1(PyObject *unicode, PyUnicode_READY(repunicode) == -1)) goto onError; + /* substract preallocated bytes */ + writer.min_size -= 1; + if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ repsize = PyBytes_Size(repunicode); - if (repsize > 1) { - str = _PyBytesWriter_Prepare(&writer, str, repsize-1); - if (str == NULL) - goto onError; - } + + str = _PyBytesWriter_Prepare(&writer, str, repsize); + if (str == NULL) + goto onError; + memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; pos = newpos; @@ -6724,11 +6723,10 @@ unicode_encode_ucs1(PyObject *unicode, have+the replacement+the rest of the string, so we won't have to check space for encodable characters) */ repsize = PyUnicode_GET_LENGTH(repunicode); - if (repsize > 1) { - str = _PyBytesWriter_Prepare(&writer, str, repsize-1); - if (str == NULL) - goto onError; - } + + str = _PyBytesWriter_Prepare(&writer, str, repsize); + if (str == NULL) + goto onError; /* check if there is anything unencodable in the replacement and copy it to the output */ -- cgit v0.12 From ce179bf6baed91ba84cc3ff647e96287c3b8e2f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 12:57:22 +0200 Subject: Add _PyBytesWriter_WriteBytes() to factorize the code --- Include/bytesobject.h | 7 +++++++ Objects/bytesobject.c | 14 ++++++++++++++ Objects/stringlib/codecs.h | 22 +++++++++++----------- Objects/unicodeobject.c | 8 +++----- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 6cd5a34..2c4c4c4 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -174,6 +174,13 @@ PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size); + +/* Write bytes. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(char*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, + char *str, + char *bytes, + Py_ssize_t size); #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 075edf8..3aa905c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3995,3 +3995,17 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) return result; } + +char* +_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, char *str, + char *bytes, Py_ssize_t size) +{ + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; + + Py_MEMCPY(str, bytes, size); + str += size; + + return str; +} diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 6842f67..7e8d928 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -388,24 +388,24 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, /* substract preallocated bytes */ writer.min_size -= max_char_size; - if (PyBytes_Check(rep)) - repsize = PyBytes_GET_SIZE(rep); - else - repsize = PyUnicode_GET_LENGTH(rep); - - p = _PyBytesWriter_Prepare(&writer, p, repsize); - if (p == NULL) - goto error; - if (PyBytes_Check(rep)) { - memcpy(p, PyBytes_AS_STRING(rep), repsize); - p += repsize; + p = _PyBytesWriter_WriteBytes(&writer, p, + PyBytes_AS_STRING(rep), + PyBytes_GET_SIZE(rep)); + if (p == NULL) + goto error; } else { /* rep is unicode */ if (PyUnicode_READY(rep) < 0) goto error; + repsize = PyUnicode_GET_LENGTH(rep); + + p = _PyBytesWriter_Prepare(&writer, p, repsize); + if (p == NULL) + goto error; + if (!PyUnicode_IS_ASCII(rep)) { raise_encode_exception(&exc, "utf-8", unicode, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0bcacd8..23b8cc7 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6706,14 +6706,12 @@ unicode_encode_ucs1(PyObject *unicode, if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ - repsize = PyBytes_Size(repunicode); - - str = _PyBytesWriter_Prepare(&writer, str, repsize); + str = _PyBytesWriter_WriteBytes(&writer, str, + PyBytes_AS_STRING(repunicode), + PyBytes_GET_SIZE(repunicode)); if (str == NULL) goto onError; - memcpy(str, PyBytes_AsString(repunicode), repsize); - str += repsize; pos = newpos; Py_DECREF(repunicode); break; -- cgit v0.12 From 6bd525b656f75c9752d39d9c4be1e1b29fa67cdb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 13:10:05 +0200 Subject: Optimize error handlers of ASCII and Latin1 encoders when the replacement string is pure ASCII: use _PyBytesWriter_WriteBytes(), don't check individual character. Cleanup unicode_encode_ucs1(): * Rename repunicode to rep * Clear rep object on error * Factorize code between bytes and unicode path --- Objects/stringlib/codecs.h | 18 +++++------- Objects/unicodeobject.c | 72 +++++++++++++++++++++++++--------------------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 7e8d928..2beb604 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -311,7 +311,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #if STRINGLIB_SIZEOF_CHAR > 1 else if (Py_UNICODE_IS_SURROGATE(ch)) { Py_ssize_t startpos, endpos, newpos; - Py_ssize_t repsize, k; + Py_ssize_t k; if (error_handler == _Py_ERROR_UNKNOWN) error_handler = get_error_handler(errors); @@ -392,20 +392,12 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, p = _PyBytesWriter_WriteBytes(&writer, p, PyBytes_AS_STRING(rep), PyBytes_GET_SIZE(rep)); - if (p == NULL) - goto error; } else { /* rep is unicode */ if (PyUnicode_READY(rep) < 0) goto error; - repsize = PyUnicode_GET_LENGTH(rep); - - p = _PyBytesWriter_Prepare(&writer, p, repsize); - if (p == NULL) - goto error; - if (!PyUnicode_IS_ASCII(rep)) { raise_encode_exception(&exc, "utf-8", unicode, @@ -415,9 +407,13 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); - memcpy(p, PyUnicode_DATA(rep), repsize); - p += repsize; + p = _PyBytesWriter_WriteBytes(&writer, p, + PyUnicode_DATA(rep), + PyUnicode_GET_LENGTH(rep)); } + + if (p == NULL) + goto error; Py_CLEAR(rep); i = newpos; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 23b8cc7..35df747 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6599,6 +6599,7 @@ unicode_encode_ucs1(PyObject *unicode, PyObject *error_handler_obj = NULL; PyObject *exc = NULL; _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; + PyObject *rep = NULL; /* output object */ _PyBytesWriter writer; @@ -6627,8 +6628,7 @@ unicode_encode_ucs1(PyObject *unicode, ++pos; } else { - PyObject *repunicode; - Py_ssize_t repsize, newpos, i; + Py_ssize_t newpos, i; /* startpos for collecting unencodable chars */ Py_ssize_t collstart = pos; Py_ssize_t collend = collstart + 1; @@ -6694,52 +6694,59 @@ unicode_encode_ucs1(PyObject *unicode, /* fallback to general error handling */ default: - repunicode = unicode_encode_call_errorhandler(errors, &error_handler_obj, - encoding, reason, unicode, &exc, - collstart, collend, &newpos); - if (repunicode == NULL || (PyUnicode_Check(repunicode) && - PyUnicode_READY(repunicode) == -1)) + rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, + encoding, reason, unicode, &exc, + collstart, collend, &newpos); + if (rep == NULL) goto onError; /* substract preallocated bytes */ writer.min_size -= 1; - if (PyBytes_Check(repunicode)) { + if (PyBytes_Check(rep)) { /* Directly copy bytes result to output. */ str = _PyBytesWriter_WriteBytes(&writer, str, - PyBytes_AS_STRING(repunicode), - PyBytes_GET_SIZE(repunicode)); + PyBytes_AS_STRING(rep), + PyBytes_GET_SIZE(rep)); if (str == NULL) goto onError; - - pos = newpos; - Py_DECREF(repunicode); - break; } + else { + assert(PyUnicode_Check(rep)); - /* need more space? (at least enough for what we - have+the replacement+the rest of the string, so - we won't have to check space for encodable characters) */ - repsize = PyUnicode_GET_LENGTH(repunicode); + if (PyUnicode_READY(rep) < 0) + goto onError; - str = _PyBytesWriter_Prepare(&writer, str, repsize); - if (str == NULL) - goto onError; + if (PyUnicode_IS_ASCII(rep)) { + /* Fast path: all characters are smaller than limit */ + assert(limit >= 128); + assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); + str = _PyBytesWriter_WriteBytes(&writer, str, + PyUnicode_DATA(rep), + PyUnicode_GET_LENGTH(rep)); + } + else { + Py_ssize_t repsize = PyUnicode_GET_LENGTH(rep); - /* check if there is anything unencodable in the replacement - and copy it to the output */ - for (i = 0; repsize-->0; ++i, ++str) { - ch = PyUnicode_READ_CHAR(repunicode, i); - if (ch >= limit) { - raise_encode_exception(&exc, encoding, unicode, - pos, pos+1, reason); - Py_DECREF(repunicode); - goto onError; + str = _PyBytesWriter_Prepare(&writer, str, repsize); + if (str == NULL) + goto onError; + + /* check if there is anything unencodable in the + replacement and copy it to the output */ + for (i = 0; repsize-->0; ++i, ++str) { + ch = PyUnicode_READ_CHAR(rep, i); + if (ch >= limit) { + raise_encode_exception(&exc, encoding, unicode, + pos, pos+1, reason); + goto onError; + } + *str = (char)ch; + } } - *str = (char)ch; } pos = newpos; - Py_DECREF(repunicode); + Py_CLEAR(rep); } /* If overallocation was disabled, ensure that it was the last @@ -6753,6 +6760,7 @@ unicode_encode_ucs1(PyObject *unicode, return _PyBytesWriter_Finish(&writer, str); onError: + Py_XDECREF(rep); _PyBytesWriter_Dealloc(&writer); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); -- cgit v0.12 From be75b8cf2310e0be0f32590a1c0df919058002e8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 22:43:24 +0200 Subject: Issue #25349: Optimize bytes % int Optimize bytes.__mod__(args) for integere formats: %d (%i, %u), %o, %x and %X. _PyBytesWriter is now used to format directly the integer into the writer buffer, instead of using a temporary bytes object. Formatting is between 30% and 50% faster on a microbenchmark. --- Include/longobject.h | 7 +++ Objects/bytesobject.c | 36 +++++++++++++++ Objects/longobject.c | 120 ++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 136 insertions(+), 27 deletions(-) diff --git a/Include/longobject.h b/Include/longobject.h index aed59ce..ab92495 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -182,6 +182,13 @@ PyAPI_FUNC(int) _PyLong_FormatWriter( int base, int alternate); +PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( + _PyBytesWriter *writer, + char *str, + PyObject *obj, + int base, + int alternate); + /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 3aa905c..fd46048 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -841,6 +841,42 @@ _PyBytes_Format(PyObject *format, PyObject *args) case 'o': case 'x': case 'X': + if (PyLong_CheckExact(v) + && width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK)) + && c != 'X') + { + /* Fast path */ + int alternate = flags & F_ALT; + int base; + + switch(c) + { + default: + assert(0 && "'type' not in [diuoxX]"); + case 'd': + case 'i': + case 'u': + base = 10; + break; + case 'o': + base = 8; + break; + case 'x': + case 'X': + base = 16; + break; + } + + /* Fast path */ + writer.min_size -= 2; /* size preallocated by "%d" */ + res = _PyLong_FormatBytesWriter(&writer, res, + v, base, alternate); + if (res == NULL) + goto error; + continue; + } + temp = formatlong(v, flags, prec, c); if (!temp) goto error; diff --git a/Objects/longobject.c b/Objects/longobject.c index 5f22455..00f5d95 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1582,7 +1582,9 @@ divrem1(PyLongObject *a, digit n, digit *prem) static int long_to_decimal_string_internal(PyObject *aa, PyObject **p_output, - _PyUnicodeWriter *writer) + _PyUnicodeWriter *writer, + _PyBytesWriter *bytes_writer, + char **bytes_str) { PyLongObject *scratch, *a; PyObject *str; @@ -1664,6 +1666,13 @@ long_to_decimal_string_internal(PyObject *aa, kind = writer->kind; str = NULL; } + else if (bytes_writer) { + *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); + if (*bytes_str == NULL) { + Py_DECREF(scratch); + return -1; + } + } else { str = PyUnicode_New(strlen, '9'); if (str == NULL) { @@ -1673,13 +1682,8 @@ long_to_decimal_string_internal(PyObject *aa, kind = PyUnicode_KIND(str); } -#define WRITE_DIGITS(TYPE) \ +#define WRITE_DIGITS(p) \ do { \ - if (writer) \ - p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + strlen; \ - else \ - p = (TYPE*)PyUnicode_DATA(str) + strlen; \ - \ /* pout[0] through pout[size-2] contribute exactly \ _PyLong_DECIMAL_SHIFT digits each */ \ for (i=0; i < size - 1; i++) { \ @@ -1699,6 +1703,16 @@ long_to_decimal_string_internal(PyObject *aa, /* and sign */ \ if (negative) \ *--p = '-'; \ + } while (0) + +#define WRITE_UNICODE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + strlen; \ + else \ + p = (TYPE*)PyUnicode_DATA(str) + strlen; \ + \ + WRITE_DIGITS(p); \ \ /* check we've counted correctly */ \ if (writer) \ @@ -1708,25 +1722,34 @@ long_to_decimal_string_internal(PyObject *aa, } while (0) /* fill the string right-to-left */ - if (kind == PyUnicode_1BYTE_KIND) { + if (bytes_writer) { + char *p = *bytes_str + strlen; + WRITE_DIGITS(p); + assert(p == *bytes_str); + } + else if (kind == PyUnicode_1BYTE_KIND) { Py_UCS1 *p; - WRITE_DIGITS(Py_UCS1); + WRITE_UNICODE_DIGITS(Py_UCS1); } else if (kind == PyUnicode_2BYTE_KIND) { Py_UCS2 *p; - WRITE_DIGITS(Py_UCS2); + WRITE_UNICODE_DIGITS(Py_UCS2); } else { Py_UCS4 *p; assert (kind == PyUnicode_4BYTE_KIND); - WRITE_DIGITS(Py_UCS4); + WRITE_UNICODE_DIGITS(Py_UCS4); } #undef WRITE_DIGITS +#undef WRITE_UNICODE_DIGITS Py_DECREF(scratch); if (writer) { writer->pos += strlen; } + else if (bytes_writer) { + (*bytes_str) += strlen; + } else { assert(_PyUnicode_CheckConsistency(str, 1)); *p_output = (PyObject *)str; @@ -1738,7 +1761,7 @@ static PyObject * long_to_decimal_string(PyObject *aa) { PyObject *v; - if (long_to_decimal_string_internal(aa, &v, NULL) == -1) + if (long_to_decimal_string_internal(aa, &v, NULL, NULL, NULL) == -1) return NULL; return v; } @@ -1750,7 +1773,8 @@ long_to_decimal_string(PyObject *aa) static int long_format_binary(PyObject *aa, int base, int alternate, - PyObject **p_output, _PyUnicodeWriter *writer) + PyObject **p_output, _PyUnicodeWriter *writer, + _PyBytesWriter *bytes_writer, char **bytes_str) { PyLongObject *a = (PyLongObject *)aa; PyObject *v; @@ -1812,6 +1836,11 @@ long_format_binary(PyObject *aa, int base, int alternate, kind = writer->kind; v = NULL; } + else if (writer) { + *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); + if (*bytes_str == NULL) + return -1; + } else { v = PyUnicode_New(sz, 'x'); if (v == NULL) @@ -1819,13 +1848,8 @@ long_format_binary(PyObject *aa, int base, int alternate, kind = PyUnicode_KIND(v); } -#define WRITE_DIGITS(TYPE) \ +#define WRITE_DIGITS(p) \ do { \ - if (writer) \ - p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + sz; \ - else \ - p = (TYPE*)PyUnicode_DATA(v) + sz; \ - \ if (size_a == 0) { \ *--p = '0'; \ } \ @@ -1860,30 +1884,50 @@ long_format_binary(PyObject *aa, int base, int alternate, } \ if (negative) \ *--p = '-'; \ + } while (0) + +#define WRITE_UNICODE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + sz; \ + else \ + p = (TYPE*)PyUnicode_DATA(v) + sz; \ + \ + WRITE_DIGITS(p); \ + \ if (writer) \ assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ else \ assert(p == (TYPE*)PyUnicode_DATA(v)); \ } while (0) - if (kind == PyUnicode_1BYTE_KIND) { + if (bytes_writer) { + char *p = *bytes_str + sz; + WRITE_DIGITS(p); + assert(p == *bytes_str); + } + else if (kind == PyUnicode_1BYTE_KIND) { Py_UCS1 *p; - WRITE_DIGITS(Py_UCS1); + WRITE_UNICODE_DIGITS(Py_UCS1); } else if (kind == PyUnicode_2BYTE_KIND) { Py_UCS2 *p; - WRITE_DIGITS(Py_UCS2); + WRITE_UNICODE_DIGITS(Py_UCS2); } else { Py_UCS4 *p; assert (kind == PyUnicode_4BYTE_KIND); - WRITE_DIGITS(Py_UCS4); + WRITE_UNICODE_DIGITS(Py_UCS4); } #undef WRITE_DIGITS +#undef WRITE_UNICODE_DIGITS if (writer) { writer->pos += sz; } + else if (bytes_writer) { + (*bytes_str) += sz; + } else { assert(_PyUnicode_CheckConsistency(v, 1)); *p_output = v; @@ -1897,9 +1941,9 @@ _PyLong_Format(PyObject *obj, int base) PyObject *str; int err; if (base == 10) - err = long_to_decimal_string_internal(obj, &str, NULL); + err = long_to_decimal_string_internal(obj, &str, NULL, NULL, NULL); else - err = long_format_binary(obj, base, 1, &str, NULL); + err = long_format_binary(obj, base, 1, &str, NULL, NULL, NULL); if (err == -1) return NULL; return str; @@ -1911,9 +1955,31 @@ _PyLong_FormatWriter(_PyUnicodeWriter *writer, int base, int alternate) { if (base == 10) - return long_to_decimal_string_internal(obj, NULL, writer); + return long_to_decimal_string_internal(obj, NULL, writer, + NULL, NULL); else - return long_format_binary(obj, base, alternate, NULL, writer); + return long_format_binary(obj, base, alternate, NULL, writer, + NULL, NULL); +} + +char* +_PyLong_FormatBytesWriter(_PyBytesWriter *writer, char *str, + PyObject *obj, + int base, int alternate) +{ + char *str2; + int res; + str2 = str; + if (base == 10) + res = long_to_decimal_string_internal(obj, NULL, NULL, + writer, &str2); + else + res = long_format_binary(obj, base, alternate, NULL, NULL, + writer, &str2); + if (res < 0) + return NULL; + assert(str2 != NULL); + return str2; } /* Table of digit values for 8-bit string -> integer conversion. -- cgit v0.12 From 0cdad1e2bc8794cdbba8a89e2a4177a7dedec313 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 22:50:36 +0200 Subject: Issue #25349: Add fast path for b'%c' % int Optimize also %% formater. --- Lib/test/test_format.py | 2 ++ Objects/bytesobject.c | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 9b13632..9924fde 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -300,6 +300,8 @@ class FormatTest(unittest.TestCase): testcommon(b"%c", 7, b"\x07") testcommon(b"%c", b"Z", b"Z") testcommon(b"%c", bytearray(b"Z"), b"Z") + testcommon(b"%5c", 65, b" A") + testcommon(b"%-5c", 65, b"A ") # %b will insert a series of bytes, either from a type that supports # the Py_buffer protocol, or something that has a __bytes__ method class FakeBytes(object): diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index fd46048..a75c54d 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -808,9 +808,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) fill = ' '; switch (c) { case '%': - pbuf = "%"; - len = 1; - break; + *res++ = '%'; + continue; case 'r': // %r is only for 2/3 code; 3 only code should use %a @@ -842,9 +841,9 @@ _PyBytes_Format(PyObject *format, PyObject *args) case 'x': case 'X': if (PyLong_CheckExact(v) - && width == -1 && prec == -1 - && !(flags & (F_SIGN | F_BLANK)) - && c != 'X') + && width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK)) + && c != 'X') { /* Fast path */ int alternate = flags & F_ALT; @@ -869,7 +868,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) } /* Fast path */ - writer.min_size -= 2; /* size preallocated by "%d" */ + writer.min_size -= 2; /* size preallocated for "%d" */ res = _PyLong_FormatBytesWriter(&writer, res, v, base, alternate); if (res == NULL) @@ -898,7 +897,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) && !(flags & (F_SIGN | F_BLANK))) { /* Fast path */ - writer.min_size -= 2; /* size preallocated by "%f" */ + writer.min_size -= 2; /* size preallocated for "%f" */ res = formatfloat(v, flags, prec, c, NULL, &writer, res); if (res == NULL) goto error; @@ -919,6 +918,11 @@ _PyBytes_Format(PyObject *format, PyObject *args) len = byte_converter(v, &onechar); if (!len) goto error; + if (width == -1) { + /* Fast path */ + *res++ = onechar; + continue; + } break; default: @@ -949,8 +953,9 @@ _PyBytes_Format(PyObject *format, PyObject *args) alloc = width; if (sign != 0 && len == width) alloc++; - if (alloc > 1) { - res = _PyBytesWriter_Prepare(&writer, res, alloc - 1); + /* 2: size preallocated for %s */ + if (alloc > 2) { + res = _PyBytesWriter_Prepare(&writer, res, alloc - 2); if (res == NULL) goto error; } -- cgit v0.12 From 0d554d7ef159761439ade414fcd028262eae656c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 10 Oct 2015 22:42:18 +0300 Subject: Issue #24164: Objects that need calling ``__new__`` with keyword arguments, can now be pickled using pickle protocols older than protocol version 4. --- Doc/whatsnew/3.6.rst | 9 ++++++ Lib/pickle.py | 17 +++++++---- Lib/test/pickletester.py | 6 ++-- Misc/NEWS | 3 ++ Modules/_pickle.c | 78 +++++++++++++++++++++++++++++++++++++++++------- Objects/typeobject.c | 16 ++-------- 6 files changed, 97 insertions(+), 32 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 50d2a6a..b735e9c 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -112,6 +112,15 @@ indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` (Contributed by Joe Jevnik in :issue:`24379`.) +pickle +------ + +Objects that need calling ``__new__`` with keyword arguments, can now be pickled +using :ref:`pickle protocols ` older than protocol version 4. +Protocol version 4 already supports this case. (Contributed by Serhiy +Storchaka in :issue:`24164`.) + + rlcomplete ---------- diff --git a/Lib/pickle.py b/Lib/pickle.py index e93057a..d41753d 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -27,6 +27,7 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache from itertools import islice +from functools import partial import sys from sys import maxsize from struct import pack, unpack @@ -544,7 +545,7 @@ class _Pickler: write = self.write func_name = getattr(func, "__name__", "") - if self.proto >= 4 and func_name == "__newobj_ex__": + if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): raise PicklingError("args[0] from {} args has no __new__" @@ -552,10 +553,16 @@ class _Pickler: if obj is not None and cls is not obj.__class__: raise PicklingError("args[0] from {} args has the wrong class" .format(func_name)) - save(cls) - save(args) - save(kwargs) - write(NEWOBJ_EX) + if self.proto >= 4: + save(cls) + save(args) + save(kwargs) + write(NEWOBJ_EX) + else: + func = partial(cls.__new__, cls, *args, **kwargs) + save(func) + save(()) + write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": # A __reduce__ implementation can direct protocol 2 or newer to # use the more efficient NEWOBJ opcode, while still diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 2ef48e6..fd641e3 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1580,16 +1580,14 @@ class AbstractPickleTests(unittest.TestCase): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): - if 2 <= proto < 4: - self.assertRaises(ValueError, self.dumps, x, proto) - continue s = self.dumps(x, proto) if proto < 1: self.assertIn(b'\nL64206', s) # LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 + elif proto < 4: + self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE else: - assert proto >= 4 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), diff --git a/Misc/NEWS b/Misc/NEWS index cb02aae..802f4b6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins Library ------- +- Issue #24164: Objects that need calling ``__new__`` with keyword arguments, + can now be pickled using pickle protocols older than protocol version 4. + - Issue #25364: zipfile now works in threads disabled builds. - Issue #25328: smtpd's SMTPChannel now correctly raises a ValueError if both diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f3b73f1..abaf4e5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -153,6 +153,9 @@ typedef struct { PyObject *codecs_encode; /* builtins.getattr, used for saving nested names with protocol < 4 */ PyObject *getattr; + /* functools.partial, used for implementing __newobj_ex__ with protocols + 2 and 3 */ + PyObject *partial; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -200,6 +203,7 @@ _Pickle_InitState(PickleState *st) PyObject *copyreg = NULL; PyObject *compat_pickle = NULL; PyObject *codecs = NULL; + PyObject *functools = NULL; builtins = PyEval_GetBuiltins(); if (builtins == NULL) @@ -314,12 +318,21 @@ _Pickle_InitState(PickleState *st) } Py_CLEAR(codecs); + functools = PyImport_ImportModule("functools"); + if (!functools) + goto error; + st->partial = PyObject_GetAttrString(functools, "partial"); + if (!st->partial) + goto error; + Py_CLEAR(functools); + return 0; error: Py_CLEAR(copyreg); Py_CLEAR(compat_pickle); Py_CLEAR(codecs); + Py_CLEAR(functools); _Pickle_ClearState(st); return -1; } @@ -3533,11 +3546,9 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) PyErr_Clear(); } else if (PyUnicode_Check(name)) { - if (self->proto >= 4) { - _Py_IDENTIFIER(__newobj_ex__); - use_newobj_ex = PyUnicode_Compare( - name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0; - } + _Py_IDENTIFIER(__newobj_ex__); + use_newobj_ex = PyUnicode_Compare( + name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0; if (!use_newobj_ex) { _Py_IDENTIFIER(__newobj__); use_newobj = PyUnicode_Compare( @@ -3581,11 +3592,58 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; } - if (save(self, cls, 0) < 0 || - save(self, args, 0) < 0 || - save(self, kwargs, 0) < 0 || - _Pickler_Write(self, &newobj_ex_op, 1) < 0) { - return -1; + if (self->proto >= 4) { + if (save(self, cls, 0) < 0 || + save(self, args, 0) < 0 || + save(self, kwargs, 0) < 0 || + _Pickler_Write(self, &newobj_ex_op, 1) < 0) { + return -1; + } + } + else { + PyObject *newargs; + PyObject *cls_new; + Py_ssize_t i; + _Py_IDENTIFIER(__new__); + + newargs = PyTuple_New(Py_SIZE(args) + 2); + if (newargs == NULL) + return -1; + + cls_new = _PyObject_GetAttrId(cls, &PyId___new__); + if (cls_new == NULL) { + Py_DECREF(newargs); + return -1; + } + PyTuple_SET_ITEM(newargs, 0, cls_new); + Py_INCREF(cls); + PyTuple_SET_ITEM(newargs, 1, cls); + for (i = 0; i < Py_SIZE(args); i++) { + PyObject *item = PyTuple_GET_ITEM(args, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newargs, i + 2, item); + } + + callable = PyObject_Call(st->partial, newargs, kwargs); + Py_DECREF(newargs); + if (callable == NULL) + return -1; + + newargs = PyTuple_New(0); + if (newargs == NULL) { + Py_DECREF(callable); + return -1; + } + + if (save(self, callable, 0) < 0 || + save(self, newargs, 0) < 0 || + _Pickler_Write(self, &reduce_op, 1) < 0) { + Py_DECREF(newargs); + Py_DECREF(callable); + return -1; + } + Py_DECREF(newargs); + Py_DECREF(callable); } } else if (use_newobj) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4b091f5..bf0d30c 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4101,7 +4101,7 @@ _PyObject_GetItemsIter(PyObject *obj, PyObject **listitems, } static PyObject * -reduce_newobj(PyObject *obj, int proto) +reduce_newobj(PyObject *obj) { PyObject *args = NULL, *kwargs = NULL; PyObject *copyreg; @@ -4153,7 +4153,7 @@ reduce_newobj(PyObject *obj, int proto) } Py_DECREF(args); } - else if (proto >= 4) { + else { _Py_IDENTIFIER(__newobj_ex__); newobj = _PyObject_GetAttrId(copyreg, &PyId___newobj_ex__); @@ -4171,16 +4171,6 @@ reduce_newobj(PyObject *obj, int proto) return NULL; } } - else { - PyErr_SetString(PyExc_ValueError, - "must use protocol 4 or greater to copy this " - "object; since __getnewargs_ex__ returned " - "keyword arguments."); - Py_DECREF(args); - Py_DECREF(kwargs); - Py_DECREF(copyreg); - return NULL; - } state = _PyObject_GetState(obj); if (state == NULL) { @@ -4225,7 +4215,7 @@ _common_reduce(PyObject *self, int proto) PyObject *copyreg, *res; if (proto >= 2) - return reduce_newobj(self, proto); + return reduce_newobj(self); copyreg = import_copyreg(); if (!copyreg) -- cgit v0.12 From a7f63009d6df9c6ae57e6835857d6c5fda024930 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 10 Oct 2015 23:56:02 -0400 Subject: Minor tweak. Make the maxlen comparisons a little more clear and consistent. --- Modules/_collectionsmodule.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 4ea9140..52f40b2 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -281,7 +281,7 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); static void deque_trim_right(dequeobject *deque) { - if (deque->maxlen != -1 && Py_SIZE(deque) > deque->maxlen) { + if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { PyObject *rv = deque_pop(deque, NULL); assert(rv != NULL); assert(Py_SIZE(deque) <= deque->maxlen); @@ -292,7 +292,7 @@ deque_trim_right(dequeobject *deque) static void deque_trim_left(dequeobject *deque) { - if (deque->maxlen != -1 && Py_SIZE(deque) > deque->maxlen) { + if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { PyObject *rv = deque_popleft(deque, NULL); assert(rv != NULL); assert(Py_SIZE(deque) <= deque->maxlen); @@ -385,7 +385,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen != -1); + int trim = (deque->maxlen >= 0); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -447,7 +447,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen != -1); + int trim = (deque->maxlen >= 0); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -686,7 +686,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) /* common case, repeating a single element */ PyObject *item = deque->leftblock->data[deque->leftindex]; - if (deque->maxlen != -1 && n > deque->maxlen) + if (deque->maxlen >= 0 && n > deque->maxlen) n = deque->maxlen; if (n > MAX_DEQUE_LEN) @@ -1355,7 +1355,7 @@ deque_repr(PyObject *deque) Py_ReprLeave(deque); return NULL; } - if (((dequeobject *)deque)->maxlen != -1) + if (((dequeobject *)deque)->maxlen >= 0) result = PyUnicode_FromFormat("deque(%R, maxlen=%zd)", aslist, ((dequeobject *)deque)->maxlen); else -- cgit v0.12 From 647dac9d6fc08b9fad14c10dabe7cc8ee48c8553 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 09:47:17 +0200 Subject: Close #25368: Fix test_eintr when Python is compiled without thread support --- Lib/test/eintrdata/eintr_tester.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 443ccd5..9de2b6b 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -52,7 +52,8 @@ class EINTRBaseTest(unittest.TestCase): cls.signal_period) # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD - faulthandler.dump_traceback_later(10 * 60, exit=True) + if hasattr(faulthandler, 'dump_traceback_later'): + faulthandler.dump_traceback_later(10 * 60, exit=True) @classmethod def stop_alarm(cls): @@ -62,7 +63,8 @@ class EINTRBaseTest(unittest.TestCase): def tearDownClass(cls): cls.stop_alarm() signal.signal(signal.SIGALRM, cls.orig_handler) - faulthandler.cancel_dump_traceback_later() + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def _sleep(cls): -- cgit v0.12 From 92f0113701c6404f9e1ec90e3260084fc6a2ef09 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 09:54:42 +0200 Subject: Close #24784: Fix compilation without thread support Add "#ifdef WITH_THREAD" around cals to: * PyGILState_Check() * _PyImport_AcquireLock() * _PyImport_ReleaseLock() --- Modules/_posixsubprocess.c | 12 ++++++++++-- Modules/socketmodule.c | 2 ++ Python/fileutils.c | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index a327fc5..800b301 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -549,7 +549,9 @@ subprocess_fork_exec(PyObject* self, PyObject *args) int need_to_reenable_gc = 0; char *const *exec_array, *const *argv = NULL, *const *envp = NULL; Py_ssize_t arg_num; +#ifdef WITH_THREAD int import_lock_held = 0; +#endif if (!PyArg_ParseTuple( args, "OOpOOOiiiiiiiiiiO:fork_exec", @@ -644,8 +646,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) preexec_fn_args_tuple = PyTuple_New(0); if (!preexec_fn_args_tuple) goto cleanup; +#ifdef WITH_THREAD _PyImport_AcquireLock(); import_lock_held = 1; +#endif } if (cwd_obj != Py_None) { @@ -688,12 +692,14 @@ subprocess_fork_exec(PyObject* self, PyObject *args) /* Capture the errno exception before errno can be clobbered. */ PyErr_SetFromErrno(PyExc_OSError); } - if (preexec_fn != Py_None && - _PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) { +#ifdef WITH_THREAD + if (preexec_fn != Py_None + && _PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); } import_lock_held = 0; +#endif /* Parent process */ if (envp) @@ -716,8 +722,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) return PyLong_FromPid(pid); cleanup: +#ifdef WITH_THREAD if (import_lock_held) _PyImport_ReleaseLock(); +#endif if (envp) _Py_FreeCharPArray(envp); if (argv) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index d9c70f8..bae9634 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -719,8 +719,10 @@ sock_call_ex(PySocketSockObject *s, int deadline_initialized = 0; int res; +#ifdef WITH_THREAD /* sock_call() must be called with the GIL held. */ assert(PyGILState_Check()); +#endif /* outer loop to retry select() when select() is interrupted by a signal or to retry select()+sock_func() on false positive (see above) */ diff --git a/Python/fileutils.c b/Python/fileutils.c index bccd321..079918c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -986,8 +986,10 @@ _Py_open_impl(const char *pathname, int flags, int gil_held) int _Py_open(const char *pathname, int flags) { +#ifdef WITH_THREAD /* _Py_open() must be called with the GIL held. */ assert(PyGILState_Check()); +#endif return _Py_open_impl(pathname, flags, 1); } @@ -1080,7 +1082,9 @@ _Py_fopen_obj(PyObject *path, const char *mode) wchar_t wmode[10]; int usize; +#ifdef WITH_THREAD assert(PyGILState_Check()); +#endif if (!PyUnicode_Check(path)) { PyErr_Format(PyExc_TypeError, @@ -1108,7 +1112,9 @@ _Py_fopen_obj(PyObject *path, const char *mode) PyObject *bytes; char *path_bytes; +#ifdef WITH_THREAD assert(PyGILState_Check()); +#endif if (!PyUnicode_FSConverter(path, &bytes)) return NULL; -- cgit v0.12 From 4967146c8d6f68480c5ca4ed73940607dd1d9286 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:03:28 +0200 Subject: Close #25369: Fix test_regrtest without thread support --- Lib/test/test_regrtest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2e33555..0f00698 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,6 +7,7 @@ Note: test_regrtest cannot be run twice in parallel. import argparse import faulthandler import getopt +import io import os.path import platform import re @@ -433,7 +434,9 @@ class ProgramsTestCase(BaseTestCase): self.tests = [self.create_test() for index in range(self.NTEST)] self.python_args = ['-Wd', '-E', '-bb'] - self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4'] + self.regrtest_args = ['-uall', '-rwW'] + if hasattr(faulthandler, 'dump_traceback_later'): + self.regrtest_args.extend(('--timeout', '3600', '-j4')) if sys.platform == 'win32': self.regrtest_args.append('-n') -- cgit v0.12 From 14b4662e18189132e343781070290bf5729452b5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:04:26 +0200 Subject: test_regrtest: catch stderr in test_nowindows() Check also that the deprecation warning is emited. --- Lib/test/test_regrtest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0f00698..16a29b1 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -5,6 +5,7 @@ Note: test_regrtest cannot be run twice in parallel. """ import argparse +import contextlib import faulthandler import getopt import io @@ -247,8 +248,11 @@ class ParseArgsTestCase(unittest.TestCase): def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.nowindows) + err = stderr.getvalue() + self.assertIn('the --nowindows (-n) option is deprecated', err) def test_forever(self): for opt in '-F', '--forever': -- cgit v0.12 From 3909e58994f929abacc20e1f32b63e30109f942b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:37:25 +0200 Subject: Close #25373: Fix regrtest --slow with interrupted test * Fix accumulate_result(): don't use time on interrupted and failed test * Add unit test for interrupted test * Add unit test on --slow with interrupted test, with and without multiprocessing --- Lib/test/libregrtest/main.py | 12 ++++++++---- Lib/test/test_regrtest.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index aa95b21..82788ad 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,10 +7,11 @@ import sys import sysconfig import tempfile import textwrap +from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, - STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) -from test.libregrtest.cmdline import _parse_args + STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, + INTERRUPTED, CHILD_ERROR) from test.libregrtest.setup import setup_tests from test import support try: @@ -87,7 +88,8 @@ class Regrtest: def accumulate_result(self, test, result): ok, test_time = result - self.test_times.append((test_time, test)) + if ok not in (CHILD_ERROR, INTERRUPTED): + self.test_times.append((test_time, test)) if ok == PASSED: self.good.append(test) elif ok == FAILED: @@ -291,10 +293,12 @@ class Regrtest: else: try: result = runtest(self.ns, test) - self.accumulate_result(test, result) except KeyboardInterrupt: + self.accumulate_result(test, (INTERRUPTED, None)) self.interrupted = True break + else: + self.accumulate_result(test, result) if self.ns.findleaks: gc.collect() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 16a29b1..ab7741f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -24,6 +24,16 @@ Py_DEBUG = hasattr(sys, 'getobjects') ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) +TEST_INTERRUPTED = textwrap.dedent(""" + from signal import SIGINT + try: + from _testcapi import raise_signal + raise_signal(SIGINT) + except ImportError: + import os + os.kill(os.getpid(), SIGINT) + """) + class ParseArgsTestCase(unittest.TestCase): """ @@ -340,16 +350,19 @@ class BaseTestCase(unittest.TestCase): return list(match.group(1) for match in parser) def check_executed_tests(self, output, tests, skipped=(), failed=(), - randomize=False): + omitted=(), randomize=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] if isinstance(failed, str): failed = [failed] + if isinstance(omitted, str): + omitted = [omitted] ntest = len(tests) nskipped = len(skipped) nfailed = len(failed) + nomitted = len(omitted) executed = self.parse_executed_tests(output) if randomize: @@ -375,7 +388,11 @@ class BaseTestCase(unittest.TestCase): regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) - good = ntest - nskipped - nfailed + if omitted: + regex = list_regex('%s test%s omitted', omitted) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed - nomitted if good: regex = r'%s test%s OK\.$' % (good, plural(good)) if not skipped and not failed and good > 1: @@ -607,6 +624,12 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + def test_interrupted(self): + code = TEST_INTERRUPTED + test = self.create_test("sigint", code=code) + output = self.run_tests(test, exitcode=1) + self.check_executed_tests(output, test, omitted=test) + def test_slow(self): # test --slow tests = [self.create_test() for index in range(3)] @@ -617,6 +640,22 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) + def test_slow_interrupted(self): + # Issue #25373: test --slow with an interrupted test + code = TEST_INTERRUPTED + test = self.create_test("sigint", code=code) + + for multiprocessing in (False, True): + if multiprocessing: + args = ("--slow", "-j2", test) + else: + args = ("--slow", test) + output = self.run_tests(*args, exitcode=1) + self.check_executed_tests(output, test, omitted=test) + regex = ('10 slowest tests:\n') + self.check_line(output, regex) + self.check_line(output, 'Test suite interrupted by signal SIGINT.') + def test_coverage(self): # test --coverage test = self.create_test() -- cgit v0.12 From e84c97656899750a8e6b01bef2cfc01b31d60220 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 11:01:02 +0200 Subject: Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). base64.b64encode() uses it to avoid a memory copy. --- Doc/library/binascii.rst | 11 ++++++++--- Lib/base64.py | 3 +-- Lib/test/test_binascii.py | 10 ++++++++++ Misc/NEWS | 3 +++ Modules/binascii.c | 21 +++++++++++++-------- Modules/clinic/binascii.c.h | 17 ++++++++++------- 6 files changed, 45 insertions(+), 20 deletions(-) diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index e3f134b..441aa57 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -52,11 +52,16 @@ The :mod:`binascii` module defines the following functions: than one line may be passed at a time. -.. function:: b2a_base64(data) +.. function:: b2a_base64(data, \*, newline=True) Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char. The length of *data* - should be at most 57 to adhere to the base64 standard. + value is the converted line, including a newline char if *newline* is + true. The length of *data* should be at most 57 to adhere to the + base64 standard. + + + .. versionchanged:: 3.6 + Added the *newline* parameter. .. function:: a2b_qp(data, header=False) diff --git a/Lib/base64.py b/Lib/base64.py index 640f787..eb925cd 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -58,8 +58,7 @@ def b64encode(s, altchars=None): The encoded byte string is returned. """ - # Strip off the trailing newline - encoded = binascii.b2a_base64(s)[:-1] + encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: assert len(altchars) == 2, repr(altchars) return encoded.translate(bytes.maketrans(b'+/', altchars)) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 8367afe..0ab46bc 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -262,6 +262,16 @@ class BinASCIITest(unittest.TestCase): # non-ASCII string self.assertRaises(ValueError, a2b, "\x80") + def test_b2a_base64_newline(self): + # Issue #25357: test newline parameter + b = self.type2test(b'hello') + self.assertEqual(binascii.b2a_base64(b), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=True), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=False), + b'aGVsbG8=') + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Misc/NEWS b/Misc/NEWS index 802f4b6..e84ad1b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins Library ------- +- Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). + base64.b64encode() uses it to avoid a memory copy. + - Issue #24164: Objects that need calling ``__new__`` with keyword arguments, can now be pickled using pickle protocols older than protocol version 4. diff --git a/Modules/binascii.c b/Modules/binascii.c index d920d23..3e26a7a 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -528,21 +528,22 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) binascii.b2a_base64 data: Py_buffer - / + * + newline: int(c_default="1") = True Base64-code line of data. [clinic start generated code]*/ static PyObject * -binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data) -/*[clinic end generated code: output=3cd61fbee2913285 input=14ec4e47371174a9]*/ +binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) +/*[clinic end generated code: output=19e1dd719a890b50 input=7b2ea6fa38d8924c]*/ { unsigned char *ascii_data, *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; PyObject *rv; - Py_ssize_t bin_len; + Py_ssize_t bin_len, out_len; bin_data = data->buf; bin_len = data->len; @@ -555,9 +556,12 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data) } /* We're lazy and allocate too much (fixed up later). - "+3" leaves room for up to two pad characters and a trailing - newline. Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ - if ( (rv=PyBytes_FromStringAndSize(NULL, bin_len*2 + 3)) == NULL ) + "+2" leaves room for up to two pad characters. + Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ + out_len = bin_len*2 + 2; + if (newline) + out_len++; + if ( (rv=PyBytes_FromStringAndSize(NULL, out_len)) == NULL ) return NULL; ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); @@ -581,7 +585,8 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data) *ascii_data++ = table_b2a_base64[(leftchar&0xf) << 2]; *ascii_data++ = BASE64_PAD; } - *ascii_data++ = '\n'; /* Append a courtesy newline */ + if (newline) + *ascii_data++ = '\n'; /* Append a courtesy newline */ if (_PyBytes_Resize(&rv, (ascii_data - diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index e348bee..46cfb8e 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -93,26 +93,29 @@ exit: } PyDoc_STRVAR(binascii_b2a_base64__doc__, -"b2a_base64($module, data, /)\n" +"b2a_base64($module, /, data, *, newline=True)\n" "--\n" "\n" "Base64-code line of data."); #define BINASCII_B2A_BASE64_METHODDEF \ - {"b2a_base64", (PyCFunction)binascii_b2a_base64, METH_O, binascii_b2a_base64__doc__}, + {"b2a_base64", (PyCFunction)binascii_b2a_base64, METH_VARARGS|METH_KEYWORDS, binascii_b2a_base64__doc__}, static PyObject * -binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data); +binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline); static PyObject * -binascii_b2a_base64(PyModuleDef *module, PyObject *arg) +binascii_b2a_base64(PyModuleDef *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; + static char *_keywords[] = {"data", "newline", NULL}; Py_buffer data = {NULL, NULL}; + int newline = 1; - if (!PyArg_Parse(arg, "y*:b2a_base64", &data)) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|$i:b2a_base64", _keywords, + &data, &newline)) goto exit; - return_value = binascii_b2a_base64_impl(module, &data); + return_value = binascii_b2a_base64_impl(module, &data, newline); exit: /* Cleanup for data */ @@ -516,4 +519,4 @@ exit: return return_value; } -/*[clinic end generated code: output=b1a3cbf7660ebaa5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b15a24350d105251 input=a9049054013a1b77]*/ -- cgit v0.12 From 20d15b5100bc12b21c561ee1ea70d8436f944d98 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 11 Oct 2015 17:52:09 +0300 Subject: Issue #24164: Fixed test_descr: __getnewargs_ex__ now is supported in protocols 2 and 3. --- Lib/test/test_descr.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index d096390..d751099 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4738,11 +4738,8 @@ class PicklingTests(unittest.TestCase): return (args, kwargs) obj = C3() for proto in protocols: - if proto >= 4: + if proto >= 2: self._check_reduce(proto, obj, args, kwargs) - elif proto >= 2: - with self.assertRaises(ValueError): - obj.__reduce_ex__(proto) class C4: def __getnewargs_ex__(self): @@ -5061,10 +5058,6 @@ class PicklingTests(unittest.TestCase): kwargs = getattr(cls, 'KWARGS', {}) obj = cls(*cls.ARGS, **kwargs) proto = pickle_copier.proto - if 2 <= proto < 4 and hasattr(cls, '__getnewargs_ex__'): - with self.assertRaises(ValueError): - pickle_copier.dumps(obj, proto) - continue objcopy = pickle_copier.copy(obj) self._assert_is_copy(obj, objcopy) # For test classes that supports this, make sure we didn't go -- cgit v0.12 From 6b1e113f9f142a0e531d0ae3d45cc31341fc9949 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 11 Oct 2015 09:43:50 -0700 Subject: Hoist the deque->maxlen lookup out of the inner-loop. --- Modules/_collectionsmodule.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 52f40b2..8cd22d7 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -385,7 +385,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen >= 0); + Py_ssize_t maxlen = deque->maxlen; /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -433,8 +433,10 @@ deque_extend(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - if (trim) - deque_trim_left(deque); + if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + PyObject *rv = deque_popleft(deque, NULL); + Py_DECREF(rv); + } } return finalize_iterator(it); } @@ -447,7 +449,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen >= 0); + Py_ssize_t maxlen = deque->maxlen; /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -495,8 +497,10 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - if (trim) - deque_trim_right(deque); + if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + PyObject *rv = deque_pop(deque, NULL); + Py_DECREF(rv); + } } return finalize_iterator(it); } -- cgit v0.12 From d96db09b570bff9e4397530082e33dd3e965a701 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 11 Oct 2015 22:34:48 -0700 Subject: Refactor the deque trim logic to eliminate the two separate trim functions. --- Modules/_collectionsmodule.c | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 8cd22d7..3c8e025 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -276,29 +276,12 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); * the limit. If it has, we get the size back down to the limit by popping an * item off of the opposite end. The methods that can trigger this are append(), * appendleft(), extend(), and extendleft(). + * + * The macro to check whether a deque needs to be trimmed uses a single + * unsigned test that returns true whenever 0 <= maxlen < Py_SIZE(deque). */ -static void -deque_trim_right(dequeobject *deque) -{ - if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { - PyObject *rv = deque_pop(deque, NULL); - assert(rv != NULL); - assert(Py_SIZE(deque) <= deque->maxlen); - Py_DECREF(rv); - } -} - -static void -deque_trim_left(dequeobject *deque) -{ - if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { - PyObject *rv = deque_popleft(deque, NULL); - assert(rv != NULL); - assert(Py_SIZE(deque) <= deque->maxlen); - Py_DECREF(rv); - } -} +#define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) static PyObject * deque_append(dequeobject *deque, PyObject *item) @@ -319,7 +302,10 @@ deque_append(dequeobject *deque, PyObject *item) Py_INCREF(item); deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - deque_trim_left(deque); + if (NEEDS_TRIM(deque, deque->maxlen)) { + PyObject *rv = deque_popleft(deque, NULL); + Py_DECREF(rv); + } Py_RETURN_NONE; } @@ -344,7 +330,10 @@ deque_appendleft(dequeobject *deque, PyObject *item) Py_INCREF(item); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - deque_trim_right(deque); + if (NEEDS_TRIM(deque, deque->maxlen)) { + PyObject *rv = deque_pop(deque, NULL); + Py_DECREF(rv); + } Py_RETURN_NONE; } @@ -433,7 +422,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + if (NEEDS_TRIM(deque, maxlen)) { PyObject *rv = deque_popleft(deque, NULL); Py_DECREF(rv); } @@ -497,7 +486,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + if (NEEDS_TRIM(deque, maxlen)) { PyObject *rv = deque_pop(deque, NULL); Py_DECREF(rv); } -- cgit v0.12 From 965362e92d50d8c4e7e0bfac46f7ac14ee26bbae Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 11 Oct 2015 22:52:54 -0700 Subject: Minor fixup. maxlen is already known. --- Modules/_collectionsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 3c8e025..cef92c0 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -399,7 +399,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (it == NULL) return NULL; - if (deque->maxlen == 0) + if (maxlen == 0) return consume_iterator(it); iternext = *Py_TYPE(it)->tp_iternext; @@ -463,7 +463,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (it == NULL) return NULL; - if (deque->maxlen == 0) + if (maxlen == 0) return consume_iterator(it); iternext = *Py_TYPE(it)->tp_iternext; -- cgit v0.12 From c29e29bed1ea1efc1a0cd3178fac96be4d763ecf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 13:12:54 +0200 Subject: Relax _PyBytesWriter API Don't require _PyBytesWriter pointer to be a "char *". Same change for _PyBytesWriter_WriteBytes() parameter. For example, binascii uses "unsigned char*". --- Include/bytesobject.h | 14 +++++++------- Objects/bytesobject.c | 15 +++++++-------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 2c4c4c4..b7a7c36 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -156,7 +156,7 @@ PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); Return a bytes object. Raise an exception and return NULL on error. */ PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - char *str); + void *str); /* Deallocate memory of a writer (clear its internal buffer). */ PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); @@ -164,22 +164,22 @@ PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); /* Allocate the buffer to write size bytes. Return the pointer to the beginning of buffer data. Raise an exception and return NULL on error. */ -PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, +PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size); /* Add *size* bytes to the buffer. str is the current pointer inside the buffer. Return the updated current pointer inside the buffer. Raise an exception and return NULL on error. */ -PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - char *str, +PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, + void *str, Py_ssize_t size); /* Write bytes. Raise an exception and return NULL on error. */ -PyAPI_FUNC(char*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - char *str, - char *bytes, +PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, + void *str, + const void *bytes, Py_ssize_t size); #endif /* Py_LIMITED_API */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a75c54d..4b31271 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3923,8 +3923,8 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) #endif } -char* -_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) +void* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) { Py_ssize_t allocated, pos; @@ -3992,7 +3992,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) /* Allocate the buffer to write size bytes. Return the pointer to the beginning of buffer data. Raise an exception and return NULL on error. */ -char* +void* _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) { /* ensure that _PyBytesWriter_Alloc() is only called once */ @@ -4011,7 +4011,7 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) } PyObject * -_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) +_PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) { Py_ssize_t pos; PyObject *result; @@ -4033,13 +4033,12 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) else { result = PyBytes_FromStringAndSize(writer->small_buffer, pos); } - return result; } -char* -_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, char *str, - char *bytes, Py_ssize_t size) +void* +_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *str, + const void *bytes, Py_ssize_t size) { str = _PyBytesWriter_Prepare(writer, str, size); if (str == NULL) -- cgit v0.12 From 6c2cdae9e6b664a0b3e204773aeb8954ae1508e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 13:29:43 +0200 Subject: Writer APIs: use empty string singletons Modify _PyBytesWriter_Finish() and _PyUnicodeWriter_Finish() to return the empty bytes/Unicode string if the string is empty. --- Objects/bytesobject.c | 23 ++++++++++++++--------- Objects/unicodeobject.c | 27 ++++++++++++++++++--------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4b31271..532051e 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -4019,19 +4019,24 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) _PyBytesWriter_CheckConsistency(writer, str); pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_small_buffer) { + if (pos == 0) { + Py_CLEAR(writer->buffer); + /* Get the empty byte string singleton */ + result = PyBytes_FromStringAndSize(NULL, 0); + } + else if (writer->use_small_buffer) { + result = PyBytes_FromStringAndSize(writer->small_buffer, pos); + } + else { + result = writer->buffer; + writer->buffer = NULL; + if (pos != writer->allocated) { - if (_PyBytes_Resize(&writer->buffer, pos)) { - assert(writer->buffer == NULL); + if (_PyBytes_Resize(&result, pos)) { + assert(result == NULL); return NULL; } } - - result = writer->buffer; - writer->buffer = NULL; - } - else { - result = PyBytes_FromStringAndSize(writer->small_buffer, pos); } return result; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 35df747..4b3746c 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13715,17 +13715,26 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) assert(PyUnicode_GET_LENGTH(str) == writer->pos); return str; } - if (PyUnicode_GET_LENGTH(writer->buffer) != writer->pos) { - PyObject *newbuffer; - newbuffer = resize_compact(writer->buffer, writer->pos); - if (newbuffer == NULL) { - Py_CLEAR(writer->buffer); - return NULL; + if (writer->pos == 0) { + Py_CLEAR(writer->buffer); + + /* Get the empty Unicode string singleton ('') */ + _Py_INCREF_UNICODE_EMPTY(); + str = unicode_empty; + } + else { + str = writer->buffer; + writer->buffer = NULL; + + if (PyUnicode_GET_LENGTH(str) != writer->pos) { + PyObject *str2; + str2 = resize_compact(str, writer->pos); + if (str2 == NULL) + return NULL; + str = str2; } - writer->buffer = newbuffer; } - str = writer->buffer; - writer->buffer = NULL; + assert(_PyUnicode_CheckConsistency(str, 1)); return unicode_result_ready(str); } -- cgit v0.12 From e9aa5950bb6ae7eab553e915402eddcb610565e4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 13:57:47 +0200 Subject: Fix compilation error in _PyBytesWriter_WriteBytes() on Windows --- Objects/bytesobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 532051e..e7ab503 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -4042,9 +4042,11 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) } void* -_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *str, +_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *ptr, const void *bytes, Py_ssize_t size) { + char *str = (char *)ptr; + str = _PyBytesWriter_Prepare(writer, str, size); if (str == NULL) return NULL; -- cgit v0.12 From d65e4f4eea278357e5aaee9f510922ef83e04143 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 14:38:24 +0200 Subject: Issue #24164: Fix test_pyclbr Ignore pickle.partial symbol which comes from functools.partial. --- Lib/test/test_pyclbr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index cab430b..48bd725 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -156,7 +156,7 @@ class PyclbrTest(TestCase): # These were once about the 10 longest modules cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator cm('cgi', ignore=('log',)) # set with = in module - cm('pickle') + cm('pickle', ignore=('partial',)) cm('aifc', ignore=('openfp', '_aifc_params')) # set with = in module cm('sre_parse', ignore=('dump', 'groups')) # from sre_constants import *; property cm('pdb') -- cgit v0.12 From 358af1352689fc10c81690a193ff5414f5f930af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 22:36:57 +0200 Subject: Issue #25353: Optimize unicode escape and raw unicode escape encoders to use the new _PyBytesWriter API. --- Modules/_pickle.c | 44 ++++++++++--------- Objects/unicodeobject.c | 113 +++++++++++++++++++++++++++++------------------- 2 files changed, 93 insertions(+), 64 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index abaf4e5..341ac0d 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2110,38 +2110,35 @@ save_bytes(PicklerObject *self, PyObject *obj) static PyObject * raw_unicode_escape(PyObject *obj) { - PyObject *repr; char *p; Py_ssize_t i, size; - size_t expandsize; void *data; unsigned int kind; + _PyBytesWriter writer; if (PyUnicode_READY(obj)) return NULL; + _PyBytesWriter_Init(&writer); + size = PyUnicode_GET_LENGTH(obj); data = PyUnicode_DATA(obj); kind = PyUnicode_KIND(obj); - if (kind == PyUnicode_4BYTE_KIND) - expandsize = 10; - else - expandsize = 6; - if ((size_t)size > (size_t)PY_SSIZE_T_MAX / expandsize) - return PyErr_NoMemory(); - repr = PyBytes_FromStringAndSize(NULL, expandsize * size); - if (repr == NULL) - return NULL; - if (size == 0) - return repr; - assert(Py_REFCNT(repr) == 1); + p = _PyBytesWriter_Alloc(&writer, size); + if (p == NULL) + goto error; + writer.overallocate = 1; - p = PyBytes_AS_STRING(repr); for (i=0; i < size; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 10-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'U'; *p++ = Py_hexdigits[(ch >> 28) & 0xf]; @@ -2153,8 +2150,13 @@ raw_unicode_escape(PyObject *obj) *p++ = Py_hexdigits[(ch >> 4) & 0xf]; *p++ = Py_hexdigits[ch & 15]; } - /* Map 16-bit characters to '\uxxxx' */ + /* Map 16-bit characters, '\\' and '\n' to '\uxxxx' */ else if (ch >= 256 || ch == '\\' || ch == '\n') { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 6-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0xf]; @@ -2166,10 +2168,12 @@ raw_unicode_escape(PyObject *obj) else *p++ = (char) ch; } - size = p - PyBytes_AS_STRING(repr); - if (_PyBytes_Resize(&repr, size) < 0) - return NULL; - return repr; + + return _PyBytesWriter_Finish(&writer, p); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } static int diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4b3746c..f5044c8 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6052,11 +6052,10 @@ PyObject * PyUnicode_AsUnicodeEscapeString(PyObject *unicode) { Py_ssize_t i, len; - PyObject *repr; char *p; int kind; void *data; - Py_ssize_t expandsize = 0; + _PyBytesWriter writer; /* Initial allocation is based on the longest-possible character escape. @@ -6072,35 +6071,28 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) } if (PyUnicode_READY(unicode) == -1) return NULL; + + _PyBytesWriter_Init(&writer); + len = PyUnicode_GET_LENGTH(unicode); kind = PyUnicode_KIND(unicode); data = PyUnicode_DATA(unicode); - switch (kind) { - case PyUnicode_1BYTE_KIND: expandsize = 4; break; - case PyUnicode_2BYTE_KIND: expandsize = 6; break; - case PyUnicode_4BYTE_KIND: expandsize = 10; break; - } - - if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); - - if (len > (PY_SSIZE_T_MAX - 2 - 1) / expandsize) - return PyErr_NoMemory(); - repr = PyBytes_FromStringAndSize(NULL, - 2 - + expandsize*len - + 1); - if (repr == NULL) - return NULL; - - p = PyBytes_AS_STRING(repr); + p = _PyBytesWriter_Alloc(&writer, len); + if (p == NULL) + goto error; + writer.overallocate = 1; for (i = 0; i < len; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Escape backslashes */ if (ch == '\\') { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = (char) ch; continue; @@ -6109,6 +6101,11 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) /* Map 21-bit characters to '\U00xxxxxx' */ else if (ch >= 0x10000) { assert(ch <= MAX_UNICODE); + + p = _PyBytesWriter_Prepare(&writer, p, 10-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'U'; *p++ = Py_hexdigits[(ch >> 28) & 0x0000000F]; @@ -6124,6 +6121,10 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) /* Map 16-bit characters to '\uxxxx' */ if (ch >= 256) { + p = _PyBytesWriter_Prepare(&writer, p, 6-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0x000F]; @@ -6134,20 +6135,37 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) /* Map special whitespace to '\t', \n', '\r' */ else if (ch == '\t') { + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 't'; } else if (ch == '\n') { + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'n'; } else if (ch == '\r') { + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'r'; } /* Map non-printable US ASCII to '\xhh' */ else if (ch < ' ' || ch >= 0x7F) { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 4-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'x'; *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; @@ -6159,10 +6177,11 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) *p++ = (char) ch; } - assert(p - PyBytes_AS_STRING(repr) > 0); - if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) - return NULL; - return repr; + return _PyBytesWriter_Finish(&writer, p); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } PyObject * @@ -6291,13 +6310,12 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, PyObject * PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) { - PyObject *repr; char *p; - char *q; - Py_ssize_t expandsize, pos; + Py_ssize_t pos; int kind; void *data; Py_ssize_t len; + _PyBytesWriter writer; if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); @@ -6305,28 +6323,29 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) } if (PyUnicode_READY(unicode) == -1) return NULL; + + _PyBytesWriter_Init(&writer); + kind = PyUnicode_KIND(unicode); data = PyUnicode_DATA(unicode); len = PyUnicode_GET_LENGTH(unicode); - /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 - bytes, and 1 byte characters 4. */ - expandsize = kind * 2 + 2; - if (len > PY_SSIZE_T_MAX / expandsize) - return PyErr_NoMemory(); - - repr = PyBytes_FromStringAndSize(NULL, expandsize * len); - if (repr == NULL) - return NULL; - if (len == 0) - return repr; + p = _PyBytesWriter_Alloc(&writer, len); + if (p == NULL) + goto error; + writer.overallocate = 1; - p = q = PyBytes_AS_STRING(repr); for (pos = 0; pos < len; pos++) { Py_UCS4 ch = PyUnicode_READ(kind, data, pos); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { assert(ch <= MAX_UNICODE); + + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 10-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'U'; *p++ = Py_hexdigits[(ch >> 28) & 0xf]; @@ -6340,6 +6359,11 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) } /* Map 16-bit characters to '\uxxxx' */ else if (ch >= 256) { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 6-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0xf]; @@ -6352,10 +6376,11 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) *p++ = (char) ch; } - assert(p > q); - if (_PyBytes_Resize(&repr, p - q) < 0) - return NULL; - return repr; + return _PyBytesWriter_Finish(&writer, p); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } PyObject * -- cgit v0.12 From eaaaf136d20e1dd5160c2083fb81313d0818ea5d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 13 Oct 2015 10:51:47 +0200 Subject: Issue #25384: Use _PyBytesWriter API in binascii This API avoids a final call to _PyBytes_Resize() for output smaller than 512 bytes. Small optimization: disable overallocation in binascii.rledecode_hqx() for the last write. --- Modules/binascii.c | 194 +++++++++++++++++++++++------------------------------ 1 file changed, 83 insertions(+), 111 deletions(-) diff --git a/Modules/binascii.c b/Modules/binascii.c index 3e26a7a..dfa5535 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -346,9 +346,10 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; - Py_ssize_t bin_len; + Py_ssize_t bin_len, out_len; + _PyBytesWriter writer; + _PyBytesWriter_Init(&writer); bin_data = data->buf; bin_len = data->len; if ( bin_len > 45 ) { @@ -358,9 +359,10 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) } /* We're lazy and allocate to much (fixed up later) */ - if ( (rv=PyBytes_FromStringAndSize(NULL, 2 + (bin_len+2)/3*4)) == NULL ) + out_len = 2 + (bin_len + 2) / 3 * 4; + ascii_data = _PyBytesWriter_Alloc(&writer, out_len); + if (ascii_data == NULL) return NULL; - ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); /* Store the length */ *ascii_data++ = ' ' + (bin_len & 077); @@ -382,12 +384,7 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) } *ascii_data++ = '\n'; /* Append a courtesy newline */ - if (_PyBytes_Resize(&rv, - (ascii_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + return _PyBytesWriter_Finish(&writer, ascii_data); } @@ -433,9 +430,9 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; Py_ssize_t ascii_len, bin_len; int quad_pos = 0; + _PyBytesWriter writer; ascii_data = data->buf; ascii_len = data->len; @@ -447,11 +444,12 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) bin_len = ((ascii_len+3)/4)*3; /* Upper bound, corrected later */ + _PyBytesWriter_Init(&writer); + /* Allocate the buffer */ - if ( (rv=PyBytes_FromStringAndSize(NULL, bin_len)) == NULL ) + bin_data = _PyBytesWriter_Alloc(&writer, bin_len); + if (bin_data == NULL) return NULL; - bin_data = (unsigned char *)PyBytes_AS_STRING(rv); - bin_len = 0; for( ; ascii_len > 0; ascii_len--, ascii_data++) { this_ch = *ascii_data; @@ -496,31 +494,17 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) if ( leftbits >= 8 ) { leftbits -= 8; *bin_data++ = (leftchar >> leftbits) & 0xff; - bin_len++; leftchar &= ((1 << leftbits) - 1); } } if (leftbits != 0) { PyErr_SetString(Error, "Incorrect padding"); - Py_DECREF(rv); + _PyBytesWriter_Dealloc(&writer); return NULL; } - /* And set string size correctly. If the result string is empty - ** (because the input was all invalid) return the shared empty - ** string instead; _PyBytes_Resize() won't do this for us. - */ - if (bin_len > 0) { - if (_PyBytes_Resize(&rv, bin_len) < 0) { - Py_CLEAR(rv); - } - } - else { - Py_DECREF(rv); - rv = PyBytes_FromStringAndSize("", 0); - } - return rv; + return _PyBytesWriter_Finish(&writer, bin_data); } @@ -542,11 +526,12 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; Py_ssize_t bin_len, out_len; + _PyBytesWriter writer; bin_data = data->buf; bin_len = data->len; + _PyBytesWriter_Init(&writer); assert(bin_len >= 0); @@ -561,9 +546,9 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) out_len = bin_len*2 + 2; if (newline) out_len++; - if ( (rv=PyBytes_FromStringAndSize(NULL, out_len)) == NULL ) + ascii_data = _PyBytesWriter_Alloc(&writer, out_len); + if (ascii_data == NULL) return NULL; - ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); for( ; bin_len > 0 ; bin_len--, bin_data++ ) { /* Shift the data into our buffer */ @@ -588,12 +573,7 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) if (newline) *ascii_data++ = '\n'; /* Append a courtesy newline */ - if (_PyBytes_Resize(&rv, - (ascii_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + return _PyBytesWriter_Finish(&writer, ascii_data); } /*[clinic input] @@ -613,12 +593,14 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; + PyObject *res; Py_ssize_t len; int done = 0; + _PyBytesWriter writer; ascii_data = data->buf; len = data->len; + _PyBytesWriter_Init(&writer); assert(len >= 0); @@ -628,9 +610,9 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) /* Allocate a string that is too big (fixed later) Add two to the initial length to prevent interning which would preclude subsequent resizing. */ - if ( (rv=PyBytes_FromStringAndSize(NULL, len+2)) == NULL ) + bin_data = _PyBytesWriter_Alloc(&writer, len + 2); + if (bin_data == NULL) return NULL; - bin_data = (unsigned char *)PyBytes_AS_STRING(rv); for( ; len > 0 ; len--, ascii_data++ ) { /* Get the byte and look it up */ @@ -639,7 +621,7 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) continue; if ( this_ch == FAIL ) { PyErr_SetString(Error, "Illegal char"); - Py_DECREF(rv); + _PyBytesWriter_Dealloc(&writer); return NULL; } if ( this_ch == DONE ) { @@ -661,21 +643,14 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) if ( leftbits && !done ) { PyErr_SetString(Incomplete, "String has incomplete number of bytes"); - Py_DECREF(rv); + _PyBytesWriter_Dealloc(&writer); return NULL; } - if (_PyBytes_Resize(&rv, - (bin_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - if (rv) { - PyObject *rrv = Py_BuildValue("Oi", rv, done); - Py_DECREF(rv); - return rrv; - } - return NULL; + res = _PyBytesWriter_Finish(&writer, bin_data); + if (res == NULL) + return NULL; + return Py_BuildValue("Ni", res, done); } @@ -693,10 +668,11 @@ binascii_rlecode_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=0905da344dbf0648 input=e1f1712447a82b09]*/ { unsigned char *in_data, *out_data; - PyObject *rv; unsigned char ch; Py_ssize_t in, inend, len; + _PyBytesWriter writer; + _PyBytesWriter_Init(&writer); in_data = data->buf; len = data->len; @@ -706,9 +682,9 @@ binascii_rlecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Worst case: output is twice as big as input (fixed later) */ - if ( (rv=PyBytes_FromStringAndSize(NULL, len*2+2)) == NULL ) + out_data = _PyBytesWriter_Alloc(&writer, len * 2 + 2); + if (out_data == NULL) return NULL; - out_data = (unsigned char *)PyBytes_AS_STRING(rv); for( in=0; inbuf; len = data->len; + _PyBytesWriter_Init(&writer); assert(len >= 0); @@ -772,9 +745,9 @@ binascii_b2a_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer that is at least large enough */ - if ( (rv=PyBytes_FromStringAndSize(NULL, len*2+2)) == NULL ) + ascii_data = _PyBytesWriter_Alloc(&writer, len * 2 + 2); + if (ascii_data == NULL) return NULL; - ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); for( ; len > 0 ; len--, bin_data++ ) { /* Shift into our buffer, and output any 6bits ready */ @@ -791,12 +764,8 @@ binascii_b2a_hqx_impl(PyModuleDef *module, Py_buffer *data) leftchar <<= (6-leftbits); *ascii_data++ = table_b2a_hqx[leftchar & 0x3f]; } - if (_PyBytes_Resize(&rv, - (ascii_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + + return _PyBytesWriter_Finish(&writer, ascii_data); } @@ -815,11 +784,12 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) { unsigned char *in_data, *out_data; unsigned char in_byte, in_repeat; - PyObject *rv; Py_ssize_t in_len, out_len, out_len_left; + _PyBytesWriter writer; in_data = data->buf; in_len = data->len; + _PyBytesWriter_Init(&writer); assert(in_len >= 0); @@ -830,45 +800,49 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer of reasonable size. Resized when needed */ - out_len = in_len*2; - if ( (rv=PyBytes_FromStringAndSize(NULL, out_len)) == NULL ) + out_len = in_len * 2; + out_data = _PyBytesWriter_Alloc(&writer, out_len); + if (out_data == NULL) return NULL; - out_len_left = out_len; - out_data = (unsigned char *)PyBytes_AS_STRING(rv); + + /* Use overallocation */ + writer.overallocate = 1; + out_len_left = writer.allocated; /* ** We need two macros here to get/put bytes and handle ** end-of-buffer for input and output strings. */ -#define INBYTE(b) \ - do { \ - if ( --in_len < 0 ) { \ - PyErr_SetString(Incomplete, ""); \ - Py_DECREF(rv); \ - return NULL; \ - } \ - b = *in_data++; \ +#define INBYTE(b) \ + do { \ + if ( --in_len < 0 ) { \ + PyErr_SetString(Incomplete, ""); \ + goto error; \ + } \ + b = *in_data++; \ } while(0) -#define OUTBYTE(b) \ - do { \ - if ( --out_len_left < 0 ) { \ - if ( out_len > PY_SSIZE_T_MAX / 2) return PyErr_NoMemory(); \ - if (_PyBytes_Resize(&rv, 2*out_len) < 0) \ - { Py_XDECREF(rv); return NULL; } \ - out_data = (unsigned char *)PyBytes_AS_STRING(rv) \ - + out_len; \ - out_len_left = out_len-1; \ - out_len = out_len * 2; \ - } \ - *out_data++ = b; \ +#define OUTBYTE(b) \ + do { \ + if ( --out_len_left < 0 ) { \ + if (in_len <= 0) { \ + /* We are done after this write, no need to \ + overallocate the buffer anymore */ \ + writer.overallocate = 0; \ + } \ + out_data = _PyBytesWriter_Prepare(&writer, out_data, 1); \ + if (out_data == NULL) \ + goto error; \ + out_len_left = writer.allocated; \ + } \ + *out_data++ = b; \ } while(0) - /* - ** Handle first byte separately (since we have to get angry - ** in case of an orphaned RLE code). - */ - INBYTE(in_byte); + /* + ** Handle first byte separately (since we have to get angry + ** in case of an orphaned RLE code). + */ + INBYTE(in_byte); if (in_byte == RUNCHAR) { INBYTE(in_repeat); @@ -877,8 +851,7 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) ** of the string only). This is a programmer error. */ PyErr_SetString(Error, "Orphaned RLE code at start"); - Py_DECREF(rv); - return NULL; + goto error; } OUTBYTE(RUNCHAR); } else { @@ -904,12 +877,11 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) OUTBYTE(in_byte); } } - if (_PyBytes_Resize(&rv, - (out_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + return _PyBytesWriter_Finish(&writer, out_data); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } -- cgit v0.12 From dbc517c2dd273f70d2608fd514477fbac459fcab Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 13 Oct 2015 21:20:14 +0300 Subject: Issue #25382: pickletools.dis() now outputs implicit memo index for the MEMOIZE opcode. --- Lib/pickletools.py | 1 + Misc/NEWS | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Lib/pickletools.py b/Lib/pickletools.py index cf5df41..43dedb3 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2440,6 +2440,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): if opcode.name == "MEMOIZE": memo_idx = len(memo) + markmsg = "(as %d)" % memo_idx else: assert arg is not None memo_idx = arg diff --git a/Misc/NEWS b/Misc/NEWS index 142360e..2700ad3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,9 @@ Core and Builtins Library ------- +- Issue #25382: pickletools.dis() now outputs implicit memo index for the + MEMOIZE opcode. + - Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). base64.b64encode() uses it to avoid a memory copy. -- cgit v0.12 From b6d84832bfa1f453d3c73d6db0982aee0e6d04fe Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 13 Oct 2015 21:26:35 +0300 Subject: Issue #24164: Document changes to __getnewargs__ and __getnewargs_ex__. --- Doc/library/pickle.rst | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index f862065..2aab909 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -488,7 +488,7 @@ methods: .. method:: object.__getnewargs_ex__() - In protocols 4 and newer, classes that implements the + In protocols 2 and newer, classes that implements the :meth:`__getnewargs_ex__` method can dictate the values passed to the :meth:`__new__` method upon unpickling. The method must return a pair ``(args, kwargs)`` where *args* is a tuple of positional arguments @@ -500,15 +500,22 @@ methods: class requires keyword-only arguments. Otherwise, it is recommended for compatibility to implement :meth:`__getnewargs__`. + .. versionchanged:: 3.6 + :meth:`__getnewargs_ex__` is now used in protocols 2 and 3. + .. method:: object.__getnewargs__() - This method serve a similar purpose as :meth:`__getnewargs_ex__` but - for protocols 2 and newer. It must return a tuple of arguments ``args`` - which will be passed to the :meth:`__new__` method upon unpickling. + This method serve a similar purpose as :meth:`__getnewargs_ex__`, but + supports only positional arguments. It must return a tuple of arguments + ``args`` which will be passed to the :meth:`__new__` method upon unpickling. + + :meth:`__getnewargs__` will not be called if :meth:`__getnewargs_ex__` is + defined. - In protocols 4 and newer, :meth:`__getnewargs__` will not be called if - :meth:`__getnewargs_ex__` is defined. + .. versionchanged:: 3.6 + Before Python 3.6, :meth:`__getnewargs__` was called instead of + :meth:`__getnewargs_ex__` in protocols 2 and 3. .. method:: object.__getstate__() -- cgit v0.12 From 03dab786b2f504791ac46a9f9b9db82e634efd05 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 00:21:35 +0200 Subject: Rewrite PyBytes_FromFormatV() using _PyBytesWriter API * Add much more unit tests on PyBytes_FromFormatV() * Remove the first loop to compute the length of the output string * Use _PyBytesWriter to handle the bytes buffer, use overallocation * Cleanup the code to make simpler and easier to review --- Lib/test/test_bytes.py | 92 ++++++++++++-- Objects/bytesobject.c | 336 ++++++++++++++++++++++++------------------------- 2 files changed, 245 insertions(+), 183 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 53a80f4..5fe193e 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -783,25 +783,93 @@ class BytesTest(BaseBytesTest, unittest.TestCase): # Test PyBytes_FromFormat() def test_from_format(self): test.support.import_module('ctypes') - from ctypes import pythonapi, py_object, c_int, c_char_p + _testcapi = test.support.import_module('_testcapi') + from ctypes import pythonapi, py_object + from ctypes import ( + c_int, c_uint, + c_long, c_ulong, + c_size_t, c_ssize_t, + c_char_p) + PyBytes_FromFormat = pythonapi.PyBytes_FromFormat PyBytes_FromFormat.restype = py_object + # basic tests self.assertEqual(PyBytes_FromFormat(b'format'), b'format') - + self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), + b'Hello world !') + + # test formatters + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), + b'c=\0') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), + b'c=@') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), + b'c=\xff') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(1), c_long(2), + c_size_t(3)), + b'd=1 ld=2 zd=3') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(-1), c_long(-2), + c_size_t(-3)), + b'd=-1 ld=-2 zd=-3') + self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', + c_uint(123), c_ulong(456), + c_size_t(789)), + b'u=123 lu=456 zu=789') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), + b'i=123') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), + b'i=-123') + self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), + b'x=abc') + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', + c_char_p(0xabcdef)), + b'ptr=0xabcdef') + self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), + b's=cstr') + + # test minimum and maximum integer values + size_max = c_size_t(-1).value + for formatstr, ctypes_type, value, py_formatter in ( + (b'%d', c_int, _testcapi.INT_MIN, str), + (b'%d', c_int, _testcapi.INT_MAX, str), + (b'%ld', c_long, _testcapi.LONG_MIN, str), + (b'%ld', c_long, _testcapi.LONG_MAX, str), + (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), + (b'%zu', c_size_t, size_max, str), + (b'%p', c_char_p, size_max, lambda value: '%#x' % value), + ): + self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), + py_formatter(value).encode('ascii')), + + # width and precision (width is currently ignored) + self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), + b'a') + self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), + b'abc') + + # '%%' formatter + self.assertEqual(PyBytes_FromFormat(b'%%'), + b'%') + self.assertEqual(PyBytes_FromFormat(b'[%%]'), + b'[%]') + self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), + b'%_') + self.assertEqual(PyBytes_FromFormat(b'%%s'), + b'%s') + + # Invalid formats and partial formatting self.assertEqual(PyBytes_FromFormat(b'%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%s'), b'%s') - self.assertEqual(PyBytes_FromFormat(b'[%%]'), b'[%]') - self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), b'%_') - - self.assertEqual(PyBytes_FromFormat(b'c:%c', c_int(255)), - b'c:\xff') - self.assertEqual(PyBytes_FromFormat(b's:%s', c_char_p(b'cstr')), - b's:cstr') + self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), + b'x=2 y=%') - # Issue #19969 + # Issue #19969: %c must raise OverflowError for values + # not in the range [0; 255] self.assertRaises(OverflowError, PyBytes_FromFormat, b'%c', c_int(-1)) self.assertRaises(OverflowError, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e7ab503..189673c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -174,190 +174,184 @@ PyBytes_FromString(const char *str) PyObject * PyBytes_FromFormatV(const char *format, va_list vargs) { - va_list count; - Py_ssize_t n = 0; - const char* f; char *s; - PyObject* string; + const char *f; + const char *p; + Py_ssize_t prec; + int longflag; + int size_tflag; + /* Longest 64-bit formatted numbers: + - "18446744073709551615\0" (21 bytes) + - "-9223372036854775808\0" (21 bytes) + Decimal takes the most space (it isn't enough for octal.) + + Longest 64-bit pointer representation: + "0xffffffffffffffff\0" (19 bytes). */ + char buffer[21]; + _PyBytesWriter writer; - Py_VA_COPY(count, vargs); - /* step 1: figure out how large a buffer we need */ - for (f = format; *f; f++) { - if (*f == '%') { - const char* p = f; - while (*++f && *f != '%' && !Py_ISALPHA(*f)) - ; - - /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since - * they don't affect the amount of space we reserve. - */ - if ((*f == 'l' || *f == 'z') && - (f[1] == 'd' || f[1] == 'u')) - ++f; - - switch (*f) { - case 'c': - { - int c = va_arg(count, int); - if (c < 0 || c > 255) { - PyErr_SetString(PyExc_OverflowError, - "PyBytes_FromFormatV(): %c format " - "expects an integer in range [0; 255]"); - return NULL; - } - n++; - break; - } - case '%': - n++; - break; - case 'd': case 'u': case 'i': case 'x': - (void) va_arg(count, int); - /* 20 bytes is enough to hold a 64-bit - integer. Decimal takes the most space. - This isn't enough for octal. */ - n += 20; - break; - case 's': - s = va_arg(count, char*); - n += strlen(s); - break; - case 'p': - (void) va_arg(count, int); - /* maximum 64-bit pointer representation: - * 0xffffffffffffffff - * so 19 characters is enough. - * XXX I count 18 -- what's the extra for? - */ - n += 19; - break; - default: - /* if we stumble upon an unknown - formatting code, copy the rest of - the format string to the output - string. (we cannot just skip the - code, since there's no way to know - what's in the argument list) */ - n += strlen(p); - goto expand; - } - } else - n++; - } - expand: - /* step 2: fill the buffer */ - /* Since we've analyzed how much space we need for the worst case, - use sprintf directly instead of the slower PyOS_snprintf. */ - string = PyBytes_FromStringAndSize(NULL, n); - if (!string) + _PyBytesWriter_Init(&writer); + + s = _PyBytesWriter_Alloc(&writer, strlen(format)); + if (s == NULL) return NULL; + writer.overallocate = 1; - s = PyBytes_AsString(string); +#define WRITE_BYTES(str) \ + do { \ + s = _PyBytesWriter_WriteBytes(&writer, s, (str), strlen(str)); \ + if (s == NULL) \ + goto error; \ + } while (0) for (f = format; *f; f++) { - if (*f == '%') { - const char* p = f++; - Py_ssize_t i; - int longflag = 0; - int size_tflag = 0; - /* parse the width.precision part (we're only - interested in the precision value, if any) */ - n = 0; - while (Py_ISDIGIT(*f)) - n = (n*10) + *f++ - '0'; - if (*f == '.') { - f++; - n = 0; - while (Py_ISDIGIT(*f)) - n = (n*10) + *f++ - '0'; - } - while (*f && *f != '%' && !Py_ISALPHA(*f)) - f++; - /* handle the long flag, but only for %ld and %lu. - others can be added when necessary. */ - if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { - longflag = 1; - ++f; + if (*f != '%') { + *s++ = *f; + continue; + } + + p = f++; + + /* ignore the width (ex: 10 in "%10s") */ + while (Py_ISDIGIT(*f)) + f++; + + /* parse the precision (ex: 10 in "%.10s") */ + prec = 0; + if (*f == '.') { + f++; + for (; Py_ISDIGIT(*f); f++) { + prec = (prec * 10) + (*f - '0'); } - /* handle the size_t flag. */ - if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { - size_tflag = 1; - ++f; + } + + while (*f && *f != '%' && !Py_ISALPHA(*f)) + f++; + + /* handle the long flag ('l'), but only for %ld and %lu. + others can be added when necessary. */ + longflag = 0; + if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { + longflag = 1; + ++f; + } + + /* handle the size_t flag ('z'). */ + size_tflag = 0; + if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { + size_tflag = 1; + ++f; + } + + /* substract bytes preallocated for the format string + (ex: 2 for "%s") */ + writer.min_size -= (f - p + 1); + + switch (*f) { + case 'c': + { + int c = va_arg(vargs, int); + if (c < 0 || c > 255) { + PyErr_SetString(PyExc_OverflowError, + "PyBytes_FromFormatV(): %c format " + "expects an integer in range [0; 255]"); + goto error; } + writer.min_size++; + *s++ = (unsigned char)c; + break; + } - switch (*f) { - case 'c': - { - int c = va_arg(vargs, int); - /* c has been checked for overflow in the first step */ - *s++ = (unsigned char)c; - break; + case 'd': + if (longflag) + sprintf(buffer, "%ld", va_arg(vargs, long)); + else if (size_tflag) + sprintf(buffer, "%" PY_FORMAT_SIZE_T "d", + va_arg(vargs, Py_ssize_t)); + else + sprintf(buffer, "%d", va_arg(vargs, int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 'u': + if (longflag) + sprintf(buffer, "%lu", + va_arg(vargs, unsigned long)); + else if (size_tflag) + sprintf(buffer, "%" PY_FORMAT_SIZE_T "u", + va_arg(vargs, size_t)); + else + sprintf(buffer, "%u", + va_arg(vargs, unsigned int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 'i': + sprintf(buffer, "%i", va_arg(vargs, int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 'x': + sprintf(buffer, "%x", va_arg(vargs, int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 's': + { + Py_ssize_t i; + + p = va_arg(vargs, char*); + i = strlen(p); + if (prec > 0 && i > prec) + i = prec; + s = _PyBytesWriter_WriteBytes(&writer, s, p, i); + if (s == NULL) + goto error; + break; + } + + case 'p': + sprintf(buffer, "%p", va_arg(vargs, void*)); + assert(strlen(buffer) < sizeof(buffer)); + /* %p is ill-defined: ensure leading 0x. */ + if (buffer[1] == 'X') + buffer[1] = 'x'; + else if (buffer[1] != 'x') { + memmove(buffer+2, buffer, strlen(buffer)+1); + buffer[0] = '0'; + buffer[1] = 'x'; } - case 'd': - if (longflag) - sprintf(s, "%ld", va_arg(vargs, long)); - else if (size_tflag) - sprintf(s, "%" PY_FORMAT_SIZE_T "d", - va_arg(vargs, Py_ssize_t)); - else - sprintf(s, "%d", va_arg(vargs, int)); - s += strlen(s); - break; - case 'u': - if (longflag) - sprintf(s, "%lu", - va_arg(vargs, unsigned long)); - else if (size_tflag) - sprintf(s, "%" PY_FORMAT_SIZE_T "u", - va_arg(vargs, size_t)); - else - sprintf(s, "%u", - va_arg(vargs, unsigned int)); - s += strlen(s); - break; - case 'i': - sprintf(s, "%i", va_arg(vargs, int)); - s += strlen(s); - break; - case 'x': - sprintf(s, "%x", va_arg(vargs, int)); - s += strlen(s); - break; - case 's': - p = va_arg(vargs, char*); - i = strlen(p); - if (n > 0 && i > n) - i = n; - Py_MEMCPY(s, p, i); - s += i; - break; - case 'p': - sprintf(s, "%p", va_arg(vargs, void*)); - /* %p is ill-defined: ensure leading 0x. */ - if (s[1] == 'X') - s[1] = 'x'; - else if (s[1] != 'x') { - memmove(s+2, s, strlen(s)+1); - s[0] = '0'; - s[1] = 'x'; - } - s += strlen(s); - break; - case '%': - *s++ = '%'; - break; - default: - strcpy(s, p); - s += strlen(s); - goto end; + WRITE_BYTES(buffer); + break; + + case '%': + writer.min_size++; + *s++ = '%'; + break; + + default: + if (*f == 0) { + /* fix min_size if we reached the end of the format string */ + writer.min_size++; } - } else - *s++ = *f; + + /* invalid format string: copy unformatted string and exit */ + WRITE_BYTES(p); + return _PyBytesWriter_Finish(&writer, s); + } } - end: - _PyBytes_Resize(&string, s - PyBytes_AS_STRING(string)); - return string; +#undef WRITE_BYTES + + return _PyBytesWriter_Finish(&writer, s); + + error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } PyObject * -- cgit v0.12 From 7ab986dd84b43742af9b504e0c7acfef8f231ab3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 02:55:12 +0200 Subject: Fix test_bytes on Windows On Windows, sprintf("%p", 0xabcdef) formats hexadecimal in uppercase and pad to 16 characters (on 64-bit system) with zeros. --- Lib/test/test_bytes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 5fe193e..947d959 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -782,7 +782,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): # Test PyBytes_FromFormat() def test_from_format(self): - test.support.import_module('ctypes') + ctypes = test.support.import_module('ctypes') _testcapi = test.support.import_module('_testcapi') from ctypes import pythonapi, py_object from ctypes import ( @@ -825,9 +825,12 @@ class BytesTest(BaseBytesTest, unittest.TestCase): b'i=-123') self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), b'x=abc') - self.assertEqual(PyBytes_FromFormat(b'ptr=%p', - c_char_p(0xabcdef)), - b'ptr=0xabcdef') + ptr = 0xabcdef + expected = [b'ptr=%#x' % ptr] + win_format = 'ptr=0x%0{}X'.format(2 * ctypes.sizeof(c_char_p)) + expected.append((win_format % ptr).encode('ascii')) + self.assertIn(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + expected) self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), b's=cstr') -- cgit v0.12 From 199c9a6f4bf6ff6fe04c4c4ca17c24bc258079f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 09:47:23 +0200 Subject: Fix long_format_binary() Issue #25399: Fix long_format_binary(), allocate bytes for the bytes writer. --- Objects/longobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 00f5d95..759116a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1836,7 +1836,7 @@ long_format_binary(PyObject *aa, int base, int alternate, kind = writer->kind; v = NULL; } - else if (writer) { + else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); if (*bytes_str == NULL) return -1; -- cgit v0.12 From 661aaccf9def380540cc1d440761159a414094d1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 09:41:48 +0200 Subject: Add use_bytearray attribute to _PyBytesWriter Issue #25399: Add a new use_bytearray attribute to _PyBytesWriter to use a bytearray buffer, instead of using a bytes object. --- Include/bytesobject.h | 12 ++++--- Objects/bytesobject.c | 93 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 73 insertions(+), 32 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index b7a7c36..fbb6322 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -128,17 +128,21 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer, A _PyBytesWriter variable must be declared at the end of variables in a function to optimize the memory allocation on the stack. */ typedef struct { - /* bytes object */ + /* bytes, bytearray or NULL (when the small buffer is used) */ PyObject *buffer; - /* Number of allocated size */ + /* Number of allocated size. */ Py_ssize_t allocated; /* Minimum number of allocated bytes, incremented by _PyBytesWriter_Prepare() */ Py_ssize_t min_size; - /* If non-zero, overallocate the buffer (default: 0). */ + /* If non-zero, use a bytearray instead of a bytes object for buffer. */ + int use_bytearray; + + /* If non-zero, overallocate the buffer (default: 0). + This flag must be zero if use_bytearray is non-zero. */ int overallocate; /* Stack buffer */ @@ -153,7 +157,7 @@ typedef struct { PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); /* Get the buffer content and reset the writer. - Return a bytes object. + Return a bytes object, or a bytearray object if use_bytearray is non-zero. Raise an exception and return NULL on error. */ PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str); diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 189673c..a1f2958 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3852,11 +3852,8 @@ bytes_iter(PyObject *seq) void _PyBytesWriter_Init(_PyBytesWriter *writer) { - writer->buffer = NULL; - writer->allocated = 0; - writer->min_size = 0; - writer->overallocate = 0; - writer->use_small_buffer = 0; + /* Set all attributes before small_buffer to 0 */ + memset(writer, 0, offsetof(_PyBytesWriter, small_buffer)); #ifdef Py_DEBUG memset(writer->small_buffer, 0xCB, sizeof(writer->small_buffer)); #endif @@ -3871,13 +3868,17 @@ _PyBytesWriter_Dealloc(_PyBytesWriter *writer) Py_LOCAL_INLINE(char*) _PyBytesWriter_AsString(_PyBytesWriter *writer) { - if (!writer->use_small_buffer) { + if (writer->use_small_buffer) { + assert(writer->buffer == NULL); + return writer->small_buffer; + } + else if (writer->use_bytearray) { assert(writer->buffer != NULL); - return PyBytes_AS_STRING(writer->buffer); + return PyByteArray_AS_STRING(writer->buffer); } else { - assert(writer->buffer == NULL); - return writer->small_buffer; + assert(writer->buffer != NULL); + return PyBytes_AS_STRING(writer->buffer); } } @@ -3897,18 +3898,28 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) #ifdef Py_DEBUG char *start, *end; - if (!writer->use_small_buffer) { + if (writer->use_small_buffer) { + assert(writer->buffer == NULL); + } + else { assert(writer->buffer != NULL); - assert(PyBytes_CheckExact(writer->buffer)); + if (writer->use_bytearray) + assert(PyByteArray_CheckExact(writer->buffer)); + else + assert(PyBytes_CheckExact(writer->buffer)); assert(Py_REFCNT(writer->buffer) == 1); } - else { - assert(writer->buffer == NULL); + + if (writer->use_bytearray) { + /* bytearray has its own overallocation algorithm, + writer overallocation must be disabled */ + assert(!writer->overallocate); } - start = _PyBytesWriter_AsString(writer); + assert(0 <= writer->allocated); assert(0 <= writer->min_size && writer->min_size <= writer->allocated); /* the last byte must always be null */ + start = _PyBytesWriter_AsString(writer); assert(start[writer->allocated] == 0); end = start + writer->allocated; @@ -3932,8 +3943,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) if (writer->min_size > PY_SSIZE_T_MAX - size) { PyErr_NoMemory(); - _PyBytesWriter_Dealloc(writer); - return NULL; + goto error; } writer->min_size += size; @@ -3950,23 +3960,38 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) pos = _PyBytesWriter_GetPos(writer, str); if (!writer->use_small_buffer) { - /* Note: Don't use a bytearray object because the conversion from - byterray to bytes requires to copy all bytes. */ - if (_PyBytes_Resize(&writer->buffer, allocated)) { - assert(writer->buffer == NULL); - return NULL; + if (writer->use_bytearray) { + if (PyByteArray_Resize(writer->buffer, allocated)) + goto error; + /* writer->allocated can be smaller than writer->buffer->ob_alloc, + but we cannot use ob_alloc because bytes may need to be moved + to use the whole buffer. bytearray uses an internal optimization + to avoid moving or copying bytes when bytes are removed at the + beginning (ex: del bytearray[:1]). */ + } + else { + if (_PyBytes_Resize(&writer->buffer, allocated)) + goto error; } } else { /* convert from stack buffer to bytes object buffer */ assert(writer->buffer == NULL); - writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); + if (writer->use_bytearray) + writer->buffer = PyByteArray_FromStringAndSize(NULL, allocated); + else + writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); if (writer->buffer == NULL) - return NULL; + goto error; if (pos != 0) { - Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), + char *dest; + if (writer->use_bytearray) + dest = PyByteArray_AS_STRING(writer->buffer); + else + dest = PyBytes_AS_STRING(writer->buffer); + Py_MEMCPY(dest, writer->small_buffer, pos); } @@ -3981,6 +4006,10 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) str = _PyBytesWriter_AsString(writer) + pos; _PyBytesWriter_CheckConsistency(writer, str); return str; + +error: + _PyBytesWriter_Dealloc(writer); + return NULL; } /* Allocate the buffer to write size bytes. @@ -4013,7 +4042,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) _PyBytesWriter_CheckConsistency(writer, str); pos = _PyBytesWriter_GetPos(writer, str); - if (pos == 0) { + if (pos == 0 && !writer->use_bytearray) { Py_CLEAR(writer->buffer); /* Get the empty byte string singleton */ result = PyBytes_FromStringAndSize(NULL, 0); @@ -4026,9 +4055,17 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) writer->buffer = NULL; if (pos != writer->allocated) { - if (_PyBytes_Resize(&result, pos)) { - assert(result == NULL); - return NULL; + if (writer->use_bytearray) { + if (PyByteArray_Resize(result, pos)) { + Py_DECREF(result); + return NULL; + } + } + else { + if (_PyBytes_Resize(&result, pos)) { + assert(result == NULL); + return NULL; + } } } } -- cgit v0.12 From 772b2b09f279fdcb01bbd703735d35bd02dd8ec1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 09:56:53 +0200 Subject: Optimize bytearray % args Issue #25399: Don't create temporary bytes objects: modify _PyBytes_Format() to create work directly on bytearray objects. * Rename _PyBytes_Format() to _PyBytes_FormatEx() just in case if something outside CPython uses it * _PyBytes_FormatEx() now uses (char*, Py_ssize_t) for the input string, so bytearray_format() doesn't need tot create a temporary input bytes object * Add use_bytearray parameter to _PyBytes_FormatEx() which is passed to _PyBytesWriter, to create a bytearray buffer instead of a bytes buffer Most formatting operations are now between 2.5 and 5 times faster. --- Include/bytesobject.h | 6 +++++- Objects/bytearrayobject.c | 22 +++++----------------- Objects/bytesobject.c | 41 +++++++++++++++++++++++------------------ 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index fbb6322..b5b37ef 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -62,7 +62,11 @@ PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *); PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject *) _PyBytes_Format(PyObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( + const char *format, + Py_ssize_t format_len, + PyObject *args, + int use_bytearray); #endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, Py_ssize_t, diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 5647b57..e535bce 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -282,26 +282,14 @@ PyByteArray_Concat(PyObject *a, PyObject *b) static PyObject * bytearray_format(PyByteArrayObject *self, PyObject *args) { - PyObject *bytes_in, *bytes_out, *res; - char *bytestring; - - if (self == NULL || !PyByteArray_Check(self) || args == NULL) { + if (self == NULL || !PyByteArray_Check(self)) { PyErr_BadInternalCall(); return NULL; } - bytestring = PyByteArray_AS_STRING(self); - bytes_in = PyBytes_FromString(bytestring); - if (bytes_in == NULL) - return NULL; - bytes_out = _PyBytes_Format(bytes_in, args); - Py_DECREF(bytes_in); - if (bytes_out == NULL) - return NULL; - res = PyByteArray_FromObject(bytes_out); - Py_DECREF(bytes_out); - if (res == NULL) - return NULL; - return res; + + return _PyBytes_FormatEx(PyByteArray_AS_STRING(self), + PyByteArray_GET_SIZE(self), + args, 1); } /* Functions stuffed into the type object */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a1f2958..20b11fb 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -568,28 +568,32 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) /* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */ PyObject * -_PyBytes_Format(PyObject *format, PyObject *args) +_PyBytes_FormatEx(const char *format, Py_ssize_t format_len, + PyObject *args, int use_bytearray) { - char *fmt, *res; + const char *fmt; + char *res; Py_ssize_t arglen, argidx; Py_ssize_t fmtcnt; int args_owned = 0; PyObject *dict = NULL; _PyBytesWriter writer; - if (format == NULL || !PyBytes_Check(format) || args == NULL) { + if (args == NULL) { PyErr_BadInternalCall(); return NULL; } - fmt = PyBytes_AS_STRING(format); - fmtcnt = PyBytes_GET_SIZE(format); + fmt = format; + fmtcnt = format_len; _PyBytesWriter_Init(&writer); + writer.use_bytearray = use_bytearray; res = _PyBytesWriter_Alloc(&writer, fmtcnt); if (res == NULL) return NULL; - writer.overallocate = 1; + if (!use_bytearray) + writer.overallocate = 1; if (PyTuple_Check(args)) { arglen = PyTuple_GET_SIZE(args); @@ -613,10 +617,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) pos = strchr(fmt + 1, '%'); if (pos != NULL) len = pos - fmt; - else { - len = PyBytes_GET_SIZE(format); - len -= (fmt - PyBytes_AS_STRING(format)); - } + else + len = format_len - (fmt - format); assert(len != 0); Py_MEMCPY(res, fmt, len); @@ -644,7 +646,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) fmt++; if (*fmt == '(') { - char *keystart; + const char *keystart; Py_ssize_t keylen; PyObject *key; int pcount = 1; @@ -924,8 +926,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) "unsupported format character '%c' (0x%x) " "at index %zd", c, c, - (Py_ssize_t)(fmt - 1 - - PyBytes_AsString(format))); + (Py_ssize_t)(fmt - 1 - format)); goto error; } @@ -1028,7 +1029,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) /* If overallocation was disabled, ensure that it was the last write. Otherwise, we missed an optimization */ - assert(writer.overallocate || fmtcnt < 0); + assert(writer.overallocate || fmtcnt < 0 || use_bytearray); } /* until end */ if (argidx < arglen && !dict) { @@ -3233,11 +3234,15 @@ bytes_methods[] = { }; static PyObject * -bytes_mod(PyObject *v, PyObject *w) +bytes_mod(PyObject *self, PyObject *args) { - if (!PyBytes_Check(v)) - Py_RETURN_NOTIMPLEMENTED; - return _PyBytes_Format(v, w); + if (self == NULL || !PyBytes_Check(self)) { + PyErr_BadInternalCall(); + return NULL; + } + + return _PyBytes_FormatEx(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + args, 0); } static PyNumberMethods bytes_as_number = { -- cgit v0.12 From ebcf9edc05c03af38c01d8aeb05494b68169756c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 10:10:00 +0200 Subject: Document latest optimizations using _PyBytesWriter --- Doc/whatsnew/3.6.rst | 17 +++++++++++++---- Misc/NEWS | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b735e9c..d1e5045 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -141,16 +141,25 @@ Optimizations ============= * The ASCII decoder is now up to 60 times as fast for error handlers: - ``surrogateescape``, ``ignore`` and ``replace``. + ``surrogateescape``, ``ignore`` and ``replace`` (Contributed + by Victor Stinner in :issue:`24870`). * The ASCII and the Latin1 encoders are now up to 3 times as fast for the error - error ``surrogateescape``. + error ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). * The UTF-8 encoder is now up to 75 times as fast for error handlers: - ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. + ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass`` (Contributed + by Victor Stinner in :issue:`25267`). * The UTF-8 decoder is now up to 15 times as fast for error handlers: - ``ignore``, ``replace`` and ``surrogateescape``. + ``ignore``, ``replace`` and ``surrogateescape`` (Contributed + by Victor Stinner in :issue:`25301`). + +* ``bytes % args`` is now up to 2 times faster. (Contributed by Victor Stinner + in :issue:`25349`). + +* ``bytearray % args`` is now between 2.5 and 5 times faster. (Contributed by + Victor Stinner in :issue:`25399`). Build and C API Changes diff --git a/Misc/NEWS b/Misc/NEWS index 2700ad3..97e5e77 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25399: Optimize bytearray % args using the new private _PyBytesWriter + API. Formatting is now between 2.5 and 5 times faster. + - Issue #25274: sys.setrecursionlimit() now raises a RecursionError if the new recursion limit is too low depending at the current recursion depth. Modify also the "lower-water mark" formula to make it monotonic. This mark is used @@ -19,6 +22,7 @@ Core and Builtins sys.stdout.fileno() fails. - Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API. + Formatting is now up to 2 times faster. - Issue #24806: Prevent builtin types that are not allowed to be subclassed from being subclassed through multiple inheritance. -- cgit v0.12 From 2bf8993db966256d564d87865ceddf0e33c02500 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 11:25:33 +0200 Subject: Optimize bytes.fromhex() and bytearray.fromhex() Issue #25401: Optimize bytes.fromhex() and bytearray.fromhex(): they are now between 2x and 3.5x faster. Changes: * Use a fast-path working on a char* string for ASCII string * Use a slow-path for non-ASCII string * Replace slow hex_digit_to_int() function with a O(1) lookup in _PyLong_DigitValue precomputed table * Use _PyBytesWriter API to handle the buffer * Add unit tests to check the error position in error messages --- Doc/whatsnew/3.6.rst | 3 ++ Include/bytesobject.h | 3 ++ Include/longobject.h | 2 +- Lib/test/test_bytes.py | 14 +++++ Misc/NEWS | 3 ++ Objects/bytearrayobject.c | 43 +--------------- Objects/bytesobject.c | 128 +++++++++++++++++++++++++++------------------- 7 files changed, 101 insertions(+), 95 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d1e5045..edacea1 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -161,6 +161,9 @@ Optimizations * ``bytearray % args`` is now between 2.5 and 5 times faster. (Contributed by Victor Stinner in :issue:`25399`). +* Optimize :meth:`bytes.fromhex` and :meth:`bytearray.fromhex`: they are now + between 2x and 3.5x faster. (Contributed by Victor Stinner in :issue:`25401`). + Build and C API Changes ======================= diff --git a/Include/bytesobject.h b/Include/bytesobject.h index b5b37ef..4046c1c 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -67,6 +67,9 @@ PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( Py_ssize_t format_len, PyObject *args, int use_bytearray); +PyAPI_FUNC(PyObject*) _PyBytes_FromHex( + PyObject *string, + int use_bytearray); #endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, Py_ssize_t, diff --git a/Include/longobject.h b/Include/longobject.h index ab92495..9574f05 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -65,7 +65,7 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); # error "void* different in size from int, long and long long" #endif /* SIZEOF_VOID_P */ -/* Used by Python/mystrtoul.c. */ +/* Used by Python/mystrtoul.c and _PyBytes_FromHex(). */ #ifndef Py_LIMITED_API PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; #endif diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 947d959..0fe33b5 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -301,6 +301,20 @@ class BaseBytesTest: self.assertRaises(ValueError, self.type2test.fromhex, '\x00') self.assertRaises(ValueError, self.type2test.fromhex, '12 \x00 34') + for data, pos in ( + # invalid first hexadecimal character + ('12 x4 56', 3), + # invalid second hexadecimal character + ('12 3x 56', 4), + # two invalid hexadecimal characters + ('12 xy 56', 3), + # test non-ASCII string + ('12 3\xff 56', 4), + ): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(data) + self.assertIn('at position %s' % pos, str(cm.exception)) + def test_hex(self): self.assertRaises(TypeError, self.type2test.hex) self.assertRaises(TypeError, self.type2test.hex, 1) diff --git a/Misc/NEWS b/Misc/NEWS index 97e5e77..312cde0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25401: Optimize bytes.fromhex() and bytearray.fromhex(): they are now + between 2x and 3.5x faster. + - Issue #25399: Optimize bytearray % args using the new private _PyBytesWriter API. Formatting is now between 2.5 and 5 times faster. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index e535bce..b270fcc 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2823,48 +2823,7 @@ static PyObject * bytearray_fromhex_impl(PyObject*cls, PyObject *string) /*[clinic end generated code: output=df3da60129b3700c input=907bbd2d34d9367a]*/ { - PyObject *newbytes; - char *buf; - Py_ssize_t hexlen, byteslen, i, j; - int top, bot; - void *data; - unsigned int kind; - - assert(PyUnicode_Check(string)); - if (PyUnicode_READY(string)) - return NULL; - kind = PyUnicode_KIND(string); - data = PyUnicode_DATA(string); - hexlen = PyUnicode_GET_LENGTH(string); - - byteslen = hexlen/2; /* This overestimates if there are spaces */ - newbytes = PyByteArray_FromStringAndSize(NULL, byteslen); - if (!newbytes) - return NULL; - buf = PyByteArray_AS_STRING(newbytes); - for (i = j = 0; i < hexlen; i += 2) { - /* skip over spaces in the input */ - while (PyUnicode_READ(kind, data, i) == ' ') - i++; - if (i >= hexlen) - break; - top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); - bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); - if (top == -1 || bot == -1) { - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", i); - goto error; - } - buf[j++] = (top << 4) + bot; - } - if (PyByteArray_Resize(newbytes, j) < 0) - goto error; - return newbytes; - - error: - Py_DECREF(newbytes); - return NULL; + return _PyBytes_FromHex(string, 1); } PyDoc_STRVAR(hex__doc__, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 20b11fb..2d4cf4b 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -30,6 +30,10 @@ static PyBytesObject *nullstring; */ #define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) +/* Forward declaration */ +Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, + char *str); + /* For PyBytes_FromString(), the parameter `str' points to a null-terminated string containing exactly `size' bytes. @@ -3078,22 +3082,6 @@ bytes_splitlines_impl(PyBytesObject*self, int keepends) ); } -static int -hex_digit_to_int(Py_UCS4 c) -{ - if (c >= 128) - return -1; - if (Py_ISDIGIT(c)) - return c - '0'; - else { - if (Py_ISUPPER(c)) - c = Py_TOLOWER(c); - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - } - return -1; -} - /*[clinic input] @classmethod bytes.fromhex @@ -3111,47 +3099,83 @@ static PyObject * bytes_fromhex_impl(PyTypeObject *type, PyObject *string) /*[clinic end generated code: output=0973acc63661bb2e input=bf4d1c361670acd3]*/ { - PyObject *newstring; + return _PyBytes_FromHex(string, 0); +} + +PyObject* +_PyBytes_FromHex(PyObject *string, int use_bytearray) +{ char *buf; - Py_ssize_t hexlen, byteslen, i, j; - int top, bot; - void *data; - unsigned int kind; + Py_ssize_t hexlen, invalid_char; + unsigned int top, bot; + Py_UCS1 *str, *end; + _PyBytesWriter writer; + + _PyBytesWriter_Init(&writer); + writer.use_bytearray = use_bytearray; assert(PyUnicode_Check(string)); if (PyUnicode_READY(string)) return NULL; - kind = PyUnicode_KIND(string); - data = PyUnicode_DATA(string); hexlen = PyUnicode_GET_LENGTH(string); - byteslen = hexlen/2; /* This overestimates if there are spaces */ - newstring = PyBytes_FromStringAndSize(NULL, byteslen); - if (!newstring) + if (!PyUnicode_IS_ASCII(string)) { + void *data = PyUnicode_DATA(string); + unsigned int kind = PyUnicode_KIND(string); + Py_ssize_t i; + + /* search for the first non-ASCII character */ + for (i = 0; i < hexlen; i++) { + if (PyUnicode_READ(kind, data, i) >= 128) + break; + } + invalid_char = i; + goto error; + } + + assert(PyUnicode_KIND(string) == PyUnicode_1BYTE_KIND); + str = PyUnicode_1BYTE_DATA(string); + + /* This overestimates if there are spaces */ + buf = _PyBytesWriter_Alloc(&writer, hexlen / 2); + if (buf == NULL) return NULL; - buf = PyBytes_AS_STRING(newstring); - for (i = j = 0; i < hexlen; i += 2) { + + end = str + hexlen; + while (str < end) { /* skip over spaces in the input */ - while (PyUnicode_READ(kind, data, i) == ' ') - i++; - if (i >= hexlen) - break; - top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); - bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); - if (top == -1 || bot == -1) { - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", i); + if (*str == ' ') { + do { + str++; + } while (*str == ' '); + if (str >= end) + break; + } + + top = _PyLong_DigitValue[*str]; + if (top >= 16) { + invalid_char = str - PyUnicode_1BYTE_DATA(string); goto error; } - buf[j++] = (top << 4) + bot; + str++; + + bot = _PyLong_DigitValue[*str]; + if (bot >= 16) { + invalid_char = str - PyUnicode_1BYTE_DATA(string); + goto error; + } + str++; + + *buf++ = (unsigned char)((top << 4) + bot); } - if (j != byteslen && _PyBytes_Resize(&newstring, j) < 0) - goto error; - return newstring; + + return _PyBytesWriter_Finish(&writer, buf); error: - Py_XDECREF(newstring); + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", invalid_char); + _PyBytesWriter_Dealloc(&writer); return NULL; } @@ -3888,7 +3912,7 @@ _PyBytesWriter_AsString(_PyBytesWriter *writer) } Py_LOCAL_INLINE(Py_ssize_t) -_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) +_PyBytesWriter_GetSize(_PyBytesWriter *writer, char *str) { char *start = _PyBytesWriter_AsString(writer); assert(str != NULL); @@ -3963,7 +3987,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) allocated += allocated / OVERALLOCATE_FACTOR; } - pos = _PyBytesWriter_GetPos(writer, str); + pos = _PyBytesWriter_GetSize(writer, str); if (!writer->use_small_buffer) { if (writer->use_bytearray) { if (PyByteArray_Resize(writer->buffer, allocated)) @@ -4041,33 +4065,33 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) PyObject * _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) { - Py_ssize_t pos; + Py_ssize_t size; PyObject *result; _PyBytesWriter_CheckConsistency(writer, str); - pos = _PyBytesWriter_GetPos(writer, str); - if (pos == 0 && !writer->use_bytearray) { + size = _PyBytesWriter_GetSize(writer, str); + if (size == 0 && !writer->use_bytearray) { Py_CLEAR(writer->buffer); /* Get the empty byte string singleton */ result = PyBytes_FromStringAndSize(NULL, 0); } else if (writer->use_small_buffer) { - result = PyBytes_FromStringAndSize(writer->small_buffer, pos); + result = PyBytes_FromStringAndSize(writer->small_buffer, size); } else { result = writer->buffer; writer->buffer = NULL; - if (pos != writer->allocated) { + if (size != writer->allocated) { if (writer->use_bytearray) { - if (PyByteArray_Resize(result, pos)) { + if (PyByteArray_Resize(result, size)) { Py_DECREF(result); return NULL; } } else { - if (_PyBytes_Resize(&result, pos)) { + if (_PyBytes_Resize(&result, size)) { assert(result == NULL); return NULL; } -- cgit v0.12 From f091033b149792e4084a479444c39636c7be2cad Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 11:59:46 +0200 Subject: Issue #25401: Remove now unused hex_digit_to_int() function --- Objects/bytearrayobject.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b270fcc..2103147 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2789,22 +2789,6 @@ bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) ); } -static int -hex_digit_to_int(Py_UCS4 c) -{ - if (c >= 128) - return -1; - if (Py_ISDIGIT(c)) - return c - '0'; - else { - if (Py_ISUPPER(c)) - c = Py_TOLOWER(c); - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - } - return -1; -} - /*[clinic input] @classmethod bytearray.fromhex -- cgit v0.12 From f6358a7e4c635a74202e73c76a01f5046e1b7c36 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 12:02:39 +0200 Subject: _PyBytesWriter_Alloc(): only use 10 bytes of the small buffer in debug mode to enhance code to detect buffer under- and overflow. --- Objects/bytesobject.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2d4cf4b..8810647 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -4053,8 +4053,20 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) writer->use_small_buffer = 1; #ifdef Py_DEBUG - /* the last byte is reserved, it must be '\0' */ writer->allocated = sizeof(writer->small_buffer) - 1; + /* In debug mode, don't use the full small buffer because it is less + efficient than bytes and bytearray objects to detect buffer underflow + and buffer overflow. Use 10 bytes of the small buffer to test also + code using the smaller buffer in debug mode. + + Don't modify the _PyBytesWriter structure (use a shorter small buffer) + in debug mode to also be able to detect stack overflow when running + tests in debug mode. The _PyBytesWriter is large (more than 512 bytes), + if Py_EnterRecursiveCall() is not used in deep C callback, we may hit a + stack overflow. */ + writer->allocated = Py_MIN(writer->allocated, 10); + /* _PyBytesWriter_CheckConsistency() requires the last byte to be 0, + to detect buffer overflow */ writer->small_buffer[writer->allocated] = 0; #else writer->allocated = sizeof(writer->small_buffer); -- cgit v0.12 From 1285e5c80562ac84ca8bc0ea39b100215d1808a1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 12:10:20 +0200 Subject: Fix compiler warnings (uninitialized variables), false alarms in fact --- Objects/longobject.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 759116a..2cc1585 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1587,7 +1587,7 @@ long_to_decimal_string_internal(PyObject *aa, char **bytes_str) { PyLongObject *scratch, *a; - PyObject *str; + PyObject *str = NULL; Py_ssize_t size, strlen, size_a, i, j; digit *pout, *pin, rem, tenpow; int negative; @@ -1664,7 +1664,6 @@ long_to_decimal_string_internal(PyObject *aa, return -1; } kind = writer->kind; - str = NULL; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); @@ -1777,7 +1776,7 @@ long_format_binary(PyObject *aa, int base, int alternate, _PyBytesWriter *bytes_writer, char **bytes_str) { PyLongObject *a = (PyLongObject *)aa; - PyObject *v; + PyObject *v = NULL; Py_ssize_t sz; Py_ssize_t size_a; enum PyUnicode_Kind kind; @@ -1834,7 +1833,6 @@ long_format_binary(PyObject *aa, int base, int alternate, if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) return -1; kind = writer->kind; - v = NULL; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); -- cgit v0.12 From 2ec8063cc960d32e244dc6a27567f66a447bbda3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:32:13 +0200 Subject: Modify _PyBytes_DecodeEscapeRecode() to use _PyBytesAPI * Don't overallocate by 400% when recode is needed: only overallocate on demand using _PyBytesWriter. * Use _PyLong_DigitValue to convert hexadecimal digit to int * Create _PyBytes_DecodeEscapeRecode() subfunction --- Include/longobject.h | 3 +- Objects/bytesobject.c | 131 ++++++++++++++++++++++++++++---------------------- 2 files changed, 75 insertions(+), 59 deletions(-) diff --git a/Include/longobject.h b/Include/longobject.h index 9574f05..eaf7a7e 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -65,7 +65,8 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); # error "void* different in size from int, long and long long" #endif /* SIZEOF_VOID_P */ -/* Used by Python/mystrtoul.c and _PyBytes_FromHex(). */ +/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), + _PyBytes_DecodeEscapeRecode(), etc. */ #ifndef Py_LIMITED_API PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; #endif diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8810647..556b480 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1068,6 +1068,42 @@ bytes_dealloc(PyObject *op) the string is UTF-8 encoded and should be re-encoded in the specified encoding. */ +static char * +_PyBytes_DecodeEscapeRecode(const char **s, const char *end, + const char *errors, const char *recode_encoding, + _PyBytesWriter *writer, char *p) +{ + PyObject *u, *w; + const char* t; + + t = *s; + /* Decode non-ASCII bytes as UTF-8. */ + while (t < end && (*t & 0x80)) + t++; + u = PyUnicode_DecodeUTF8(*s, t - *s, errors); + if (u == NULL) + return NULL; + + /* Recode them in target encoding. */ + w = PyUnicode_AsEncodedString(u, recode_encoding, errors); + Py_DECREF(u); + if (w == NULL) + return NULL; + assert(PyBytes_Check(w)); + + /* Append bytes to output buffer. */ + writer->min_size--; /* substract 1 preallocated byte */ + p = _PyBytesWriter_WriteBytes(writer, p, + PyBytes_AS_STRING(w), + PyBytes_GET_SIZE(w)); + Py_DECREF(w); + if (p == NULL) + return NULL; + + *s = t; + return p; +} + PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, @@ -1075,54 +1111,42 @@ PyObject *PyBytes_DecodeEscape(const char *s, const char *recode_encoding) { int c; - char *p, *buf; + char *p; const char *end; - PyObject *v; - Py_ssize_t newlen = recode_encoding ? 4*len:len; - v = PyBytes_FromStringAndSize((char *)NULL, newlen); - if (v == NULL) + _PyBytesWriter writer; + + _PyBytesWriter_Init(&writer); + + p = _PyBytesWriter_Alloc(&writer, len); + if (p == NULL) return NULL; - p = buf = PyBytes_AsString(v); + writer.overallocate = 1; + end = s + len; while (s < end) { if (*s != '\\') { non_esc: - if (recode_encoding && (*s & 0x80)) { - PyObject *u, *w; - char *r; - const char* t; - Py_ssize_t rn; - t = s; - /* Decode non-ASCII bytes as UTF-8. */ - while (t < end && (*t & 0x80)) t++; - u = PyUnicode_DecodeUTF8(s, t - s, errors); - if(!u) goto failed; - - /* Recode them in target encoding. */ - w = PyUnicode_AsEncodedString( - u, recode_encoding, errors); - Py_DECREF(u); - if (!w) goto failed; - - /* Append bytes to output buffer. */ - assert(PyBytes_Check(w)); - r = PyBytes_AS_STRING(w); - rn = PyBytes_GET_SIZE(w); - Py_MEMCPY(p, r, rn); - p += rn; - Py_DECREF(w); - s = t; - } else { + if (!(recode_encoding && (*s & 0x80))) { *p++ = *s++; } + else { + /* non-ASCII character and need to recode */ + p = _PyBytes_DecodeEscapeRecode(&s, end, + errors, recode_encoding, + &writer, p); + if (p == NULL) + goto failed; + } continue; } + s++; - if (s==end) { + if (s == end) { PyErr_SetString(PyExc_ValueError, "Trailing \\ in string"); goto failed; } + switch (*s++) { /* XXX This assumes ASCII! */ case '\n': break; @@ -1147,28 +1171,18 @@ PyObject *PyBytes_DecodeEscape(const char *s, *p++ = c; break; case 'x': - if (s+1 < end && Py_ISXDIGIT(s[0]) && Py_ISXDIGIT(s[1])) { - unsigned int x = 0; - c = Py_CHARMASK(*s); - s++; - if (Py_ISDIGIT(c)) - x = c - '0'; - else if (Py_ISLOWER(c)) - x = 10 + c - 'a'; - else - x = 10 + c - 'A'; - x = x << 4; - c = Py_CHARMASK(*s); - s++; - if (Py_ISDIGIT(c)) - x += c - '0'; - else if (Py_ISLOWER(c)) - x += 10 + c - 'a'; - else - x += 10 + c - 'A'; - *p++ = x; - break; + if (s+1 < end) { + int digit1, digit2; + digit1 = _PyLong_DigitValue[Py_CHARMASK(s[0])]; + digit2 = _PyLong_DigitValue[Py_CHARMASK(s[1])]; + if (digit1 < 16 && digit2 < 16) { + *p++ = (unsigned char)((digit1 << 4) + digit2); + s += 2; + break; + } } + /* invalid hexadecimal digits */ + if (!errors || strcmp(errors, "strict") == 0) { PyErr_Format(PyExc_ValueError, "invalid \\x escape at position %d", @@ -1190,6 +1204,7 @@ PyObject *PyBytes_DecodeEscape(const char *s, if (s < end && Py_ISXDIGIT(s[0])) s++; /* and a hexdigit */ break; + default: *p++ = '\\'; s--; @@ -1197,11 +1212,11 @@ PyObject *PyBytes_DecodeEscape(const char *s, UTF-8 bytes may follow. */ } } - if (p-buf < newlen) - _PyBytes_Resize(&v, p - buf); - return v; + + return _PyBytesWriter_Finish(&writer, p); + failed: - Py_DECREF(v); + _PyBytesWriter_Dealloc(&writer); return NULL; } -- cgit v0.12 From f2eafa323bc16e11dd4564ec68f27e8ea8b9c254 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:44:29 +0200 Subject: Split PyBytes_FromObject() into subfunctions --- Objects/bytesobject.c | 185 +++++++++++++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 71 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 556b480..1aae9e3 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3384,88 +3384,99 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return PyBytes_FromObject(x); } -PyObject * -PyBytes_FromObject(PyObject *x) +static PyObject* +_PyBytes_FromBuffer(PyObject *x) { - PyObject *new, *it; - Py_ssize_t i, size; + PyObject *new; + Py_buffer view; - if (x == NULL) { - PyErr_BadInternalCall(); + if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) return NULL; - } - if (PyBytes_CheckExact(x)) { - Py_INCREF(x); - return x; - } + new = PyBytes_FromStringAndSize(NULL, view.len); + if (!new) + goto fail; + if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, + &view, view.len, 'C') < 0) + goto fail; + PyBuffer_Release(&view); + return new; - /* Use the modern buffer interface */ - if (PyObject_CheckBuffer(x)) { - Py_buffer view; - if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) - return NULL; - new = PyBytes_FromStringAndSize(NULL, view.len); - if (!new) - goto fail; - if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, - &view, view.len, 'C') < 0) - goto fail; - PyBuffer_Release(&view); - return new; - fail: - Py_XDECREF(new); - PyBuffer_Release(&view); - return NULL; - } - if (PyUnicode_Check(x)) { - PyErr_SetString(PyExc_TypeError, - "cannot convert unicode object to bytes"); +fail: + Py_XDECREF(new); + PyBuffer_Release(&view); + return NULL; +} + +static PyObject* +_PyBytes_FromList(PyObject *x) +{ + PyObject *new; + Py_ssize_t i; + Py_ssize_t value; + char *str; + + new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); + if (new == NULL) return NULL; - } + str = ((PyBytesObject *)new)->ob_sval; - if (PyList_CheckExact(x)) { - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - for (i = 0; i < Py_SIZE(x); i++) { - Py_ssize_t value = PyNumber_AsSsize_t( - PyList_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) { - Py_DECREF(new); - return NULL; - } - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - Py_DECREF(new); - return NULL; - } - ((PyBytesObject *)new)->ob_sval[i] = (char) value; + for (i = 0; i < Py_SIZE(x); i++) { + value = PyNumber_AsSsize_t(PyList_GET_ITEM(x, i), PyExc_ValueError); + if (value == -1 && PyErr_Occurred()) + goto error; + + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "bytes must be in range(0, 256)"); + goto error; } - return new; + *str++ = (char) value; } - if (PyTuple_CheckExact(x)) { - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - for (i = 0; i < Py_SIZE(x); i++) { - Py_ssize_t value = PyNumber_AsSsize_t( - PyTuple_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) { - Py_DECREF(new); - return NULL; - } - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - Py_DECREF(new); - return NULL; - } - ((PyBytesObject *)new)->ob_sval[i] = (char) value; + return new; + +error: + Py_DECREF(new); + return NULL; +} + +static PyObject* +_PyBytes_FromTuple(PyObject *x) +{ + PyObject *new; + Py_ssize_t i; + Py_ssize_t value; + char *str; + + new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); + if (new == NULL) + return NULL; + str = ((PyBytesObject *)new)->ob_sval; + + for (i = 0; i < Py_SIZE(x); i++) { + value = PyNumber_AsSsize_t(PyTuple_GET_ITEM(x, i), PyExc_ValueError); + if (value == -1 && PyErr_Occurred()) + goto error; + + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "bytes must be in range(0, 256)"); + goto error; } - return new; + *str++ = (char) value; } + return new; + +error: + Py_DECREF(new); + return NULL; +} + +static PyObject * +_PyBytes_FromIterator(PyObject *x) +{ + PyObject *new, *it; + Py_ssize_t i, size; /* For iterator version, create a string object and resize as needed */ size = PyObject_LengthHint(x, 64); @@ -3533,6 +3544,38 @@ PyBytes_FromObject(PyObject *x) return NULL; } +PyObject * +PyBytes_FromObject(PyObject *x) +{ + if (x == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + if (PyBytes_CheckExact(x)) { + Py_INCREF(x); + return x; + } + + /* Use the modern buffer interface */ + if (PyObject_CheckBuffer(x)) + return _PyBytes_FromBuffer(x); + + if (PyList_CheckExact(x)) + return _PyBytes_FromList(x); + + if (PyTuple_CheckExact(x)) + return _PyBytes_FromTuple(x); + + if (PyUnicode_Check(x)) { + PyErr_SetString(PyExc_TypeError, + "cannot convert unicode object to bytes"); + return NULL; + } + + return _PyBytes_FromIterator(x); +} + static PyObject * str_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { -- cgit v0.12 From 3c50ce39bfe0b5b905432b195c5bb1cf939f4273 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:50:40 +0200 Subject: Factorize _PyBytes_FromList() and _PyBytes_FromTuple() code using a C macro --- Objects/bytesobject.c | 89 ++++++++++++++++++++------------------------------- 1 file changed, 35 insertions(+), 54 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 1aae9e3..e54f299 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3408,68 +3408,49 @@ fail: return NULL; } +#define _PyBytes_FROM_LIST_BODY(x, GET_ITEM) \ + do { \ + PyObject *bytes; \ + Py_ssize_t i; \ + Py_ssize_t value; \ + char *str; \ + PyObject *item; \ + \ + bytes = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); \ + if (bytes == NULL) \ + return NULL; \ + str = ((PyBytesObject *)bytes)->ob_sval; \ + \ + for (i = 0; i < Py_SIZE(x); i++) { \ + item = GET_ITEM((x), i); \ + value = PyNumber_AsSsize_t(item, PyExc_ValueError); \ + if (value == -1 && PyErr_Occurred()) \ + goto error; \ + \ + if (value < 0 || value >= 256) { \ + PyErr_SetString(PyExc_ValueError, \ + "bytes must be in range(0, 256)"); \ + goto error; \ + } \ + *str++ = (char) value; \ + } \ + return bytes; \ + \ + error: \ + Py_DECREF(bytes); \ + return NULL; \ + } while (0) + static PyObject* _PyBytes_FromList(PyObject *x) { - PyObject *new; - Py_ssize_t i; - Py_ssize_t value; - char *str; - - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - str = ((PyBytesObject *)new)->ob_sval; - - for (i = 0; i < Py_SIZE(x); i++) { - value = PyNumber_AsSsize_t(PyList_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) - goto error; - - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - goto error; - } - *str++ = (char) value; - } - return new; - -error: - Py_DECREF(new); - return NULL; + _PyBytes_FROM_LIST_BODY(x, PyList_GET_ITEM); } static PyObject* _PyBytes_FromTuple(PyObject *x) { - PyObject *new; - Py_ssize_t i; - Py_ssize_t value; - char *str; - - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - str = ((PyBytesObject *)new)->ob_sval; - - for (i = 0; i < Py_SIZE(x); i++) { - value = PyNumber_AsSsize_t(PyTuple_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) - goto error; - - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - goto error; - } - *str++ = (char) value; - } - return new; - -error: - Py_DECREF(new); - return NULL; + _PyBytes_FROM_LIST_BODY(x, PyTuple_GET_ITEM); } static PyObject * -- cgit v0.12 From c5c3ba4becf98b37b4dd92333c144b0d95faef03 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:56:47 +0200 Subject: Add _PyBytesWriter_Resize() function This function gives a control to the buffer size without using min_size. --- Include/bytesobject.h | 19 ++++++++++++++++++- Objects/bytesobject.c | 48 ++++++++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 4046c1c..8469112 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -178,7 +178,9 @@ PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size); -/* Add *size* bytes to the buffer. +/* Ensure that the buffer is large enough to write *size* bytes. + Add size to the writer minimum size (min_size attribute). + str is the current pointer inside the buffer. Return the updated current pointer inside the buffer. Raise an exception and return NULL on error. */ @@ -186,6 +188,21 @@ PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size); +/* Resize the buffer to make it larger. + The new buffer may be larger than size bytes because of overallocation. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. + + Note: size must be greater than the number of allocated bytes in the writer. + + This function doesn't use the writer minimum size (min_size attribute). + + See also _PyBytesWriter_Prepare(). + */ +PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, + void *str, + Py_ssize_t size); + /* Write bytes. Raise an exception and return NULL on error. */ PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e54f299..ae7b1ea 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3997,29 +3997,14 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) } void* -_PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) +_PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size) { Py_ssize_t allocated, pos; _PyBytesWriter_CheckConsistency(writer, str); - assert(size >= 0); - - if (size == 0) { - /* nothing to do */ - return str; - } - - if (writer->min_size > PY_SSIZE_T_MAX - size) { - PyErr_NoMemory(); - goto error; - } - writer->min_size += size; - - allocated = writer->allocated; - if (writer->min_size <= allocated) - return str; + assert(writer->allocated < size); - allocated = writer->min_size; + allocated = size; if (writer->overallocate && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { /* overallocate to limit the number of realloc() */ @@ -4080,6 +4065,33 @@ error: return NULL; } +void* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) +{ + Py_ssize_t new_min_size; + + _PyBytesWriter_CheckConsistency(writer, str); + assert(size >= 0); + + if (size == 0) { + /* nothing to do */ + return str; + } + + if (writer->min_size > PY_SSIZE_T_MAX - size) { + PyErr_NoMemory(); + _PyBytesWriter_Dealloc(writer); + return NULL; + } + new_min_size = writer->min_size + size; + + if (new_min_size > writer->allocated) + str = _PyBytesWriter_Resize(writer, str, new_min_size); + + writer->min_size = new_min_size; + return str; +} + /* Allocate the buffer to write size bytes. Return the pointer to the beginning of buffer data. Raise an exception and return NULL on error. */ -- cgit v0.12 From c3d2bc19e47892e19c9ad6a048047964dbb78cb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 14:15:49 +0200 Subject: Use _PyBytesWriter in _PyBytes_FromIterator() --- Objects/bytesobject.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ae7b1ea..c10dbdf 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3456,23 +3456,23 @@ _PyBytes_FromTuple(PyObject *x) static PyObject * _PyBytes_FromIterator(PyObject *x) { - PyObject *new, *it; + char *str; + PyObject *it; Py_ssize_t i, size; + _PyBytesWriter writer; + + _PyBytesWriter_Init(&writer); /* For iterator version, create a string object and resize as needed */ size = PyObject_LengthHint(x, 64); if (size == -1 && PyErr_Occurred()) return NULL; - /* Allocate an extra byte to prevent PyBytes_FromStringAndSize() from - returning a shared empty bytes string. This required because we - want to call _PyBytes_Resize() the returned object, which we can - only do on bytes objects with refcount == 1. */ - if (size == 0) - size = 1; - new = PyBytes_FromStringAndSize(NULL, size); - if (new == NULL) + + str = _PyBytesWriter_Alloc(&writer, size); + if (str == NULL) return NULL; - assert(Py_REFCNT(new) == 1); + writer.overallocate = 1; + size = writer.allocated; /* Get the iterator */ it = PyObject_GetIter(x); @@ -3507,21 +3507,20 @@ _PyBytes_FromIterator(PyObject *x) /* Append the byte */ if (i >= size) { - size = 2 * size + 1; - if (_PyBytes_Resize(&new, size) < 0) - goto error; + str = _PyBytesWriter_Resize(&writer, str, size+1); + if (str == NULL) + return NULL; + size = writer.allocated; } - ((PyBytesObject *)new)->ob_sval[i] = (char) value; + *str++ = (char) value; } - _PyBytes_Resize(&new, i); - - /* Clean up and return success */ Py_DECREF(it); - return new; + + return _PyBytesWriter_Finish(&writer, str); error: + _PyBytesWriter_Dealloc(&writer); Py_XDECREF(it); - Py_XDECREF(new); return NULL; } -- cgit v0.12 From 1bfe9305859cf280464ed1f40fc9e2793b0f03b7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:02:35 +0200 Subject: Issue #25384: Fix binascii.rledecode_hqx() Fix usage of _PyBytesWriter API. Use the new _PyBytesWriter_Resize() function instead of _PyBytesWriter_Prepare(). --- Lib/test/test_binascii.py | 16 +++++++++++++++- Modules/binascii.c | 11 +++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 0ab46bc..fbc933e4 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -159,11 +159,25 @@ class BinASCIITest(unittest.TestCase): # Then calculate the hexbin4 binary-to-ASCII translation rle = binascii.rlecode_hqx(self.data) a = binascii.b2a_hqx(self.type2test(rle)) + b, _ = binascii.a2b_hqx(self.type2test(a)) res = binascii.rledecode_hqx(b) - self.assertEqual(res, self.rawdata) + def test_rle(self): + # test repetition with a repetition longer than the limit of 255 + data = (b'a' * 100 + b'b' + b'c' * 300) + + encoded = binascii.rlecode_hqx(data) + self.assertEqual(encoded, + (b'a\x90d' # 'a' * 100 + b'b' # 'b' + b'c\x90\xff' # 'c' * 255 + b'c\x90-')) # 'c' * 45 + + decoded = binascii.rledecode_hqx(encoded) + self.assertEqual(decoded, data) + def test_hex(self): # test hexlification s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' diff --git a/Modules/binascii.c b/Modules/binascii.c index dfa5535..a1070b7 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -800,14 +800,15 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer of reasonable size. Resized when needed */ - out_len = in_len * 2; + out_len = in_len; out_data = _PyBytesWriter_Alloc(&writer, out_len); if (out_data == NULL) return NULL; /* Use overallocation */ writer.overallocate = 1; - out_len_left = writer.allocated; + out_len = writer.allocated; + out_len_left = out_len; /* ** We need two macros here to get/put bytes and handle @@ -830,10 +831,12 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) overallocate the buffer anymore */ \ writer.overallocate = 0; \ } \ - out_data = _PyBytesWriter_Prepare(&writer, out_data, 1); \ + out_data = _PyBytesWriter_Resize(&writer, out_data, \ + writer.allocated + 1); \ if (out_data == NULL) \ goto error; \ - out_len_left = writer.allocated; \ + out_len_left = writer.allocated - out_len - 1; \ + out_len = writer.allocated; \ } \ *out_data++ = b; \ } while(0) -- cgit v0.12 From f9c9a3fedfcf9da646cd0183b93bc00459cff164 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:20:07 +0200 Subject: Refactor binascii.rledecode_hqx() Rewrite the code to handle the output buffer. --- Modules/binascii.c | 53 +++++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/Modules/binascii.c b/Modules/binascii.c index a1070b7..ccd81fa 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -784,7 +784,7 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) { unsigned char *in_data, *out_data; unsigned char in_byte, in_repeat; - Py_ssize_t in_len, out_len, out_len_left; + Py_ssize_t in_len; _PyBytesWriter writer; in_data = data->buf; @@ -800,15 +800,12 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer of reasonable size. Resized when needed */ - out_len = in_len; - out_data = _PyBytesWriter_Alloc(&writer, out_len); + out_data = _PyBytesWriter_Alloc(&writer, in_len); if (out_data == NULL) return NULL; /* Use overallocation */ writer.overallocate = 1; - out_len = writer.allocated; - out_len_left = out_len; /* ** We need two macros here to get/put bytes and handle @@ -823,24 +820,6 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) b = *in_data++; \ } while(0) -#define OUTBYTE(b) \ - do { \ - if ( --out_len_left < 0 ) { \ - if (in_len <= 0) { \ - /* We are done after this write, no need to \ - overallocate the buffer anymore */ \ - writer.overallocate = 0; \ - } \ - out_data = _PyBytesWriter_Resize(&writer, out_data, \ - writer.allocated + 1); \ - if (out_data == NULL) \ - goto error; \ - out_len_left = writer.allocated - out_len - 1; \ - out_len = writer.allocated; \ - } \ - *out_data++ = b; \ - } while(0) - /* ** Handle first byte separately (since we have to get angry ** in case of an orphaned RLE code). @@ -849,6 +828,10 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) if (in_byte == RUNCHAR) { INBYTE(in_repeat); + /* only 1 byte will be written, but 2 bytes were preallocated: + substract 1 byte to prevent overallocation */ + writer.min_size--; + if (in_repeat != 0) { /* Note Error, not Incomplete (which is at the end ** of the string only). This is a programmer error. @@ -856,9 +839,9 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) PyErr_SetString(Error, "Orphaned RLE code at start"); goto error; } - OUTBYTE(RUNCHAR); + *out_data++ = RUNCHAR; } else { - OUTBYTE(in_byte); + *out_data++ = in_byte; } while( in_len > 0 ) { @@ -866,18 +849,32 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) if (in_byte == RUNCHAR) { INBYTE(in_repeat); + /* only 1 byte will be written, but 2 bytes were preallocated: + substract 1 byte to prevent overallocation */ + writer.min_size--; + if ( in_repeat == 0 ) { /* Just an escaped RUNCHAR value */ - OUTBYTE(RUNCHAR); + *out_data++ = RUNCHAR; } else { /* Pick up value and output a sequence of it */ in_byte = out_data[-1]; + + /* enlarge the buffer if needed */ + if (in_repeat > 1) { + /* -1 because we already preallocated 1 byte */ + out_data = _PyBytesWriter_Prepare(&writer, out_data, + in_repeat - 1); + if (out_data == NULL) + goto error; + } + while ( --in_repeat > 0 ) - OUTBYTE(in_byte); + *out_data++ = in_byte; } } else { /* Normal byte */ - OUTBYTE(in_byte); + *out_data++ = in_byte; } } return _PyBytesWriter_Finish(&writer, out_data); -- cgit v0.12 From 83ff8a68324dfe54af8baf7e67f81b2db567ae7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:28:59 +0200 Subject: test_bytes: new try to fix test on '%p' formatter on Windows --- Lib/test/test_bytes.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 0fe33b5..87799df 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -839,12 +839,22 @@ class BytesTest(BaseBytesTest, unittest.TestCase): b'i=-123') self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), b'x=abc') + + sizeof_ptr = ctypes.sizeof(c_char_p) + + if os.name == 'nt': + # Windows (MSCRT) + ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + def ptr_formatter(ptr): + return (ptr_format % ptr) + else: + # UNIX (glibc) + def ptr_formatter(ptr): + return '%#x' % ptr + ptr = 0xabcdef - expected = [b'ptr=%#x' % ptr] - win_format = 'ptr=0x%0{}X'.format(2 * ctypes.sizeof(c_char_p)) - expected.append((win_format % ptr).encode('ascii')) - self.assertIn(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), - expected) + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + ('ptr=' + ptr_formatter(ptr)).encode('ascii')) self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), b's=cstr') @@ -859,7 +869,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), (b'%zu', c_size_t, size_max, str), - (b'%p', c_char_p, size_max, lambda value: '%#x' % value), + (b'%p', c_char_p, size_max, ptr_formatter), ): self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), py_formatter(value).encode('ascii')), -- cgit v0.12 From 91108f049ffb35b4b9f0353a41d9c6097b37046d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 18:25:31 +0200 Subject: Issue #25210: Change error message of do_richcompare() Don't add parenthesis to type names. Add also quotes around the type names. Before: TypeError: unorderable types: int() < NoneType() After: TypeError: '<' not supported between instances of 'int' and 'NoneType' --- Doc/howto/argparse.rst | 3 ++- Doc/library/enum.rst | 2 +- Doc/library/pathlib.rst | 2 +- Objects/object.c | 5 ++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 510d1d4..9c111b4 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -547,7 +547,8 @@ And this is what it gives: Traceback (most recent call last): File "prog.py", line 11, in if args.verbosity >= 2: - TypeError: unorderable types: NoneType() >= int() + TypeError: '>=' not supported between instances of 'NoneType' and 'int' + * First output went well, and fixes the bug we had before. That is, we want any value >= 2 to be as verbose as possible. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 18519f0..0fbbf5a 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -257,7 +257,7 @@ members are not integers (but see `IntEnum`_ below):: >>> Color.red < Color.blue Traceback (most recent call last): File "", line 1, in - TypeError: unorderable types: Color() < Color() + TypeError: '<' not supported between instances of 'Color' and 'Color' Equality comparisons are defined though:: diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2f06544..ff5196d 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -195,7 +195,7 @@ Paths of a different flavour compare unequal and cannot be ordered:: >>> PureWindowsPath('foo') < PurePosixPath('foo') Traceback (most recent call last): File "", line 1, in - TypeError: unorderable types: PureWindowsPath() < PurePosixPath() + TypeError: '<' not supported between instances of 'PureWindowsPath' and 'PurePosixPath' Operators diff --git a/Objects/object.c b/Objects/object.c index 6fc4df1..e1718ea 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -686,11 +686,10 @@ do_richcompare(PyObject *v, PyObject *w, int op) res = (v != w) ? Py_True : Py_False; break; default: - /* XXX Special-case None so it doesn't show as NoneType() */ PyErr_Format(PyExc_TypeError, - "unorderable types: %.100s() %s %.100s()", - v->ob_type->tp_name, + "'%s' not supported between instances of '%.100s' and '%.100s'", opstrings[op], + v->ob_type->tp_name, w->ob_type->tp_name); return NULL; } -- cgit v0.12 From 1286d14500e0cf30e900ecf0b597ee1c308e0dfe Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 14 Oct 2015 23:16:57 -0700 Subject: Improve variable names and constant expressions --- Modules/_collectionsmodule.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index cef92c0..06f85e9 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -303,8 +303,8 @@ deque_append(dequeobject *deque, PyObject *item) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *rv = deque_popleft(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_popleft(deque, NULL); + Py_DECREF(olditem); } Py_RETURN_NONE; } @@ -331,8 +331,8 @@ deque_appendleft(dequeobject *deque, PyObject *item) deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *rv = deque_pop(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_pop(deque, NULL); + Py_DECREF(olditem); } Py_RETURN_NONE; } @@ -423,8 +423,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *rv = deque_popleft(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_popleft(deque, NULL); + Py_DECREF(olditem); } } return finalize_iterator(it); @@ -487,8 +487,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *rv = deque_pop(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_pop(deque, NULL); + Py_DECREF(olditem); } } return finalize_iterator(it); @@ -1283,6 +1283,7 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) PyObject *item; Py_ssize_t index; Py_ssize_t indexlo = deque->leftindex; + Py_ssize_t indexhigh; for (b = deque->leftblock; b != deque->rightblock; b = b->rightlink) { for (index = indexlo; index < BLOCKLEN ; index++) { @@ -1291,7 +1292,8 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) } indexlo = 0; } - for (index = indexlo; index <= deque->rightindex; index++) { + indexhigh = deque->rightindex; + for (index = indexlo; index <= indexhigh; index++) { item = b->data[index]; Py_VISIT(item); } -- cgit v0.12 From bc00341105afed2255ecd139ca6ca381a482b1f9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 14 Oct 2015 23:33:23 -0700 Subject: Use unsigned division --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 06f85e9..bc225a5 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1482,7 +1482,7 @@ deque_sizeof(dequeobject *deque, void *unused) Py_ssize_t blocks; res = sizeof(dequeobject); - blocks = (deque->leftindex + Py_SIZE(deque) + BLOCKLEN - 1) / BLOCKLEN; + blocks = (size_t)(deque->leftindex + Py_SIZE(deque) + BLOCKLEN - 1) / BLOCKLEN; assert(deque->leftindex + Py_SIZE(deque) - 1 == (blocks - 1) * BLOCKLEN + deque->rightindex); res += blocks * sizeof(block); -- cgit v0.12 From a4b13d00200053e5985c089af45be34cdbffd1eb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 15 Oct 2015 08:05:31 -0700 Subject: Rewrap comment. --- Modules/_collectionsmodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index bc225a5..85a0793 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -272,10 +272,10 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); /* The deque's size limit is d.maxlen. The limit can be zero or positive. * If there is no limit, then d.maxlen == -1. * - * After an item is added to a deque, we check to see if the size has grown past - * the limit. If it has, we get the size back down to the limit by popping an - * item off of the opposite end. The methods that can trigger this are append(), - * appendleft(), extend(), and extendleft(). + * After an item is added to a deque, we check to see if the size has + * grown past the limit. If it has, we get the size back down to the limit + * by popping an item off of the opposite end. The methods that can + * trigger this are append(), appendleft(), extend(), and extendleft(). * * The macro to check whether a deque needs to be trimmed uses a single * unsigned test that returns true whenever 0 <= maxlen < Py_SIZE(deque). -- cgit v0.12 From 1eca237c2f3d5c150cf9184fa7c9d9b137d1187c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 15 Oct 2015 23:25:53 -0700 Subject: Remove old Todo entry that isn't going to happen. --- Modules/_collectionsmodule.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 85a0793..1acbf86 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -156,12 +156,6 @@ freeblock(block *b) } } -/* XXX Todo: - If aligned memory allocations become available, make the - deque object 64 byte aligned so that all of the fields - can be retrieved or updated in a single cache line. -*/ - static PyObject * deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { -- cgit v0.12 From c0d91aff9a3b91307b26e8b7c34dfbf27bbdd43a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 16 Oct 2015 12:21:37 -0700 Subject: Upgrade the imp module's deprecation to DeprecationWarning. --- Doc/whatsnew/3.6.rst | 3 +++ Lib/imp.py | 2 +- Lib/pkgutil.py | 2 +- Misc/NEWS | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index edacea1..b374280 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -228,6 +228,9 @@ Changes in the Python API now raises :exc:`ValueError` for out-of-range values, rather than returning :const:`None`. See :issue:`20059`. +* The :mod:`imp` module now raises a :exc:`DeprecationWarning` instead of + :exc:`PendingDeprecationWarning`. + Changes in the C API -------------------- diff --git a/Lib/imp.py b/Lib/imp.py index f6fff44..b339952 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -30,7 +30,7 @@ import warnings warnings.warn("the imp module is deprecated in favour of importlib; " "see the module's documentation for alternative uses", - PendingDeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=2) # DEPRECATED SEARCH_ERROR = 0 diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index fc4a074..203d515 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -180,7 +180,7 @@ iter_importer_modules.register( def _import_imp(): global imp with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) imp = importlib.import_module('imp') class ImpImporter: diff --git a/Misc/NEWS b/Misc/NEWS index bcc85a7..6e3e2c3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -63,6 +63,8 @@ Core and Builtins Library ------- +- Move the imp module from a PendingDeprecationWarning to DeprecationWarning. + - Issue #25407: Remove mentions of the formatter module being removed in Python 3.6. -- cgit v0.12 From 9b63868f77c6782bba6b3ceed8e2058b471b6fd5 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 16 Oct 2015 15:14:27 -0700 Subject: Issue #25154: Deprecate the pyvenv script. This was done so as to move users to `python3 -m venv` which prevents confusion over which Python interpreter will be used in the virtual environment when more than one is installed. --- Doc/whatsnew/3.6.rst | 5 ++++- Misc/NEWS | 6 ++++++ Tools/scripts/pyvenv | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b374280..22513cf8 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -197,7 +197,10 @@ Deprecated functions and types of the C API Deprecated features ------------------- -* None yet. +* The ``pyvenv`` script has been deprecated in favour of ``python3 -m venv``. + This prevents confusion as to what Python interpreter ``pyvenv`` is + connected to and thus what Python interpreter will be used by the virtual + environment. See :issue:`25154`. Removed diff --git a/Misc/NEWS b/Misc/NEWS index 6e3e2c3..7d2c79f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -289,6 +289,12 @@ Windows - Issue #25022: Removed very outdated PC/example_nt/ directory. +Tools/Demos +----------- + +- Issue #25154: The pyvenv script has been deprecated in favour of + `python3 -m venv`. + What's New in Python 3.5.1 release candidate 1? =============================================== diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv index 978d691..1043efc 100755 --- a/Tools/scripts/pyvenv +++ b/Tools/scripts/pyvenv @@ -1,6 +1,12 @@ #!/usr/bin/env python3 if __name__ == '__main__': import sys + import pathlib + + executable = pathlib.Path(sys.executable or 'python3').name + print('WARNING: the pyenv script is deprecated in favour of ' + '`{} -m venv`'.format(exeutable, file=sys.stderr)) + rc = 1 try: import venv -- cgit v0.12 From 67317742162dd5b9728672b2ff7ed21e2aa7d2fa Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 16 Oct 2015 20:45:53 -0400 Subject: Issue 25422: Add tests for multi-line string tokenization. Also remove truncated tokens. --- Lib/test/test_tokenize.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 3b17ca6..b74396f 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -24,8 +24,7 @@ class TokenizeTest(TestCase): if type == ENDMARKER: break type = tok_name[type] - result.append(" %(type)-10.10s %(token)-13.13r %(start)s %(end)s" % - locals()) + result.append(f" {type:10} {token!r:13} {start} {end}") self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) @@ -132,18 +131,18 @@ def k(x): self.check_tokenize("x = 0xfffffffffff", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '0xffffffffff (1, 4) (1, 17) + NUMBER '0xfffffffffff' (1, 4) (1, 17) """) self.check_tokenize("x = 123141242151251616110", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '123141242151 (1, 4) (1, 25) + NUMBER '123141242151251616110' (1, 4) (1, 25) """) self.check_tokenize("x = -15921590215012591", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) - NUMBER '159215902150 (1, 5) (1, 22) + NUMBER '15921590215012591' (1, 5) (1, 22) """) def test_float(self): @@ -307,6 +306,33 @@ def k(x): OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) """) + # Check 0, 1, and 2 character string prefixes. + self.check_tokenize(r'"a\ +de\ +fg"', """\ + STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3) + """) + self.check_tokenize(r'u"a\ +de"', """\ + STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3) + """) + self.check_tokenize(r'rb"a\ +d"', """\ + STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2) + """) + self.check_tokenize(r'"""a\ +b"""', """\ + STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'u"""a\ +b"""', """\ + STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'rb"""a\ +b\ +c"""', """\ + STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) + """) def test_function(self): self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\ @@ -505,7 +531,7 @@ def k(x): # Methods self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\ OP '@' (1, 0) (1, 1) - NAME 'staticmethod (1, 1) (1, 13) + NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) -- cgit v0.12 From 20151f50f6942f32584ece6e751e1edb59699007 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 16 Oct 2015 22:47:29 -0700 Subject: Issue #25414: Remove unnecessary tests that can never succeed. --- Modules/_collectionsmodule.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 1acbf86..ceba037 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -110,12 +110,6 @@ static PyTypeObject deque_type; #define CHECK_NOT_END(link) #endif -/* To prevent len from overflowing PY_SSIZE_T_MAX, we refuse to - allocate new blocks if the current len is nearing overflow. -*/ - -#define MAX_DEQUE_LEN (PY_SSIZE_T_MAX - 3*BLOCKLEN) - /* A simple freelisting scheme is used to minimize calls to the memory allocator. It accommodates common use cases where new blocks are being added at about the same rate as old blocks are being freed. @@ -128,11 +122,6 @@ static block *freeblocks[MAXFREEBLOCKS]; static block * newblock(Py_ssize_t len) { block *b; - if (len >= MAX_DEQUE_LEN) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more blocks to the deque"); - return NULL; - } if (numfreeblocks) { numfreeblocks--; return freeblocks[numfreeblocks]; @@ -676,9 +665,6 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) if (deque->maxlen >= 0 && n > deque->maxlen) n = deque->maxlen; - if (n > MAX_DEQUE_LEN) - return PyErr_NoMemory(); - deque->state++; for (i = 0 ; i < n-1 ; ) { if (deque->rightindex == BLOCKLEN - 1) { @@ -709,7 +695,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } - if ((size_t)size > MAX_DEQUE_LEN / (size_t)n) { + if ((size_t)size > PY_SSIZE_T_MAX / (size_t)n) { return PyErr_NoMemory(); } -- cgit v0.12 From 6dc9ce19230e102ff6a64d998304292799949418 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 20 Oct 2015 01:07:53 -0400 Subject: Remove double 'error'. --- Doc/whatsnew/3.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 22513cf8..b500550 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -144,7 +144,7 @@ Optimizations ``surrogateescape``, ``ignore`` and ``replace`` (Contributed by Victor Stinner in :issue:`24870`). -* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error +* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). * The UTF-8 encoder is now up to 75 times as fast for error handlers: -- cgit v0.12 From 0f43bb160e65b4acbee898896b4fd4cc1420807f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 20 Oct 2015 00:03:33 -0700 Subject: Only update the state variable once per iteration. --- Modules/_collectionsmodule.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ceba037..aa879be 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -269,7 +269,6 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); static PyObject * deque_append(dequeobject *deque, PyObject *item) { - deque->state++; if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) @@ -288,6 +287,8 @@ deque_append(dequeobject *deque, PyObject *item) if (NEEDS_TRIM(deque, deque->maxlen)) { PyObject *olditem = deque_popleft(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } Py_RETURN_NONE; } @@ -297,7 +298,6 @@ PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); static PyObject * deque_appendleft(dequeobject *deque, PyObject *item) { - deque->state++; if (deque->leftindex == 0) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) @@ -316,6 +316,8 @@ deque_appendleft(dequeobject *deque, PyObject *item) if (NEEDS_TRIM(deque, deque->maxlen)) { PyObject *olditem = deque_pop(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } Py_RETURN_NONE; } @@ -387,7 +389,6 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - deque->state++; if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) { @@ -408,6 +409,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (NEEDS_TRIM(deque, maxlen)) { PyObject *olditem = deque_popleft(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } } return finalize_iterator(it); @@ -451,7 +454,6 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - deque->state++; if (deque->leftindex == 0) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) { @@ -472,6 +474,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (NEEDS_TRIM(deque, maxlen)) { PyObject *olditem = deque_pop(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } } return finalize_iterator(it); -- cgit v0.12 From 6a4d52e86521cfabd0646daacd31ba08e6098a78 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 22 Oct 2015 07:49:36 +0300 Subject: Issue #25210: Add some basic tests for the new exception message --- Lib/test/test_richcmp.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 1582caa..58729a9 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -253,6 +253,31 @@ class MiscTest(unittest.TestCase): self.assertTrue(a != b) self.assertTrue(a < b) + def test_exception_message(self): + class Spam: + pass + + tests = [ + (lambda: 42 < None, r"'<' .* of 'int' and 'NoneType'"), + (lambda: None < 42, r"'<' .* of 'NoneType' and 'int'"), + (lambda: 42 > None, r"'>' .* of 'int' and 'NoneType'"), + (lambda: "foo" < None, r"'<' .* of 'str' and 'NoneType'"), + (lambda: "foo" >= 666, r"'>=' .* of 'str' and 'int'"), + (lambda: 42 <= None, r"'<=' .* of 'int' and 'NoneType'"), + (lambda: 42 >= None, r"'>=' .* of 'int' and 'NoneType'"), + (lambda: 42 < [], r"'<' .* of 'int' and 'list'"), + (lambda: () > [], r"'>' .* of 'tuple' and 'list'"), + (lambda: None >= None, r"'>=' .* of 'NoneType' and 'NoneType'"), + (lambda: Spam() < 42, r"'<' .* of 'Spam' and 'int'"), + (lambda: 42 < Spam(), r"'<' .* of 'int' and 'Spam'"), + (lambda: Spam() <= Spam(), r"'<=' .* of 'Spam' and 'Spam'"), + ] + for i, test in enumerate(tests): + with self.subTest(test=i): + with self.assertRaisesRegex(TypeError, test[1]): + test[0]() + + class DictTest(unittest.TestCase): def test_dicts(self): -- cgit v0.12 From db41fd43cf60ed22e3c3b2f55101b48f9ae3d215 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 22 Oct 2015 22:48:16 -0700 Subject: Removed unused parameter --- Modules/_collectionsmodule.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index aa879be..ba8238c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -120,7 +120,7 @@ static Py_ssize_t numfreeblocks = 0; static block *freeblocks[MAXFREEBLOCKS]; static block * -newblock(Py_ssize_t len) { +newblock(void) { block *b; if (numfreeblocks) { numfreeblocks--; @@ -156,7 +156,7 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (deque == NULL) return NULL; - b = newblock(0); + b = newblock(); if (b == NULL) { Py_DECREF(deque); return NULL; @@ -270,7 +270,7 @@ static PyObject * deque_append(dequeobject *deque, PyObject *item) { if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) return NULL; b->leftlink = deque->rightblock; @@ -299,7 +299,7 @@ static PyObject * deque_appendleft(dequeobject *deque, PyObject *item) { if (deque->leftindex == 0) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) return NULL; b->rightlink = deque->leftblock; @@ -390,7 +390,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) { Py_DECREF(item); Py_DECREF(it); @@ -455,7 +455,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque->leftindex == 0) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) { Py_DECREF(item); Py_DECREF(it); @@ -586,7 +586,7 @@ deque_clear(dequeobject *deque) adversary could cause it to never terminate). */ - b = newblock(0); + b = newblock(); if (b == NULL) { PyErr_Clear(); goto alternate_method; @@ -672,7 +672,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) deque->state++; for (i = 0 ; i < n-1 ; ) { if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(Py_SIZE(deque) + i); + block *b = newblock(); if (b == NULL) { Py_SIZE(deque) += i; return NULL; @@ -786,7 +786,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) while (n > 0) { if (leftindex == 0) { if (b == NULL) { - b = newblock(len); + b = newblock(); if (b == NULL) goto done; } @@ -830,7 +830,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) while (n < 0) { if (rightindex == BLOCKLEN - 1) { if (b == NULL) { - b = newblock(len); + b = newblock(); if (b == NULL) goto done; } -- cgit v0.12 From 5b6f3644e67d6bc597ff246a1e75902d36d5699e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 23 Oct 2015 13:24:03 -0700 Subject: Issue #24633: Removes automatic rename of site-packages/README since README.txt is now committed. --- Tools/msi/lib/lib_files.wxs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index fa79a8d..804ab01 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -64,16 +64,10 @@ - - - - - - -- cgit v0.12 From f1c47e47514e1cf1006dc21de386af7ee25f8db5 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 25 Oct 2015 17:40:31 -0700 Subject: Issue #25154: Make the file argument apply to the print function and not str.format call. --- Tools/scripts/pyvenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv index 1043efc..1eed3ff 100755 --- a/Tools/scripts/pyvenv +++ b/Tools/scripts/pyvenv @@ -5,7 +5,7 @@ if __name__ == '__main__': executable = pathlib.Path(sys.executable or 'python3').name print('WARNING: the pyenv script is deprecated in favour of ' - '`{} -m venv`'.format(exeutable, file=sys.stderr)) + '`{} -m venv`'.format(exeutable), file=sys.stderr) rc = 1 try: -- cgit v0.12 From 1c8222c80a8a534bd9357aafc3fba7b6927efc15 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Mon, 26 Oct 2015 04:37:55 -0400 Subject: Issue 25311: Add support for f-strings to tokenize.py. Also added some comments to explain what's happening, since it's not so obvious. --- Lib/test/test_tokenize.py | 17 +++++++ Lib/tokenize.py | 118 ++++++++++++++++++++++++++-------------------- 2 files changed, 84 insertions(+), 51 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index b74396f..90438e7 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -333,6 +333,23 @@ b\ c"""', """\ STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) """) + self.check_tokenize('f"abc"', """\ + STRING 'f"abc"' (1, 0) (1, 6) + """) + self.check_tokenize('fR"a{b}c"', """\ + STRING 'fR"a{b}c"' (1, 0) (1, 9) + """) + self.check_tokenize('f"""abc"""', """\ + STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10) + """) + self.check_tokenize(r'f"abc\ +def"', """\ + STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4) + """) + self.check_tokenize(r'Rf"abc\ +def"', """\ + STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4) + """) def test_function(self): self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\ diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 65d06e5..2237c3a 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -29,6 +29,7 @@ from codecs import lookup, BOM_UTF8 import collections from io import TextIOWrapper from itertools import chain +import itertools as _itertools import re import sys from token import * @@ -131,7 +132,28 @@ Floatnumber = group(Pointfloat, Expfloat) Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) -StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?' +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permuations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = set(['']) + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) # Tail end of ' string. Single = r"[^'\\]*(?:\\.[^'\\]*)*'" @@ -169,50 +191,25 @@ ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) -def _compile(expr): - return re.compile(expr, re.UNICODE) - -endpats = {"'": Single, '"': Double, - "'''": Single3, '"""': Double3, - "r'''": Single3, 'r"""': Double3, - "b'''": Single3, 'b"""': Double3, - "R'''": Single3, 'R"""': Double3, - "B'''": Single3, 'B"""': Double3, - "br'''": Single3, 'br"""': Double3, - "bR'''": Single3, 'bR"""': Double3, - "Br'''": Single3, 'Br"""': Double3, - "BR'''": Single3, 'BR"""': Double3, - "rb'''": Single3, 'rb"""': Double3, - "Rb'''": Single3, 'Rb"""': Double3, - "rB'''": Single3, 'rB"""': Double3, - "RB'''": Single3, 'RB"""': Double3, - "u'''": Single3, 'u"""': Double3, - "U'''": Single3, 'U"""': Double3, - 'r': None, 'R': None, 'b': None, 'B': None, - 'u': None, 'U': None} - -triple_quoted = {} -for t in ("'''", '"""', - "r'''", 'r"""', "R'''", 'R"""', - "b'''", 'b"""', "B'''", 'B"""', - "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""', - "rb'''", 'rb"""', "rB'''", 'rB"""', - "Rb'''", 'Rb"""', "RB'''", 'RB"""', - "u'''", 'u"""', "U'''", 'U"""', - ): - triple_quoted[t] = t -single_quoted = {} -for t in ("'", '"', - "r'", 'r"', "R'", 'R"', - "b'", 'b"', "B'", 'B"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"' , - "rb'", 'rb"', "rB'", 'rB"', - "Rb'", 'Rb"', "RB'", 'RB"' , - "u'", 'u"', "U'", 'U"', - ): - single_quoted[t] = t +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) tabsize = 8 @@ -626,6 +623,7 @@ def _tokenize(readline, encoding): yield stashed stashed = None yield TokenInfo(COMMENT, token, spos, epos, line) + elif token in triple_quoted: endprog = _compile(endpats[token]) endmatch = endprog.match(line, pos) @@ -638,19 +636,37 @@ def _tokenize(readline, encoding): contstr = line[start:] contline = line break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come afer + # triple quote checking (above). + elif (initial in single_quoted or + token[:2] in single_quoted or + token[:3] in single_quoted): if token[-1] == '\n': # continued string strstart = (lnum, start) - endprog = _compile(endpats[initial] or - endpats[token[1]] or - endpats[token[2]]) + # Again, using the first 3 chars of the + # token. This is looking for the matching end + # regex for the correct type of quote + # character. So it's really looking for + # endpats["'"] or endpats['"'], by trying to + # skip string prefix characters, if any. + endprog = _compile(endpats.get(initial) or + endpats.get(token[1]) or + endpats.get(token[2])) contstr, needcont = line[start:], 1 contline = line break else: # ordinary string yield TokenInfo(STRING, token, spos, epos, line) + elif initial.isidentifier(): # ordinary name if token in ('async', 'await'): if async_def: -- cgit v0.12 From dd1e670758794b1dccdd42aa1f8ccddabdd406c2 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 26 Oct 2015 17:11:04 -0700 Subject: Fix a variable typo by switching to a f-string. --- Tools/scripts/pyvenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv index 1eed3ff..1fb42c6 100755 --- a/Tools/scripts/pyvenv +++ b/Tools/scripts/pyvenv @@ -5,7 +5,7 @@ if __name__ == '__main__': executable = pathlib.Path(sys.executable or 'python3').name print('WARNING: the pyenv script is deprecated in favour of ' - '`{} -m venv`'.format(exeutable), file=sys.stderr) + f'`{executable} -m venv`', file=sys.stderr) rc = 1 try: -- cgit v0.12 From 213b4056a64a670587419c8a15a724faeb789221 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 30 Oct 2015 14:41:06 -0700 Subject: Issue #25487: Fix tests not updated when the imp module moved to a DeprecationWarning. Thanks to Martin Panter for finding the tests. --- Lib/modulefinder.py | 2 +- Lib/test/test_imp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 50f2462..4a2f1b5 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -10,7 +10,7 @@ import types import struct import warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) import imp # XXX Clean up once str8's cstor matches bytes. diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index ee9ee1a..efb0384 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -12,7 +12,7 @@ from test import support import unittest import warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) import imp -- cgit v0.12 From 9785261d68f62ce33968d15c7a0f10f1f8209807 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 1 Nov 2015 17:14:27 +0200 Subject: Issue #18973: Command-line interface of the calendar module now uses argparse instead of optparse. --- Lib/calendar.py | 81 ++++++++++++++++++++++++++--------------------- Lib/test/test_calendar.py | 8 ++--- Misc/NEWS | 3 ++ 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/Lib/calendar.py b/Lib/calendar.py index 02050ea..4ff154c 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -605,51 +605,63 @@ def timegm(tuple): def main(args): - import optparse - parser = optparse.OptionParser(usage="usage: %prog [options] [year [month]]") - parser.add_option( + import argparse + parser = argparse.ArgumentParser() + textgroup = parser.add_argument_group('text only arguments') + htmlgroup = parser.add_argument_group('html only arguments') + textgroup.add_argument( "-w", "--width", - dest="width", type="int", default=2, - help="width of date column (default 2, text only)" + type=int, default=2, + help="width of date column (default 2)" ) - parser.add_option( + textgroup.add_argument( "-l", "--lines", - dest="lines", type="int", default=1, - help="number of lines for each week (default 1, text only)" + type=int, default=1, + help="number of lines for each week (default 1)" ) - parser.add_option( + textgroup.add_argument( "-s", "--spacing", - dest="spacing", type="int", default=6, - help="spacing between months (default 6, text only)" + type=int, default=6, + help="spacing between months (default 6)" ) - parser.add_option( + textgroup.add_argument( "-m", "--months", - dest="months", type="int", default=3, - help="months per row (default 3, text only)" + type=int, default=3, + help="months per row (default 3)" ) - parser.add_option( + htmlgroup.add_argument( "-c", "--css", - dest="css", default="calendar.css", - help="CSS to use for page (html only)" + default="calendar.css", + help="CSS to use for page" ) - parser.add_option( + parser.add_argument( "-L", "--locale", - dest="locale", default=None, + default=None, help="locale to be used from month and weekday names" ) - parser.add_option( + parser.add_argument( "-e", "--encoding", - dest="encoding", default=None, - help="Encoding to use for output." + default=None, + help="encoding to use for output" ) - parser.add_option( + parser.add_argument( "-t", "--type", - dest="type", default="text", + default="text", choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "year", + nargs='?', type=int, + help="year number (1-9999)" + ) + parser.add_argument( + "month", + nargs='?', type=int, + help="month number (1-12, text only)" + ) - (options, args) = parser.parse_args(args) + options = parser.parse_args(args[1:]) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -667,10 +679,10 @@ def main(args): encoding = sys.getdefaultencoding() optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write - if len(args) == 1: + if options.year is None: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif len(args) == 2: - write(cal.formatyearpage(int(args[1]), **optdict)) + elif options.month is None: + write(cal.formatyearpage(options.year, **optdict)) else: parser.error("incorrect number of arguments") sys.exit(1) @@ -680,18 +692,15 @@ def main(args): else: cal = TextCalendar() optdict = dict(w=options.width, l=options.lines) - if len(args) != 3: + if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if len(args) == 1: + if options.year is None: result = cal.formatyear(datetime.date.today().year, **optdict) - elif len(args) == 2: - result = cal.formatyear(int(args[1]), **optdict) - elif len(args) == 3: - result = cal.formatmonth(int(args[1]), int(args[2]), **optdict) + elif options.month is None: + result = cal.formatyear(options.year, **optdict) else: - parser.error("incorrect number of arguments") - sys.exit(1) + result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: result = result.encode(options.encoding) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 80ed632..d9d3128 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -702,19 +702,19 @@ class CommandLineTestCase(unittest.TestCase): def assertFailure(self, *args): rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) - self.assertIn(b'Usage:', stderr) + self.assertIn(b'usage:', stderr) self.assertEqual(rc, 2) def test_help(self): stdout = self.run_ok('-h') - self.assertIn(b'Usage:', stdout) + self.assertIn(b'usage:', stdout) self.assertIn(b'calendar.py', stdout) self.assertIn(b'--help', stdout) def test_illegal_arguments(self): self.assertFailure('-z') - #self.assertFailure('spam') - #self.assertFailure('2004', 'spam') + self.assertFailure('spam') + self.assertFailure('2004', 'spam') self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): diff --git a/Misc/NEWS b/Misc/NEWS index b67a3a5..5722a3a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,9 @@ Core and Builtins Library ------- +- Issue #18973: Command-line interface of the calendar module now uses argparse + instead of optparse. + - Issue #25510: fileinput.FileInput.readline() now returns b'' instead of '' at the end if the FileInput was opened with binary mode. Patch by Ryosuke Ito. -- cgit v0.12 From 67b97b8f8dc792aa7e1130a52a0ccc5069c400ed Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 1 Nov 2015 23:57:37 -0500 Subject: Move the initial start-search out of the main loop so it can be factored-out later. --- Modules/_collectionsmodule.c | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ba8238c..c12d43e 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1017,11 +1017,12 @@ deque_len(dequeobject *deque) static PyObject * deque_index(dequeobject *deque, PyObject *args) { - Py_ssize_t i, start=0, stop=Py_SIZE(deque); + Py_ssize_t i, n, start=0, stop=Py_SIZE(deque); PyObject *v, *item; block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; + int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, _PyEval_SliceIndex, &start, @@ -1039,22 +1040,32 @@ deque_index(dequeobject *deque, PyObject *args) } if (stop > Py_SIZE(deque)) stop = Py_SIZE(deque); + if (start > stop) + start = stop; + assert(0 <= start && start <= stop && stop <= Py_SIZE(deque)); - for (i=0 ; i= start) { - int cmp; - CHECK_NOT_END(b); - item = b->data[index]; - cmp = PyObject_RichCompareBool(item, v, Py_EQ); - if (cmp > 0) - return PyLong_FromSsize_t(i); - else if (cmp < 0) - return NULL; - if (start_state != deque->state) { - PyErr_SetString(PyExc_RuntimeError, - "deque mutated during iteration"); - return NULL; - } + /* XXX Replace this loop with faster code from deque_item() */ + for (i=0 ; irightlink; + index = 0; + } + } + + n = stop - i; + while (n--) { + CHECK_NOT_END(b); + item = b->data[index]; + cmp = PyObject_RichCompareBool(item, v, Py_EQ); + if (cmp > 0) + return PyLong_FromSsize_t(stop - (n + 1)); + else if (cmp < 0) + return NULL; + if (start_state != deque->state) { + PyErr_SetString(PyExc_RuntimeError, + "deque mutated during iteration"); + return NULL; } index++; if (index == BLOCKLEN) { -- cgit v0.12 From a1fc949b5ab8911a803eee691e6eea55cec43eeb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 2 Nov 2015 00:39:56 -0500 Subject: Issue #24379: Revert the operator.subscript patch (dccc4e63aef5) pending resolution of the related refcnt leak. --- Doc/library/operator.rst | 15 --------------- Doc/whatsnew/3.6.rst | 8 -------- Lib/operator.py | 28 +--------------------------- Lib/test/test_operator.py | 33 --------------------------------- 4 files changed, 1 insertion(+), 83 deletions(-) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 0695391..c01e63b 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -333,21 +333,6 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] -.. data:: subscript - - A helper to turn subscript notation into indexing objects. This can be - used to create item access patterns ahead of time to pass them into - various subscriptable objects. - - For example: - - * ``subscript[5] == 5`` - * ``subscript[3:7:2] == slice(3, 7, 2)`` - * ``subscript[5, 8] == (5, 8)`` - - .. versionadded:: 3.6 - - .. function:: methodcaller(name[, args...]) Return a callable object that calls the method *name* on its operand. If diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b500550..0d387bb 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -104,14 +104,6 @@ directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) -operator --------- - -New object :data:`operator.subscript` makes it easier to create complex -indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` -(Contributed by Joe Jevnik in :issue:`24379`.) - - pickle ------ diff --git a/Lib/operator.py b/Lib/operator.py index bc2a947..0e2e53e 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -17,7 +17,7 @@ __all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'subscript', 'truediv', 'truth', 'xor'] + 'setitem', 'sub', 'truediv', 'truth', 'xor'] from builtins import abs as _abs @@ -408,32 +408,6 @@ def ixor(a, b): return a -@object.__new__ # create a singleton instance -class subscript: - """ - A helper to turn subscript notation into indexing objects. This can be - used to create item access patterns ahead of time to pass them into - various subscriptable objects. - - For example: - subscript[5] == 5 - subscript[3:7:2] == slice(3, 7, 2) - subscript[5, 8] == (5, 8) - """ - __slots__ = () - - def __new__(cls): - raise TypeError("cannot create '{}' instances".format(cls.__name__)) - - @staticmethod - def __getitem__(key): - return key - - @staticmethod - def __reduce__(): - return 'subscript' - - try: from _operator import * except ImportError: diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 27501c2..54fd1f4 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -596,38 +596,5 @@ class CCOperatorPickleTestCase(OperatorPickleTestCase, unittest.TestCase): module2 = c_operator -class SubscriptTestCase: - def test_subscript(self): - subscript = self.module.subscript - self.assertIsNone(subscript[None]) - self.assertEqual(subscript[0], 0) - self.assertEqual(subscript[0:1:2], slice(0, 1, 2)) - self.assertEqual( - subscript[0, ..., :2, ...], - (0, Ellipsis, slice(2), Ellipsis), - ) - - def test_pickle(self): - from operator import subscript - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - self.assertIs( - pickle.loads(pickle.dumps(subscript, proto)), - subscript, - ) - - def test_singleton(self): - with self.assertRaises(TypeError): - type(self.module.subscript)() - - def test_immutable(self): - with self.assertRaises(AttributeError): - self.module.subscript.attr = None - - -class PySubscriptTestCase(SubscriptTestCase, PyOperatorTestCase): - pass - - if __name__ == "__main__": unittest.main() -- cgit v0.12 From df8f5b56c9dc5e950f39ed0221212256a39c1835 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 2 Nov 2015 07:27:40 -0500 Subject: Minor cleanup. --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c9e4568..1ca6c72 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1060,7 +1060,7 @@ deque_index(dequeobject *deque, PyObject *args) cmp = PyObject_RichCompareBool(item, v, Py_EQ); if (cmp > 0) return PyLong_FromSsize_t(stop - (n + 1)); - else if (cmp < 0) + if (cmp < 0) return NULL; if (start_state != deque->state) { PyErr_SetString(PyExc_RuntimeError, -- cgit v0.12 From 2753a096e0365bf93e1fd3a3e20735f598c14e91 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Nov 2015 14:34:51 +0100 Subject: locale.delocalize(): only call localeconv() once --- Lib/locale.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/locale.py b/Lib/locale.py index ceaa6d8..074f6e0 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -303,12 +303,16 @@ def str(val): def delocalize(string): "Parses a string as a normalized number according to the locale settings." + + conv = localeconv() + #First, get rid of the grouping - ts = localeconv()['thousands_sep'] + ts = conv['thousands_sep'] if ts: string = string.replace(ts, '') + #next, replace the decimal point with a dot - dd = localeconv()['decimal_point'] + dd = conv['decimal_point'] if dd: string = string.replace(dd, '.') return string -- cgit v0.12 From a78c7954d549fba967a842143b4e3050ff488468 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 3 Nov 2015 12:45:05 -0500 Subject: Issue 25483: Add an opcode to make f-string formatting more robust. --- Include/ceval.h | 8 ++ Include/opcode.h | 1 + Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 2 + Python/ceval.c | 57 ++++++++++ Python/compile.c | 98 ++++++----------- Python/importlib_external.h | 206 +++++++++++++++++------------------ Python/opcode_targets.h | 2 +- 8 files changed, 207 insertions(+), 170 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index b5373a9..d194044 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -206,6 +206,14 @@ PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif +/* Masks and values used by FORMAT_VALUE opcode. */ +#define FVC_MASK 0x3 +#define FVC_NONE 0x0 +#define FVC_STR 0x1 +#define FVC_REPR 0x2 +#define FVC_ASCII 0x3 +#define FVS_MASK 0x4 +#define FVS_HAVE_SPEC 0x4 #ifdef __cplusplus } diff --git a/Include/opcode.h b/Include/opcode.h index 3f917fb..b265368 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -122,6 +122,7 @@ extern "C" { #define BUILD_TUPLE_UNPACK 152 #define BUILD_SET_UNPACK 153 #define SETUP_ASYNC_WITH 154 +#define FORMAT_VALUE 155 /* EXCEPT_HANDLER is a special, implicit block type which is created when entering an except handler. It is not an opcode but we define it here diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index e0a1200..c58a62a 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -223,12 +223,13 @@ _code_type = type(_write_atomic.__code__) # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations) # Python 3.5b2 3340 (fix dictionary display evaluation order #11205) # Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). -MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3360).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index 4c826a7..71ce672 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -214,4 +214,6 @@ def_op('BUILD_MAP_UNPACK_WITH_CALL', 151) def_op('BUILD_TUPLE_UNPACK', 152) def_op('BUILD_SET_UNPACK', 153) +def_op('FORMAT_VALUE', 155) + del def_op, name_op, jrel_op, jabs_op diff --git a/Python/ceval.c b/Python/ceval.c index f9e8ef8..60db703 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3363,6 +3363,63 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + TARGET(FORMAT_VALUE) { + /* Handles f-string value formatting. */ + PyObject *result; + PyObject *fmt_spec; + PyObject *value; + PyObject *(*conv_fn)(PyObject *); + int which_conversion = oparg & FVC_MASK; + int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; + + fmt_spec = have_fmt_spec ? POP() : NULL; + value = TOP(); + + /* See if any conversion is specified. */ + switch (which_conversion) { + case FVC_STR: conv_fn = PyObject_Str; break; + case FVC_REPR: conv_fn = PyObject_Repr; break; + case FVC_ASCII: conv_fn = PyObject_ASCII; break; + + /* Must be 0 (meaning no conversion), since only four + values are allowed by (oparg & FVC_MASK). */ + default: conv_fn = NULL; break; + } + + /* If there's a conversion function, call it and replace + value with that result. Otherwise, just use value, + without conversion. */ + if (conv_fn) { + result = conv_fn(value); + Py_DECREF(value); + if (!result) { + Py_XDECREF(fmt_spec); + goto error; + } + value = result; + } + + /* If value is a unicode object, and there's no fmt_spec, + then we know the result of format(value) is value + itself. In that case, skip calling format(). I plan to + move this optimization in to PyObject_Format() + itself. */ + if (PyUnicode_CheckExact(value) && fmt_spec == NULL) { + /* Do nothing, just transfer ownership to result. */ + result = value; + } else { + /* Actually call format(). */ + result = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_XDECREF(fmt_spec); + if (!result) + goto error; + } + + SET_TOP(result); + DISPATCH(); + } + TARGET(EXTENDED_ARG) { opcode = NEXTOP(); oparg = oparg<<16 | NEXTARG(); diff --git a/Python/compile.c b/Python/compile.c index 5b53de3..030afed 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1067,6 +1067,10 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return 1; case GET_YIELD_FROM_ITER: return 0; + case FORMAT_VALUE: + /* If there's a fmt_spec on the stack, we go from 2->1, + else 1->1. */ + return (oparg & FVS_MASK) == FVS_HAVE_SPEC ? -1 : 0; default: return PY_INVALID_STACK_EFFECT; } @@ -3241,83 +3245,47 @@ compiler_joined_str(struct compiler *c, expr_ty e) return 1; } -/* Note that this code uses the builtin functions format(), str(), - repr(), and ascii(). You can break this code, or make it do odd - things, by redefining those functions. */ +/* Used to implement f-strings. Format a single value. */ static int compiler_formatted_value(struct compiler *c, expr_ty e) { - PyObject *conversion_name = NULL; - - static PyObject *format_string; - static PyObject *str_string; - static PyObject *repr_string; - static PyObject *ascii_string; - - if (!format_string) { - format_string = PyUnicode_InternFromString("format"); - if (!format_string) - return 0; - } - - if (!str_string) { - str_string = PyUnicode_InternFromString("str"); - if (!str_string) - return 0; - } - - if (!repr_string) { - repr_string = PyUnicode_InternFromString("repr"); - if (!repr_string) - return 0; - } - if (!ascii_string) { - ascii_string = PyUnicode_InternFromString("ascii"); - if (!ascii_string) - return 0; - } + /* Our oparg encodes 2 pieces of information: the conversion + character, and whether or not a format_spec was provided. + + Convert the conversion char to 2 bits: + None: 000 0x0 FVC_NONE + !s : 001 0x1 FVC_STR + !r : 010 0x2 FVC_REPR + !a : 011 0x3 FVC_ASCII + + next bit is whether or not we have a format spec: + yes : 100 0x4 + no : 000 0x0 + */ - ADDOP_NAME(c, LOAD_GLOBAL, format_string, names); + int oparg; - /* If needed, convert via str, repr, or ascii. */ - if (e->v.FormattedValue.conversion != -1) { - switch (e->v.FormattedValue.conversion) { - case 's': - conversion_name = str_string; - break; - case 'r': - conversion_name = repr_string; - break; - case 'a': - conversion_name = ascii_string; - break; - default: - PyErr_SetString(PyExc_SystemError, - "Unrecognized conversion character"); - return 0; - } - ADDOP_NAME(c, LOAD_GLOBAL, conversion_name, names); - } - - /* Evaluate the value. */ + /* Evaluate the expression to be formatted. */ VISIT(c, expr, e->v.FormattedValue.value); - /* If needed, convert via str, repr, or ascii. */ - if (conversion_name) { - /* Call the function we previously pushed. */ - ADDOP_I(c, CALL_FUNCTION, 1); + switch (e->v.FormattedValue.conversion) { + case 's': oparg = FVC_STR; break; + case 'r': oparg = FVC_REPR; break; + case 'a': oparg = FVC_ASCII; break; + case -1: oparg = FVC_NONE; break; + default: + PyErr_SetString(PyExc_SystemError, + "Unrecognized conversion character"); + return 0; } - - /* If we have a format spec, use format(value, format_spec). Otherwise, - use the single argument form. */ if (e->v.FormattedValue.format_spec) { + /* Evaluate the format spec, and update our opcode arg. */ VISIT(c, expr, e->v.FormattedValue.format_spec); - ADDOP_I(c, CALL_FUNCTION, 2); - } else { - /* No format spec specified, call format(value). */ - ADDOP_I(c, CALL_FUNCTION, 1); + oparg |= FVS_HAVE_SPEC; } + /* And push our opcode and oparg */ + ADDOP_I(c, FORMAT_VALUE, oparg); return 1; } diff --git a/Python/importlib_external.h b/Python/importlib_external.h index c4b6dc9..dd322a9 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -258,7 +258,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,5,0,0,0,218,13,95,119,114,105,116,101,95,97,116, 111,109,105,99,99,0,0,0,115,26,0,0,0,0,5,24, 1,9,1,33,1,3,3,21,1,20,1,20,1,13,1,3, - 1,17,1,13,1,5,1,114,55,0,0,0,105,22,13,0, + 1,17,1,13,1,5,1,114,55,0,0,0,105,32,13,0, 0,233,2,0,0,0,114,13,0,0,0,115,2,0,0,0, 13,10,90,11,95,95,112,121,99,97,99,104,101,95,95,122, 4,111,112,116,45,122,3,46,112,121,122,4,46,112,121,99, @@ -368,7 +368,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,90,15,97,108,109,111,115,116,95,102,105,108,101,110,97, 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, 0,218,17,99,97,99,104,101,95,102,114,111,109,95,115,111, - 117,114,99,101,243,0,0,0,115,46,0,0,0,0,18,12, + 117,114,99,101,244,0,0,0,115,46,0,0,0,0,18,12, 1,9,1,7,1,12,1,6,1,12,1,18,1,18,1,24, 1,12,1,12,1,12,1,36,1,12,1,18,1,9,2,12, 1,12,1,12,1,12,1,21,1,21,1,114,79,0,0,0, @@ -447,7 +447,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,95, - 99,97,99,104,101,31,1,0,0,115,44,0,0,0,0,9, + 99,97,99,104,101,32,1,0,0,115,44,0,0,0,0,9, 18,1,12,1,18,1,18,1,12,1,9,1,15,1,15,1, 12,1,9,1,15,1,12,1,22,1,15,1,9,1,12,1, 22,1,12,1,9,1,12,1,19,1,114,85,0,0,0,99, @@ -484,7 +484,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,36,0,0,0,90,9,101,120,116,101,110,115,105, 111,110,218,11,115,111,117,114,99,101,95,112,97,116,104,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,15, - 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,64, + 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,65, 1,0,0,115,20,0,0,0,0,7,18,1,4,1,24,1, 35,1,4,1,3,1,16,1,19,1,21,1,114,91,0,0, 0,99,1,0,0,0,0,0,0,0,1,0,0,0,11,0, @@ -499,7 +499,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,79,0,0,0,114,66,0,0,0,114,74,0,0, 0,41,1,218,8,102,105,108,101,110,97,109,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,103, - 101,116,95,99,97,99,104,101,100,83,1,0,0,115,16,0, + 101,116,95,99,97,99,104,101,100,84,1,0,0,115,16,0, 0,0,0,1,21,1,3,1,14,1,13,1,8,1,21,1, 4,2,114,95,0,0,0,99,1,0,0,0,0,0,0,0, 2,0,0,0,11,0,0,0,67,0,0,0,115,60,0,0, @@ -514,7 +514,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,39,0,0,0,114,41,0,0,0,114,40,0,0,0,41, 2,114,35,0,0,0,114,42,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,10,95,99,97,108, - 99,95,109,111,100,101,95,1,0,0,115,12,0,0,0,0, + 99,95,109,111,100,101,96,1,0,0,115,12,0,0,0,0, 2,3,1,19,1,13,1,11,3,10,1,114,97,0,0,0, 99,1,0,0,0,0,0,0,0,3,0,0,0,11,0,0, 0,3,0,0,0,115,84,0,0,0,100,1,0,135,0,0, @@ -554,7 +554,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,115,90,6,107,119,97,114,103,115,41,1,218,6,109,101, 116,104,111,100,114,4,0,0,0,114,5,0,0,0,218,19, 95,99,104,101,99,107,95,110,97,109,101,95,119,114,97,112, - 112,101,114,115,1,0,0,115,12,0,0,0,0,1,12,1, + 112,101,114,116,1,0,0,115,12,0,0,0,0,1,12,1, 12,1,15,1,6,1,25,1,122,40,95,99,104,101,99,107, 95,110,97,109,101,46,60,108,111,99,97,108,115,62,46,95, 99,104,101,99,107,95,110,97,109,101,95,119,114,97,112,112, @@ -573,7 +573,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,97,116,116,114,218,8,95,95,100,105,99,116,95,95,218, 6,117,112,100,97,116,101,41,3,90,3,110,101,119,90,3, 111,108,100,114,52,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,5,95,119,114,97,112,126,1, + 0,0,114,5,0,0,0,218,5,95,119,114,97,112,127,1, 0,0,115,8,0,0,0,0,1,25,1,15,1,29,1,122, 26,95,99,104,101,99,107,95,110,97,109,101,46,60,108,111, 99,97,108,115,62,46,95,119,114,97,112,41,3,218,10,95, @@ -581,7 +581,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,97,109,101,69,114,114,111,114,41,3,114,102,0,0,0, 114,103,0,0,0,114,113,0,0,0,114,4,0,0,0,41, 1,114,102,0,0,0,114,5,0,0,0,218,11,95,99,104, - 101,99,107,95,110,97,109,101,107,1,0,0,115,14,0,0, + 101,99,107,95,110,97,109,101,108,1,0,0,115,14,0,0, 0,0,8,21,7,3,1,13,1,13,2,17,5,13,1,114, 116,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, 0,4,0,0,0,67,0,0,0,115,84,0,0,0,124,0, @@ -611,7 +611,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,112,111,114,116,105,111,110,115,218,3,109,115,103,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,17, 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, - 109,135,1,0,0,115,10,0,0,0,0,10,21,1,24,1, + 109,136,1,0,0,115,10,0,0,0,0,10,21,1,24,1, 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,252, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, @@ -698,7 +698,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,11,115,111,117,114,99,101,95,115,105,122,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,25,95,118, 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, - 95,104,101,97,100,101,114,152,1,0,0,115,76,0,0,0, + 95,104,101,97,100,101,114,153,1,0,0,115,76,0,0,0, 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, 16,1,16,1,12,1,18,1,16,1,18,1,18,1,15,1, 16,1,15,1,18,1,15,1,16,1,12,1,12,1,3,1, @@ -729,7 +729,7 @@ const unsigned char _Py_M__importlib_external[] = { 5,114,53,0,0,0,114,98,0,0,0,114,89,0,0,0, 114,90,0,0,0,218,4,99,111,100,101,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,109, - 112,105,108,101,95,98,121,116,101,99,111,100,101,207,1,0, + 112,105,108,101,95,98,121,116,101,99,111,100,101,208,1,0, 0,115,16,0,0,0,0,2,15,1,15,1,16,1,12,1, 16,1,4,2,18,1,114,141,0,0,0,114,59,0,0,0, 99,3,0,0,0,0,0,0,0,4,0,0,0,3,0,0, @@ -749,7 +749,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,117,109,112,115,41,4,114,140,0,0,0,114,126,0,0, 0,114,134,0,0,0,114,53,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,100, - 101,95,116,111,95,98,121,116,101,99,111,100,101,219,1,0, + 101,95,116,111,95,98,121,116,101,99,111,100,101,220,1,0, 0,115,10,0,0,0,0,3,12,1,19,1,19,1,22,1, 114,144,0,0,0,99,1,0,0,0,0,0,0,0,5,0, 0,0,4,0,0,0,67,0,0,0,115,89,0,0,0,100, @@ -778,7 +778,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,101,110,99,111,100,105,110,103,90,15,110,101,119,108, 105,110,101,95,100,101,99,111,100,101,114,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,13,100,101,99,111, - 100,101,95,115,111,117,114,99,101,229,1,0,0,115,10,0, + 100,101,95,115,111,117,114,99,101,230,1,0,0,115,10,0, 0,0,0,5,12,1,18,1,15,1,18,1,114,149,0,0, 0,114,120,0,0,0,218,26,115,117,98,109,111,100,117,108, 101,95,115,101,97,114,99,104,95,108,111,99,97,116,105,111, @@ -843,7 +843,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,105,120,101,115,114,153,0,0,0,90,7,100,105,114,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,23,115,112,101,99,95,102,114,111,109,95,102,105, - 108,101,95,108,111,99,97,116,105,111,110,246,1,0,0,115, + 108,101,95,108,111,99,97,116,105,111,110,247,1,0,0,115, 60,0,0,0,0,12,12,4,6,1,15,2,3,1,19,1, 13,1,5,8,24,1,9,3,12,1,22,1,21,1,15,1, 9,1,5,2,4,3,12,2,15,1,3,1,19,1,13,1, @@ -883,7 +883,7 @@ const unsigned char _Py_M__importlib_external[] = { 72,75,69,89,95,76,79,67,65,76,95,77,65,67,72,73, 78,69,41,2,218,3,99,108,115,218,3,107,101,121,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,218,14,95, - 111,112,101,110,95,114,101,103,105,115,116,114,121,68,2,0, + 111,112,101,110,95,114,101,103,105,115,116,114,121,69,2,0, 0,115,8,0,0,0,0,2,3,1,23,1,13,1,122,36, 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, @@ -910,7 +910,7 @@ const unsigned char _Py_M__importlib_external[] = { 121,95,107,101,121,114,165,0,0,0,90,4,104,107,101,121, 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, - 99,104,95,114,101,103,105,115,116,114,121,75,2,0,0,115, + 99,104,95,114,101,103,105,115,116,114,121,76,2,0,0,115, 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, @@ -935,7 +935,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,101,116,114,171,0,0,0,114,120,0,0,0,114,160,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,9,102,105,110,100,95,115,112,101, - 99,90,2,0,0,115,26,0,0,0,0,2,15,1,12,1, + 99,91,2,0,0,115,26,0,0,0,0,2,15,1,12,1, 4,1,3,1,14,1,13,1,9,1,22,1,21,1,9,1, 15,1,9,1,122,31,87,105,110,100,111,119,115,82,101,103, 105,115,116,114,121,70,105,110,100,101,114,46,102,105,110,100, @@ -954,7 +954,7 @@ const unsigned char _Py_M__importlib_external[] = { 175,0,0,0,114,120,0,0,0,41,4,114,164,0,0,0, 114,119,0,0,0,114,35,0,0,0,114,158,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 102,105,110,100,95,109,111,100,117,108,101,106,2,0,0,115, + 102,105,110,100,95,109,111,100,117,108,101,107,2,0,0,115, 8,0,0,0,0,7,18,1,12,1,7,2,122,33,87,105, 110,100,111,119,115,82,101,103,105,115,116,114,121,70,105,110, 100,101,114,46,102,105,110,100,95,109,111,100,117,108,101,41, @@ -963,7 +963,7 @@ const unsigned char _Py_M__importlib_external[] = { 167,0,0,0,218,11,99,108,97,115,115,109,101,116,104,111, 100,114,166,0,0,0,114,172,0,0,0,114,175,0,0,0, 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,162,0,0,0,56,2, + 4,0,0,0,114,5,0,0,0,114,162,0,0,0,57,2, 0,0,115,20,0,0,0,12,2,6,3,6,3,6,2,6, 2,18,7,18,15,3,1,21,15,3,1,114,162,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, @@ -1001,7 +1001,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,0,0,0,114,119,0,0,0,114,94,0,0,0,90,13, 102,105,108,101,110,97,109,101,95,98,97,115,101,90,9,116, 97,105,108,95,110,97,109,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,153,0,0,0,125,2,0,0, + 0,0,114,5,0,0,0,114,153,0,0,0,126,2,0,0, 115,8,0,0,0,0,3,25,1,22,1,19,1,122,24,95, 76,111,97,100,101,114,66,97,115,105,99,115,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, @@ -1012,7 +1012,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,110,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, 5,0,0,0,218,13,99,114,101,97,116,101,95,109,111,100, - 117,108,101,133,2,0,0,115,0,0,0,0,122,27,95,76, + 117,108,101,134,2,0,0,115,0,0,0,0,122,27,95,76, 111,97,100,101,114,66,97,115,105,99,115,46,99,114,101,97, 116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,0, 0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,80, @@ -1033,7 +1033,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,99,114,111,0,0,0,41,3,114,100,0,0,0,218,6, 109,111,100,117,108,101,114,140,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,11,101,120,101,99, - 95,109,111,100,117,108,101,136,2,0,0,115,10,0,0,0, + 95,109,111,100,117,108,101,137,2,0,0,115,10,0,0,0, 0,2,18,1,12,1,9,1,15,1,122,25,95,76,111,97, 100,101,114,66,97,115,105,99,115,46,101,120,101,99,95,109, 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1043,13 +1043,13 @@ const unsigned char _Py_M__importlib_external[] = { 95,109,111,100,117,108,101,95,115,104,105,109,41,2,114,100, 0,0,0,114,119,0,0,0,114,4,0,0,0,114,4,0, 0,0,114,5,0,0,0,218,11,108,111,97,100,95,109,111, - 100,117,108,101,144,2,0,0,115,2,0,0,0,0,1,122, + 100,117,108,101,145,2,0,0,115,2,0,0,0,0,1,122, 25,95,76,111,97,100,101,114,66,97,115,105,99,115,46,108, 111,97,100,95,109,111,100,117,108,101,78,41,8,114,105,0, 0,0,114,104,0,0,0,114,106,0,0,0,114,107,0,0, 0,114,153,0,0,0,114,180,0,0,0,114,185,0,0,0, 114,187,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,178,0,0,0,120,2, + 4,0,0,0,114,5,0,0,0,114,178,0,0,0,121,2, 0,0,115,10,0,0,0,12,3,6,2,12,8,12,3,12, 8,114,178,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,4,0,0,0,64,0,0,0,115,106,0,0,0, @@ -1077,7 +1077,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,78,41,1,218,7,73,79,69,114,114,111,114, 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, - 104,95,109,116,105,109,101,150,2,0,0,115,2,0,0,0, + 104,95,109,116,105,109,101,151,2,0,0,115,2,0,0,0, 0,6,122,23,83,111,117,114,99,101,76,111,97,100,101,114, 46,112,97,116,104,95,109,116,105,109,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, @@ -1112,7 +1112,7 @@ const unsigned char _Py_M__importlib_external[] = { 10,32,32,32,32,32,32,32,32,114,126,0,0,0,41,1, 114,190,0,0,0,41,2,114,100,0,0,0,114,35,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,10,112,97,116,104,95,115,116,97,116,115,158,2,0,0, + 218,10,112,97,116,104,95,115,116,97,116,115,159,2,0,0, 115,2,0,0,0,0,11,122,23,83,111,117,114,99,101,76, 111,97,100,101,114,46,112,97,116,104,95,115,116,97,116,115, 99,4,0,0,0,0,0,0,0,4,0,0,0,3,0,0, @@ -1136,7 +1136,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,100,0,0,0,114,90,0,0,0,90,10,99,97,99,104, 101,95,112,97,116,104,114,53,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,15,95,99,97,99, - 104,101,95,98,121,116,101,99,111,100,101,171,2,0,0,115, + 104,101,95,98,121,116,101,99,111,100,101,172,2,0,0,115, 2,0,0,0,0,8,122,28,83,111,117,114,99,101,76,111, 97,100,101,114,46,95,99,97,99,104,101,95,98,121,116,101, 99,111,100,101,99,3,0,0,0,0,0,0,0,3,0,0, @@ -1153,7 +1153,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,115,46,10,32,32,32,32,32,32,32,32,78,114,4,0, 0,0,41,3,114,100,0,0,0,114,35,0,0,0,114,53, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,192,0,0,0,181,2,0,0,115,0,0,0,0, + 0,0,114,192,0,0,0,182,2,0,0,115,0,0,0,0, 122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,115, 101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0, 5,0,0,0,16,0,0,0,67,0,0,0,115,105,0,0, @@ -1175,7 +1175,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,41,5,114,100,0,0,0,114,119,0,0,0,114,35,0, 0,0,114,147,0,0,0,218,3,101,120,99,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,103,101,116, - 95,115,111,117,114,99,101,188,2,0,0,115,14,0,0,0, + 95,115,111,117,114,99,101,189,2,0,0,115,14,0,0,0, 0,2,15,1,3,1,19,1,18,1,9,1,31,1,122,23, 83,111,117,114,99,101,76,111,97,100,101,114,46,103,101,116, 95,115,111,117,114,99,101,218,9,95,111,112,116,105,109,105, @@ -1197,7 +1197,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,111,109,112,105,108,101,41,4,114,100,0,0,0,114,53, 0,0,0,114,35,0,0,0,114,197,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,14,115,111, - 117,114,99,101,95,116,111,95,99,111,100,101,198,2,0,0, + 117,114,99,101,95,116,111,95,99,111,100,101,199,2,0,0, 115,4,0,0,0,0,5,21,1,122,27,83,111,117,114,99, 101,76,111,97,100,101,114,46,115,111,117,114,99,101,95,116, 111,95,99,111,100,101,99,2,0,0,0,0,0,0,0,10, @@ -1259,7 +1259,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,218,10,98,121,116,101,115,95,100,97,116,97,114,147, 0,0,0,90,11,99,111,100,101,95,111,98,106,101,99,116, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 181,0,0,0,206,2,0,0,115,78,0,0,0,0,7,15, + 181,0,0,0,207,2,0,0,115,78,0,0,0,0,7,15, 1,6,1,3,1,16,1,13,1,11,2,3,1,19,1,13, 1,5,2,16,1,3,1,19,1,13,1,5,2,3,1,9, 1,12,1,13,1,19,1,5,2,12,1,7,1,15,1,6, @@ -1271,7 +1271,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,193,0,0,0,114,192,0,0,0,114,196,0, 0,0,114,200,0,0,0,114,181,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,188,0,0,0,148,2,0,0,115,14,0,0,0,12,2, + 114,188,0,0,0,149,2,0,0,115,14,0,0,0,12,2, 12,8,12,13,12,10,12,7,12,10,18,8,114,188,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, 0,0,0,0,0,0,115,112,0,0,0,101,0,0,90,1, @@ -1300,7 +1300,7 @@ const unsigned char _Py_M__importlib_external[] = { 46,78,41,2,114,98,0,0,0,114,35,0,0,0,41,3, 114,100,0,0,0,114,119,0,0,0,114,35,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,179, - 0,0,0,7,3,0,0,115,4,0,0,0,0,3,9,1, + 0,0,0,8,3,0,0,115,4,0,0,0,0,3,9,1, 122,19,70,105,108,101,76,111,97,100,101,114,46,95,95,105, 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, @@ -1309,7 +1309,7 @@ const unsigned char _Py_M__importlib_external[] = { 83,41,1,78,41,2,218,9,95,95,99,108,97,115,115,95, 95,114,111,0,0,0,41,2,114,100,0,0,0,218,5,111, 116,104,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,95,101,113,95,95,13,3,0,0,115, + 0,0,0,218,6,95,95,101,113,95,95,14,3,0,0,115, 4,0,0,0,0,1,18,1,122,17,70,105,108,101,76,111, 97,100,101,114,46,95,95,101,113,95,95,99,1,0,0,0, 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, @@ -1318,7 +1318,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,78,41,3,218,4,104,97,115,104,114,98,0,0,0,114, 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,8,95,95,104,97, - 115,104,95,95,17,3,0,0,115,2,0,0,0,0,1,122, + 115,104,95,95,18,3,0,0,115,2,0,0,0,0,1,122, 19,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, 115,104,95,95,99,2,0,0,0,0,0,0,0,2,0,0, 0,3,0,0,0,3,0,0,0,115,22,0,0,0,116,0, @@ -1333,7 +1333,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,117,112,101,114,114,204,0,0,0,114,187,0,0,0,41, 2,114,100,0,0,0,114,119,0,0,0,41,1,114,205,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,187,0,0, - 0,20,3,0,0,115,2,0,0,0,0,10,122,22,70,105, + 0,21,3,0,0,115,2,0,0,0,0,10,122,22,70,105, 108,101,76,111,97,100,101,114,46,108,111,97,100,95,109,111, 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, 0,1,0,0,0,67,0,0,0,115,7,0,0,0,124,0, @@ -1343,7 +1343,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100, 101,114,46,41,1,114,35,0,0,0,41,2,114,100,0,0, 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,151,0,0,0,32,3,0,0,115,2, + 114,5,0,0,0,114,151,0,0,0,33,3,0,0,115,2, 0,0,0,0,3,122,23,70,105,108,101,76,111,97,100,101, 114,46,103,101,116,95,102,105,108,101,110,97,109,101,99,2, 0,0,0,0,0,0,0,3,0,0,0,9,0,0,0,67, @@ -1356,14 +1356,14 @@ const unsigned char _Py_M__importlib_external[] = { 78,41,3,114,49,0,0,0,114,50,0,0,0,90,4,114, 101,97,100,41,3,114,100,0,0,0,114,35,0,0,0,114, 54,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,194,0,0,0,37,3,0,0,115,4,0,0, + 0,0,0,114,194,0,0,0,38,3,0,0,115,4,0,0, 0,0,2,21,1,122,19,70,105,108,101,76,111,97,100,101, 114,46,103,101,116,95,100,97,116,97,41,11,114,105,0,0, 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, 114,179,0,0,0,114,207,0,0,0,114,209,0,0,0,114, 116,0,0,0,114,187,0,0,0,114,151,0,0,0,114,194, 0,0,0,114,4,0,0,0,114,4,0,0,0,41,1,114, - 205,0,0,0,114,5,0,0,0,114,204,0,0,0,2,3, + 205,0,0,0,114,5,0,0,0,114,204,0,0,0,3,3, 0,0,115,14,0,0,0,12,3,6,2,12,6,12,4,12, 3,24,12,18,5,114,204,0,0,0,99,0,0,0,0,0, 0,0,0,0,0,0,0,4,0,0,0,64,0,0,0,115, @@ -1387,7 +1387,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,105,109,101,90,7,115,116,95,115,105,122,101,41,3,114, 100,0,0,0,114,35,0,0,0,114,202,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,191,0, - 0,0,47,3,0,0,115,4,0,0,0,0,2,12,1,122, + 0,0,48,3,0,0,115,4,0,0,0,0,2,12,1,122, 27,83,111,117,114,99,101,70,105,108,101,76,111,97,100,101, 114,46,112,97,116,104,95,115,116,97,116,115,99,4,0,0, 0,0,0,0,0,5,0,0,0,5,0,0,0,67,0,0, @@ -1397,7 +1397,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,101,41,2,114,97,0,0,0,114,192,0,0,0,41,5, 114,100,0,0,0,114,90,0,0,0,114,89,0,0,0,114, 53,0,0,0,114,42,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,193,0,0,0,52,3,0, + 0,0,0,114,5,0,0,0,114,193,0,0,0,53,3,0, 0,115,4,0,0,0,0,2,12,1,122,32,83,111,117,114, 99,101,70,105,108,101,76,111,97,100,101,114,46,95,99,97, 99,104,101,95,98,121,116,101,99,111,100,101,114,214,0,0, @@ -1436,7 +1436,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,53,0,0,0,114,214,0,0,0,218,6,112,97,114, 101,110,116,114,94,0,0,0,114,27,0,0,0,114,23,0, 0,0,114,195,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,192,0,0,0,57,3,0,0,115, + 0,114,5,0,0,0,114,192,0,0,0,58,3,0,0,115, 42,0,0,0,0,2,18,1,6,2,22,1,18,1,17,2, 19,1,15,1,3,1,17,1,13,2,7,1,18,3,9,1, 10,1,27,1,3,1,16,1,20,1,18,2,12,1,122,25, @@ -1445,7 +1445,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, 114,191,0,0,0,114,193,0,0,0,114,192,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,212,0,0,0,43,3,0,0,115,8,0,0, + 0,0,0,114,212,0,0,0,44,3,0,0,115,8,0,0, 0,12,2,6,2,12,5,12,5,114,212,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, 0,0,0,115,46,0,0,0,101,0,0,90,1,0,100,0, @@ -1467,7 +1467,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,135,0,0,0,114,141,0,0,0,41,5,114,100,0, 0,0,114,119,0,0,0,114,35,0,0,0,114,53,0,0, 0,114,203,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,92,3,0,0,115,8, + 114,5,0,0,0,114,181,0,0,0,93,3,0,0,115,8, 0,0,0,0,1,15,1,15,1,24,1,122,29,83,111,117, 114,99,101,108,101,115,115,70,105,108,101,76,111,97,100,101, 114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0, @@ -1477,13 +1477,13 @@ const unsigned char _Py_M__importlib_external[] = { 32,105,115,32,110,111,32,115,111,117,114,99,101,32,99,111, 100,101,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,98,3,0,0,115,2,0, + 5,0,0,0,114,196,0,0,0,99,3,0,0,115,2,0, 0,0,0,2,122,31,83,111,117,114,99,101,108,101,115,115, 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,115, 111,117,114,99,101,78,41,6,114,105,0,0,0,114,104,0, 0,0,114,106,0,0,0,114,107,0,0,0,114,181,0,0, 0,114,196,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,88, + 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,89, 3,0,0,115,6,0,0,0,12,2,6,2,12,6,114,217, 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, 3,0,0,0,64,0,0,0,115,136,0,0,0,101,0,0, @@ -1508,7 +1508,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,124,0,0,95,1,0,100,0,0,83,41,1,78,41,2, 114,98,0,0,0,114,35,0,0,0,41,3,114,100,0,0, 0,114,98,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,115, + 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,116, 3,0,0,115,4,0,0,0,0,1,9,1,122,28,69,120, 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, 114,46,95,95,105,110,105,116,95,95,99,2,0,0,0,0, @@ -1518,7 +1518,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,0,107,2,0,83,41,1,78,41,2,114,205,0,0,0, 114,111,0,0,0,41,2,114,100,0,0,0,114,206,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,207,0,0,0,119,3,0,0,115,4,0,0,0,0,1, + 114,207,0,0,0,120,3,0,0,115,4,0,0,0,0,1, 18,1,122,26,69,120,116,101,110,115,105,111,110,70,105,108, 101,76,111,97,100,101,114,46,95,95,101,113,95,95,99,1, 0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,67, @@ -1527,7 +1527,7 @@ const unsigned char _Py_M__importlib_external[] = { 65,83,41,1,78,41,3,114,208,0,0,0,114,98,0,0, 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,209,0,0, - 0,123,3,0,0,115,2,0,0,0,0,1,122,28,69,120, + 0,124,3,0,0,115,2,0,0,0,0,1,122,28,69,120, 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, 114,46,95,95,104,97,115,104,95,95,99,2,0,0,0,0, 0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115, @@ -1544,7 +1544,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, 0,0,0,114,35,0,0,0,41,3,114,100,0,0,0,114, 158,0,0,0,114,184,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,126,3,0, + 0,0,0,114,5,0,0,0,114,180,0,0,0,127,3,0, 0,115,10,0,0,0,0,2,6,1,15,1,9,1,16,1, 122,33,69,120,116,101,110,115,105,111,110,70,105,108,101,76, 111,97,100,101,114,46,99,114,101,97,116,101,95,109,111,100, @@ -1562,7 +1562,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, 0,0,0,114,35,0,0,0,41,2,114,100,0,0,0,114, 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,185,0,0,0,134,3,0,0,115,6,0,0, + 0,0,0,114,185,0,0,0,135,3,0,0,115,6,0,0, 0,0,2,19,1,9,1,122,31,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,101,120,101, 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, @@ -1581,7 +1581,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,41,2,114,22,0,0,0,218,6,115,117,102,102,105, 120,41,1,218,9,102,105,108,101,95,110,97,109,101,114,4, 0,0,0,114,5,0,0,0,250,9,60,103,101,110,101,120, - 112,114,62,143,3,0,0,115,2,0,0,0,6,1,122,49, + 112,114,62,144,3,0,0,115,2,0,0,0,6,1,122,49, 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, 100,101,114,46,105,115,95,112,97,99,107,97,103,101,46,60, 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, @@ -1589,7 +1589,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,121,218,18,69,88,84,69,78,83,73,79,78,95,83,85, 70,70,73,88,69,83,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,41,1,114,220,0,0,0,114,5, - 0,0,0,114,153,0,0,0,140,3,0,0,115,6,0,0, + 0,0,0,114,153,0,0,0,141,3,0,0,115,6,0,0, 0,0,2,19,1,18,1,122,30,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, @@ -1600,7 +1600,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,116,32,99,114,101,97,116,101,32,97,32,99,111,100,101, 32,111,98,106,101,99,116,46,78,114,4,0,0,0,41,2, 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,181,0,0,0,146,3, + 4,0,0,0,114,5,0,0,0,114,181,0,0,0,147,3, 0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,110, 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,103, 101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,0, @@ -1611,7 +1611,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 196,0,0,0,150,3,0,0,115,2,0,0,0,0,2,122, + 196,0,0,0,151,3,0,0,115,2,0,0,0,0,2,122, 30,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, @@ -1622,7 +1622,7 @@ const unsigned char _Py_M__importlib_external[] = { 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,151,0,0,0,154,3,0,0,115,2,0,0,0,0,3, + 114,151,0,0,0,155,3,0,0,115,2,0,0,0,0,3, 122,32,69,120,116,101,110,115,105,111,110,70,105,108,101,76, 111,97,100,101,114,46,103,101,116,95,102,105,108,101,110,97, 109,101,78,41,14,114,105,0,0,0,114,104,0,0,0,114, @@ -1631,7 +1631,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,153,0,0,0,114,181,0,0,0,114,196,0,0, 0,114,116,0,0,0,114,151,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 218,0,0,0,107,3,0,0,115,20,0,0,0,12,6,6, + 218,0,0,0,108,3,0,0,115,20,0,0,0,12,6,6, 2,12,4,12,4,12,3,12,8,12,6,12,6,12,4,12, 4,114,218,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,2,0,0,0,64,0,0,0,115,130,0,0,0, @@ -1675,7 +1675,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,95,102,105,110,100,101,114,41,4,114,100,0,0,0,114, 98,0,0,0,114,35,0,0,0,218,11,112,97,116,104,95, 102,105,110,100,101,114,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,179,0,0,0,167,3,0,0,115,8, + 114,5,0,0,0,114,179,0,0,0,168,3,0,0,115,8, 0,0,0,0,1,9,1,9,1,21,1,122,23,95,78,97, 109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,110, 105,116,95,95,99,1,0,0,0,0,0,0,0,4,0,0, @@ -1694,7 +1694,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,216,0,0,0,218,3,100,111,116,90,2,109, 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, 218,23,95,102,105,110,100,95,112,97,114,101,110,116,95,112, - 97,116,104,95,110,97,109,101,115,173,3,0,0,115,8,0, + 97,116,104,95,110,97,109,101,115,174,3,0,0,115,8,0, 0,0,0,2,27,1,12,2,4,3,122,38,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,95,102,105,110,100, 95,112,97,114,101,110,116,95,112,97,116,104,95,110,97,109, @@ -1707,7 +1707,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,114,100,0,0,0,90,18,112,97,114,101,110,116,95,109, 111,100,117,108,101,95,110,97,109,101,90,14,112,97,116,104, 95,97,116,116,114,95,110,97,109,101,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,227,0,0,0,183,3, + 4,0,0,0,114,5,0,0,0,114,227,0,0,0,184,3, 0,0,115,4,0,0,0,0,1,18,1,122,31,95,78,97, 109,101,115,112,97,99,101,80,97,116,104,46,95,103,101,116, 95,112,97,114,101,110,116,95,112,97,116,104,99,1,0,0, @@ -1725,7 +1725,7 @@ const unsigned char _Py_M__importlib_external[] = { 150,0,0,0,114,226,0,0,0,41,3,114,100,0,0,0, 90,11,112,97,114,101,110,116,95,112,97,116,104,114,158,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,187, + 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,188, 3,0,0,115,16,0,0,0,0,2,18,1,15,1,21,3, 27,1,9,1,12,1,9,1,122,27,95,78,97,109,101,115, 112,97,99,101,80,97,116,104,46,95,114,101,99,97,108,99, @@ -1734,7 +1734,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, 1,78,41,2,218,4,105,116,101,114,114,234,0,0,0,41, 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,200, + 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,201, 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,95,95,105,116,101, 114,95,95,99,1,0,0,0,0,0,0,0,1,0,0,0, @@ -1742,7 +1742,7 @@ const unsigned char _Py_M__importlib_external[] = { 124,0,0,106,1,0,131,0,0,131,1,0,83,41,1,78, 41,2,114,31,0,0,0,114,234,0,0,0,41,1,114,100, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,7,95,95,108,101,110,95,95,203,3,0,0,115, + 0,0,218,7,95,95,108,101,110,95,95,204,3,0,0,115, 2,0,0,0,0,1,122,22,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,95,108,101,110,95,95,99,1, 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, @@ -1751,7 +1751,7 @@ const unsigned char _Py_M__importlib_external[] = { 109,101,115,112,97,99,101,80,97,116,104,40,123,33,114,125, 41,41,2,114,47,0,0,0,114,226,0,0,0,41,1,114, 100,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,95,114,101,112,114,95,95,206,3,0, + 0,0,0,218,8,95,95,114,101,112,114,95,95,207,3,0, 0,115,2,0,0,0,0,1,122,23,95,78,97,109,101,115, 112,97,99,101,80,97,116,104,46,95,95,114,101,112,114,95, 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, @@ -1759,7 +1759,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,106,0,0,131,0,0,107,6,0,83,41,1,78,41,1, 114,234,0,0,0,41,2,114,100,0,0,0,218,4,105,116, 101,109,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,209, + 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,210, 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,95,95,99,111,110, 116,97,105,110,115,95,95,99,2,0,0,0,0,0,0,0, @@ -1768,7 +1768,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,100,0,0,83,41,1,78,41,2,114,226,0,0,0,114, 157,0,0,0,41,2,114,100,0,0,0,114,239,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 157,0,0,0,212,3,0,0,115,2,0,0,0,0,1,122, + 157,0,0,0,213,3,0,0,115,2,0,0,0,0,1,122, 21,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, 97,112,112,101,110,100,78,41,13,114,105,0,0,0,114,104, 0,0,0,114,106,0,0,0,114,107,0,0,0,114,179,0, @@ -1776,7 +1776,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,236,0,0,0,114,237,0,0,0,114,238,0,0,0, 114,240,0,0,0,114,157,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,224, - 0,0,0,160,3,0,0,115,20,0,0,0,12,5,6,2, + 0,0,0,161,3,0,0,115,20,0,0,0,12,5,6,2, 12,6,12,10,12,4,12,13,12,3,12,3,12,3,12,3, 114,224,0,0,0,99,0,0,0,0,0,0,0,0,0,0, 0,0,3,0,0,0,64,0,0,0,115,118,0,0,0,101, @@ -1795,7 +1795,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,114,224,0,0,0,114,226,0,0,0,41,4,114,100,0, 0,0,114,98,0,0,0,114,35,0,0,0,114,230,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,218,3,0,0,115,2,0,0,0,0,1, + 114,179,0,0,0,219,3,0,0,115,2,0,0,0,0,1, 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, @@ -1812,21 +1812,21 @@ const unsigned char _Py_M__importlib_external[] = { 115,112,97,99,101,41,62,41,2,114,47,0,0,0,114,105, 0,0,0,41,2,114,164,0,0,0,114,184,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 109,111,100,117,108,101,95,114,101,112,114,221,3,0,0,115, + 109,111,100,117,108,101,95,114,101,112,114,222,3,0,0,115, 2,0,0,0,0,7,122,28,95,78,97,109,101,115,112,97, 99,101,76,111,97,100,101,114,46,109,111,100,117,108,101,95, 114,101,112,114,99,2,0,0,0,0,0,0,0,2,0,0, 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, 0,83,41,2,78,84,114,4,0,0,0,41,2,114,100,0, 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,153,0,0,0,230,3,0,0,115, + 0,114,5,0,0,0,114,153,0,0,0,231,3,0,0,115, 2,0,0,0,0,1,122,27,95,78,97,109,101,115,112,97, 99,101,76,111,97,100,101,114,46,105,115,95,112,97,99,107, 97,103,101,99,2,0,0,0,0,0,0,0,2,0,0,0, 1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,0, 83,41,2,78,114,30,0,0,0,114,4,0,0,0,41,2, 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,196,0,0,0,233,3, + 4,0,0,0,114,5,0,0,0,114,196,0,0,0,234,3, 0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,101, 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, @@ -1836,7 +1836,7 @@ const unsigned char _Py_M__importlib_external[] = { 60,115,116,114,105,110,103,62,114,183,0,0,0,114,198,0, 0,0,84,41,1,114,199,0,0,0,41,2,114,100,0,0, 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,236,3,0,0,115,2, + 114,5,0,0,0,114,181,0,0,0,237,3,0,0,115,2, 0,0,0,0,1,122,25,95,78,97,109,101,115,112,97,99, 101,76,111,97,100,101,114,46,103,101,116,95,99,111,100,101, 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, @@ -1846,14 +1846,14 @@ const unsigned char _Py_M__importlib_external[] = { 108,101,32,99,114,101,97,116,105,111,110,46,78,114,4,0, 0,0,41,2,114,100,0,0,0,114,158,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,180,0, - 0,0,239,3,0,0,115,0,0,0,0,122,30,95,78,97, + 0,0,240,3,0,0,115,0,0,0,0,122,30,95,78,97, 109,101,115,112,97,99,101,76,111,97,100,101,114,46,99,114, 101,97,116,101,95,109,111,100,117,108,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, 115,4,0,0,0,100,0,0,83,41,1,78,114,4,0,0, 0,41,2,114,100,0,0,0,114,184,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,185,0,0, - 0,242,3,0,0,115,2,0,0,0,0,1,122,28,95,78, + 0,243,3,0,0,115,2,0,0,0,0,1,122,28,95,78, 97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,101, 120,101,99,95,109,111,100,117,108,101,99,2,0,0,0,0, 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, @@ -1871,7 +1871,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,123,33,114,125,41,4,114,114,0,0,0,114,129,0,0, 0,114,226,0,0,0,114,186,0,0,0,41,2,114,100,0, 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,187,0,0,0,245,3,0,0,115, + 0,114,5,0,0,0,114,187,0,0,0,246,3,0,0,115, 6,0,0,0,0,7,9,1,10,1,122,28,95,78,97,109, 101,115,112,97,99,101,76,111,97,100,101,114,46,108,111,97, 100,95,109,111,100,117,108,101,78,41,12,114,105,0,0,0, @@ -1880,7 +1880,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,181,0,0,0,114,180,0,0,0,114,185,0, 0,0,114,187,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,241,0,0,0, - 217,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, + 218,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, 3,12,3,12,3,12,3,12,3,114,241,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,64, 0,0,0,115,160,0,0,0,101,0,0,90,1,0,100,0, @@ -1917,7 +1917,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,114,95,99,97,99,104,101,218,6,118,97,108,117,101,115, 114,108,0,0,0,114,244,0,0,0,41,2,114,164,0,0, 0,218,6,102,105,110,100,101,114,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,244,0,0,0,7,4,0, + 0,0,0,114,5,0,0,0,114,244,0,0,0,8,4,0, 0,115,6,0,0,0,0,4,22,1,15,1,122,28,80,97, 116,104,70,105,110,100,101,114,46,105,110,118,97,108,105,100, 97,116,101,95,99,97,99,104,101,115,99,2,0,0,0,0, @@ -1943,7 +1943,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,99,0,0,0,41,3,114,164,0,0,0,114,35,0,0, 0,90,4,104,111,111,107,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,11,95,112,97,116,104,95,104,111, - 111,107,115,15,4,0,0,115,16,0,0,0,0,7,25,1, + 111,107,115,16,4,0,0,115,16,0,0,0,0,7,25,1, 16,1,16,1,3,1,14,1,13,1,12,2,122,22,80,97, 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,104, 111,111,107,115,99,2,0,0,0,0,0,0,0,3,0,0, @@ -1975,7 +1975,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,249,0,0,0,41,3,114,164,0,0,0,114, 35,0,0,0,114,247,0,0,0,114,4,0,0,0,114,4, 0,0,0,114,5,0,0,0,218,20,95,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,32,4, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,33,4, 0,0,115,22,0,0,0,0,8,12,1,3,1,16,1,13, 3,9,1,3,1,17,1,13,1,15,1,18,1,122,31,80, 97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95, @@ -1995,7 +1995,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,119,0,0,0,114,247,0,0,0,114,120,0, 0,0,114,121,0,0,0,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,16,95,108,101, - 103,97,99,121,95,103,101,116,95,115,112,101,99,54,4,0, + 103,97,99,121,95,103,101,116,95,115,112,101,99,55,4,0, 0,115,18,0,0,0,0,4,15,1,24,2,15,1,6,1, 12,1,16,1,18,1,9,1,122,27,80,97,116,104,70,105, 110,100,101,114,46,95,108,101,103,97,99,121,95,103,101,116, @@ -2031,7 +2031,7 @@ const unsigned char _Py_M__importlib_external[] = { 109,101,115,112,97,99,101,95,112,97,116,104,90,5,101,110, 116,114,121,114,247,0,0,0,114,158,0,0,0,114,121,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,9,95,103,101,116,95,115,112,101,99,69,4,0,0, + 0,218,9,95,103,101,116,95,115,112,101,99,70,4,0,0, 115,40,0,0,0,0,5,6,1,13,1,21,1,3,1,15, 1,12,1,15,1,21,2,18,1,12,1,3,1,15,1,4, 1,9,1,12,1,12,5,17,2,18,1,9,1,122,20,80, @@ -2059,7 +2059,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,6,114,164,0,0,0,114,119,0,0,0,114,35,0,0, 0,114,174,0,0,0,114,158,0,0,0,114,254,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 175,0,0,0,101,4,0,0,115,26,0,0,0,0,4,12, + 175,0,0,0,102,4,0,0,115,26,0,0,0,0,4,12, 1,9,1,21,1,12,1,4,1,15,1,9,1,6,3,9, 1,24,1,4,2,7,2,122,20,80,97,116,104,70,105,110, 100,101,114,46,102,105,110,100,95,115,112,101,99,99,3,0, @@ -2081,7 +2081,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,114,175,0,0,0,114,120,0,0,0,41,4,114,164,0, 0,0,114,119,0,0,0,114,35,0,0,0,114,158,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,176,0,0,0,123,4,0,0,115,8,0,0,0,0,8, + 114,176,0,0,0,124,4,0,0,115,8,0,0,0,0,8, 18,1,12,1,4,1,122,22,80,97,116,104,70,105,110,100, 101,114,46,102,105,110,100,95,109,111,100,117,108,101,41,12, 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, @@ -2089,7 +2089,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,251,0,0,0,114,252,0,0,0,114,255,0, 0,0,114,175,0,0,0,114,176,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,243,0,0,0,3,4,0,0,115,22,0,0,0,12,2, + 114,243,0,0,0,4,4,0,0,115,22,0,0,0,12,2, 6,2,18,8,18,17,18,22,18,15,3,1,18,31,3,1, 21,21,3,1,114,243,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,133, @@ -2138,7 +2138,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,0,86,1,113,3,0,100,0,0,83,41,1,78,114,4, 0,0,0,41,2,114,22,0,0,0,114,219,0,0,0,41, 1,114,120,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,152,4,0,0,115,2,0,0,0,6,0, + 114,221,0,0,0,153,4,0,0,115,2,0,0,0,6,0, 122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105, 110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60, 103,101,110,101,120,112,114,62,114,58,0,0,0,114,29,0, @@ -2151,7 +2151,7 @@ const unsigned char _Py_M__importlib_external[] = { 108,111,97,100,101,114,95,100,101,116,97,105,108,115,90,7, 108,111,97,100,101,114,115,114,160,0,0,0,114,4,0,0, 0,41,1,114,120,0,0,0,114,5,0,0,0,114,179,0, - 0,0,146,4,0,0,115,16,0,0,0,0,4,6,1,19, + 0,0,147,4,0,0,115,16,0,0,0,0,4,6,1,19, 1,36,1,9,2,15,1,9,1,12,1,122,19,70,105,108, 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, @@ -2161,7 +2161,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,114,121,32,109,116,105,109,101,46,114,29,0,0,0,78, 114,87,0,0,0,41,1,114,2,1,0,0,41,1,114,100, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,244,0,0,0,160,4,0,0,115,2,0,0,0, + 0,0,114,244,0,0,0,161,4,0,0,115,2,0,0,0, 0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105, 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, 99,2,0,0,0,0,0,0,0,3,0,0,0,2,0,0, @@ -2185,7 +2185,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,114,175,0,0,0,114,120,0,0,0,114,150,0,0,0, 41,3,114,100,0,0,0,114,119,0,0,0,114,158,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,117,0,0,0,166,4,0,0,115,8,0,0,0,0,7, + 114,117,0,0,0,167,4,0,0,115,8,0,0,0,0,7, 15,1,12,1,10,1,122,22,70,105,108,101,70,105,110,100, 101,114,46,102,105,110,100,95,108,111,97,100,101,114,99,6, 0,0,0,0,0,0,0,7,0,0,0,7,0,0,0,67, @@ -2196,7 +2196,7 @@ const unsigned char _Py_M__importlib_external[] = { 161,0,0,0,41,7,114,100,0,0,0,114,159,0,0,0, 114,119,0,0,0,114,35,0,0,0,90,4,115,109,115,108, 114,174,0,0,0,114,120,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,255,0,0,0,178,4, + 4,0,0,0,114,5,0,0,0,114,255,0,0,0,179,4, 0,0,115,6,0,0,0,0,1,15,1,18,1,122,20,70, 105,108,101,70,105,110,100,101,114,46,95,103,101,116,95,115, 112,101,99,78,99,3,0,0,0,0,0,0,0,14,0,0, @@ -2260,7 +2260,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,90,13,105,110,105,116,95,102,105,108,101,110,97,109, 101,90,9,102,117,108,108,95,112,97,116,104,114,158,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,175,0,0,0,183,4,0,0,115,70,0,0,0,0,3, + 114,175,0,0,0,184,4,0,0,115,70,0,0,0,0,3, 6,1,19,1,3,1,34,1,13,1,11,1,15,1,10,1, 9,2,9,1,9,1,15,2,9,1,6,2,12,1,18,1, 22,1,10,1,15,1,12,1,32,4,12,2,22,1,22,1, @@ -2296,7 +2296,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,106,0,0,131,0,0,146,2,0,113,6,0,83,114,4, 0,0,0,41,1,114,88,0,0,0,41,2,114,22,0,0, 0,90,2,102,110,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,2, + 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,3, 5,0,0,115,2,0,0,0,9,0,122,41,70,105,108,101, 70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,99, 104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,116, @@ -2314,7 +2314,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,98,0,0,0,114,231,0,0,0,114,219,0, 0,0,90,8,110,101,119,95,110,97,109,101,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,7,1,0,0, - 229,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, + 230,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, 1,22,3,11,3,18,1,18,7,9,1,13,1,24,1,6, 1,27,2,6,1,17,1,9,1,18,1,122,22,70,105,108, 101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97, @@ -2352,7 +2352,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,99,0,0,0,41,1,114,35,0,0,0,41, 2,114,164,0,0,0,114,6,1,0,0,114,4,0,0,0, 114,5,0,0,0,218,24,112,97,116,104,95,104,111,111,107, - 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,14, + 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,15, 5,0,0,115,6,0,0,0,0,2,12,1,18,1,122,54, 70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95, 104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97, @@ -2360,7 +2360,7 @@ const unsigned char _Py_M__importlib_external[] = { 70,105,110,100,101,114,114,4,0,0,0,41,3,114,164,0, 0,0,114,6,1,0,0,114,12,1,0,0,114,4,0,0, 0,41,2,114,164,0,0,0,114,6,1,0,0,114,5,0, - 0,0,218,9,112,97,116,104,95,104,111,111,107,4,5,0, + 0,0,218,9,112,97,116,104,95,104,111,111,107,5,5,0, 0,115,4,0,0,0,0,10,21,6,122,20,70,105,108,101, 70,105,110,100,101,114,46,112,97,116,104,95,104,111,111,107, 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, @@ -2369,7 +2369,7 @@ const unsigned char _Py_M__importlib_external[] = { 105,108,101,70,105,110,100,101,114,40,123,33,114,125,41,41, 2,114,47,0,0,0,114,35,0,0,0,41,1,114,100,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,238,0,0,0,22,5,0,0,115,2,0,0,0,0, + 0,114,238,0,0,0,23,5,0,0,115,2,0,0,0,0, 1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95, 114,101,112,114,95,95,41,15,114,105,0,0,0,114,104,0, 0,0,114,106,0,0,0,114,107,0,0,0,114,179,0,0, @@ -2377,7 +2377,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,117,0,0,0,114,255,0,0,0,114,175,0,0,0,114, 7,1,0,0,114,177,0,0,0,114,13,1,0,0,114,238, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,0,1,0,0,137,4,0,0, + 0,0,114,5,0,0,0,114,0,1,0,0,138,4,0,0, 115,20,0,0,0,12,7,6,2,12,14,12,4,6,2,12, 12,12,5,15,46,12,31,18,18,114,0,1,0,0,99,4, 0,0,0,0,0,0,0,6,0,0,0,11,0,0,0,67, @@ -2403,7 +2403,7 @@ const unsigned char _Py_M__importlib_external[] = { 90,8,112,97,116,104,110,97,109,101,90,9,99,112,97,116, 104,110,97,109,101,114,120,0,0,0,114,158,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,14, - 95,102,105,120,95,117,112,95,109,111,100,117,108,101,28,5, + 95,102,105,120,95,117,112,95,109,111,100,117,108,101,29,5, 0,0,115,34,0,0,0,0,2,15,1,15,1,6,1,6, 1,12,1,12,1,18,2,15,1,6,1,21,1,3,1,10, 1,10,1,10,1,14,1,13,2,114,18,1,0,0,99,0, @@ -2424,7 +2424,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,74,0,0,0,41,3,90,10,101,120,116,101,110,115, 105,111,110,115,90,6,115,111,117,114,99,101,90,8,98,121, 116,101,99,111,100,101,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,155,0,0,0,51,5,0,0,115,8, + 114,5,0,0,0,114,155,0,0,0,52,5,0,0,115,8, 0,0,0,0,5,18,1,12,1,12,1,114,155,0,0,0, 99,1,0,0,0,0,0,0,0,12,0,0,0,12,0,0, 0,67,0,0,0,115,70,2,0,0,124,0,0,97,0,0, @@ -2486,7 +2486,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,0,100,1,0,83,41,2,114,29,0,0,0,78,41,1, 114,31,0,0,0,41,2,114,22,0,0,0,114,77,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,87,5,0,0,115,2,0,0,0,6,0, + 114,221,0,0,0,88,5,0,0,115,2,0,0,0,6,0, 122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115, 62,46,60,103,101,110,101,120,112,114,62,114,59,0,0,0, 122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,117, @@ -2516,7 +2516,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, 111,100,117,108,101,90,13,119,105,110,114,101,103,95,109,111, 100,117,108,101,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,115,101,116,117,112,62,5,0,0,115, + 0,0,0,218,6,95,115,101,116,117,112,63,5,0,0,115, 82,0,0,0,0,8,6,1,9,1,9,3,13,1,13,1, 15,1,18,2,13,1,20,3,33,1,19,2,31,1,10,1, 15,1,13,1,4,2,3,1,15,1,5,1,13,1,12,2, @@ -2542,7 +2542,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,243,0,0,0,114,212,0,0,0,41,2,114,26,1,0, 0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,97, 100,101,114,115,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,105,110,115,116,97,108,108,130,5,0, + 0,0,0,218,8,95,105,110,115,116,97,108,108,131,5,0, 0,115,16,0,0,0,0,2,10,1,9,1,28,1,15,1, 16,1,16,4,9,1,114,29,1,0,0,41,3,122,3,119, 105,110,114,1,0,0,0,114,2,0,0,0,41,56,114,107, @@ -2571,7 +2571,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,4,0,0,0,114,5,0,0,0,218,8,60, 109,111,100,117,108,101,62,8,0,0,0,115,98,0,0,0, 6,17,6,3,12,12,12,5,12,5,12,6,12,12,12,10, - 12,9,12,5,12,7,15,22,15,110,22,1,18,2,6,1, + 12,9,12,5,12,7,15,22,15,111,22,1,18,2,6,1, 6,2,9,2,9,2,10,2,21,44,12,33,12,19,12,12, 12,12,12,28,12,17,21,55,21,12,18,10,12,14,9,3, 12,1,15,65,19,64,19,28,22,110,19,41,25,45,25,16, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 19259e1..3872256 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -154,7 +154,7 @@ static void *opcode_targets[256] = { &&TARGET_BUILD_TUPLE_UNPACK, &&TARGET_BUILD_SET_UNPACK, &&TARGET_SETUP_ASYNC_WITH, - &&_unknown_opcode, + &&TARGET_FORMAT_VALUE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, -- cgit v0.12 From 281d5321a3bdee1fd7091e439855c0358371216c Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 3 Nov 2015 13:09:01 -0500 Subject: Issue 25483: Update dis.rst with FORMAT_VALUE opcode description. --- Doc/library/dis.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1bcb3a4..c6e1656 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -989,6 +989,25 @@ the more significant byte last. arguments. +.. opcode:: FORMAT_VALUE (flags) + + Used for implementing formatted literal strings (f-strings). Pops + an optional *fmt_spec* from the stack, then a required *value*. + *flags* is interpreted as follows: + + * ``(flags & 0x03) == 0x00``: *value* is formattedd as-is. + * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before + formatting it. + * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before + formatting it. + * ``(flags & 0x03) == 0x03``: call :func:`ascii` on *value* before + formatting it. + * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use + it, else use an empty *fmt_spec*. + + Formatting is performed using the :c:func:`PyObject_Format` function. + + .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between -- cgit v0.12 From 9ce52e3bdac2b393c8de5b0da3b1a537dec1928c Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 3 Nov 2015 16:30:49 -0500 Subject: Issue 25483: Fix doc typo and added versionadded. Thanks, Berker Peksag. --- Doc/library/dis.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index c6e1656..ef601e6 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -995,7 +995,7 @@ the more significant byte last. an optional *fmt_spec* from the stack, then a required *value*. *flags* is interpreted as follows: - * ``(flags & 0x03) == 0x00``: *value* is formattedd as-is. + * ``(flags & 0x03) == 0x00``: *value* is formatted as-is. * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before formatting it. * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before @@ -1007,6 +1007,8 @@ the more significant byte last. Formatting is performed using the :c:func:`PyObject_Format` function. + .. versionadded:: 3.6 + .. opcode:: HAVE_ARGUMENT -- cgit v0.12 From 4a91d2138199d28cd906997854cc4632f156bb05 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 3 Nov 2015 22:00:26 -0500 Subject: Neaten-up the inner-loop logic. --- Modules/_collectionsmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 1ca6c72..0e59470 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1053,13 +1053,13 @@ deque_index(dequeobject *deque, PyObject *args) } } - n = stop - i; - while (n--) { + n = stop - i + 1; + while (--n) { CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); if (cmp > 0) - return PyLong_FromSsize_t(stop - (n + 1)); + return PyLong_FromSsize_t(stop - n); if (cmp < 0) return NULL; if (start_state != deque->state) { -- cgit v0.12 From 45550178ef6102aad81b75462df828e663f3c640 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2015 09:03:53 +0100 Subject: regrtest: display progress every 30 seconds (instead of 60 seconds) when running tests in multiprocessing mode (-jN). --- Lib/test/libregrtest/runtest_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 4473db4..0ca7dd7 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,7 +21,7 @@ from test.libregrtest.setup import setup_tests PROGRESS_MIN_TIME = 30.0 # seconds # Display the running tests if nothing happened last N seconds -PROGRESS_UPDATE = 60.0 # seconds +PROGRESS_UPDATE = 30.0 # seconds def run_test_in_subprocess(testname, ns): -- cgit v0.12 From a3a3d732ec951e272a763f4539439f67d42ef38e Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 4 Nov 2015 07:11:13 -0500 Subject: For FORMAT_VALUE opcode, make it clear that the result of PyObject_Format is pushed on the stack. --- Doc/library/dis.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index ef601e6..7222636 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1005,7 +1005,8 @@ the more significant byte last. * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use it, else use an empty *fmt_spec*. - Formatting is performed using the :c:func:`PyObject_Format` function. + Formatting is performed using :c:func:`PyObject_Format`. The + result is pushed on the stack. .. versionadded:: 3.6 -- cgit v0.12 From cf01b68b889f637c6fb3f11029df3a96b840333d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2015 11:21:38 +0100 Subject: sysmodule.c: reuse Py_STRINGIFY() macro --- Python/sysmodule.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 334f5d0..e0aa233 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1643,15 +1643,11 @@ make_version_info(void) /* sys.implementation values */ #define NAME "cpython" const char *_PySys_ImplName = NAME; -#define QUOTE(arg) #arg -#define STRIFY(name) QUOTE(name) -#define MAJOR STRIFY(PY_MAJOR_VERSION) -#define MINOR STRIFY(PY_MINOR_VERSION) +#define MAJOR Py_STRINGIFY(PY_MAJOR_VERSION) +#define MINOR Py_STRINGIFY(PY_MINOR_VERSION) #define TAG NAME "-" MAJOR MINOR const char *_PySys_ImplCacheTag = TAG; #undef NAME -#undef QUOTE -#undef STRIFY #undef MAJOR #undef MINOR #undef TAG -- cgit v0.12 From e20310fa19188f764b84fbdc40d32d6933f98af9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2015 13:56:58 +0100 Subject: Issue #25556: Add assertions to PyObject_GetItem() to ensure that an exception is raised when it returns NULL. Simplify also ceval.c: rely on the fact that PyObject_GetItem() raised an exception when it returns NULL. --- Objects/abstract.c | 11 ++++++++--- Python/ceval.c | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 63fcf15..2c1c76e 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -141,8 +141,11 @@ PyObject_GetItem(PyObject *o, PyObject *key) return null_error(); m = o->ob_type->tp_as_mapping; - if (m && m->mp_subscript) - return m->mp_subscript(o, key); + if (m && m->mp_subscript) { + PyObject *item = m->mp_subscript(o, key); + assert((item != NULL) ^ (PyErr_Occurred() != NULL)); + return item; + } if (o->ob_type->tp_as_sequence) { if (PyIndex_Check(key)) { @@ -1526,8 +1529,10 @@ PySequence_GetItem(PyObject *s, Py_ssize_t i) if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); - if (l < 0) + if (l < 0) { + assert(PyErr_Occurred()); return NULL; + } i += l; } } diff --git a/Python/ceval.c b/Python/ceval.c index 67ea388..7f9ef6f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2307,7 +2307,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { v = PyObject_GetItem(locals, name); - if (v == NULL && _PyErr_OCCURRED()) { + if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); @@ -2426,7 +2426,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { value = PyObject_GetItem(locals, name); - if (value == NULL && PyErr_Occurred()) { + if (value == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); -- cgit v0.12 From 12b2538ab8e44d69d9ed2c8b329812130db9e6bc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 Nov 2015 17:43:42 +0200 Subject: Reuse Py_STRINGIFY() macro in sre_lib.h and dynload_win.c. --- Modules/sre_lib.h | 6 ++---- Python/dynload_win.c | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 6ad2ab7..78f7ac7 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -367,14 +367,12 @@ SRE(info)(SRE_STATE* state, SRE_CODE* pattern) #define RETURN_ON_FAILURE(i) \ do { RETURN_ON_ERROR(i); if (i == 0) RETURN_FAILURE; } while (0) -#define SFY(x) #x - #define DATA_STACK_ALLOC(state, type, ptr) \ do { \ alloc_pos = state->data_stack_base; \ TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \ "(%" PY_FORMAT_SIZE_T "d)\n", \ - SFY(type), alloc_pos, sizeof(type))); \ + Py_STRINGIFY(type), alloc_pos, sizeof(type))); \ if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ if (j < 0) return j; \ @@ -387,7 +385,7 @@ do { \ #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ do { \ - TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \ + TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", Py_STRINGIFY(type), pos)); \ ptr = (type*)(state->data_stack+pos); \ } while (0) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index f2c796e..9fb0525 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -24,12 +24,10 @@ void _Py_DeactivateActCtx(ULONG_PTR cookie); #define PYD_DEBUG_SUFFIX "" #endif -#define STRINGIZE2(x) #x -#define STRINGIZE(x) STRINGIZE2(x) #ifdef PYD_PLATFORM_TAG -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" STRINGIZE(PY_MAJOR_VERSION) STRINGIZE(PY_MINOR_VERSION) "-" PYD_PLATFORM_TAG ".pyd" +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) "-" PYD_PLATFORM_TAG ".pyd" #else -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" STRINGIZE(PY_MAJOR_VERSION) STRINGIZE(PY_MINOR_VERSION) ".pyd" +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) ".pyd" #endif #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -- cgit v0.12 From c106c68aeb5814583a622587e63504f0c93c5140 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Nov 2015 17:01:48 +0100 Subject: Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node when compiling AST from Python objects. --- Include/Python-ast.h | 5 +++-- Misc/NEWS | 3 +++ Parser/asdl_c.py | 12 ++++++++++-- Python/Python-ast.c | 31 +++++++++++++++++++++++++++++-- Python/ast.c | 9 +++------ 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index ea6679c..175e380 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -602,8 +602,9 @@ excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq * arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg, asdl_seq * defaults, PyArena *arena); -#define arg(a0, a1, a2) _Py_arg(a0, a1, a2) -arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena); +#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4) +arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset, + PyArena *arena); #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) diff --git a/Misc/NEWS b/Misc/NEWS index e42df93..68de958 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node + when compiling AST from Python objects. + - Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that was mutated by direct calls of dict methods. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 3128078..eedd89b 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -275,7 +275,9 @@ class PrototypeVisitor(EmitVisitor): def visitProduct(self, prod, name): self.emit_function(name, get_c_type(name), - self.get_args(prod.fields), [], union=False) + self.get_args(prod.fields), + self.get_args(prod.attributes), + union=False) class FunctionVisitor(PrototypeVisitor): @@ -329,7 +331,8 @@ class FunctionVisitor(PrototypeVisitor): self.emit(s, depth, reflow) for argtype, argname, opt in args: emit("p->%s = %s;" % (argname, argname), 1) - assert not attrs + for argtype, argname, opt in attrs: + emit("p->%s = %s;" % (argname, argname), 1) class PickleVisitor(EmitVisitor): @@ -452,10 +455,15 @@ class Obj2ModVisitor(PickleVisitor): self.emit("PyObject* tmp = NULL;", 1) for f in prod.fields: self.visitFieldDeclaration(f, name, prod=prod, depth=1) + for a in prod.attributes: + self.visitFieldDeclaration(a, name, prod=prod, depth=1) self.emit("", 0) for f in prod.fields: self.visitField(f, name, prod=prod, depth=1) + for a in prod.attributes: + self.visitField(a, name, prod=prod, depth=1) args = [f.name for f in prod.fields] + args.extend([a.name for a in prod.attributes]) self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1) self.emit("return 0;", 1) self.emit("failed:", 0) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index a2e9816..07d9b3e 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -2425,7 +2425,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * } arg_ty -arg(identifier arg, expr_ty annotation, PyArena *arena) +arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena + *arena) { arg_ty p; if (!arg) { @@ -2438,6 +2439,8 @@ arg(identifier arg, expr_ty annotation, PyArena *arena) return NULL; p->arg = arg; p->annotation = annotation; + p->lineno = lineno; + p->col_offset = col_offset; return p; } @@ -7247,6 +7250,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) PyObject* tmp = NULL; identifier arg; expr_ty annotation; + int lineno; + int col_offset; if (_PyObject_HasAttrId(obj, &PyId_arg)) { int res; @@ -7269,7 +7274,29 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) } else { annotation = NULL; } - *out = arg(arg, annotation, arena); + if (_PyObject_HasAttrId(obj, &PyId_lineno)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); + if (tmp == NULL) goto failed; + res = obj2ast_int(tmp, &lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg"); + return 1; + } + if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); + if (tmp == NULL) goto failed; + res = obj2ast_int(tmp, &col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg"); + return 1; + } + *out = arg(arg, annotation, lineno, col_offset, arena); return 0; failed: Py_XDECREF(tmp); diff --git a/Python/ast.c b/Python/ast.c index 5a7a745..77ebc83 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1181,11 +1181,9 @@ ast_for_arg(struct compiling *c, const node *n) return NULL; } - ret = arg(name, annotation, c->c_arena); + ret = arg(name, annotation, LINENO(n), n->n_col_offset, c->c_arena); if (!ret) return NULL; - ret->lineno = LINENO(n); - ret->col_offset = n->n_col_offset; return ret; } @@ -1241,11 +1239,10 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start, goto error; if (forbidden_name(c, argname, ch, 0)) goto error; - arg = arg(argname, annotation, c->c_arena); + arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset, + c->c_arena); if (!arg) goto error; - arg->lineno = LINENO(ch); - arg->col_offset = ch->n_col_offset; asdl_seq_SET(kwonlyargs, j++, arg); i += 2; /* the name and the comma */ break; -- cgit v0.12 From fad85aadb0e168b7bde414694e448f34bb38c8ef Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 Nov 2015 15:42:38 +0200 Subject: Issue #25558: Use compile-time asserts. --- Include/pymacro.h | 4 ++++ Modules/_ctypes/_ctypes.c | 2 +- Modules/_datetimemodule.c | 6 +++--- Modules/_pickle.c | 2 +- Modules/pyexpat.c | 3 ++- Python/pytime.c | 25 ++++++++++++------------- Python/random.c | 2 +- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 3f6f5dc..49929e5 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -36,6 +36,10 @@ #define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#define Py_BUILD_ASSERT(cond) do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) + /* Get the number of elements in a visible array This does not work on pointers, or arrays declared as [], or function diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b9fd82e..ac4323a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2386,7 +2386,7 @@ unique_key(CDataObject *target, Py_ssize_t index) char *cp = string; size_t bytes_left; - assert(sizeof(string) - 1 > sizeof(Py_ssize_t) * 2); + Py_BUILD_ASSERT(sizeof(string) - 1 > sizeof(Py_ssize_t) * 2); cp += sprintf(cp, "%x", Py_SAFE_DOWNCAST(index, Py_ssize_t, int)); while (target->b_base) { bytes_left = sizeof(string) - (cp - string) - 1; diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 94336cf..55988c5 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5329,19 +5329,19 @@ PyInit__datetime(void) /* A 4-year cycle has an extra leap day over what we'd get from * pasting together 4 single years. */ - assert(DI4Y == 4 * 365 + 1); + Py_BUILD_ASSERT(DI4Y == 4 * 365 + 1); assert(DI4Y == days_before_year(4+1)); /* Similarly, a 400-year cycle has an extra leap day over what we'd * get from pasting together 4 100-year cycles. */ - assert(DI400Y == 4 * DI100Y + 1); + Py_BUILD_ASSERT(DI400Y == 4 * DI100Y + 1); assert(DI400Y == days_before_year(400+1)); /* OTOH, a 100-year cycle has one fewer leap day than we'd get from * pasting together 25 4-year cycles. */ - assert(DI100Y == 25 * DI4Y - 1); + Py_BUILD_ASSERT(DI100Y == 25 * DI4Y - 1); assert(DI100Y == days_before_year(100+1)); one = PyLong_FromLong(1); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0e3a68e..06882d0 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -874,7 +874,7 @@ _write_size64(char *out, size_t value) { size_t i; - assert(sizeof(size_t) <= 8); + Py_BUILD_ASSERT(sizeof(size_t) <= 8); for (i = 0; i < sizeof(size_t); i++) { out[i] = (unsigned char)((value >> (8 * i)) & 0xff); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9a6da73..b45e3da 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -747,7 +747,8 @@ pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyObject *data, s += MAX_CHUNK_SIZE; slen -= MAX_CHUNK_SIZE; } - assert(MAX_CHUNK_SIZE < INT_MAX && slen < INT_MAX); + Py_BUILD_ASSERT(MAX_CHUNK_SIZE <= INT_MAX); + assert(slen <= INT_MAX); rc = XML_Parse(self->itself, s, (int)slen, isfinal); done: diff --git a/Python/pytime.c b/Python/pytime.c index 53611b1..8a6cb55 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -43,7 +43,7 @@ _PyLong_AsTime_t(PyObject *obj) val = PyLong_AsLongLong(obj); #else long val; - assert(sizeof(time_t) <= sizeof(long)); + Py_BUILD_ASSERT(sizeof(time_t) <= sizeof(long)); val = PyLong_AsLong(obj); #endif if (val == -1 && PyErr_Occurred()) { @@ -60,7 +60,7 @@ _PyLong_FromTime_t(time_t t) #if defined(HAVE_LONG_LONG) && SIZEOF_TIME_T == SIZEOF_LONG_LONG return PyLong_FromLongLong((PY_LONG_LONG)t); #else - assert(sizeof(time_t) <= sizeof(long)); + Py_BUILD_ASSERT(sizeof(time_t) <= sizeof(long)); return PyLong_FromLong((long)t); #endif } @@ -209,6 +209,8 @@ _PyTime_FromSeconds(int seconds) /* ensure that integer overflow cannot happen, int type should have 32 bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_MS takes 30 bits). */ + Py_BUILD_ASSERT(INT_MAX <= _PyTime_MAX / SEC_TO_NS); + Py_BUILD_ASSERT(INT_MIN >= _PyTime_MIN / SEC_TO_NS); assert((t >= 0 && t <= _PyTime_MAX / SEC_TO_NS) || (t < 0 && t >= _PyTime_MIN / SEC_TO_NS)); t *= SEC_TO_NS; @@ -219,7 +221,7 @@ _PyTime_t _PyTime_FromNanoseconds(PY_LONG_LONG ns) { _PyTime_t t; - assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); t = Py_SAFE_DOWNCAST(ns, PY_LONG_LONG, _PyTime_t); return t; } @@ -231,7 +233,7 @@ _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts, int raise) _PyTime_t t; int res = 0; - assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); t = (_PyTime_t)ts->tv_sec; if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { @@ -253,7 +255,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) _PyTime_t t; int res = 0; - assert(sizeof(tv->tv_sec) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(tv->tv_sec) <= sizeof(_PyTime_t)); t = (_PyTime_t)tv->tv_sec; if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { @@ -304,12 +306,12 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, else { #ifdef HAVE_LONG_LONG PY_LONG_LONG sec; - assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); sec = PyLong_AsLongLong(obj); #else long sec; - assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); sec = PyLong_AsLong(obj); #endif @@ -364,10 +366,10 @@ PyObject * _PyTime_AsNanosecondsObject(_PyTime_t t) { #ifdef HAVE_LONG_LONG - assert(sizeof(PY_LONG_LONG) >= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) >= sizeof(_PyTime_t)); return PyLong_FromLongLong((PY_LONG_LONG)t); #else - assert(sizeof(long) >= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(long) >= sizeof(_PyTime_t)); return PyLong_FromLong((long)t); #endif } @@ -650,7 +652,7 @@ pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) assert(info == NULL || raise); ticks = GetTickCount64(); - assert(sizeof(ticks) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(ticks) <= sizeof(_PyTime_t)); t = (_PyTime_t)ticks; if (_PyTime_check_mul_overflow(t, MS_TO_NS)) { @@ -774,8 +776,5 @@ _PyTime_Init(void) if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) < 0) return -1; - /* check that _PyTime_FromSeconds() cannot overflow */ - assert(INT_MAX <= _PyTime_MAX / SEC_TO_NS); - assert(INT_MIN >= _PyTime_MIN / SEC_TO_NS); return 0; } diff --git a/Python/random.c b/Python/random.c index 772bfef..9b42d54 100644 --- a/Python/random.c +++ b/Python/random.c @@ -379,7 +379,7 @@ _PyRandom_Init(void) char *env; unsigned char *secret = (unsigned char *)&_Py_HashSecret.uc; Py_ssize_t secret_size = sizeof(_Py_HashSecret_t); - assert(secret_size == sizeof(_Py_HashSecret.uc)); + Py_BUILD_ASSERT(sizeof(_Py_HashSecret_t) == sizeof(_Py_HashSecret.uc)); if (_Py_HashSecret_Initialized) return; -- cgit v0.12 From 29a2f7c6b38e5a6ed891aa72af38974a1ff2d372 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 Nov 2015 18:06:24 +0200 Subject: Issue #25263: Trying to fix test_use on Windows. --- Lib/tkinter/test/test_tkinter/test_widgets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index 7171667..5a01a5a 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -91,9 +91,10 @@ class ToplevelTest(AbstractToplevelTest, unittest.TestCase): widget = self.create() self.assertEqual(widget['use'], '') parent = self.create(container=True) - wid = parent.winfo_id() - widget2 = self.create(use=wid) - self.assertEqual(int(widget2['use']), wid) + wid = hex(parent.winfo_id()) + with self.subTest(wid=wid): + widget2 = self.create(use=wid) + self.assertEqual(widget2['use'], wid) @add_standard_options(StandardOptionsTests) -- cgit v0.12 From b9d672491d5082c541bf267eb7bb99fdc6529324 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 9 Nov 2015 14:43:31 +0200 Subject: Issue #25263: Trying to fix test_use on Windows. Avoid possible weird behavior of WideInt convertion. "winfo id" always returns string hexadecimal representation. --- Lib/tkinter/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 12085a9..46f86f9 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -845,8 +845,7 @@ class Misc: self.tk.call('winfo', 'height', self._w)) def winfo_id(self): """Return identifier ID for this widget.""" - return self.tk.getint( - self.tk.call('winfo', 'id', self._w)) + return int(self.tk.call('winfo', 'id', self._w), 0) def winfo_interps(self, displayof=0): """Return the name of all Tcl interpreters for this display.""" args = ('winfo', 'interps') + self._displayof(displayof) -- cgit v0.12 From c826ab06c824933652fec3e902ffa75f13ea5c6a Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 11 Nov 2015 22:41:43 -0600 Subject: Fix typos in whatsnew. Reported on docs@ by Igor Zhun. --- Doc/whatsnew/3.6.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 0d387bb..3949dff 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -98,8 +98,8 @@ Improved Modules datetime -------- -:meth:`datetime.stftime ` and -:meth:`date.stftime ` methods now support ISO 8601 date +:meth:`datetime.strftime ` and +:meth:`date.strftime ` methods now support ISO 8601 date directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) -- cgit v0.12 From 8adaec56714c0ca902862d8b6279e1e3c969d664 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 13 Nov 2015 22:17:08 +0000 Subject: Issue #25498: Add NEWS entry for 3.6 --- Misc/NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 96c40b3..ce8aea1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -85,6 +85,10 @@ Core and Builtins Library ------- +- Issue #25498: Fix a crash when garbage-collecting ctypes objects created + by wrapping a memoryview. This was a regression made in 3.5a1. Based + on patch by Eryksun. + - Issue #25584: Added "escape" to the __all__ list in the glob module. - Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. -- cgit v0.12 From 6fe39266c8eb0b3e3f3544f6ad2214c15449e086 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 13 Nov 2015 23:54:02 +0000 Subject: Issue #25590: Complete attribute names even if they are not yet created --- Doc/whatsnew/3.6.rst | 4 ++++ Lib/rlcompleter.py | 8 +++++--- Lib/test/test_rlcompleter.py | 8 ++++++++ Misc/NEWS | 4 ++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3949dff..89d69cc 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -120,6 +120,10 @@ Private and special attribute names now are omitted unless the prefix starts with underscores. A space or a colon can be added after completed keyword. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) +Names of most attributes listed by :func:`dir` are now completed. +Previously, names of properties and slots which were not yet created on +an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) + urllib.robotparser ------------------ diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index d368876..02e1fa5 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -160,12 +160,14 @@ class Completer: for word in words: if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): + match = "%s.%s" % (expr, word) try: val = getattr(thisobject, word) except Exception: - continue # Exclude properties that are not set - word = self._callable_postfix(val, "%s.%s" % (expr, word)) - matches.append(word) + pass # Include even if attribute not set + else: + match = self._callable_postfix(val, match) + matches.append(match) if matches or not noprefix: break if noprefix == '_': diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index fee39bc..8ff75c7 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -92,6 +92,14 @@ class TestRlcompleter(unittest.TestCase): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertEqual(f.calls, 1) + def test_uncreated_attr(self): + # Attributes like properties and slots should be completed even when + # they haven't been created on an instance + class Foo: + __slots__ = ("bar",) + completer = rlcompleter.Completer(dict(f=Foo())) + self.assertEqual(completer.complete('f.', 0), 'f.bar') + def test_complete(self): completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') diff --git a/Misc/NEWS b/Misc/NEWS index f95fff0..f00b4e7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -85,6 +85,10 @@ Core and Builtins Library ------- +- Issue #25590: In the Readline completer, only call getattr() once per + attribute. Also complete names of attributes such as properties and slots + which are listed by dir() but not yet created on an instance. + - Issue #25498: Fix a crash when garbage-collecting ctypes objects created by wrapping a memoryview. This was a regression made in 3.5a1. Based on patch by Eryksun. -- cgit v0.12 From fad4b600744cde16e952a49af95ad6e520d05ca2 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 01:29:17 +0000 Subject: Adjust grammar and punctuation in whatsnew/3.6.rst --- Doc/whatsnew/3.6.rst | 22 +++++++++++----------- Misc/NEWS | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 89d69cc..b2066c5 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -98,8 +98,8 @@ Improved Modules datetime -------- -:meth:`datetime.strftime ` and -:meth:`date.strftime ` methods now support ISO 8601 date +The :meth:`datetime.strftime() ` and +:meth:`date.strftime() ` methods now support ISO 8601 date directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) @@ -107,17 +107,17 @@ directives ``%G``, ``%u`` and ``%V``. pickle ------ -Objects that need calling ``__new__`` with keyword arguments, can now be pickled +Objects that need calling ``__new__`` with keyword arguments can now be pickled using :ref:`pickle protocols ` older than protocol version 4. Protocol version 4 already supports this case. (Contributed by Serhiy Storchaka in :issue:`24164`.) -rlcomplete ----------- +rlcompleter +----------- Private and special attribute names now are omitted unless the prefix starts -with underscores. A space or a colon can be added after completed keyword. +with underscores. A space or a colon is added after some completed keywords. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) Names of most attributes listed by :func:`dir` are now completed. @@ -128,7 +128,7 @@ an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) urllib.robotparser ------------------ -:class:`~urllib.robotparser.RobotFileParser` now supports ``Crawl-delay`` and +:class:`~urllib.robotparser.RobotFileParser` now supports the ``Crawl-delay`` and ``Request-rate`` extensions. (Contributed by Nikolay Bogoychev in :issue:`16099`.) @@ -136,18 +136,18 @@ urllib.robotparser Optimizations ============= -* The ASCII decoder is now up to 60 times as fast for error handlers: +* The ASCII decoder is now up to 60 times as fast for error handlers ``surrogateescape``, ``ignore`` and ``replace`` (Contributed by Victor Stinner in :issue:`24870`). * The ASCII and the Latin1 encoders are now up to 3 times as fast for the - error ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). + error handler ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). -* The UTF-8 encoder is now up to 75 times as fast for error handlers: +* The UTF-8 encoder is now up to 75 times as fast for error handlers ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass`` (Contributed by Victor Stinner in :issue:`25267`). -* The UTF-8 decoder is now up to 15 times as fast for error handlers: +* The UTF-8 decoder is now up to 15 times as fast for error handlers ``ignore``, ``replace`` and ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25301`). diff --git a/Misc/NEWS b/Misc/NEWS index 0a11ebf..8f33a5b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -191,10 +191,10 @@ Library - Issue #25203: Failed readline.set_completer_delims() no longer left the module in inconsistent state. -- Issue #25011: rlcomplete now omits private and special attribute names unless +- Issue #25011: rlcompleter now omits private and special attribute names unless the prefix starts with underscores. -- Issue #25209: rlcomplete now can add a space or a colon after completed keyword. +- Issue #25209: rlcompleter now can add a space or a colon after completed keyword. - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. -- cgit v0.12 From 63c1ebb67b4716720ad6e807d8f6a19042af83eb Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 08:54:30 +0000 Subject: Issue #25168: Temporary timezone and cache debugging --- Lib/test/datetimetester.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 11deffc..332d94b 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1979,7 +1979,16 @@ class TestDateTime(TestDate): seconds = tzseconds hours, minutes = divmod(seconds//60, 60) dtstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) - dt = strptime(dtstr, "%z %Z") + try: + dt = strptime(dtstr, "%z %Z") + except ValueError: + import os + self.fail( + "Issue #25168 strptime() failure info:\n" + f"_TimeRE_cache['Z']={_strptime._TimeRE_cache['Z']!r}\n" + f"TZ={os.environ.get('TZ')!r}, or {os.getenv('TZ')!r} via getenv()\n" + f"_regex_cache={_strptime._regex_cache!r}\n" + ) self.assertEqual(dt.utcoffset(), timedelta(seconds=tzseconds)) self.assertEqual(dt.tzname(), tzname) # Can produce inconsistent datetime -- cgit v0.12 From d226d308a3856e67c654ff3d5924be6d24568388 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 11:47:00 +0000 Subject: Issue #23883: Add test.support.check__all__() and test gettext.__all__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patches by Jacek Kołodziej. --- Doc/library/test.rst | 42 ++++++++++++++++++++++++++++++ Lib/test/support/__init__.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_gettext.py | 6 +++++ Lib/test/test_support.py | 22 ++++++++++++++++ Misc/ACKS | 1 + 5 files changed, 132 insertions(+) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 85cab3b..797afa5 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -580,6 +580,48 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.5 +.. function:: check__all__(test_case, module, name_of_module=None, extra=(), blacklist=()) + + Assert that the ``__all__`` variable of *module* contains all public names. + + The module's public names (its API) are detected automatically + based on whether they match the public name convention and were defined in + *module*. + + The *name_of_module* argument can specify (as a string or tuple thereof) what + module(s) an API could be defined in in order to be detected as a public + API. One case for this is when *module* imports part of its public API from + other modules, possibly a C backend (like ``csv`` and its ``_csv``). + + The *extra* argument can be a set of names that wouldn't otherwise be automatically + detected as "public", like objects without a proper ``__module__`` + attribute. If provided, it will be added to the automatically detected ones. + + The *blacklist* argument can be a set of names that must not be treated as part of + the public API even though their names indicate otherwise. + + Example use:: + + import bar + import foo + import unittest + from test import support + + class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, foo) + + class OtherTestCase(unittest.TestCase): + def test__all__(self): + extra = {'BAR_CONST', 'FOO_CONST'} + blacklist = {'baz'} # Undocumented name. + # bar imports part of its API from _bar. + support.check__all__(self, bar, ('bar', '_bar'), + extra=extra, blacklist=blacklist) + + .. versionadded:: 3.6 + + The :mod:`test.support` module defines the following classes: .. class:: TransientResource(exc, **kwargs) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 359d6dd..728b459 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -26,6 +26,7 @@ import sys import sysconfig import tempfile import time +import types import unittest import urllib.error import warnings @@ -89,6 +90,7 @@ __all__ = [ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", + "check__all__", # sys "is_jython", "check_impl_detail", # network @@ -2199,6 +2201,65 @@ def detect_api_mismatch(ref_api, other_api, *, ignore=()): return missing_items +def check__all__(test_case, module, name_of_module=None, extra=(), + blacklist=()): + """Assert that the __all__ variable of 'module' contains all public names. + + The module's public names (its API) are detected automatically based on + whether they match the public name convention and were defined in + 'module'. + + The 'name_of_module' argument can specify (as a string or tuple thereof) + what module(s) an API could be defined in in order to be detected as a + public API. One case for this is when 'module' imports part of its public + API from other modules, possibly a C backend (like 'csv' and its '_csv'). + + The 'extra' argument can be a set of names that wouldn't otherwise be + automatically detected as "public", like objects without a proper + '__module__' attriubute. If provided, it will be added to the + automatically detected ones. + + The 'blacklist' argument can be a set of names that must not be treated + as part of the public API even though their names indicate otherwise. + + Usage: + import bar + import foo + import unittest + from test import support + + class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, foo) + + class OtherTestCase(unittest.TestCase): + def test__all__(self): + extra = {'BAR_CONST', 'FOO_CONST'} + blacklist = {'baz'} # Undocumented name. + # bar imports part of its API from _bar. + support.check__all__(self, bar, ('bar', '_bar'), + extra=extra, blacklist=blacklist) + + """ + + if name_of_module is None: + name_of_module = (module.__name__, ) + elif isinstance(name_of_module, str): + name_of_module = (name_of_module, ) + + expected = set(extra) + + for name in dir(module): + if name.startswith('_') or name in blacklist: + continue + obj = getattr(module, name) + if (getattr(obj, '__module__', None) in name_of_module or + (not hasattr(obj, '__module__') and + not isinstance(obj, types.ModuleType))): + expected.add(name) + test_case.assertCountEqual(module.__all__, expected) + + class SuppressCrashReport: """Try to prevent a crash report from popping up. diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index de610c7..3a94383 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -440,6 +440,12 @@ class GettextCacheTestCase(GettextBaseTest): self.assertEqual(t.__class__, DummyGNUTranslations) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'c2py', 'ENOENT'} + support.check__all__(self, gettext, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 2c00417..a9ba460 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -312,6 +312,28 @@ class TestSupport(unittest.TestCase): self.OtherClass, self.RefClass, ignore=ignore) self.assertEqual(set(), missing_items) + def test_check__all__(self): + extra = {'tempdir'} + blacklist = {'template'} + support.check__all__(self, + tempfile, + extra=extra, + blacklist=blacklist) + + extra = {'TextTestResult', 'installHandler'} + blacklist = {'load_tests', "TestProgram", "BaseTestSuite"} + + support.check__all__(self, + unittest, + ("unittest.result", "unittest.case", + "unittest.suite", "unittest.loader", + "unittest.main", "unittest.runner", + "unittest.signals"), + extra=extra, + blacklist=blacklist) + + self.assertRaises(AssertionError, support.check__all__, self, unittest) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Misc/ACKS b/Misc/ACKS index 6341b23..934cba2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -765,6 +765,7 @@ Damon Kohler Marko Kohtala Vajrasky Kok Guido Kollerie +Jacek Kołodziej Jacek Konieczny Марк Коренберг Arkady Koplyarov -- cgit v0.12 From 19e69c5a2067fe6322ead88733ebbca77673010b Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 12:46:42 +0000 Subject: =?UTF-8?q?Issue=20#23883:=20Add=20missing=20APIs=20to=20=5F=5Fall?= =?UTF-8?q?=5F=5F;=20patch=20by=20Jacek=20Ko=C5=82odziej?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/csv.py | 11 ++++++----- Lib/enum.py | 2 +- Lib/ftplib.py | 3 ++- Lib/logging/__init__.py | 5 +++-- Lib/optparse.py | 3 ++- Lib/test/test_csv.py | 6 ++++++ Lib/test/test_enum.py | 7 +++++++ Lib/test/test_ftplib.py | 11 ++++++++++- Lib/test/test_logging.py | 14 +++++++++++++- Lib/test/test_optparse.py | 7 +++++++ Lib/test/test_pickletools.py | 33 +++++++++++++++++++++++++++++++++ Lib/test/test_threading.py | 8 ++++++++ Lib/test/test_wave.py | 7 +++++++ Lib/threading.py | 8 +++++--- Lib/wave.py | 2 +- 15 files changed, 111 insertions(+), 16 deletions(-) diff --git a/Lib/csv.py b/Lib/csv.py index ca40e5e..90461db 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -13,11 +13,12 @@ from _csv import Dialect as _Dialect from io import StringIO -__all__ = [ "QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", - "Error", "Dialect", "__doc__", "excel", "excel_tab", - "field_size_limit", "reader", "writer", - "register_dialect", "get_dialect", "list_dialects", "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter" ] +__all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", + "Error", "Dialect", "__doc__", "excel", "excel_tab", + "field_size_limit", "reader", "writer", + "register_dialect", "get_dialect", "list_dialects", "Sniffer", + "unregister_dialect", "__version__", "DictReader", "DictWriter", + "unix_dialect"] class Dialect: """Describe a CSV dialect. diff --git a/Lib/enum.py b/Lib/enum.py index 8d04e2d..35a9c77 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -8,7 +8,7 @@ except ImportError: from collections import OrderedDict -__all__ = ['Enum', 'IntEnum', 'unique'] +__all__ = ['EnumMeta', 'Enum', 'IntEnum', 'unique'] def _is_descriptor(obj): diff --git a/Lib/ftplib.py b/Lib/ftplib.py index c416d85..2afa19d 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -42,7 +42,8 @@ import socket import warnings from socket import _GLOBAL_DEFAULT_TIMEOUT -__all__ = ["FTP"] +__all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", + "all_errors"] # Magic number from MSG_OOB = 0x1 # Process data out of band diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 104b0be..369d2c3 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -33,8 +33,9 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig', 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', - 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning', - 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort'] + 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', + 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', + 'lastResort', 'raiseExceptions'] try: import threading diff --git a/Lib/optparse.py b/Lib/optparse.py index 432a2eb..d239ea2 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -38,7 +38,8 @@ __all__ = ['Option', 'OptionError', 'OptionConflictError', 'OptionValueError', - 'BadOptionError'] + 'BadOptionError', + 'check_choice'] __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 8e9c2b4..4763bbb 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1084,5 +1084,11 @@ class TestUnicode(unittest.TestCase): self.assertEqual(fileobj.read(), expected) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {'__doc__', '__version__'} + support.check__all__(self, csv, ('csv', '_csv'), extra=extra) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0f7b769..e4e6c2b 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -6,6 +6,7 @@ from collections import OrderedDict from enum import Enum, IntEnum, EnumMeta, unique from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL +from test import support # for pickle tests try: @@ -1708,5 +1709,11 @@ class TestStdLib(unittest.TestCase): if failed: self.fail("result does not equal expected, see print above") + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, enum) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index aef66da..9d8de21 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -1049,10 +1049,19 @@ class TestTimeouts(TestCase): ftp.close() +class MiscTestCase(TestCase): + def test__all__(self): + blacklist = {'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', + 'Error', 'parse150', 'parse227', 'parse229', 'parse257', + 'print_line', 'ftpcp', 'test'} + support.check__all__(self, ftplib, blacklist=blacklist) + + def test_main(): tests = [TestFTPClass, TestTimeouts, TestIPv6Environment, - TestTLS_FTPClassMixin, TestTLS_FTPClass] + TestTLS_FTPClassMixin, TestTLS_FTPClass, + MiscTestCase] thread_info = support.threading_setup() try: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 95575bf..9c4344f 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4159,6 +4159,17 @@ class NTEventLogHandlerTest(BaseTest): msg = 'Record not found in event log, went back %d records' % GO_BACK self.assertTrue(found, msg=msg) + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'logThreads', 'logMultiprocessing', + 'logProcesses', 'currentframe', + 'PercentStyle', 'StrFormatStyle', 'StringTemplateStyle', + 'Filterer', 'PlaceHolder', 'Manager', 'RootLogger', + 'root'} + support.check__all__(self, logging, blacklist=blacklist) + + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. @@ -4175,7 +4186,8 @@ def test_main(): RotatingFileHandlerTest, LastResortTest, LogRecordTest, ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, - UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest) + UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, + MiscTestCase) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 7621c24..91a0319 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -16,6 +16,7 @@ from io import StringIO from test import support +import optparse from optparse import make_option, Option, \ TitledHelpFormatter, OptionParser, OptionGroup, \ SUPPRESS_USAGE, OptionError, OptionConflictError, \ @@ -1650,6 +1651,12 @@ class TestParseNumber(BaseTest): "option -l: invalid integer value: '0x12x'") +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'} + support.check__all__(self, optparse, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index bbe6875..80221f0 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -4,6 +4,7 @@ import pickletools from test import support from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests +import unittest class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -59,8 +60,40 @@ class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): self.assertNotIn(pickle.BINPUT, pickled2) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'bytes_types', + 'UP_TO_NEWLINE', 'TAKEN_FROM_ARGUMENT1', + 'TAKEN_FROM_ARGUMENT4', 'TAKEN_FROM_ARGUMENT4U', + 'TAKEN_FROM_ARGUMENT8U', 'ArgumentDescriptor', + 'read_uint1', 'read_uint2', 'read_int4', 'read_uint4', + 'read_uint8', 'read_stringnl', 'read_stringnl_noescape', + 'read_stringnl_noescape_pair', 'read_string1', + 'read_string4', 'read_bytes1', 'read_bytes4', + 'read_bytes8', 'read_unicodestringnl', + 'read_unicodestring1', 'read_unicodestring4', + 'read_unicodestring8', 'read_decimalnl_short', + 'read_decimalnl_long', 'read_floatnl', 'read_float8', + 'read_long1', 'read_long4', + 'uint1', 'uint2', 'int4', 'uint4', 'uint8', 'stringnl', + 'stringnl_noescape', 'stringnl_noescape_pair', 'string1', + 'string4', 'bytes1', 'bytes4', 'bytes8', + 'unicodestringnl', 'unicodestring1', 'unicodestring4', + 'unicodestring8', 'decimalnl_short', 'decimalnl_long', + 'floatnl', 'float8', 'long1', 'long4', + 'StackObject', + 'pyint', 'pylong', 'pyinteger_or_bool', 'pybool', 'pyfloat', + 'pybytes_or_str', 'pystring', 'pybytes', 'pyunicode', + 'pynone', 'pytuple', 'pylist', 'pydict', 'pyset', + 'pyfrozenset', 'anyobject', 'markobject', 'stackslice', + 'OpcodeInfo', 'opcodes', 'code2op', + } + support.check__all__(self, pickletools, blacklist=blacklist) + + def test_main(): support.run_unittest(OptimizedPickleTests) + support.run_unittest(MiscTestCase) support.run_doctest(pickletools) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3b11bf6..45564f7 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -18,6 +18,7 @@ import os import subprocess from test import lock_tests +from test import support # Between fork() and exec(), only async-safe functions are allowed (issues @@ -1098,5 +1099,12 @@ class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): class BarrierTests(lock_tests.BarrierTests): barriertype = staticmethod(threading.Barrier) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {"ThreadError"} + blacklist = {'currentThread', 'activeCount'} + support.check__all__(self, threading, ('threading', '_thread'), + extra=extra, blacklist=blacklist) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 3eff773..a67a8b0 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,6 +1,7 @@ from test.support import TESTFN import unittest from test import audiotests +from test import support from audioop import byteswap import sys import wave @@ -103,5 +104,11 @@ class WavePCM32Test(WaveTest, unittest.TestCase): frames = byteswap(frames, 4) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'WAVE_FORMAT_PCM'} + support.check__all__(self, wave, blacklist=blacklist) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/threading.py b/Lib/threading.py index 828019d..2bf33c1 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -22,9 +22,11 @@ except ImportError: # with the multiprocessing module, which doesn't provide the old # Java inspired names. -__all__ = ['active_count', 'Condition', 'current_thread', 'enumerate', 'Event', - 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', - 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size'] +__all__ = ['get_ident', 'active_count', 'Condition', 'current_thread', + 'enumerate', 'main_thread', 'TIMEOUT_MAX', + 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', + 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', + 'setprofile', 'settrace', 'local', 'stack_size'] # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread diff --git a/Lib/wave.py b/Lib/wave.py index 8a101e3..f71f7e5 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -73,7 +73,7 @@ is destroyed. import builtins -__all__ = ["open", "openfp", "Error"] +__all__ = ["open", "openfp", "Error", "Wave_read", "Wave_write"] class Error(Exception): pass -- cgit v0.12 From 28a465c9e0d4f49eb2f344d8cbb986f61a47efbe Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 12:52:08 +0000 Subject: Issue #23883: Add news listing modules with new exported APIs --- Doc/whatsnew/3.6.rst | 6 ++++++ Misc/NEWS | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b2066c5..172b253 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -230,6 +230,12 @@ Changes in the Python API * The :mod:`imp` module now raises a :exc:`DeprecationWarning` instead of :exc:`PendingDeprecationWarning`. +* The following modules have had missing APIs added to their :attr:`__all__` + attributes to match the documented APIs: :mod:`csv`, :mod:`enum`, + :mod:`ftplib`, :mod:`logging`, :mod:`optparse`, :mod:`threading` and + :mod:`wave`. This means they will export new symbols when ``import *`` + is used. See :issue:`23883`. + Changes in the C API -------------------- diff --git a/Misc/NEWS b/Misc/NEWS index 8f33a5b..4a64eb4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -85,6 +85,11 @@ Core and Builtins Library ------- +- Issue #23883: Added missing APIs to __all__ to match the documented APIs + for the following modules: csv, enum, ftplib, logging, optparse, threading + and wave. Also added a test.support.check__all__() helper. Patches by + Jacek Kołodziej. + - Issue #25590: In the Readline completer, only call getattr() once per attribute. Also complete names of attributes such as properties and slots which are listed by dir() but not yet created on an instance. -- cgit v0.12 From 413fdcea21908055cb8acad28a94b8f72eb2ffec Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 14 Nov 2015 15:42:17 +0200 Subject: Issue #24821: Refactor STRINGLIB(fastsearch_memchr_1char) and split it on STRINGLIB(find_char) and STRINGLIB(rfind_char) that can be used independedly without special preconditions. --- Objects/bytearrayobject.c | 19 +++--- Objects/bytesobject.c | 19 +++--- Objects/stringlib/fastsearch.h | 150 ++++++++++++++++++++++++----------------- Objects/unicodeobject.c | 33 +++++---- 4 files changed, 121 insertions(+), 100 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 2103147..131e04d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1159,16 +1159,15 @@ bytearray_find_internal(PyByteArrayObject *self, PyObject *args, int dir) ADJUST_INDICES(start, end, len); if (end - start < sub_len) res = -1; - else if (sub_len == 1 -#ifndef HAVE_MEMRCHR - && dir > 0 -#endif - ) { - unsigned char needle = *sub; - int mode = (dir > 0) ? FAST_SEARCH : FAST_RSEARCH; - res = stringlib_fastsearch_memchr_1char( - PyByteArray_AS_STRING(self) + start, end - start, - needle, needle, mode); + else if (sub_len == 1) { + if (dir > 0) + res = stringlib_find_char( + PyByteArray_AS_STRING(self) + start, end - start, + *sub); + else + res = stringlib_rfind_char( + PyByteArray_AS_STRING(self) + start, end - start, + *sub); if (res >= 0) res += start; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index c10dbdf..6a9e062 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1937,16 +1937,15 @@ bytes_find_internal(PyBytesObject *self, PyObject *args, int dir) ADJUST_INDICES(start, end, len); if (end - start < sub_len) res = -1; - else if (sub_len == 1 -#ifndef HAVE_MEMRCHR - && dir > 0 -#endif - ) { - unsigned char needle = *sub; - int mode = (dir > 0) ? FAST_SEARCH : FAST_RSEARCH; - res = stringlib_fastsearch_memchr_1char( - PyBytes_AS_STRING(self) + start, end - start, - needle, needle, mode); + else if (sub_len == 1) { + if (dir > 0) + res = stringlib_find_char( + PyBytes_AS_STRING(self) + start, end - start, + *sub); + else + res = stringlib_rfind_char( + PyBytes_AS_STRING(self) + start, end - start, + *sub); if (res >= 0) res += start; } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index cda68e7..98165ad 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -32,52 +32,98 @@ #define STRINGLIB_BLOOM(mask, ch) \ ((mask & (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1))))) - Py_LOCAL_INLINE(Py_ssize_t) -STRINGLIB(fastsearch_memchr_1char)(const STRINGLIB_CHAR* s, Py_ssize_t n, - STRINGLIB_CHAR ch, unsigned char needle, - int mode) +STRINGLIB(find_char)(const STRINGLIB_CHAR* s, Py_ssize_t n, STRINGLIB_CHAR ch) { - if (mode == FAST_SEARCH) { - const STRINGLIB_CHAR *ptr = s; - const STRINGLIB_CHAR *e = s + n; - while (ptr < e) { - void *candidate = memchr((const void *) ptr, needle, (e - ptr) * sizeof(STRINGLIB_CHAR)); - if (candidate == NULL) - return -1; - ptr = (const STRINGLIB_CHAR *) _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); - if (sizeof(STRINGLIB_CHAR) == 1 || *ptr == ch) - return (ptr - s); - /* False positive */ - ptr++; - } + const STRINGLIB_CHAR *p, *e; + + p = s; + e = s + n; + if (n > 10) { +#if STRINGLIB_SIZEOF_CHAR == 1 + p = memchr(s, ch, n); + if (p != NULL) + return (p - s); return -1; +#else + /* use memchr if we can choose a needle without two many likely + false positives */ + unsigned char needle = ch & 0xff; + /* If looking for a multiple of 256, we'd have too + many false positives looking for the '\0' byte in UCS2 + and UCS4 representations. */ + if (needle != 0) { + while (p < e) { + void *candidate = memchr(p, needle, + (e - p) * sizeof(STRINGLIB_CHAR)); + if (candidate == NULL) + return -1; + p = (const STRINGLIB_CHAR *) + _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); + if (*p == ch) + return (p - s); + /* False positive */ + p++; + } + return -1; + } +#endif } + while (p < e) { + if (*p == ch) + return (p - s); + p++; + } + return -1; +} + +Py_LOCAL_INLINE(Py_ssize_t) +STRINGLIB(rfind_char)(const STRINGLIB_CHAR* s, Py_ssize_t n, STRINGLIB_CHAR ch) +{ + const STRINGLIB_CHAR *p; #ifdef HAVE_MEMRCHR /* memrchr() is a GNU extension, available since glibc 2.1.91. it doesn't seem as optimized as memchr(), but is still quite - faster than our hand-written loop in FASTSEARCH below */ - else if (mode == FAST_RSEARCH) { - while (n > 0) { - const STRINGLIB_CHAR *found; - void *candidate = memrchr((const void *) s, needle, n * sizeof(STRINGLIB_CHAR)); - if (candidate == NULL) - return -1; - found = (const STRINGLIB_CHAR *) _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); - n = found - s; - if (sizeof(STRINGLIB_CHAR) == 1 || *found == ch) - return n; - /* False positive */ - } + faster than our hand-written loop below */ + + if (n > 10) { +#if STRINGLIB_SIZEOF_CHAR == 1 + p = memrchr(s, ch, n); + if (p != NULL) + return (p - s); return -1; - } +#else + /* use memrchr if we can choose a needle without two many likely + false positives */ + unsigned char needle = ch & 0xff; + /* If looking for a multiple of 256, we'd have too + many false positives looking for the '\0' byte in UCS2 + and UCS4 representations. */ + if (needle != 0) { + while (n > 0) { + void *candidate = memrchr(s, needle, + n * sizeof(STRINGLIB_CHAR)); + if (candidate == NULL) + return -1; + p = (const STRINGLIB_CHAR *) + _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); + n = p - s; + if (*p == ch) + return n; + /* False positive */ + } + return -1; + } #endif - else { - assert(0); /* Should never get here */ - return 0; } - -#undef DO_MEMCHR +#endif /* HAVE_MEMRCHR */ + p = s + n; + while (p > s) { + p--; + if (*p == ch) + return (p - s); + } + return -1; } Py_LOCAL_INLINE(Py_ssize_t) @@ -99,25 +145,11 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, if (m <= 0) return -1; /* use special case for 1-character strings */ - if (n > 10 && (mode == FAST_SEARCH -#ifdef HAVE_MEMRCHR - || mode == FAST_RSEARCH -#endif - )) { - /* use memchr if we can choose a needle without two many likely - false positives */ - unsigned char needle; - needle = p[0] & 0xff; -#if STRINGLIB_SIZEOF_CHAR > 1 - /* If looking for a multiple of 256, we'd have too - many false positives looking for the '\0' byte in UCS2 - and UCS4 representations. */ - if (needle != 0) -#endif - return STRINGLIB(fastsearch_memchr_1char) - (s, n, p[0], needle, mode); - } - if (mode == FAST_COUNT) { + if (mode == FAST_SEARCH) + return STRINGLIB(find_char)(s, n, p[0]); + else if (mode == FAST_RSEARCH) + return STRINGLIB(rfind_char)(s, n, p[0]); + else { /* FAST_COUNT */ for (i = 0; i < n; i++) if (s[i] == p[0]) { count++; @@ -125,14 +157,6 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, return maxcount; } return count; - } else if (mode == FAST_SEARCH) { - for (i = 0; i < n; i++) - if (s[i] == p[0]) - return i; - } else { /* FAST_RSEARCH */ - for (i = n - 1; i > -1; i--) - if (s[i] == p[0]) - return i; } return -1; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 18a30e2..c2b6f64 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -811,27 +811,26 @@ Py_LOCAL_INLINE(Py_ssize_t) findchar(const void *s, int kind, Py_ssize_t size, Py_UCS4 ch, int direction) { - int mode = (direction == 1) ? FAST_SEARCH : FAST_RSEARCH; - switch (kind) { case PyUnicode_1BYTE_KIND: - { - Py_UCS1 ch1 = (Py_UCS1) ch; - if (ch1 == ch) - return ucs1lib_fastsearch((Py_UCS1 *) s, size, &ch1, 1, 0, mode); - else - return -1; - } + if ((Py_UCS1) ch != ch) + return -1; + if (direction > 0) + return ucs1lib_find_char((Py_UCS1 *) s, size, (Py_UCS1) ch); + else + return ucs1lib_rfind_char((Py_UCS1 *) s, size, (Py_UCS1) ch); case PyUnicode_2BYTE_KIND: - { - Py_UCS2 ch2 = (Py_UCS2) ch; - if (ch2 == ch) - return ucs2lib_fastsearch((Py_UCS2 *) s, size, &ch2, 1, 0, mode); - else - return -1; - } + if ((Py_UCS2) ch != ch) + return -1; + if (direction > 0) + return ucs2lib_find_char((Py_UCS2 *) s, size, (Py_UCS2) ch); + else + return ucs2lib_rfind_char((Py_UCS2 *) s, size, (Py_UCS2) ch); case PyUnicode_4BYTE_KIND: - return ucs4lib_fastsearch((Py_UCS4 *) s, size, &ch, 1, 0, mode); + if (direction > 0) + return ucs4lib_find_char((Py_UCS4 *) s, size, ch); + else + return ucs4lib_rfind_char((Py_UCS4 *) s, size, ch); default: assert(0); return -1; -- cgit v0.12 From fca22327ca9086a17902b3a21f2b8af45f7a4892 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 16 Nov 2015 09:22:19 +0000 Subject: Issue #20220: Revert time zone test debugging, revision 139c18943d9b --- Lib/test/datetimetester.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f814f23..7374608 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2006,16 +2006,7 @@ class TestDateTime(TestDate): seconds = tzseconds hours, minutes = divmod(seconds//60, 60) dtstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) - try: - dt = strptime(dtstr, "%z %Z") - except ValueError: - import os - self.fail( - "Issue #25168 strptime() failure info:\n" - f"_TimeRE_cache['Z']={_strptime._TimeRE_cache['Z']!r}\n" - f"TZ={os.environ.get('TZ')!r}, or {os.getenv('TZ')!r} via getenv()\n" - f"_regex_cache={_strptime._regex_cache!r}\n" - ) + dt = strptime(dtstr, "%z %Z") self.assertEqual(dt.utcoffset(), timedelta(seconds=tzseconds)) self.assertEqual(dt.tzname(), tzname) # Can produce inconsistent datetime -- cgit v0.12 From 9ba97df69c9b056d8531bb8ed54378ca81d761de Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Nov 2015 12:15:07 +0100 Subject: Closes #25645: Fix a reference leak introduced by change bc5894a3a0e6 of the issue #24164. --- Modules/_pickle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 06882d0..a6f414c 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -193,6 +193,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->import_mapping_3to2); Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); + Py_CLEAR(st->partial); } /* Initialize the given pickle module state. */ -- cgit v0.12 From 6019c8ced03fcb7acde715bf39aea18d1e44cba8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 17 Nov 2015 08:28:07 -0800 Subject: Issue #25629: Move set fill/used updates out of inner loop --- Objects/setobject.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 03bb230..083cbea 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -254,9 +254,10 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) Internal routine used by set_table_resize() to insert an item which is known to be absent from the set. This routine also assumes that the set contains no deleted entries. Besides the performance benefit, -using set_insert_clean() in set_table_resize() is dangerous (SF bug #1456209). -Note that no refcounts are changed by this routine; if needed, the caller -is responsible for incref'ing `key`. +there is also safety benefit since using set_add_entry() risks making +a callback in the middle of a set_table_resize(), see issue 1456209. +The caller is responsible for updating the key's reference count and +the setobject's fill and used fields. */ static void set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) @@ -285,8 +286,6 @@ set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) found_null: entry->key = key; entry->hash = hash; - so->fill++; - so->used++; } /* ======== End logic for probing the hash table ========================== */ @@ -356,8 +355,8 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) /* Make the set empty, using the new table. */ assert(newtable != oldtable); memset(newtable, 0, sizeof(setentry) * newsize); - so->fill = 0; - so->used = 0; + so->fill = oldused; + so->used = oldused; so->mask = newsize - 1; so->table = newtable; @@ -676,6 +675,8 @@ set_merge(PySetObject *so, PyObject *otherset) /* If our table is empty, we can use set_insert_clean() */ if (so->fill == 0) { + so->fill = other->used; + so->used = other->used; for (i = 0; i <= other->mask; i++, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { -- cgit v0.12 From 66f6238fca7286a3917eab0e9fafd7aa6f8e90a4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 17 Nov 2015 20:58:43 -0800 Subject: Add assertion to verify the pre-condition in the comments. --- Objects/setobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/setobject.c b/Objects/setobject.c index 083cbea..7fde01f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -269,6 +269,7 @@ set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) size_t i = (size_t)hash & mask; size_t j; + assert(so->fill == so->used); while (1) { entry = &table[i]; if (entry->key == NULL) -- cgit v0.12 From 8b4ee344ef43552a37c4e6fe342eb4a8c4b1a10d Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 20 Nov 2015 02:37:57 +0000 Subject: Issue #25583: Add news to 3.6 section --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 72d2073..1e18892 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -95,6 +95,9 @@ Core and Builtins Library ------- +- Issue #25583: Avoid incorrect errors raised by os.makedirs(exist_ok=True) + when the OS gives priority to errors such as EACCES over EEXIST. + - Issue #25593: Change semantics of EventLoop.stop() in asyncio. - Issue #6973: When we know a subprocess.Popen process has died, do -- cgit v0.12 From b4efc963d628475c79f02ecf43a4ec564b46428e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Nov 2015 09:24:02 +0100 Subject: Issue #25557: Refactor _PyDict_LoadGlobal() Don't fallback to PyDict_GetItemWithError() if the hash is unknown: compute the hash instead. Add also comments to explain the optimization a little bit. --- Objects/dictobject.c | 55 +++++++++++++++++++++++++++------------------------- Python/ceval.c | 11 +++++++++-- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 624ae9b..4a72c9a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1165,39 +1165,42 @@ _PyDict_GetItemIdWithError(PyObject *dp, struct _Py_Identifier *key) return PyDict_GetItemWithError(dp, kv); } -/* Fast version of global value lookup. +/* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. + * + * Raise an exception and return NULL if an error occurred (ex: computing the + * key hash failed, key comparison failed, ...). Return NULL if the key doesn't + * exist. Return the value if the key exists. */ PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) { - PyObject *x; - if (PyUnicode_CheckExact(key)) { - PyObject **value_addr; - Py_hash_t hash = ((PyASCIIObject *)key)->hash; - if (hash != -1) { - PyDictKeyEntry *e; - e = globals->ma_keys->dk_lookup(globals, key, hash, &value_addr); - if (e == NULL) { - return NULL; - } - x = *value_addr; - if (x != NULL) - return x; - e = builtins->ma_keys->dk_lookup(builtins, key, hash, &value_addr); - if (e == NULL) { - return NULL; - } - x = *value_addr; - return x; - } + Py_hash_t hash; + PyDictKeyEntry *entry; + PyObject **value_addr; + PyObject *value; + + if (!PyUnicode_CheckExact(key) || + (hash = ((PyASCIIObject *) key)->hash) == -1) + { + hash = PyObject_Hash(key); + if (hash == -1) + return NULL; } - x = PyDict_GetItemWithError((PyObject *)globals, key); - if (x != NULL) - return x; - if (PyErr_Occurred()) + + /* namespace 1: globals */ + entry = globals->ma_keys->dk_lookup(globals, key, hash, &value_addr); + if (entry == NULL) return NULL; - return PyDict_GetItemWithError((PyObject *)builtins, key); + value = *value_addr; + if (value != NULL) + return value; + + /* namespace 2: builtins */ + entry = builtins->ma_keys->dk_lookup(builtins, key, hash, &value_addr); + if (entry == NULL) + return NULL; + return *value_addr; } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the diff --git a/Python/ceval.c b/Python/ceval.c index 7f9ef6f..1d65649 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2347,26 +2347,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyObject *name = GETITEM(names, oparg); PyObject *v; if (PyDict_CheckExact(f->f_globals) - && PyDict_CheckExact(f->f_builtins)) { + && PyDict_CheckExact(f->f_builtins)) + { v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals, (PyDictObject *)f->f_builtins, name); if (v == NULL) { - if (!_PyErr_OCCURRED()) + if (!_PyErr_OCCURRED()) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ format_exc_check_arg(PyExc_NameError, NAME_ERROR_MSG, name); + } goto error; } Py_INCREF(v); } else { /* Slow-path if globals or builtins is not a dict */ + + /* namespace 1: globals */ v = PyObject_GetItem(f->f_globals, name); if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); + /* namespace 2: builtins */ v = PyObject_GetItem(f->f_builtins, name); if (v == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyError)) -- cgit v0.12 From 748dad5b6a54370c72029afea9e7baa683d1ffe8 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Fri, 20 Nov 2015 13:12:26 -0800 Subject: Close 25594: advise against accessing Enum members from other members --- Doc/library/enum.rst | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 0fbbf5a..a76f5a3 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -730,18 +730,24 @@ member instances. Finer Points ^^^^^^^^^^^^ -Enum members are instances of an Enum class, and even though they are -accessible as `EnumClass.member`, they are not accessible directly from -the member:: - - >>> Color.red - - >>> Color.red.blue - Traceback (most recent call last): +:class:`Enum` members are instances of an :class:`Enum` class, and even +though they are accessible as `EnumClass.member`, they should not be accessed +directly from the member as that lookup may fail or, worse, return something +besides the :class:`Enum` member you looking for:: + + >>> class FieldTypes(Enum): + ... name = 0 + ... value = 1 + ... size = 2 ... - AttributeError: 'Color' object has no attribute 'blue' + >>> FieldTypes.value.size + + >>> FieldTypes.size.value + 2 + +.. versionchanged:: 3.5 -Likewise, the :attr:`__members__` is only available on the class. +The :attr:`__members__` attribute is only available on the class. If you give your :class:`Enum` subclass extra methods, like the `Planet`_ class above, those methods will show up in a :func:`dir` of the member, -- cgit v0.12 From 1aa10278a77e76e6d0cacae2d79429c1b3e2c447 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 21 Nov 2015 10:57:47 +0000 Subject: Issue #25626: Add news to 3.6 section --- Misc/NEWS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 0e25ad2..39226f5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -95,6 +95,13 @@ Core and Builtins Library ------- +- Issue #25626: Change three zlib functions to accept sizes that fit in + Py_ssize_t, but internally cap those sizes to UINT_MAX. This resolves a + regression in 3.5 where GzipFile.read() failed to read chunks larger than 2 + or 4 GiB. The change affects the zlib.Decompress.decompress() max_length + parameter, the zlib.decompress() bufsize parameter, and the + zlib.Decompress.flush() length parameter. + - Issue #25583: Avoid incorrect errors raised by os.makedirs(exist_ok=True) when the OS gives priority to errors such as EACCES over EEXIST. -- cgit v0.12 From dc0965551e84de92744be587fcb33471a8d565b6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 22 Nov 2015 15:18:40 +0100 Subject: Issue #25694: Install test.libregrtest to be able to run tests on the installed Python --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 823def3..c6c534e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1167,6 +1167,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ test/cjkencodings test/decimaltestdata test/xmltestdata \ test/eintrdata \ test/imghdrdata \ + test/libregrtest \ test/subprocessdata test/sndhdrdata test/support \ test/tracedmodules test/encoded_modules \ test/test_import \ -- cgit v0.12 From da43ee4316a508d031c2c68aeb0f7797294d09ff Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 22 Nov 2015 15:18:54 +0100 Subject: Issue #25694: Fix test_regrtest for installed Python --- Lib/test/test_regrtest.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ab7741f..59f8c9d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -14,6 +14,7 @@ import platform import re import subprocess import sys +import sysconfig import textwrap import unittest from test import libregrtest @@ -306,7 +307,7 @@ class BaseTestCase(unittest.TestCase): TESTNAME_REGEX = r'test_[a-z0-9_]+' def setUp(self): - self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test') + self.testdir = os.path.realpath(os.path.dirname(__file__)) # When test_regrtest is interrupted by CTRL+c, it can leave # temporary test files @@ -330,8 +331,13 @@ class BaseTestCase(unittest.TestCase): self.addCleanup(support.unlink, path) # Use 'x' mode to ensure that we do not override existing tests - with open(path, 'x', encoding='utf-8') as fp: - fp.write(code) + try: + with open(path, 'x', encoding='utf-8') as fp: + fp.write(code) + except PermissionError as exc: + if not sysconfig.is_python_build(): + self.skipTest("cannot write %s: %s" % (path, exc)) + raise return name def regex_search(self, regex, output): @@ -471,7 +477,7 @@ class ProgramsTestCase(BaseTestCase): def test_script_regrtest(self): # Lib/test/regrtest.py - script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py') + script = os.path.join(self.testdir, 'regrtest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) @@ -503,10 +509,12 @@ class ProgramsTestCase(BaseTestCase): def test_script_autotest(self): # Lib/test/autotest.py - script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py') + script = os.path.join(self.testdir, 'autotest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) + @unittest.skipUnless(sysconfig.is_python_build(), + 'run_tests.py script is not installed') def test_tools_script_run_tests(self): # Tools/scripts/run_tests.py script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') @@ -516,6 +524,8 @@ class ProgramsTestCase(BaseTestCase): proc = self.run_command(args) self.check_output(proc.stdout) + @unittest.skipUnless(sysconfig.is_python_build(), + 'test.bat script is not installed') @unittest.skipUnless(sys.platform == 'win32', 'Windows only') def test_tools_buildbot_test(self): # Tools\buildbot\test.bat -- cgit v0.12 From 745f6b3a5db0a9b1cdae7b20c7be2c3c8a3240a7 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 00:20:00 +0000 Subject: Issue #25663: Update rlcompleter test for new 3.6 behaviour --- Lib/test/test_rlcompleter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 34a5eff..d9e4de6 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -124,9 +124,10 @@ class TestRlcompleter(unittest.TestCase): completer = rlcompleter.Completer(namespace) self.assertEqual(completer.complete('False', 0), 'False') self.assertIsNone(completer.complete('False', 1)) # No duplicates - self.assertEqual(completer.complete('assert', 0), 'assert') + # Space or colon added due to being a reserved keyword + self.assertEqual(completer.complete('assert', 0), 'assert ') self.assertIsNone(completer.complete('assert', 1)) - self.assertEqual(completer.complete('try', 0), 'try') + self.assertEqual(completer.complete('try', 0), 'try:') self.assertIsNone(completer.complete('try', 1)) # No opening bracket "(" because we overrode the built-in class self.assertEqual(completer.complete('memoryview', 0), 'memoryview') -- cgit v0.12 From 33623b145df4d41f49ab3ae9e781135fb87fd98d Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 22:12:05 +0000 Subject: Issue #25695: Defer creation of TESTDIRN until the test case is run --- Lib/test/test_support.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index a9ba460..f86ea91 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -9,13 +9,11 @@ import errno from test import support TESTFN = support.TESTFN -TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) class TestSupport(unittest.TestCase): def setUp(self): support.unlink(TESTFN) - support.rmtree(TESTDIRN) tearDown = setUp def test_import_module(self): @@ -48,6 +46,10 @@ class TestSupport(unittest.TestCase): support.unlink(TESTFN) def test_rmtree(self): + TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) + self.addCleanup(support.rmtree, TESTDIRN) + support.rmtree(TESTDIRN) + os.mkdir(TESTDIRN) os.mkdir(os.path.join(TESTDIRN, TESTDIRN)) support.rmtree(TESTDIRN) -- cgit v0.12 From f65dd1d4db088ab2a12430f1b6d43e8f1ed04e18 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 23:00:37 +0000 Subject: Issue #25576: Apply fix to new urlopen() doc string --- Lib/urllib/request.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index e6abf34..57d0dea 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -149,13 +149,8 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *data* should be a buffer in the standard application/x-www-form-urlencoded format. The urllib.parse.urlencode() function takes a mapping or sequence - of 2-tuples and returns a string in this format. It should be encoded to - bytes before being used as the data parameter. The charset parameter in - Content-Type header may be used to specify the encoding. If charset - parameter is not sent with the Content-Type header, the server following - the HTTP 1.1 recommendation may assume that the data is encoded in - ISO-8859-1 encoding. It is advisable to use charset parameter with encoding - used in Content-Type header with the Request. + of 2-tuples and returns an ASCII text string in this format. It should be + encoded to bytes before being used as the data parameter. urllib.request module uses HTTP/1.1 and includes a "Connection:close" header in its HTTP requests. -- cgit v0.12 From dde0815c359fc321b0e7a94f885132e2e77534a1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 25 Nov 2015 15:28:13 +0200 Subject: Issue #7990: dir() on ElementTree.Element now lists properties: "tag", "text", "tail" and "attrib". Original patch by Santoso Wijaya. --- Lib/test/test_xml_etree.py | 10 +-- Misc/NEWS | 3 + Modules/_elementtree.c | 170 +++++++++++++++++++++++++-------------------- 3 files changed, 103 insertions(+), 80 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 57d8e4d..0293201 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -182,10 +182,12 @@ class ElementTreeTest(unittest.TestCase): def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") - self.assertTrue(hasattr(element, "tag"), msg="no tag member") - self.assertTrue(hasattr(element, "attrib"), msg="no attrib member") - self.assertTrue(hasattr(element, "text"), msg="no text member") - self.assertTrue(hasattr(element, "tail"), msg="no tail member") + direlem = dir(element) + for attr in 'tag', 'attrib', 'text', 'tail': + self.assertTrue(hasattr(element, attr), + msg='no %s member' % attr) + self.assertIn(attr, direlem, + msg='no %s visible by dir' % attr) check_string(element.tag) check_mapping(element.attrib) diff --git a/Misc/NEWS b/Misc/NEWS index ac1744b..56a4d2d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -95,6 +95,9 @@ Core and Builtins Library ------- +- Issue #7990: dir() on ElementTree.Element now lists properties: "tag", + "text", "tail" and "attrib". Original patch by Santoso Wijaya. + - Issue #25725: Fixed a reference leak in pickle.loads() when unpickling invalid data including tuple instructions. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 744e833..f03e136 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1870,94 +1870,92 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } static PyObject* -element_getattro(ElementObject* self, PyObject* nameobj) +element_tag_getter(ElementObject *self, void *closure) { - PyObject* res; - char *name = ""; + PyObject *res = self->tag; + Py_INCREF(res); + return res; +} - if (PyUnicode_Check(nameobj)) - name = _PyUnicode_AsString(nameobj); +static PyObject* +element_text_getter(ElementObject *self, void *closure) +{ + PyObject *res = element_get_text(self); + Py_XINCREF(res); + return res; +} - if (name == NULL) - return NULL; +static PyObject* +element_tail_getter(ElementObject *self, void *closure) +{ + PyObject *res = element_get_tail(self); + Py_XINCREF(res); + return res; +} - /* handle common attributes first */ - if (strcmp(name, "tag") == 0) { - res = self->tag; - Py_INCREF(res); - return res; - } else if (strcmp(name, "text") == 0) { - res = element_get_text(self); - Py_XINCREF(res); - return res; +static PyObject* +element_attrib_getter(ElementObject *self, void *closure) +{ + PyObject *res; + if (!self->extra) { + if (create_extra(self, NULL) < 0) + return NULL; } + res = element_get_attrib(self); + Py_XINCREF(res); + return res; +} - /* methods */ - res = PyObject_GenericGetAttr((PyObject*) self, nameobj); - if (res) - return res; - - /* less common attributes */ - if (strcmp(name, "tail") == 0) { - PyErr_Clear(); - res = element_get_tail(self); - } else if (strcmp(name, "attrib") == 0) { - PyErr_Clear(); - if (!self->extra) { - if (create_extra(self, NULL) < 0) - return NULL; - } - res = element_get_attrib(self); +/* macro for setter validation */ +#define _VALIDATE_ATTR_VALUE(V) \ + if ((V) == NULL) { \ + PyErr_SetString( \ + PyExc_AttributeError, \ + "can't delete element attribute"); \ + return -1; \ } - if (!res) - return NULL; - - Py_INCREF(res); - return res; +static int +element_tag_setter(ElementObject *self, PyObject *value, void *closure) +{ + _VALIDATE_ATTR_VALUE(value); + Py_INCREF(value); + Py_DECREF(self->tag); + self->tag = value; + return 0; } static int -element_setattro(ElementObject* self, PyObject* nameobj, PyObject* value) +element_text_setter(ElementObject *self, PyObject *value, void *closure) { - char *name = ""; + _VALIDATE_ATTR_VALUE(value); + Py_INCREF(value); + Py_DECREF(JOIN_OBJ(self->text)); + self->text = value; + return 0; +} - if (value == NULL) { - PyErr_SetString(PyExc_AttributeError, - "can't delete attribute"); - return -1; - } - if (PyUnicode_Check(nameobj)) - name = _PyUnicode_AsString(nameobj); - if (name == NULL) - return -1; +static int +element_tail_setter(ElementObject *self, PyObject *value, void *closure) +{ + _VALIDATE_ATTR_VALUE(value); + Py_INCREF(value); + Py_DECREF(JOIN_OBJ(self->tail)); + self->tail = value; + return 0; +} - if (strcmp(name, "tag") == 0) { - Py_DECREF(self->tag); - self->tag = value; - Py_INCREF(self->tag); - } else if (strcmp(name, "text") == 0) { - Py_DECREF(JOIN_OBJ(self->text)); - self->text = value; - Py_INCREF(self->text); - } else if (strcmp(name, "tail") == 0) { - Py_DECREF(JOIN_OBJ(self->tail)); - self->tail = value; - Py_INCREF(self->tail); - } else if (strcmp(name, "attrib") == 0) { - if (!self->extra) { - if (create_extra(self, NULL) < 0) - return -1; - } - Py_DECREF(self->extra->attrib); - self->extra->attrib = value; - Py_INCREF(self->extra->attrib); - } else { - PyErr_SetString(PyExc_AttributeError, - "Can't set arbitrary attributes on Element"); - return -1; +static int +element_attrib_setter(ElementObject *self, PyObject *value, void *closure) +{ + _VALIDATE_ATTR_VALUE(value); + if (!self->extra) { + if (create_extra(self, NULL) < 0) + return -1; } - + Py_INCREF(value); + Py_DECREF(self->extra->attrib); + self->extra->attrib = value; return 0; } @@ -3770,6 +3768,26 @@ static PyMappingMethods element_as_mapping = { (objobjargproc) element_ass_subscr, }; +static PyGetSetDef element_getsetlist[] = { + {"tag", + (getter)element_tag_getter, + (setter)element_tag_setter, + "A string identifying what kind of data this element represents"}, + {"text", + (getter)element_text_getter, + (setter)element_text_setter, + "A string of text directly after the start tag, or None"}, + {"tail", + (getter)element_tail_getter, + (setter)element_tail_setter, + "A string of text directly after the end tag, or None"}, + {"attrib", + (getter)element_attrib_getter, + (setter)element_attrib_setter, + "A dictionary containing the element's attributes"}, + {NULL}, +}; + static PyTypeObject Element_Type = { PyVarObject_HEAD_INIT(NULL, 0) "xml.etree.ElementTree.Element", sizeof(ElementObject), 0, @@ -3786,8 +3804,8 @@ static PyTypeObject Element_Type = { 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - (getattrofunc)element_getattro, /* tp_getattro */ - (setattrofunc)element_setattro, /* tp_setattro */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ @@ -3800,7 +3818,7 @@ static PyTypeObject Element_Type = { 0, /* tp_iternext */ element_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + element_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ -- cgit v0.12 From aa785555b1f8ab2c82b4103c5e9ed2d46af66726 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 26 Nov 2015 02:36:26 +0000 Subject: Issue #25622: Rename to PythonValuesTestCase and enable for non-Windows --- Lib/ctypes/test/test_values.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index 9551e7a..c7c78ce 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -28,8 +28,7 @@ class ValuesTestCase(unittest.TestCase): ctdll = CDLL(_ctypes_test.__file__) self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") -@unittest.skipUnless(sys.platform == 'win32', 'Windows-specific test') -class Win_ValuesTestCase(unittest.TestCase): +class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" def test_optimizeflag(self): @@ -76,7 +75,7 @@ class Win_ValuesTestCase(unittest.TestCase): if entry.name in bootstrap_expected: bootstrap_seen.append(entry.name) self.assertTrue(entry.size, - "{} was reported as having no size".format(entry.name)) + "{!r} was reported as having no size".format(entry.name)) continue items.append((entry.name, entry.size)) -- cgit v0.12 From 4f09806e662928c5524ab5d792d73297c50494b3 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 28 Nov 2015 12:24:52 -0500 Subject: #25485: Add context manager support to Telnet class. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Stéphane Wirtel. --- Doc/library/telnetlib.rst | 11 +++++++++++ Doc/whatsnew/3.6.rst | 7 +++++++ Lib/telnetlib.py | 15 ++++++++++----- Lib/test/test_telnetlib.py | 5 +++++ Misc/NEWS | 2 ++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst index 4040f72..ce406ca 100644 --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -43,6 +43,17 @@ Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). :exc:`EOFError` when the end of the connection is read, because they can return an empty string for other reasons. See the individual descriptions below. + A :class:`Telnet` object is a context manager and can be used in a + :keyword:`with` statement. When the :keyword:`with` block ends, the + :meth:`close` method is called:: + + >>> from telnetlib import Telnet + >>> with Telnet('localhost', 23) as tn: + ... tn.interact() + ... + + .. versionchanged:: 3.6 Context manager support added + .. seealso:: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 172b253..fc32fb5 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -125,6 +125,13 @@ Previously, names of properties and slots which were not yet created on an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) +telnetlib +--------- + +:class:`~telnetlib.Telnet` is now a context manager (contributed by +Stéphane Wirtel in :issue:`25485`). + + urllib.robotparser ------------------ diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py index 72dabc7..b0863b1 100644 --- a/Lib/telnetlib.py +++ b/Lib/telnetlib.py @@ -637,6 +637,12 @@ class Telnet: raise EOFError return (-1, None, text) + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + def test(): """Test program for telnetlib. @@ -660,11 +666,10 @@ def test(): port = int(portstr) except ValueError: port = socket.getservbyname(portstr, 'tcp') - tn = Telnet() - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - tn.close() + with Telnet() as tn: + tn.set_debuglevel(debuglevel) + tn.open(host, port, timeout=0.5) + tn.interact() if __name__ == '__main__': test() diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 524bba3..23029e0 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -42,6 +42,11 @@ class GeneralTests(TestCase): telnet = telnetlib.Telnet(HOST, self.port) telnet.sock.close() + def testContextManager(self): + with telnetlib.Telnet(HOST, self.port) as tn: + self.assertIsNotNone(tn.get_socket()) + self.assertIsNone(tn.get_socket()) + def testTimeoutDefault(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) diff --git a/Misc/NEWS b/Misc/NEWS index c3cce96..2cb06e9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25485: telnetlib.Telnet is now a context manager. + - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside __getattr__. -- cgit v0.12 From 40062a1127a9678669bb0efec89d015bbbed1836 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 28 Nov 2015 22:38:24 +0000 Subject: Issue #25754: Allow test_rlcompleter to be run multiple times --- Lib/test/test_rlcompleter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d9e4de6..208c054 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch import builtins import rlcompleter @@ -72,12 +73,12 @@ class TestRlcompleter(unittest.TestCase): self.assertIn('CompleteMe.__name__', matches) self.assertIn('CompleteMe.__new__(', matches) - CompleteMe.me = CompleteMe - self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), - ['CompleteMe.me.me.spam']) - self.assertEqual(self.completer.attr_matches('egg.s'), - ['egg.{}('.format(x) for x in dir(str) - if x.startswith('s')]) + with patch.object(CompleteMe, "me", CompleteMe, create=True): + self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), + ['CompleteMe.me.me.spam']) + self.assertEqual(self.completer.attr_matches('egg.s'), + ['egg.{}('.format(x) for x in dir(str) + if x.startswith('s')]) def test_excessive_getattr(self): # Ensure getattr() is invoked no more than once per attribute -- cgit v0.12 From b4ce1fc31be5614d527d77c55018281ebbcd70ab Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 30 Nov 2015 03:18:29 +0000 Subject: Issue #5319: New Py_FinalizeEx() API to exit with status 120 on failure --- Doc/c-api/init.rst | 33 ++++++++++++++++++---------- Doc/c-api/intro.rst | 4 ++-- Doc/c-api/sys.rst | 14 +++++++----- Doc/extending/embedding.rst | 6 +++-- Doc/includes/run-func.c | 4 +++- Doc/library/sys.rst | 7 +++++- Doc/whatsnew/3.6.rst | 6 +++-- Include/pylifecycle.h | 1 + Lib/test/test_cmd_line.py | 3 ++- Misc/NEWS | 3 +++ Misc/SpecialBuilds.txt | 8 +++---- Modules/main.c | 8 +++++-- PC/bdist_wininst/install.c | 18 +++++++++------ PC/python3.def | 1 + Python/frozenmain.c | 4 +++- Python/pylifecycle.c | 52 +++++++++++++++++++++++++++++++------------- Python/pystate.c | 2 +- Tools/scripts/combinerefs.py | 4 ++-- 18 files changed, 120 insertions(+), 58 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 81823bf..465147c 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -25,7 +25,7 @@ Initializing and finalizing the interpreter triple: module; search; path single: PySys_SetArgv() single: PySys_SetArgvEx() - single: Py_Finalize() + single: Py_FinalizeEx() Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; with the @@ -34,7 +34,7 @@ Initializing and finalizing the interpreter modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time - (without calling :c:func:`Py_Finalize` first). There is no return value; it is a + (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal error if the initialization fails. @@ -48,19 +48,20 @@ Initializing and finalizing the interpreter .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_Finalize` is called, this returns false until + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until :c:func:`Py_Initialize` is called again. -.. c:function:: void Py_Finalize() +.. c:function:: int Py_FinalizeEx() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). There is no return - value; errors during finalization are ignored. + time (without calling :c:func:`Py_Initialize` again first). Normally the + return value is 0. If there were errors during finalization + (flushing buffered data), -1 is returned. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -79,7 +80,15 @@ Initializing and finalizing the interpreter freed. Some memory allocated by extension modules may not be freed. Some extensions may not work properly if their initialization routine is called more than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_Finalize` more than once. + :c:func:`Py_FinalizeEx` more than once. + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. Process-wide parameters @@ -107,7 +116,7 @@ Process-wide parameters Note that :data:`sys.stderr` always uses the "backslashreplace" error handler, regardless of this (or any other) setting. - If :c:func:`Py_Finalize` is called, this function will need to be called + If :c:func:`Py_FinalizeEx` is called, this function will need to be called again in order to affect subsequent calls to :c:func:`Py_Initialize`. Returns 0 if successful, a nonzero value on error (e.g. calling after the @@ -918,7 +927,7 @@ using the following functions: entry.) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: Py_Initialize() Extension modules are shared between (sub-)interpreters as follows: the first @@ -928,7 +937,7 @@ using the following functions: and filled with the contents of this copy; the extension's ``init`` function is not called. Note that this is different from what happens when an extension is imported after the interpreter has been completely re-initialized by calling - :c:func:`Py_Finalize` and :c:func:`Py_Initialize`; in that case, the extension's + :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that case, the extension's ``initmodule`` function *is* called again. .. index:: single: close() (in module os) @@ -936,14 +945,14 @@ using the following functions: .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_Finalize() + .. index:: single: Py_FinalizeEx() Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the discussion of thread states below. When the call returns, the current thread state is *NULL*. All thread states associated with this interpreter are destroyed. (The global interpreter lock must be held before calling this function and is still held - when it returns.) :c:func:`Py_Finalize` will destroy all sub-interpreters that + when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that haven't been explicitly destroyed at that point. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index bc3a752..95cbef5 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -578,9 +578,9 @@ Sometimes, it is desirable to "uninitialize" Python. For instance, the application may want to start over (make another call to :c:func:`Py_Initialize`) or the application is simply done with its use of Python and wants to free memory allocated by Python. This can be accomplished -by calling :c:func:`Py_Finalize`. The function :c:func:`Py_IsInitialized` returns +by calling :c:func:`Py_FinalizeEx`. The function :c:func:`Py_IsInitialized` returns true if Python is currently in the initialized state. More information about -these functions is given in a later chapter. Notice that :c:func:`Py_Finalize` +these functions is given in a later chapter. Notice that :c:func:`Py_FinalizeEx` does *not* free all memory allocated by the Python interpreter, e.g. memory allocated by extension modules currently cannot be released. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 3d83b27..9ba6496 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -212,20 +212,24 @@ Process Control .. c:function:: void Py_Exit(int status) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: exit() - Exit the current process. This calls :c:func:`Py_Finalize` and then calls the - standard C library function ``exit(status)``. + Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the + standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` + indicates an error, the exit status is set to 120. + + .. versionchanged:: 3.6 + Errors from finalization no longer ignored. .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: cleanup functions - Register a cleanup function to be called by :c:func:`Py_Finalize`. The cleanup + Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup function will be called with no arguments and should return no value. At most 32 cleanup functions can be registered. When the registration is successful, :c:func:`Py_AtExit` returns ``0``; on failure, it returns ``-1``. The cleanup diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index acd60ae..1546b1a 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -67,7 +67,9 @@ perform some operation on a file. :: Py_Initialize(); PyRun_SimpleString("from time import time,ctime\n" "print('Today is', ctime(time()))\n"); - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + exit(120); + } PyMem_RawFree(program); return 0; } @@ -76,7 +78,7 @@ The :c:func:`Py_SetProgramName` function should be called before :c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time libraries. Next, the Python interpreter is initialized with :c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script -that prints the date and time. Afterwards, the :c:func:`Py_Finalize` call shuts +that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts the interpreter down, followed by the end of the program. In a real program, you may want to get the Python script from another source, perhaps a text-editor routine, a file, or a database. Getting the Python code from a file can better diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c index 986d670..ead7bdd 100644 --- a/Doc/includes/run-func.c +++ b/Doc/includes/run-func.c @@ -63,6 +63,8 @@ main(int argc, char *argv[]) fprintf(stderr, "Failed to load \"%s\"\n", argv[1]); return 1; } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + return 120; + } return 0; } diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index f6325cc..97b5899 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -255,7 +255,7 @@ always available. (defaulting to zero), or another type of object. If it is an integer, zero is considered "successful termination" and any nonzero value is considered "abnormal termination" by shells and the like. Most systems require it to be - in the range 0-127, and produce undefined results otherwise. Some systems + in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command line syntax errors and 1 for all other kind of errors. If another type of @@ -268,6 +268,11 @@ always available. the process when called from the main thread, and the exception is not intercepted. + .. versionchanged:: 3.6 + If an error occurs in the cleanup after the Python interpreter + has caught :exc:`SystemExit` (such as an error flushing buffered data + in the standard streams), the exit status is changed to 120. + .. data:: flags diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index fc32fb5..f97c70f 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -171,7 +171,8 @@ Optimizations Build and C API Changes ======================= -* None yet. +* New :c:func:`Py_FinalizeEx` API which indicates if flushing buffered data + failed (:issue:`5319`). Deprecated @@ -247,4 +248,5 @@ Changes in the Python API Changes in the C API -------------------- -* None yet. +* :c:func:`Py_Exit` (and the main interpreter) now override the exit status + with 120 if flushing buffered data failed. See :issue:`5319`. diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index ccdebe2..e96eb70 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -27,6 +27,7 @@ PyAPI_FUNC(void) Py_InitializeEx(int); PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int); #endif PyAPI_FUNC(void) Py_Finalize(void); +PyAPI_FUNC(int) Py_FinalizeEx(void); PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 0feb63f..b410608 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -348,8 +348,9 @@ class CmdLineTest(unittest.TestCase): test.support.SuppressCrashReport().__enter__() sys.stdout.write('x') os.close(sys.stdout.fileno())""" - rc, out, err = assert_python_ok('-c', code) + rc, out, err = assert_python_failure('-c', code) self.assertEqual(b'', out) + self.assertEqual(120, rc) self.assertRegex(err.decode('ascii', 'ignore'), 'Exception ignored in.*\nOSError: .*') diff --git a/Misc/NEWS b/Misc/NEWS index 2cb06e9..ccbb3fb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #5319: New Py_FinalizeEx() API allowing Python to set an exit status + of 120 on failure to flush buffered streams. + - Issue #25485: telnetlib.Telnet is now a context manager. - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside diff --git a/Misc/SpecialBuilds.txt b/Misc/SpecialBuilds.txt index 3004174..4b673fd 100644 --- a/Misc/SpecialBuilds.txt +++ b/Misc/SpecialBuilds.txt @@ -65,9 +65,9 @@ sys.getobjects(max[, type]) simply by virtue of being in the list. envvar PYTHONDUMPREFS - If this envvar exists, Py_Finalize() arranges to print a list of all + If this envvar exists, Py_FinalizeEx() arranges to print a list of all still-live heap objects. This is printed twice, in different formats, - before and after Py_Finalize has cleaned up everything it can clean up. The + before and after Py_FinalizeEx has cleaned up everything it can clean up. The first output block produces the repr() of each object so is more informative; however, a lot of stuff destined to die is still alive then. The second output block is much harder to work with (repr() can't be invoked @@ -144,7 +144,7 @@ Special gimmicks: envvar PYTHONMALLOCSTATS If this envvar exists, a report of pymalloc summary statistics is printed to - stderr whenever a new arena is allocated, and also by Py_Finalize(). + stderr whenever a new arena is allocated, and also by Py_FinalizeEx(). Changed in 2.5: The number of extra bytes allocated is 4*sizeof(size_t). Before it was 16 on all boxes, reflecting that Python couldn't make use of @@ -179,7 +179,7 @@ Each type object grows three new members: */ int tp_maxalloc; -Allocation and deallocation code keeps these counts up to date. Py_Finalize() +Allocation and deallocation code keeps these counts up to date. Py_FinalizeEx() displays a summary of the info returned by sys.getcounts() (see below), along with assorted other special allocation counts (like the number of tuple allocations satisfied by a tuple free-list, the number of 1-character strings diff --git a/Modules/main.c b/Modules/main.c index 2a9ea28..0fbdb69 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -654,7 +654,7 @@ Py_Main(int argc, wchar_t **argv) Py_SetProgramName(wbuf); /* Don't free wbuf, the argument to Py_SetProgramName - * must remain valid until the Py_Finalize is called. + * must remain valid until Py_FinalizeEx is called. */ } else { Py_SetProgramName(argv[0]); @@ -785,7 +785,11 @@ Py_Main(int argc, wchar_t **argv) sts = PyRun_AnyFileFlags(stdin, "", &cf) != 0; } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + /* Value unlikely to be confused with a non-error exit status or + other special meaning */ + sts = 120; + } #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c index f39b381..16eeb35 100644 --- a/PC/bdist_wininst/install.c +++ b/PC/bdist_wininst/install.c @@ -709,7 +709,7 @@ static int prepare_script_environment(HINSTANCE hPython) * 1 if the Python-dll does not export the functions we need * 2 if no install-script is specified in pathname * 3 if the install-script file could not be opened - * the return value of PyRun_SimpleString() otherwise, + * the return value of PyRun_SimpleString() or Py_FinalizeEx() otherwise, * which is 0 if everything is ok, -1 if an exception had occurred * in the install-script. */ @@ -722,7 +722,7 @@ do_run_installscript(HINSTANCE hPython, char *pathname, int argc, char **argv) DECLPROC(hPython, void, Py_Initialize, (void)); DECLPROC(hPython, int, PySys_SetArgv, (int, wchar_t **)); DECLPROC(hPython, int, PyRun_SimpleString, (char *)); - DECLPROC(hPython, void, Py_Finalize, (void)); + DECLPROC(hPython, int, Py_FinalizeEx, (void)); DECLPROC(hPython, PyObject *, Py_BuildValue, (char *, ...)); DECLPROC(hPython, PyObject *, PyCFunction_New, (PyMethodDef *, PyObject *)); @@ -730,7 +730,7 @@ do_run_installscript(HINSTANCE hPython, char *pathname, int argc, char **argv) DECLPROC(hPython, PyObject *, PyErr_Format, (PyObject *, char *)); if (!Py_Initialize || !PySys_SetArgv - || !PyRun_SimpleString || !Py_Finalize) + || !PyRun_SimpleString || !Py_FinalizeEx) return 1; if (!Py_BuildValue || !PyArg_ParseTuple || !PyErr_Format) @@ -777,7 +777,9 @@ do_run_installscript(HINSTANCE hPython, char *pathname, int argc, char **argv) } } } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + result = -1; + } close(fh); return result; @@ -839,11 +841,11 @@ static int do_run_simple_script(HINSTANCE hPython, char *script) int rc; DECLPROC(hPython, void, Py_Initialize, (void)); DECLPROC(hPython, void, Py_SetProgramName, (wchar_t *)); - DECLPROC(hPython, void, Py_Finalize, (void)); + DECLPROC(hPython, int, Py_FinalizeEx, (void)); DECLPROC(hPython, int, PyRun_SimpleString, (char *)); DECLPROC(hPython, void, PyErr_Print, (void)); - if (!Py_Initialize || !Py_SetProgramName || !Py_Finalize || + if (!Py_Initialize || !Py_SetProgramName || !Py_FinalizeEx || !PyRun_SimpleString || !PyErr_Print) return -1; @@ -853,7 +855,9 @@ static int do_run_simple_script(HINSTANCE hPython, char *script) rc = PyRun_SimpleString(script); if (rc) PyErr_Print(); - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + rc = -1; + } return rc; } diff --git a/PC/python3.def b/PC/python3.def index e146e8f..e8d2d8c 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -648,6 +648,7 @@ EXPORTS Py_FatalError=python36.Py_FatalError Py_FileSystemDefaultEncoding=python36.Py_FileSystemDefaultEncoding DATA Py_Finalize=python36.Py_Finalize + Py_FinalizeEx=python36.Py_FinalizeEx Py_GetBuildInfo=python36.Py_GetBuildInfo Py_GetCompiler=python36.Py_GetCompiler Py_GetCopyright=python36.Py_GetCopyright diff --git a/Python/frozenmain.c b/Python/frozenmain.c index de8bd35..769b33d 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -99,7 +99,9 @@ Py_FrozenMain(int argc, char **argv) #ifdef MS_WINDOWS PyWinFreeze_ExeTerm(); #endif - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + sts = 120; + } error: PyMem_RawFree(argv_copy); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 857a543..b7f6ec8 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -154,8 +154,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) return 0; } -/* Global initializations. Can be undone by Py_Finalize(). Don't - call this twice without an intervening Py_Finalize() call. When +/* Global initializations. Can be undone by Py_FinalizeEx(). Don't + call this twice without an intervening Py_FinalizeEx() call. When initializations fail, a fatal error is issued and the function does not return. On return, the first thread and interpreter state have been created. @@ -327,11 +327,11 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) (void) PyThreadState_Swap(tstate); #ifdef WITH_THREAD - /* We can't call _PyEval_FiniThreads() in Py_Finalize because + /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because destroying the GIL might fail when it is being referenced from another running thread (see issue #9901). Instead we destroy the previously created GIL here, which ensures - that we can call Py_Initialize / Py_Finalize multiple times. */ + that we can call Py_Initialize / Py_FinalizeEx multiple times. */ _PyEval_FiniThreads(); /* Auto-thread-state API */ @@ -477,28 +477,35 @@ file_is_closed(PyObject *fobj) return r > 0; } -static void +static int flush_std_files(void) { PyObject *fout = _PySys_GetObjectId(&PyId_stdout); PyObject *ferr = _PySys_GetObjectId(&PyId_stderr); PyObject *tmp; + int status = 0; if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); - if (tmp == NULL) + if (tmp == NULL) { PyErr_WriteUnraisable(fout); + status = -1; + } else Py_DECREF(tmp); } if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) { tmp = _PyObject_CallMethodId(ferr, &PyId_flush, ""); - if (tmp == NULL) + if (tmp == NULL) { PyErr_Clear(); + status = -1; + } else Py_DECREF(tmp); } + + return status; } /* Undo the effect of Py_Initialize(). @@ -515,14 +522,15 @@ flush_std_files(void) */ -void -Py_Finalize(void) +int +Py_FinalizeEx(void) { PyInterpreterState *interp; PyThreadState *tstate; + int status = 0; if (!initialized) - return; + return status; wait_for_thread_shutdown(); @@ -547,7 +555,9 @@ Py_Finalize(void) initialized = 0; /* Flush sys.stdout and sys.stderr */ - flush_std_files(); + if (flush_std_files() < 0) { + status = -1; + } /* Disable signal handling */ PyOS_FiniInterrupts(); @@ -576,7 +586,9 @@ Py_Finalize(void) PyImport_Cleanup(); /* Flush sys.stdout and sys.stderr (again, in case more was printed) */ - flush_std_files(); + if (flush_std_files() < 0) { + status = -1; + } /* Collect final garbage. This disposes of cycles created by * class definitions, for example. @@ -696,6 +708,13 @@ Py_Finalize(void) #endif call_ll_exitfuncs(); + return status; +} + +void +Py_Finalize(void) +{ + Py_FinalizeEx(); } /* Create and initialize a new interpreter and thread, and return the @@ -803,7 +822,7 @@ handle_error: frames, and that it is its interpreter's only remaining thread. It is a fatal error to violate these constraints. - (Py_Finalize() doesn't have these constraints -- it zaps + (Py_FinalizeEx() doesn't have these constraints -- it zaps everything, regardless.) Locking: as above. @@ -1016,7 +1035,8 @@ create_stdio(PyObject* io, mode = "rb"; buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi", fd, mode, buffering, - Py_None, Py_None, Py_None, 0); + Py_None, Py_None, /* encoding, errors */ + Py_None, 0); /* newline, closefd */ if (buf == NULL) goto error; @@ -1450,7 +1470,9 @@ call_ll_exitfuncs(void) void Py_Exit(int sts) { - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + sts = 120; + } exit(sts); } diff --git a/Python/pystate.c b/Python/pystate.c index 7e0267a..50edfbb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -686,7 +686,7 @@ PyThreadState_IsCurrent(PyThreadState *tstate) } /* Internal initialization/finalization functions called by - Py_Initialize/Py_Finalize + Py_Initialize/Py_FinalizeEx */ void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) diff --git a/Tools/scripts/combinerefs.py b/Tools/scripts/combinerefs.py index e10e49a..7ca9526 100755 --- a/Tools/scripts/combinerefs.py +++ b/Tools/scripts/combinerefs.py @@ -6,7 +6,7 @@ combinerefs path A helper for analyzing PYTHONDUMPREFS output. When the PYTHONDUMPREFS envar is set in a debug build, at Python shutdown -time Py_Finalize() prints the list of all live objects twice: first it +time Py_FinalizeEx() prints the list of all live objects twice: first it prints the repr() of each object while the interpreter is still fully intact. After cleaning up everything it can, it prints all remaining live objects again, but the second time just prints their addresses, refcounts, and type @@ -41,7 +41,7 @@ CAUTION: If object is a container type, it may not actually contain all the objects shown in the repr: the repr was captured from the first output block, and some of the containees may have been released since then. For example, it's common for the line showing the dict of interned strings to display -strings that no longer exist at the end of Py_Finalize; this can be recognized +strings that no longer exist at the end of Py_FinalizeEx; this can be recognized (albeit painfully) because such containees don't have a line of their own. The objects are listed in allocation order, with most-recently allocated -- cgit v0.12 From b28c3275c720b6908bebf9d545b90fc8789cf1fe Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Tue, 1 Dec 2015 19:59:53 +1100 Subject: Issue #25177: Fixed problem with the mean of very small and very large numbers. --- Lib/statistics.py | 185 +++++++++++++--------- Lib/test/test_statistics.py | 363 ++++++++++++++++++++++++++++++++++++++------ Misc/NEWS | 4 + 3 files changed, 433 insertions(+), 119 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 9203cf1..518f546 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -104,6 +104,8 @@ import math from fractions import Fraction from decimal import Decimal +from itertools import groupby + # === Exceptions === @@ -115,86 +117,102 @@ class StatisticsError(ValueError): # === Private utilities === def _sum(data, start=0): - """_sum(data [, start]) -> value + """_sum(data [, start]) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. - Return a high-precision sum of the given numeric data. If optional - argument ``start`` is given, it is added to the total. If ``data`` is - empty, ``start`` (defaulting to 0) is returned. + If optional argument ``start`` is given, it is added to the total. + If ``data`` is empty, ``start`` (defaulting to 0) is returned. Examples -------- >>> _sum([3, 2.25, 4.5, -0.5, 1.0], 0.75) - 11.0 + (, Fraction(11, 1), 5) Some sources of round-off error will be avoided: >>> _sum([1e50, 1, -1e50] * 1000) # Built-in sum returns zero. - 1000.0 + (, Fraction(1000, 1), 3000) Fractions and Decimals are also supported: >>> from fractions import Fraction as F >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - Fraction(63, 20) + (, Fraction(63, 20), 4) >>> from decimal import Decimal as D >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] >>> _sum(data) - Decimal('0.6963') + (, Fraction(6963, 10000), 4) Mixed types are currently treated as an error, except that int is allowed. """ - # We fail as soon as we reach a value that is not an int or the type of - # the first value which is not an int. E.g. _sum([int, int, float, int]) - # is okay, but sum([int, int, float, Fraction]) is not. - allowed_types = {int, type(start)} + count = 0 n, d = _exact_ratio(start) - partials = {d: n} # map {denominator: sum of numerators} - # Micro-optimizations. - exact_ratio = _exact_ratio + partials = {d: n} partials_get = partials.get - # Add numerators for each denominator. - for x in data: - _check_type(type(x), allowed_types) - n, d = exact_ratio(x) - partials[d] = partials_get(d, 0) + n - # Find the expected result type. If allowed_types has only one item, it - # will be int; if it has two, use the one which isn't int. - assert len(allowed_types) in (1, 2) - if len(allowed_types) == 1: - assert allowed_types.pop() is int - T = int - else: - T = (allowed_types - {int}).pop() + T = _coerce(int, type(start)) + for typ, values in groupby(data, type): + T = _coerce(T, typ) # or raise TypeError + for n,d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n if None in partials: - assert issubclass(T, (float, Decimal)) - assert not math.isfinite(partials[None]) - return T(partials[None]) - total = Fraction() - for d, n in sorted(partials.items()): - total += Fraction(n, d) - if issubclass(T, int): - assert total.denominator == 1 - return T(total.numerator) - if issubclass(T, Decimal): - return T(total.numerator)/total.denominator - return T(total) - - -def _check_type(T, allowed): - if T not in allowed: - if len(allowed) == 1: - allowed.add(T) - else: - types = ', '.join([t.__name__ for t in allowed] + [T.__name__]) - raise TypeError("unsupported mixed types: %s" % types) + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + # FIXME is this faster if we sum them in order of the denominator? + total = sum(Fraction(n, d) for d, n in sorted(partials.items())) + return (T, total, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) def _exact_ratio(x): - """Convert Real number x exactly to (numerator, denominator) pair. + """Return Real number x to exact (numerator, denominator) pair. >>> _exact_ratio(0.25) (1, 4) @@ -202,29 +220,31 @@ def _exact_ratio(x): x is expected to be an int, Fraction, Decimal or float. """ try: + # Optimise the common case of floats. We expect that the most often + # used numeric type will be builtin floats, so try to make this as + # fast as possible. + if type(x) is float: + return x.as_integer_ratio() try: - # int, Fraction + # x may be an int, Fraction, or Integral ABC. return (x.numerator, x.denominator) except AttributeError: - # float try: + # x may be a float subclass. return x.as_integer_ratio() except AttributeError: - # Decimal try: + # x may be a Decimal. return _decimal_to_ratio(x) except AttributeError: - msg = "can't convert type '{}' to numerator/denominator" - raise TypeError(msg.format(type(x).__name__)) from None + # Just give up? + pass except (OverflowError, ValueError): - # INF or NAN - if __debug__: - # Decimal signalling NANs cannot be converted to float :-( - if isinstance(x, Decimal): - assert not x.is_finite() - else: - assert not math.isfinite(x) + # float NAN or INF. + assert not math.isfinite(x) return (x, None) + msg = "can't convert type '{}' to numerator/denominator" + raise TypeError(msg.format(type(x).__name__)) # FIXME This is faster than Fraction.from_decimal, but still too slow. @@ -239,7 +259,7 @@ def _decimal_to_ratio(d): sign, digits, exp = d.as_tuple() if exp in ('F', 'n', 'N'): # INF, NAN, sNAN assert not d.is_finite() - raise ValueError + return (d, None) num = 0 for digit in digits: num = num*10 + digit @@ -253,6 +273,24 @@ def _decimal_to_ratio(d): return (num, den) +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + if issubclass(T, int) and value.denominator != 1: + T = float + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator)/T(value.denominator) + else: + raise + + def _counts(data): # Generate a table of sorted (value, frequency) pairs. table = collections.Counter(iter(data)).most_common() @@ -290,7 +328,9 @@ def mean(data): n = len(data) if n < 1: raise StatisticsError('mean requires at least one data point') - return _sum(data)/n + T, total, count = _sum(data) + assert count == n + return _convert(total/n, T) # FIXME: investigate ways to calculate medians without sorting? Quickselect? @@ -460,12 +500,14 @@ def _ss(data, c=None): """ if c is None: c = mean(data) - ss = _sum((x-c)**2 for x in data) + T, total, count = _sum((x-c)**2 for x in data) # The following sum should mathematically equal zero, but due to rounding # error may not. - ss -= _sum((x-c) for x in data)**2/len(data) - assert not ss < 0, 'negative sum of square deviations: %f' % ss - return ss + U, total2, count2 = _sum((x-c) for x in data) + assert T == U and count == count2 + total -= total2**2/len(data) + assert not total < 0, 'negative sum of square deviations: %f' % total + return (T, total) def variance(data, xbar=None): @@ -511,8 +553,8 @@ def variance(data, xbar=None): n = len(data) if n < 2: raise StatisticsError('variance requires at least two data points') - ss = _ss(data, xbar) - return ss/(n-1) + T, ss = _ss(data, xbar) + return _convert(ss/(n-1), T) def pvariance(data, mu=None): @@ -560,7 +602,8 @@ def pvariance(data, mu=None): if n < 1: raise StatisticsError('pvariance requires at least one data point') ss = _ss(data, mu) - return ss/n + T, ss = _ss(data, mu) + return _convert(ss/n, T) def stdev(data, xbar=None): diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 758a481..0089ae8 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -21,6 +21,37 @@ import statistics # === Helper functions and class === +def _nan_equal(a, b): + """Return True if a and b are both the same kind of NAN. + + >>> _nan_equal(Decimal('NAN'), Decimal('NAN')) + True + >>> _nan_equal(Decimal('sNAN'), Decimal('sNAN')) + True + >>> _nan_equal(Decimal('NAN'), Decimal('sNAN')) + False + >>> _nan_equal(Decimal(42), Decimal('NAN')) + False + + >>> _nan_equal(float('NAN'), float('NAN')) + True + >>> _nan_equal(float('NAN'), 0.5) + False + + >>> _nan_equal(float('NAN'), Decimal('NAN')) + False + + NAN payloads are not compared. + """ + if type(a) is not type(b): + return False + if isinstance(a, float): + return math.isnan(a) and math.isnan(b) + aexp = a.as_tuple()[2] + bexp = b.as_tuple()[2] + return (aexp == bexp) and (aexp in ('n', 'N')) # Both NAN or both sNAN. + + def _calc_errors(actual, expected): """Return the absolute and relative errors between two numbers. @@ -675,15 +706,60 @@ class ExactRatioTest(unittest.TestCase): self.assertEqual(_exact_ratio(D("12.345")), (12345, 1000)) self.assertEqual(_exact_ratio(D("-1.98")), (-198, 100)) + def test_inf(self): + INF = float("INF") + class MyFloat(float): + pass + class MyDecimal(Decimal): + pass + for inf in (INF, -INF): + for type_ in (float, MyFloat, Decimal, MyDecimal): + x = type_(inf) + ratio = statistics._exact_ratio(x) + self.assertEqual(ratio, (x, None)) + self.assertEqual(type(ratio[0]), type_) + self.assertTrue(math.isinf(ratio[0])) + + def test_float_nan(self): + NAN = float("NAN") + class MyFloat(float): + pass + for nan in (NAN, MyFloat(NAN)): + ratio = statistics._exact_ratio(nan) + self.assertTrue(math.isnan(ratio[0])) + self.assertIs(ratio[1], None) + self.assertEqual(type(ratio[0]), type(nan)) + + def test_decimal_nan(self): + NAN = Decimal("NAN") + sNAN = Decimal("sNAN") + class MyDecimal(Decimal): + pass + for nan in (NAN, MyDecimal(NAN), sNAN, MyDecimal(sNAN)): + ratio = statistics._exact_ratio(nan) + self.assertTrue(_nan_equal(ratio[0], nan)) + self.assertIs(ratio[1], None) + self.assertEqual(type(ratio[0]), type(nan)) + class DecimalToRatioTest(unittest.TestCase): # Test _decimal_to_ratio private function. - def testSpecialsRaise(self): - # Test that NANs and INFs raise ValueError. - # Non-special values are covered by _exact_ratio above. - for d in (Decimal('NAN'), Decimal('sNAN'), Decimal('INF')): - self.assertRaises(ValueError, statistics._decimal_to_ratio, d) + def test_infinity(self): + # Test that INFs are handled correctly. + inf = Decimal('INF') + self.assertEqual(statistics._decimal_to_ratio(inf), (inf, None)) + self.assertEqual(statistics._decimal_to_ratio(-inf), (-inf, None)) + + def test_nan(self): + # Test that NANs are handled correctly. + for nan in (Decimal('NAN'), Decimal('sNAN')): + num, den = statistics._decimal_to_ratio(nan) + # Because NANs always compare non-equal, we cannot use assertEqual. + # Nor can we use an identity test, as we don't guarantee anything + # about the object identity. + self.assertTrue(_nan_equal(num, nan)) + self.assertIs(den, None) def test_sign(self): # Test sign is calculated correctly. @@ -718,25 +794,181 @@ class DecimalToRatioTest(unittest.TestCase): self.assertEqual(t, (147000, 1)) -class CheckTypeTest(unittest.TestCase): - # Test _check_type private function. +class IsFiniteTest(unittest.TestCase): + # Test _isfinite private function. + + def test_finite(self): + # Test that finite numbers are recognised as finite. + for x in (5, Fraction(1, 3), 2.5, Decimal("5.5")): + self.assertTrue(statistics._isfinite(x)) - def test_allowed(self): - # Test that a type which should be allowed is allowed. - allowed = set([int, float]) - statistics._check_type(int, allowed) - statistics._check_type(float, allowed) + def test_infinity(self): + # Test that INFs are not recognised as finite. + for x in (float("inf"), Decimal("inf")): + self.assertFalse(statistics._isfinite(x)) + + def test_nan(self): + # Test that NANs are not recognised as finite. + for x in (float("nan"), Decimal("NAN"), Decimal("sNAN")): + self.assertFalse(statistics._isfinite(x)) + + +class CoerceTest(unittest.TestCase): + # Test that private function _coerce correctly deals with types. + + # The coercion rules are currently an implementation detail, although at + # some point that should change. The tests and comments here define the + # correct implementation. + + # Pre-conditions of _coerce: + # + # - The first time _sum calls _coerce, the + # - coerce(T, S) will never be called with bool as the first argument; + # this is a pre-condition, guarded with an assertion. + + # + # - coerce(T, T) will always return T; we assume T is a valid numeric + # type. Violate this assumption at your own risk. + # + # - Apart from as above, bool is treated as if it were actually int. + # + # - coerce(int, X) and coerce(X, int) return X. + # - + def test_bool(self): + # bool is somewhat special, due to the pre-condition that it is + # never given as the first argument to _coerce, and that it cannot + # be subclassed. So we test it specially. + for T in (int, float, Fraction, Decimal): + self.assertIs(statistics._coerce(T, bool), T) + class MyClass(T): pass + self.assertIs(statistics._coerce(MyClass, bool), MyClass) + + def assertCoerceTo(self, A, B): + """Assert that type A coerces to B.""" + self.assertIs(statistics._coerce(A, B), B) + self.assertIs(statistics._coerce(B, A), B) + + def check_coerce_to(self, A, B): + """Checks that type A coerces to B, including subclasses.""" + # Assert that type A is coerced to B. + self.assertCoerceTo(A, B) + # Subclasses of A are also coerced to B. + class SubclassOfA(A): pass + self.assertCoerceTo(SubclassOfA, B) + # A, and subclasses of A, are coerced to subclasses of B. + class SubclassOfB(B): pass + self.assertCoerceTo(A, SubclassOfB) + self.assertCoerceTo(SubclassOfA, SubclassOfB) + + def assertCoerceRaises(self, A, B): + """Assert that coercing A to B, or vice versa, raises TypeError.""" + self.assertRaises(TypeError, statistics._coerce, (A, B)) + self.assertRaises(TypeError, statistics._coerce, (B, A)) + + def check_type_coercions(self, T): + """Check that type T coerces correctly with subclasses of itself.""" + assert T is not bool + # Coercing a type with itself returns the same type. + self.assertIs(statistics._coerce(T, T), T) + # Coercing a type with a subclass of itself returns the subclass. + class U(T): pass + class V(T): pass + class W(U): pass + for typ in (U, V, W): + self.assertCoerceTo(T, typ) + self.assertCoerceTo(U, W) + # Coercing two subclasses that aren't parent/child is an error. + self.assertCoerceRaises(U, V) + self.assertCoerceRaises(V, W) - def test_not_allowed(self): - # Test that a type which should not be allowed raises. - allowed = set([int, float]) - self.assertRaises(TypeError, statistics._check_type, Decimal, allowed) + def test_int(self): + # Check that int coerces correctly. + self.check_type_coercions(int) + for typ in (float, Fraction, Decimal): + self.check_coerce_to(int, typ) - def test_add_to_allowed(self): - # Test that a second type will be added to the allowed set. - allowed = set([int]) - statistics._check_type(float, allowed) - self.assertEqual(allowed, set([int, float])) + def test_fraction(self): + # Check that Fraction coerces correctly. + self.check_type_coercions(Fraction) + self.check_coerce_to(Fraction, float) + + def test_decimal(self): + # Check that Decimal coerces correctly. + self.check_type_coercions(Decimal) + + def test_float(self): + # Check that float coerces correctly. + self.check_type_coercions(float) + + def test_non_numeric_types(self): + for bad_type in (str, list, type(None), tuple, dict): + for good_type in (int, float, Fraction, Decimal): + self.assertCoerceRaises(good_type, bad_type) + + def test_incompatible_types(self): + # Test that incompatible types raise. + for T in (float, Fraction): + class MySubclass(T): pass + self.assertCoerceRaises(T, Decimal) + self.assertCoerceRaises(MySubclass, Decimal) + + +class ConvertTest(unittest.TestCase): + # Test private _convert function. + + def check_exact_equal(self, x, y): + """Check that x equals y, and has the same type as well.""" + self.assertEqual(x, y) + self.assertIs(type(x), type(y)) + + def test_int(self): + # Test conversions to int. + x = statistics._convert(Fraction(71), int) + self.check_exact_equal(x, 71) + class MyInt(int): pass + x = statistics._convert(Fraction(17), MyInt) + self.check_exact_equal(x, MyInt(17)) + + def test_fraction(self): + # Test conversions to Fraction. + x = statistics._convert(Fraction(95, 99), Fraction) + self.check_exact_equal(x, Fraction(95, 99)) + class MyFraction(Fraction): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(71, 13), MyFraction) + self.check_exact_equal(x, MyFraction(71, 13)) + + def test_float(self): + # Test conversions to float. + x = statistics._convert(Fraction(-1, 2), float) + self.check_exact_equal(x, -0.5) + class MyFloat(float): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(9, 8), MyFloat) + self.check_exact_equal(x, MyFloat(1.125)) + + def test_decimal(self): + # Test conversions to Decimal. + x = statistics._convert(Fraction(1, 40), Decimal) + self.check_exact_equal(x, Decimal("0.025")) + class MyDecimal(Decimal): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(-15, 16), MyDecimal) + self.check_exact_equal(x, MyDecimal("-0.9375")) + + def test_inf(self): + for INF in (float('inf'), Decimal('inf')): + for inf in (INF, -INF): + x = statistics._convert(inf, type(inf)) + self.check_exact_equal(x, inf) + + def test_nan(self): + for nan in (float('nan'), Decimal('NAN'), Decimal('sNAN')): + x = statistics._convert(nan, type(nan)) + self.assertTrue(_nan_equal(x, nan)) # === Tests for public functions === @@ -874,52 +1106,71 @@ class UnivariateTypeMixin: self.assertIs(type(result), kind) -class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): +class TestSumCommon(UnivariateCommonMixin, UnivariateTypeMixin): + # Common test cases for statistics._sum() function. + + # This test suite looks only at the numeric value returned by _sum, + # after conversion to the appropriate type. + def setUp(self): + def simplified_sum(*args): + T, value, n = statistics._sum(*args) + return statistics._coerce(value, T) + self.func = simplified_sum + + +class TestSum(NumericTestCase): # Test cases for statistics._sum() function. + # These tests look at the entire three value tuple returned by _sum. + def setUp(self): self.func = statistics._sum def test_empty_data(self): # Override test for empty data. for data in ([], (), iter([])): - self.assertEqual(self.func(data), 0) - self.assertEqual(self.func(data, 23), 23) - self.assertEqual(self.func(data, 2.3), 2.3) + self.assertEqual(self.func(data), (int, Fraction(0), 0)) + self.assertEqual(self.func(data, 23), (int, Fraction(23), 0)) + self.assertEqual(self.func(data, 2.3), (float, Fraction(2.3), 0)) def test_ints(self): - self.assertEqual(self.func([1, 5, 3, -4, -8, 20, 42, 1]), 60) - self.assertEqual(self.func([4, 2, 3, -8, 7], 1000), 1008) + self.assertEqual(self.func([1, 5, 3, -4, -8, 20, 42, 1]), + (int, Fraction(60), 8)) + self.assertEqual(self.func([4, 2, 3, -8, 7], 1000), + (int, Fraction(1008), 5)) def test_floats(self): - self.assertEqual(self.func([0.25]*20), 5.0) - self.assertEqual(self.func([0.125, 0.25, 0.5, 0.75], 1.5), 3.125) + self.assertEqual(self.func([0.25]*20), + (float, Fraction(5.0), 20)) + self.assertEqual(self.func([0.125, 0.25, 0.5, 0.75], 1.5), + (float, Fraction(3.125), 4)) def test_fractions(self): - F = Fraction - self.assertEqual(self.func([Fraction(1, 1000)]*500), Fraction(1, 2)) + self.assertEqual(self.func([Fraction(1, 1000)]*500), + (Fraction, Fraction(1, 2), 500)) def test_decimals(self): D = Decimal data = [D("0.001"), D("5.246"), D("1.702"), D("-0.025"), D("3.974"), D("2.328"), D("4.617"), D("2.843"), ] - self.assertEqual(self.func(data), Decimal("20.686")) + self.assertEqual(self.func(data), + (Decimal, Decimal("20.686"), 8)) def test_compare_with_math_fsum(self): # Compare with the math.fsum function. # Ideally we ought to get the exact same result, but sometimes # we differ by a very slight amount :-( data = [random.uniform(-100, 1000) for _ in range(1000)] - self.assertApproxEqual(self.func(data), math.fsum(data), rel=2e-16) + self.assertApproxEqual(float(self.func(data)[1]), math.fsum(data), rel=2e-16) def test_start_argument(self): # Test that the optional start argument works correctly. data = [random.uniform(1, 1000) for _ in range(100)] - t = self.func(data) - self.assertEqual(t+42, self.func(data, 42)) - self.assertEqual(t-23, self.func(data, -23)) - self.assertEqual(t+1e20, self.func(data, 1e20)) + t = self.func(data)[1] + self.assertEqual(t+42, self.func(data, 42)[1]) + self.assertEqual(t-23, self.func(data, -23)[1]) + self.assertEqual(t+Fraction(1e20), self.func(data, 1e20)[1]) def test_strings_fail(self): # Sum of strings should fail. @@ -934,7 +1185,7 @@ class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): def test_mixed_sum(self): # Mixed input types are not (currently) allowed. # Check that mixed data types fail. - self.assertRaises(TypeError, self.func, [1, 2.0, Fraction(1, 2)]) + self.assertRaises(TypeError, self.func, [1, 2.0, Decimal(1)]) # And so does mixed start argument. self.assertRaises(TypeError, self.func, [1, 2.0], Decimal(1)) @@ -942,11 +1193,14 @@ class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): class SumTortureTest(NumericTestCase): def test_torture(self): # Tim Peters' torture test for sum, and variants of same. - self.assertEqual(statistics._sum([1, 1e100, 1, -1e100]*10000), 20000.0) - self.assertEqual(statistics._sum([1e100, 1, 1, -1e100]*10000), 20000.0) - self.assertApproxEqual( - statistics._sum([1e-100, 1, 1e-100, -1]*10000), 2.0e-96, rel=5e-16 - ) + self.assertEqual(statistics._sum([1, 1e100, 1, -1e100]*10000), + (float, Fraction(20000.0), 40000)) + self.assertEqual(statistics._sum([1e100, 1, 1, -1e100]*10000), + (float, Fraction(20000.0), 40000)) + T, num, count = statistics._sum([1e-100, 1, 1e-100, -1]*10000) + self.assertIs(T, float) + self.assertEqual(count, 40000) + self.assertApproxEqual(float(num), 2.0e-96, rel=5e-16) class SumSpecialValues(NumericTestCase): @@ -955,7 +1209,7 @@ class SumSpecialValues(NumericTestCase): def test_nan(self): for type_ in (float, Decimal): nan = type_('nan') - result = statistics._sum([1, nan, 2]) + result = statistics._sum([1, nan, 2])[1] self.assertIs(type(result), type_) self.assertTrue(math.isnan(result)) @@ -968,10 +1222,10 @@ class SumSpecialValues(NumericTestCase): def do_test_inf(self, inf): # Adding a single infinity gives infinity. - result = statistics._sum([1, 2, inf, 3]) + result = statistics._sum([1, 2, inf, 3])[1] self.check_infinity(result, inf) # Adding two infinities of the same sign also gives infinity. - result = statistics._sum([1, 2, inf, 3, inf, 4]) + result = statistics._sum([1, 2, inf, 3, inf, 4])[1] self.check_infinity(result, inf) def test_float_inf(self): @@ -987,7 +1241,7 @@ class SumSpecialValues(NumericTestCase): def test_float_mismatched_infs(self): # Test that adding two infinities of opposite sign gives a NAN. inf = float('inf') - result = statistics._sum([1, 2, inf, 3, -inf, 4]) + result = statistics._sum([1, 2, inf, 3, -inf, 4])[1] self.assertTrue(math.isnan(result)) def test_decimal_extendedcontext_mismatched_infs_to_nan(self): @@ -995,7 +1249,7 @@ class SumSpecialValues(NumericTestCase): inf = Decimal('inf') data = [1, 2, inf, 3, -inf, 4] with decimal.localcontext(decimal.ExtendedContext): - self.assertTrue(math.isnan(statistics._sum(data))) + self.assertTrue(math.isnan(statistics._sum(data)[1])) def test_decimal_basiccontext_mismatched_infs_to_nan(self): # Test adding Decimal INFs with opposite sign raises InvalidOperation. @@ -1111,6 +1365,19 @@ class TestMean(NumericTestCase, AverageMixin, UnivariateTypeMixin): d = Decimal('1e4') self.assertEqual(statistics.mean([d]), d) + def test_regression_25177(self): + # Regression test for issue 25177. + # Ensure very big and very small floats don't overflow. + # See http://bugs.python.org/issue25177. + self.assertEqual(statistics.mean( + [8.988465674311579e+307, 8.98846567431158e+307]), + 8.98846567431158e+307) + big = 8.98846567431158e+307 + tiny = 5e-324 + for n in (2, 3, 5, 200): + self.assertEqual(statistics.mean([big]*n), big) + self.assertEqual(statistics.mean([tiny]*n), tiny) + class TestMedian(NumericTestCase, AverageMixin): # Common tests for median and all median.* functions. diff --git a/Misc/NEWS b/Misc/NEWS index 649c891..fdefafc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -107,6 +107,10 @@ Core and Builtins Library ------- +- Issue #25177: Fixed problem with the mean of very small and very large + numbers. As a side effect, statistics.mean and statistics.variance should + be significantly faster. + - Issue #25718: Fixed copying object with state with boolean value is false. - Issue #10131: Fixed deep copying of minidom documents. Based on patch -- cgit v0.12 From 65ca88e4e0b3d26e625d1aecd7c0fe0beb5e3b1d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 4 Dec 2015 15:19:42 -0800 Subject: Issue #25771: Tweak ValueError message when package isn't specified for importlib.util.resolve_name() but is needed. Thanks to Martin Panter for the bug report. --- Lib/importlib/util.py | 4 ++-- Misc/NEWS | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 1dbff26..39cb0f7 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -22,8 +22,8 @@ def resolve_name(name, package): if not name.startswith('.'): return name elif not package: - raise ValueError('{!r} is not a relative name ' - '(no leading dot)'.format(name)) + raise ValueError(f'no package specified for {repr(name)} ' + '(required for relative module names)') level = 0 for character in name: if character != '.': diff --git a/Misc/NEWS b/Misc/NEWS index f87ed38..f4e7e54 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,9 @@ Core and Builtins Library ------- +- Issue #25771: Tweak the exception message for importlib.util.resolve_name() + when 'package' isn't specified but necessary. + - Issue #6478: _strptime's regexp cache now is reset after changing timezone with time.tzset(). -- cgit v0.12 From da0f2a1f5250a36c6067e035dcc9064b00b20bf5 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 04:16:45 +0000 Subject: Issue #25764: Attempt to debug and skip OS X setrlimit() failure --- Lib/test/test_subprocess.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 0448d64..b68def5 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1516,10 +1516,16 @@ class POSIXProcessTestCase(BaseTestCase): # The internal code did not preserve the previous exception when # re-enabling garbage collection try: - from resource import getrlimit, setrlimit, RLIMIT_NPROC + from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIM_INFINITY except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) + try: + setrlimit(RLIMIT_NPROC, limits) + except ValueError as err: + # Seems to happen on AMD64 Snow Leop and x86-64 Yosemite buildbots + print(f"Setting NPROC to {limits!r}: {err!r}, RLIM_INFINITY={RLIM_INFINITY!r}") + self.skipTest("Setting existing NPROC limit failed") [_, hard] = limits setrlimit(RLIMIT_NPROC, (0, hard)) self.addCleanup(setrlimit, RLIMIT_NPROC, limits) -- cgit v0.12 From aa0780801c7a53760296de5c7c9ee0fe6562b6bc Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 05:42:18 +0000 Subject: Issue #25764: OS X now failing on the second setrlimit() call --- Lib/test/test_subprocess.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index b68def5..ed1b213 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1520,14 +1520,17 @@ class POSIXProcessTestCase(BaseTestCase): except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) + [_, hard] = limits try: setrlimit(RLIMIT_NPROC, limits) + setrlimit(RLIMIT_NPROC, (0, hard)) except ValueError as err: - # Seems to happen on AMD64 Snow Leop and x86-64 Yosemite buildbots - print(f"Setting NPROC to {limits!r}: {err!r}, RLIM_INFINITY={RLIM_INFINITY!r}") - self.skipTest("Setting existing NPROC limit failed") - [_, hard] = limits - setrlimit(RLIMIT_NPROC, (0, hard)) + # Seems to happen on various OS X buildbots + print( + f"Setting NPROC failed: {err!r}, limits={limits!r}, " + f"RLIM_INFINITY={RLIM_INFINITY!r}, " + f"getrlimit() -> {getrlimit(RLIMIT_NPROC)!r}") + self.skipTest("Setting NPROC limit failed") self.addCleanup(setrlimit, RLIMIT_NPROC, limits) # Forking should raise EAGAIN, translated to BlockingIOError with self.assertRaises(BlockingIOError): -- cgit v0.12 From 0d559cf72a5c80e7deda214022deb56ec8db1707 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 10:18:25 +0000 Subject: Issue #25764: Remove test debugging --- Lib/test/test_subprocess.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 81c2672..b32ef97 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1518,21 +1518,12 @@ class POSIXProcessTestCase(BaseTestCase): # The internal code did not preserve the previous exception when # re-enabling garbage collection try: - from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIM_INFINITY + from resource import getrlimit, setrlimit, RLIMIT_NPROC except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) [_, hard] = limits - try: - setrlimit(RLIMIT_NPROC, limits) - setrlimit(RLIMIT_NPROC, (0, hard)) - except ValueError as err: - # Seems to happen on various OS X buildbots - print( - f"Setting NPROC failed: {err!r}, limits={limits!r}, " - f"RLIM_INFINITY={RLIM_INFINITY!r}, " - f"getrlimit() -> {getrlimit(RLIMIT_NPROC)!r}") - self.skipTest("Setting NPROC limit failed") + setrlimit(RLIMIT_NPROC, (0, hard)) self.addCleanup(setrlimit, RLIMIT_NPROC, limits) # Forking should raise EAGAIN, translated to BlockingIOError with self.assertRaises(BlockingIOError): -- cgit v0.12 From 59fb6342a4ffdc23e1269a418734f4fc0f984873 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Dec 2015 22:01:35 +0200 Subject: Issue #25761: Improved detecting errors in broken pickle data. --- Lib/pickle.py | 88 +++++++++++++++---------------------- Lib/test/pickletester.py | 17 +++----- Lib/test/test_pickle.py | 5 --- Misc/NEWS | 2 + Modules/_pickle.c | 111 +++++++++++++++++++++++++++++------------------ 5 files changed, 113 insertions(+), 110 deletions(-) diff --git a/Lib/pickle.py b/Lib/pickle.py index 53978fb..a60b1b7 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1031,7 +1031,7 @@ class _Unpickler: self._unframer = _Unframer(self._file_read, self._file_readline) self.read = self._unframer.read self.readline = self._unframer.readline - self.mark = object() # any new unique object + self.metastack = [] self.stack = [] self.append = self.stack.append self.proto = 0 @@ -1047,20 +1047,12 @@ class _Unpickler: except _Stop as stopinst: return stopinst.value - # Return largest index k such that self.stack[k] is self.mark. - # If the stack doesn't contain a mark, eventually raises IndexError. - # This could be sped by maintaining another stack, of indices at which - # the mark appears. For that matter, the latter stack would suffice, - # and we wouldn't need to push mark objects on self.stack at all. - # Doing so is probably a good thing, though, since if the pickle is - # corrupt (or hostile) we may get a clue from finding self.mark embedded - # in unpickled objects. - def marker(self): - stack = self.stack - mark = self.mark - k = len(stack)-1 - while stack[k] is not mark: k = k-1 - return k + # Return a list of items pushed in the stack after last MARK instruction. + def pop_mark(self): + items = self.stack + self.stack = self.metastack.pop() + self.append = self.stack.append + return items def persistent_load(self, pid): raise UnpicklingError("unsupported persistent id encountered") @@ -1237,8 +1229,8 @@ class _Unpickler: dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode def load_tuple(self): - k = self.marker() - self.stack[k:] = [tuple(self.stack[k+1:])] + items = self.pop_mark() + self.append(tuple(items)) dispatch[TUPLE[0]] = load_tuple def load_empty_tuple(self): @@ -1270,21 +1262,20 @@ class _Unpickler: dispatch[EMPTY_SET[0]] = load_empty_set def load_frozenset(self): - k = self.marker() - self.stack[k:] = [frozenset(self.stack[k+1:])] + items = self.pop_mark() + self.append(frozenset(items)) dispatch[FROZENSET[0]] = load_frozenset def load_list(self): - k = self.marker() - self.stack[k:] = [self.stack[k+1:]] + items = self.pop_mark() + self.append(items) dispatch[LIST[0]] = load_list def load_dict(self): - k = self.marker() - items = self.stack[k+1:] + items = self.pop_mark() d = {items[i]: items[i+1] for i in range(0, len(items), 2)} - self.stack[k:] = [d] + self.append(d) dispatch[DICT[0]] = load_dict # INST and OBJ differ only in how they get a class object. It's not @@ -1292,9 +1283,7 @@ class _Unpickler: # previously diverged and grew different bugs. # klass is the class to instantiate, and k points to the topmost mark # object, following which are the arguments for klass.__init__. - def _instantiate(self, klass, k): - args = tuple(self.stack[k+1:]) - del self.stack[k:] + def _instantiate(self, klass, args): if (args or not isinstance(klass, type) or hasattr(klass, "__getinitargs__")): try: @@ -1310,14 +1299,14 @@ class _Unpickler: module = self.readline()[:-1].decode("ascii") name = self.readline()[:-1].decode("ascii") klass = self.find_class(module, name) - self._instantiate(klass, self.marker()) + self._instantiate(klass, self.pop_mark()) dispatch[INST[0]] = load_inst def load_obj(self): # Stack is ... markobject classobject arg1 arg2 ... - k = self.marker() - klass = self.stack.pop(k+1) - self._instantiate(klass, k) + args = self.pop_mark() + cls = args.pop(0) + self._instantiate(cls, args) dispatch[OBJ[0]] = load_obj def load_newobj(self): @@ -1402,12 +1391,14 @@ class _Unpickler: dispatch[REDUCE[0]] = load_reduce def load_pop(self): - del self.stack[-1] + if self.stack: + del self.stack[-1] + else: + self.pop_mark() dispatch[POP[0]] = load_pop def load_pop_mark(self): - k = self.marker() - del self.stack[k:] + self.pop_mark() dispatch[POP_MARK[0]] = load_pop_mark def load_dup(self): @@ -1463,17 +1454,14 @@ class _Unpickler: dispatch[APPEND[0]] = load_append def load_appends(self): - stack = self.stack - mark = self.marker() - list_obj = stack[mark - 1] - items = stack[mark + 1:] + items = self.pop_mark() + list_obj = self.stack[-1] if isinstance(list_obj, list): list_obj.extend(items) else: append = list_obj.append for item in items: append(item) - del stack[mark:] dispatch[APPENDS[0]] = load_appends def load_setitem(self): @@ -1485,27 +1473,21 @@ class _Unpickler: dispatch[SETITEM[0]] = load_setitem def load_setitems(self): - stack = self.stack - mark = self.marker() - dict = stack[mark - 1] - for i in range(mark + 1, len(stack), 2): - dict[stack[i]] = stack[i + 1] - - del stack[mark:] + items = self.pop_mark() + dict = self.stack[-1] + for i in range(0, len(items), 2): + dict[items[i]] = items[i + 1] dispatch[SETITEMS[0]] = load_setitems def load_additems(self): - stack = self.stack - mark = self.marker() - set_obj = stack[mark - 1] - items = stack[mark + 1:] + items = self.pop_mark() + set_obj = self.stack[-1] if isinstance(set_obj, set): set_obj.update(items) else: add = set_obj.add for item in items: add(item) - del stack[mark:] dispatch[ADDITEMS[0]] = load_additems def load_build(self): @@ -1533,7 +1515,9 @@ class _Unpickler: dispatch[BUILD[0]] = load_build def load_mark(self): - self.append(self.mark) + self.metastack.append(self.stack) + self.stack = [] + self.append = self.stack.append dispatch[MARK[0]] = load_mark def load_stop(self): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 608c35a..217aa3d 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1000,7 +1000,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'0', # POP b'1', # POP_MARK b'2', # DUP - # b'(2', # PyUnpickler doesn't raise + b'(2', b'R', # REDUCE b')R', b'a', # APPEND @@ -1009,7 +1009,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'Nb', b'd', # DICT b'e', # APPENDS - # b'(e', # PyUnpickler raises AttributeError + b'(e', b'ibuiltins\nlist\n', # INST b'l', # LIST b'o', # OBJ @@ -1022,7 +1022,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'NNs', b't', # TUPLE b'u', # SETITEMS - # b'(u', # PyUnpickler doesn't raise + b'(u', b'}(Nu', b'\x81', # NEWOBJ b')\x81', @@ -1033,7 +1033,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N\x87', b'NN\x87', b'\x90', # ADDITEMS - # b'(\x90', # PyUnpickler raises AttributeError + b'(\x90', b'\x91', # FROZENSET b'\x92', # NEWOBJ_EX b')}\x92', @@ -1046,7 +1046,7 @@ class AbstractUnpickleTests(unittest.TestCase): def test_bad_mark(self): badpickles = [ - # b'N(.', # STOP + b'N(.', # STOP b'N(2', # DUP b'cbuiltins\nlist\n)(R', # REDUCE b'cbuiltins\nlist\n()R', @@ -1081,7 +1081,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N(\x94', # MEMOIZE ] for p in badpickles: - self.check_unpickling_error(self.bad_mark_errors, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_truncated_data(self): self.check_unpickling_error(EOFError, b'') @@ -2581,11 +2581,6 @@ class AbstractPickleModuleTests(unittest.TestCase): self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) - def test_bad_input(self): - # Test issue4298 - s = bytes([0x58, 0, 0, 0, 0x54]) - self.assertRaises(EOFError, pickle.loads, s) - class AbstractPersistentPicklerTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index bd38cfb..6b97315 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -33,8 +33,6 @@ class PyUnpicklerTests(AbstractUnpickleTests): unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) - bad_mark_errors = (IndexError, pickle.UnpicklingError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -69,8 +67,6 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) - bad_mark_errors = (pickle.UnpicklingError, IndexError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -132,7 +128,6 @@ if has_c_implementation: class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) - bad_mark_errors = (EOFError,) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError) diff --git a/Misc/NEWS b/Misc/NEWS index 026a19a..578d2a3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,8 @@ Core and Builtins Library ------- +- Issue #25761: Improved detecting errors in broken pickle data. + - Issue #25717: Restore the previous behaviour of tolerating most fstat() errors when opening files. This was a regression in 3.5a1, and stopped anonymous temporary files from working in special cases. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 3ddf6a0..38598c5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -370,18 +370,12 @@ _Pickle_FastCall(PyObject *func, PyObject *obj) /*************************************************************************/ -static int -stack_underflow(void) -{ - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "unpickling stack underflow"); - return -1; -} - /* Internal data type used as the unpickling stack. */ typedef struct { PyObject_VAR_HEAD PyObject **data; + int mark_set; /* is MARK set? */ + Py_ssize_t fence; /* position of top MARK or 0 */ Py_ssize_t allocated; /* number of slots in data allocated */ } Pdata; @@ -412,6 +406,8 @@ Pdata_New(void) if (!(self = PyObject_New(Pdata, &Pdata_Type))) return NULL; Py_SIZE(self) = 0; + self->mark_set = 0; + self->fence = 0; self->allocated = 8; self->data = PyMem_MALLOC(self->allocated * sizeof(PyObject *)); if (self->data) @@ -429,8 +425,7 @@ Pdata_clear(Pdata *self, Py_ssize_t clearto) { Py_ssize_t i = Py_SIZE(self); - if (clearto < 0) - return stack_underflow(); + assert(clearto >= self->fence); if (clearto >= i) return 0; @@ -466,6 +461,17 @@ Pdata_grow(Pdata *self) return -1; } +static int +Pdata_stack_underflow(Pdata *self) +{ + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, + self->mark_set ? + "unexpected MARK found" : + "unpickling stack underflow"); + return -1; +} + /* D is a Pdata*. Pop the topmost element and store it into V, which * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError * is raised and V is set to NULL. @@ -473,9 +479,8 @@ Pdata_grow(Pdata *self) static PyObject * Pdata_pop(Pdata *self) { - if (Py_SIZE(self) == 0) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "bad pickle data"); + if (Py_SIZE(self) <= self->fence) { + Pdata_stack_underflow(self); return NULL; } return self->data[--Py_SIZE(self)]; @@ -507,6 +512,10 @@ Pdata_poptuple(Pdata *self, Py_ssize_t start) PyObject *tuple; Py_ssize_t len, i, j; + if (start < self->fence) { + Pdata_stack_underflow(self); + return NULL; + } len = Py_SIZE(self) - start; tuple = PyTuple_New(len); if (tuple == NULL) @@ -4585,13 +4594,19 @@ find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name) static Py_ssize_t marker(UnpicklerObject *self) { - PickleState *st = _Pickle_GetGlobalState(); + Py_ssize_t mark; + if (self->num_marks < 1) { + PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "could not find MARK"); return -1; } - return self->marks[--self->num_marks]; + mark = self->marks[--self->num_marks]; + self->stack->mark_set = self->num_marks != 0; + self->stack->fence = self->num_marks ? + self->marks[self->num_marks - 1] : 0; + return mark; } static int @@ -5052,7 +5067,7 @@ load_counted_tuple(UnpicklerObject *self, int len) PyObject *tuple; if (Py_SIZE(self->stack) < len) - return stack_underflow(); + return Pdata_stack_underflow(self->stack); tuple = Pdata_poptuple(self->stack, Py_SIZE(self->stack) - len); if (tuple == NULL) @@ -5134,6 +5149,12 @@ load_dict(UnpicklerObject *self) if ((dict = PyDict_New()) == NULL) return -1; + if ((j - i) % 2 != 0) { + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "odd number of items for DICT"); + return -1; + } + for (k = i + 1; k < j; k += 2) { key = self->stack->data[k - 1]; value = self->stack->data[k]; @@ -5201,7 +5222,7 @@ load_obj(UnpicklerObject *self) return -1; if (Py_SIZE(self->stack) - i < 1) - return stack_underflow(); + return Pdata_stack_underflow(self->stack); args = Pdata_poptuple(self->stack, i + 1); if (args == NULL) @@ -5518,12 +5539,15 @@ load_pop(UnpicklerObject *self) */ if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) { self->num_marks--; - } else if (len > 0) { + self->stack->mark_set = self->num_marks != 0; + self->stack->fence = self->num_marks ? + self->marks[self->num_marks - 1] : 0; + } else if (len <= self->stack->fence) + return Pdata_stack_underflow(self->stack); + else { len--; Py_DECREF(self->stack->data[len]); Py_SIZE(self->stack) = len; - } else { - return stack_underflow(); } return 0; } @@ -5545,10 +5569,10 @@ static int load_dup(UnpicklerObject *self) { PyObject *last; - Py_ssize_t len; + Py_ssize_t len = Py_SIZE(self->stack); - if ((len = Py_SIZE(self->stack)) <= 0) - return stack_underflow(); + if (len <= self->stack->fence) + return Pdata_stack_underflow(self->stack); last = self->stack->data[len - 1]; PDATA_APPEND(self->stack, last, -1); return 0; @@ -5731,8 +5755,8 @@ load_put(UnpicklerObject *self) return -1; if (len < 2) return bad_readline(); - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; key = PyLong_FromString(s, NULL, 10); @@ -5760,8 +5784,8 @@ load_binput(UnpicklerObject *self) if (_Unpickler_Read(self, &s, 1) < 0) return -1; - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = Py_CHARMASK(s[0]); @@ -5779,8 +5803,8 @@ load_long_binput(UnpicklerObject *self) if (_Unpickler_Read(self, &s, 4) < 0) return -1; - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = calc_binsize(s, 4); @@ -5798,8 +5822,8 @@ load_memoize(UnpicklerObject *self) { PyObject *value; - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; return _Unpickler_MemoPut(self, self->memo_len, value); @@ -5813,8 +5837,8 @@ do_append(UnpicklerObject *self, Py_ssize_t x) Py_ssize_t len, i; len = Py_SIZE(self->stack); - if (x > len || x <= 0) - return stack_underflow(); + if (x > len || x <= self->stack->fence) + return Pdata_stack_underflow(self->stack); if (len == x) /* nothing to do */ return 0; @@ -5863,8 +5887,8 @@ do_append(UnpicklerObject *self, Py_ssize_t x) static int load_append(UnpicklerObject *self) { - if (Py_SIZE(self->stack) - 1 <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) - 1 <= self->stack->fence) + return Pdata_stack_underflow(self->stack); return do_append(self, Py_SIZE(self->stack) - 1); } @@ -5886,8 +5910,8 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) int status = 0; len = Py_SIZE(self->stack); - if (x > len || x <= 0) - return stack_underflow(); + if (x > len || x <= self->stack->fence) + return Pdata_stack_underflow(self->stack); if (len == x) /* nothing to do */ return 0; if ((len - x) % 2 != 0) { @@ -5940,8 +5964,8 @@ load_additems(UnpicklerObject *self) if (mark < 0) return -1; len = Py_SIZE(self->stack); - if (mark > len || mark <= 0) - return stack_underflow(); + if (mark > len || mark <= self->stack->fence) + return Pdata_stack_underflow(self->stack); if (len == mark) /* nothing to do */ return 0; @@ -5996,8 +6020,8 @@ load_build(UnpicklerObject *self) /* Stack is ... instance, state. We want to leave instance at * the stack top, possibly mutated via instance.__setstate__(state). */ - if (Py_SIZE(self->stack) < 2) - return stack_underflow(); + if (Py_SIZE(self->stack) - 2 < self->stack->fence) + return Pdata_stack_underflow(self->stack); PDATA_POP(self->stack, state); if (state == NULL) @@ -6133,7 +6157,8 @@ load_mark(UnpicklerObject *self) self->marks_size = (Py_ssize_t)alloc; } - self->marks[self->num_marks++] = Py_SIZE(self->stack); + self->stack->mark_set = 1; + self->marks[self->num_marks++] = self->stack->fence = Py_SIZE(self->stack); return 0; } @@ -6216,6 +6241,8 @@ load(UnpicklerObject *self) char *s = NULL; self->num_marks = 0; + self->stack->mark_set = 0; + self->stack->fence = 0; self->proto = 0; if (Py_SIZE(self->stack)) Pdata_clear(self->stack, 0); -- cgit v0.12 From 9ec5e25f26a490510bb5da5c26a276cd30a263a0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Dec 2015 02:31:11 +0200 Subject: Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. ElementTree.XMLParser._setevents now accepts any objects with the append method, not just a list. --- Lib/xml/etree/ElementTree.py | 92 ++++++++++++++--------------------------- Misc/NEWS | 2 + Modules/_elementtree.c | 35 +++++++++------- Modules/clinic/_elementtree.c.h | 7 ++-- 4 files changed, 56 insertions(+), 80 deletions(-) diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 62b5d3a..a58ef31 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -95,6 +95,7 @@ import sys import re import warnings import io +import collections import contextlib from . import ElementPath @@ -1198,16 +1199,37 @@ def iterparse(source, events=None, parser=None): Returns an iterator providing (event, elem) pairs. """ + # Use the internal, undocumented _parser argument for now; When the + # parser argument of iterparse is removed, this can be killed. + pullparser = XMLPullParser(events=events, _parser=parser) + def iterator(): + try: + while True: + yield from pullparser.read_events() + # load event buffer + data = source.read(16 * 1024) + if not data: + break + pullparser.feed(data) + root = pullparser._close_and_return_root() + yield from pullparser.read_events() + it.root = root + finally: + if close_source: + source.close() + + class IterParseIterator(collections.Iterator): + __next__ = iterator().__next__ + it = IterParseIterator() + it.root = None + del iterator, IterParseIterator + close_source = False if not hasattr(source, "read"): source = open(source, "rb") close_source = True - try: - return _IterParseIterator(source, events, parser, close_source) - except: - if close_source: - source.close() - raise + + return it class XMLPullParser: @@ -1217,9 +1239,7 @@ class XMLPullParser: # upon in user code. It will be removed in a future release. # See http://bugs.python.org/issue17741 for more details. - # _elementtree.c expects a list, not a deque - self._events_queue = [] - self._index = 0 + self._events_queue = collections.deque() self._parser = _parser or XMLParser(target=TreeBuilder()) # wire up the parser for event reporting if events is None: @@ -1257,64 +1277,14 @@ class XMLPullParser: retrieved from the iterator. """ events = self._events_queue - while True: - index = self._index - try: - event = events[self._index] - # Avoid retaining references to past events - events[self._index] = None - except IndexError: - break - index += 1 - # Compact the list in a O(1) amortized fashion - # As noted above, _elementree.c needs a list, not a deque - if index * 2 >= len(events): - events[:index] = [] - self._index = 0 - else: - self._index = index + while events: + event = events.popleft() if isinstance(event, Exception): raise event else: yield event -class _IterParseIterator: - - def __init__(self, source, events, parser, close_source=False): - # Use the internal, undocumented _parser argument for now; When the - # parser argument of iterparse is removed, this can be killed. - self._parser = XMLPullParser(events=events, _parser=parser) - self._file = source - self._close_file = close_source - self.root = self._root = None - - def __next__(self): - try: - while 1: - for event in self._parser.read_events(): - return event - if self._parser._parser is None: - break - # load event buffer - data = self._file.read(16 * 1024) - if data: - self._parser.feed(data) - else: - self._root = self._parser._close_and_return_root() - self.root = self._root - except: - if self._close_file: - self._file.close() - raise - if self._close_file: - self._file.close() - raise StopIteration - - def __iter__(self): - return self - - def XML(text, parser=None): """Parse XML document from string constant. diff --git a/Misc/NEWS b/Misc/NEWS index 578d2a3..d235f64 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,8 @@ Core and Builtins Library ------- +- Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. + - Issue #25761: Improved detecting errors in broken pickle data. - Issue #25717: Restore the previous behaviour of tolerating most fstat() diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 9026585..168c7d6 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2289,7 +2289,7 @@ typedef struct { PyObject *element_factory; /* element tracing */ - PyObject *events; /* list of events, or NULL if not collecting */ + PyObject *events_append; /* the append method of the list of events, or NULL */ PyObject *start_event_obj; /* event objects (NULL to ignore) */ PyObject *end_event_obj; PyObject *start_ns_event_obj; @@ -2324,7 +2324,7 @@ treebuilder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } t->index = 0; - t->events = NULL; + t->events_append = NULL; t->start_event_obj = t->end_event_obj = NULL; t->start_ns_event_obj = t->end_ns_event_obj = NULL; } @@ -2374,7 +2374,7 @@ treebuilder_gc_clear(TreeBuilderObject *self) Py_CLEAR(self->start_ns_event_obj); Py_CLEAR(self->end_event_obj); Py_CLEAR(self->start_event_obj); - Py_CLEAR(self->events); + Py_CLEAR(self->events_append); Py_CLEAR(self->stack); Py_CLEAR(self->data); Py_CLEAR(self->last); @@ -2455,13 +2455,14 @@ treebuilder_append_event(TreeBuilderObject *self, PyObject *action, PyObject *node) { if (action != NULL) { - PyObject *res = PyTuple_Pack(2, action, node); - if (res == NULL) + PyObject *res; + PyObject *event = PyTuple_Pack(2, action, node); + if (event == NULL) return -1; - if (PyList_Append(self->events, res) < 0) { - Py_DECREF(res); + res = PyObject_CallFunctionObjArgs(self->events_append, event, NULL); + Py_DECREF(event); + if (res == NULL) return -1; - } Py_DECREF(res); } return 0; @@ -3039,7 +3040,7 @@ expat_start_ns_handler(XMLParserObject* self, const XML_Char* prefix, if (PyErr_Occurred()) return; - if (!target->events || !target->start_ns_event_obj) + if (!target->events_append || !target->start_ns_event_obj) return; if (!uri) @@ -3062,7 +3063,7 @@ expat_end_ns_handler(XMLParserObject* self, const XML_Char* prefix_in) if (PyErr_Occurred()) return; - if (!target->events) + if (!target->events_append) return; treebuilder_append_event(target, target->end_ns_event_obj, Py_None); @@ -3551,7 +3552,7 @@ _elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, /*[clinic input] _elementtree.XMLParser._setevents - events_queue: object(subclass_of='&PyList_Type') + events_queue: object events_to_report: object = None / @@ -3561,12 +3562,12 @@ static PyObject * _elementtree_XMLParser__setevents_impl(XMLParserObject *self, PyObject *events_queue, PyObject *events_to_report) -/*[clinic end generated code: output=1440092922b13ed1 input=59db9742910c6174]*/ +/*[clinic end generated code: output=1440092922b13ed1 input=abf90830a1c3b0fc]*/ { /* activate element event reporting */ Py_ssize_t i, seqlen; TreeBuilderObject *target; - PyObject *events_seq; + PyObject *events_append, *events_seq; if (!TreeBuilder_CheckExact(self->target)) { PyErr_SetString( @@ -3579,9 +3580,11 @@ _elementtree_XMLParser__setevents_impl(XMLParserObject *self, target = (TreeBuilderObject*) self->target; - Py_INCREF(events_queue); - Py_XDECREF(target->events); - target->events = events_queue; + events_append = PyObject_GetAttrString(events_queue, "append"); + if (events_append == NULL) + return NULL; + Py_XDECREF(target->events_append); + target->events_append = events_append; /* clear out existing events */ Py_CLEAR(target->start_event_obj); diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 86b4c4c..92e98cf 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -668,12 +668,13 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *args) PyObject *events_queue; PyObject *events_to_report = Py_None; - if (!PyArg_ParseTuple(args, "O!|O:_setevents", - &PyList_Type, &events_queue, &events_to_report)) + if (!PyArg_UnpackTuple(args, "_setevents", + 1, 2, + &events_queue, &events_to_report)) goto exit; return_value = _elementtree_XMLParser__setevents_impl(self, events_queue, events_to_report); exit: return return_value; } -/*[clinic end generated code: output=25b8bf7e7f2151ca input=a9049054013a1b77]*/ +/*[clinic end generated code: output=19d94e2d2726d3aa input=a9049054013a1b77]*/ -- cgit v0.12 From 3ac5380d249ea572ae9f73c7ab65180c2c06a8d4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Dec 2015 11:32:00 +0200 Subject: Issue #25761: Fixed reference leak added in previous changeset (5c670af0100f). --- Modules/_pickle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 38598c5..b3d9b91 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5152,6 +5152,7 @@ load_dict(UnpicklerObject *self) if ((j - i) % 2 != 0) { PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "odd number of items for DICT"); + Py_DECREF(dict); return -1; } -- cgit v0.12 From 2300bf29e69ec9ecee563134f8cdfdab93b2a7c0 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 7 Dec 2015 20:45:16 -0800 Subject: Only update the arr variable when PyObject_RichCompareBool() has been called. --- Modules/_heapqmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 28604af..1c37c75 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -78,6 +78,7 @@ siftup(PyListObject *heap, Py_ssize_t pos) if (cmp < 0) return -1; childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */ + arr = _PyList_ITEMS(heap); /* arr may have changed */ if (endpos != PyList_GET_SIZE(heap)) { PyErr_SetString(PyExc_RuntimeError, "list changed size during iteration"); @@ -85,7 +86,6 @@ siftup(PyListObject *heap, Py_ssize_t pos) } } /* Move the smaller child up. */ - arr = _PyList_ITEMS(heap); tmp1 = arr[childpos]; tmp2 = arr[pos]; arr[childpos] = tmp2; @@ -432,6 +432,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) if (cmp < 0) return -1; childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */ + arr = _PyList_ITEMS(heap); /* arr may have changed */ if (endpos != PyList_GET_SIZE(heap)) { PyErr_SetString(PyExc_RuntimeError, "list changed size during iteration"); @@ -439,7 +440,6 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) } } /* Move the smaller child up. */ - arr = _PyList_ITEMS(heap); tmp1 = arr[childpos]; tmp2 = arr[pos]; arr[childpos] = tmp2; -- cgit v0.12 From 36ff997988bdf2a72f362e60b7b03f439506411c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 10 Dec 2015 09:51:53 +0200 Subject: Issue #25638: Optimized ElementTree parsing; it is now 10% faster. --- Misc/NEWS | 1 + Modules/_elementtree.c | 29 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 67cd7bb..372c355 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -110,6 +110,7 @@ Library ------- - Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. + Optimized ElementTree parsing; it is now 10% faster. - Issue #25761: Improved detecting errors in broken pickle data. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 3cf3d59..c483d87 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2491,10 +2491,17 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, self->data = NULL; } - if (self->element_factory && self->element_factory != Py_None) { - node = PyObject_CallFunction(self->element_factory, "OO", tag, attrib); - } else { + if (!self->element_factory || self->element_factory == Py_None) { node = create_new_element(tag, attrib); + } else if (attrib == Py_None) { + attrib = PyDict_New(); + if (!attrib) + return NULL; + node = PyObject_CallFunction(self->element_factory, "OO", tag, attrib); + Py_DECREF(attrib); + } + else { + node = PyObject_CallFunction(self->element_factory, "OO", tag, attrib); } if (!node) { return NULL; @@ -2959,12 +2966,8 @@ expat_start_handler(XMLParserObject* self, const XML_Char* tag_in, attrib_in += 2; } } else { - /* Pass an empty dictionary on */ - attrib = PyDict_New(); - if (!attrib) { - Py_DECREF(tag); - return; - } + Py_INCREF(Py_None); + attrib = Py_None; } if (TreeBuilder_CheckExact(self->target)) { @@ -2973,6 +2976,14 @@ expat_start_handler(XMLParserObject* self, const XML_Char* tag_in, tag, attrib); } else if (self->handle_start) { + if (attrib == Py_None) { + Py_DECREF(attrib); + attrib = PyDict_New(); + if (!attrib) { + Py_DECREF(tag); + return; + } + } res = PyObject_CallFunction(self->handle_start, "OO", tag, attrib); } else res = NULL; -- cgit v0.12 From 5088f6005ff5e6bb9fbc1bdfd9777b827c6aab0f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Dec 2015 18:45:01 -0800 Subject: Hoist constant expressions (so->table and so->mask) out of the inner-loop. --- Objects/setobject.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 7fde01f..b00e85f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -260,16 +260,13 @@ The caller is responsible for updating the key's reference count and the setobject's fill and used fields. */ static void -set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) +set_insert_clean(setentry *table, size_t mask, PyObject *key, Py_hash_t hash) { - setentry *table = so->table; setentry *entry; size_t perturb = hash; - size_t mask = (size_t)so->mask; size_t i = (size_t)hash & mask; size_t j; - assert(so->fill == so->used); while (1) { entry = &table[i]; if (entry->key == NULL) @@ -285,8 +282,8 @@ set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) i = (i * 5 + 1 + perturb) & mask; } found_null: - entry->key = key; entry->hash = hash; + entry->key = key; } /* ======== End logic for probing the hash table ========================== */ @@ -304,6 +301,8 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry *oldtable, *newtable, *entry; Py_ssize_t oldfill = so->fill; Py_ssize_t oldused = so->used; + Py_ssize_t oldmask = so->mask; + size_t newmask; int is_oldtable_malloced; setentry small_copy[PySet_MINSIZE]; @@ -363,18 +362,17 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) /* Copy the data over; this is refcount-neutral for active entries; dummy entries aren't copied over, of course */ + newmask = (size_t)so->mask; if (oldfill == oldused) { - for (entry = oldtable; oldused > 0; entry++) { + for (entry = oldtable; entry <= oldtable + oldmask; entry++) { if (entry->key != NULL) { - oldused--; - set_insert_clean(so, entry->key, entry->hash); + set_insert_clean(newtable, newmask, entry->key, entry->hash); } } } else { - for (entry = oldtable; oldused > 0; entry++) { + for (entry = oldtable; entry <= oldtable + oldmask; entry++) { if (entry->key != NULL && entry->key != dummy) { - oldused--; - set_insert_clean(so, entry->key, entry->hash); + set_insert_clean(newtable, newmask, entry->key, entry->hash); } } } @@ -676,13 +674,15 @@ set_merge(PySetObject *so, PyObject *otherset) /* If our table is empty, we can use set_insert_clean() */ if (so->fill == 0) { + setentry *newtable = so->table; + size_t newmask = (size_t)so->mask; so->fill = other->used; so->used = other->used; for (i = 0; i <= other->mask; i++, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { Py_INCREF(key); - set_insert_clean(so, key, other_entry->hash); + set_insert_clean(newtable, newmask, key, other_entry->hash); } } return 0; -- cgit v0.12 From 86d322f0204797b0a2fe46e33bce240335270c6f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Dec 2015 19:27:17 -0800 Subject: Undo inadvertent line swap --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index b00e85f..4723b58 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -282,8 +282,8 @@ set_insert_clean(setentry *table, size_t mask, PyObject *key, Py_hash_t hash) i = (i * 5 + 1 + perturb) & mask; } found_null: - entry->hash = hash; entry->key = key; + entry->hash = hash; } /* ======== End logic for probing the hash table ========================== */ -- cgit v0.12 From e4495877dd08b4d0568bb8a75ae5c69b3d91ff60 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 15 Dec 2015 00:42:30 -0800 Subject: Minor tweek. Counting down rather than up reduces register pressure. --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 4723b58..45aa200 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -678,7 +678,7 @@ set_merge(PySetObject *so, PyObject *otherset) size_t newmask = (size_t)so->mask; so->fill = other->used; so->used = other->used; - for (i = 0; i <= other->mask; i++, other_entry++) { + for (i = other->mask + 1; i > 0 ; i--, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { Py_INCREF(key); -- cgit v0.12 From 129a41449071928cc7afcb4789cfae22b08607b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Dec 2015 11:29:59 +0100 Subject: Issue #25868: Try to make test_eintr.test_sigwaitinfo() more reliable especially on slow buildbots Use a pipe to synchronize the parent and the child processes. --- Lib/test/eintrdata/eintr_tester.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index ee6e75b..531d576 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -377,10 +377,10 @@ class SignalEINTRTest(EINTRBaseTest): @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), 'need signal.sigwaitinfo()') def test_sigwaitinfo(self): - # Issue #25277: The sleep is a weak synchronization between the parent - # and the child process. If the sleep is too low, the test hangs on - # slow or highly loaded systems. - self.sleep_time = 2.0 + # Issue #25277, #25868: give a few miliseconds to the parent process + # between os.write() and signal.sigwaitinfo() to works around a race + # condition + self.sleep_time = 0.100 signum = signal.SIGUSR1 pid = os.getpid() @@ -388,18 +388,28 @@ class SignalEINTRTest(EINTRBaseTest): old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) + rpipe, wpipe = os.pipe() + code = '\n'.join(( 'import os, time', 'pid = %s' % os.getpid(), 'signum = %s' % int(signum), 'sleep_time = %r' % self.sleep_time, + 'rpipe = %r' % rpipe, + 'os.read(rpipe, 1)', + 'os.close(rpipe)', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) t0 = time.monotonic() - proc = self.subprocess(code) + proc = self.subprocess(code, pass_fds=(rpipe,)) + os.close(rpipe) with kill_on_error(proc): + # sync child-parent + os.write(wpipe, b'x') + os.close(wpipe) + # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 -- cgit v0.12 From 9b3a2eec1c64466cd2e2004b7732cf2b5a79a644 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 10:03:13 +0200 Subject: Issues #25890, #25891, #25892: Removed unused variables in Windows code. Reported by Alexander Riccio. --- Modules/posixmodule.c | 2 -- Objects/unicodeobject.c | 1 - 2 files changed, 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0e423f9..81c3c48 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3463,7 +3463,6 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) char *bufptr = namebuf; /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; - PyObject *po = NULL; wchar_t *wnamebuf = NULL; if (!path->narrow) { @@ -12380,7 +12379,6 @@ enable_symlink() HANDLE tok; TOKEN_PRIVILEGES tok_priv; LUID luid; - int meth_idx = 0; if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ALL_ACCESS, &tok)) return 0; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 895a4e8..ad8f505 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -7332,7 +7332,6 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes, BOOL usedDefaultChar = FALSE; BOOL *pusedDefaultChar = &usedDefaultChar; int outsize; - PyObject *exc = NULL; wchar_t *p; Py_ssize_t size; const DWORD flags = encode_code_page_flags(code_page, NULL); -- cgit v0.12 From 8bc2b4d5227871d25eeaac91cd64387cc27d6d74 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 10:06:58 +0200 Subject: Issue #25890: Removed yet one unused variable. --- Modules/posixmodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 81c3c48..7ec1f47 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3460,7 +3460,6 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) BOOL result; WIN32_FIND_DATA FileData; char namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ - char *bufptr = namebuf; /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; wchar_t *wnamebuf = NULL; -- cgit v0.12 From f8ed0044f62b4ce0c21ccb1873f6bf656cd8ac01 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 10:19:30 +0200 Subject: Issue #25889: Got rid of warning about mixing signed/unsigned char pointers. --- PC/launcher.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PC/launcher.c b/PC/launcher.c index e414237..a94241b 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1106,7 +1106,7 @@ maybe_handle_shebang(wchar_t ** argv, wchar_t * cmdline) */ FILE * fp; errno_t rc = _wfopen_s(&fp, *argv, L"rb"); - unsigned char buffer[BUFSIZE]; + char buffer[BUFSIZE]; wchar_t shebang_line[BUFSIZE + 1]; size_t read; char *p; @@ -1128,7 +1128,8 @@ maybe_handle_shebang(wchar_t ** argv, wchar_t * cmdline) fclose(fp); if ((read >= 4) && (buffer[3] == '\n') && (buffer[2] == '\r')) { - ip = find_by_magic((buffer[1] << 8 | buffer[0]) & 0xFFFF); + ip = find_by_magic((((unsigned char)buffer[1]) << 8 | + (unsigned char)buffer[0]) & 0xFFFF); if (ip != NULL) { debug(L"script file is compiled against Python %ls\n", ip->version); -- cgit v0.12 From 42bb126f0aa1de4817babb8c56b9ad6df8bab458 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 13:23:33 +0200 Subject: Issue #25899: Converted Objects/listsort.txt to UTF-8. Original patch by Chris Angelico. --- Objects/listsort.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 832e4f2..fef982f 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -486,7 +486,7 @@ sub-run, yet finding such very efficiently when they exist. I first learned about the galloping strategy in a related context; see: "Adaptive Set Intersections, Unions, and Differences" (2000) - Erik D. Demaine, Alejandro L�pez-Ortiz, J. Ian Munro + Erik D. Demaine, Alejandro López-Ortiz, J. Ian Munro and its followup(s). An earlier paper called the same strategy "exponential search": -- cgit v0.12 From a254921cd4c1ca22d725872601292c48b7caa20a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 19 Dec 2015 09:43:14 +0200 Subject: Issue #22227: The TarFile iterator is reimplemented using generator. This implementation is simpler that using class. --- Lib/tarfile.py | 68 ++++++++++++++++++++++------------------------------------ Misc/NEWS | 3 +++ 2 files changed, 29 insertions(+), 42 deletions(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 9d22c8e..80b6e35 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2372,9 +2372,32 @@ class TarFile(object): """Provide an iterator object. """ if self._loaded: - return iter(self.members) - else: - return TarIter(self) + yield from self.members + return + + # Yield items using TarFile's next() method. + # When all members have been read, set TarFile as _loaded. + index = 0 + # Fix for SF #1100429: Under rare circumstances it can + # happen that getmembers() is called during iteration, + # which will have already exhausted the next() method. + if self.firstmember is not None: + tarinfo = self.next() + index += 1 + yield tarinfo + + while True: + if index < len(self.members): + tarinfo = self.members[index] + elif not self._loaded: + tarinfo = self.next() + if not tarinfo: + self._loaded = True + return + else: + return + index += 1 + yield tarinfo def _dbg(self, level, msg): """Write debugging output to sys.stderr. @@ -2395,45 +2418,6 @@ class TarFile(object): if not self._extfileobj: self.fileobj.close() self.closed = True -# class TarFile - -class TarIter: - """Iterator Class. - - for tarinfo in TarFile(...): - suite... - """ - - def __init__(self, tarfile): - """Construct a TarIter object. - """ - self.tarfile = tarfile - self.index = 0 - def __iter__(self): - """Return iterator object. - """ - return self - def __next__(self): - """Return the next item using TarFile's next() method. - When all members have been read, set TarFile as _loaded. - """ - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will cause TarIter to stop prematurely. - - if self.index == 0 and self.tarfile.firstmember is not None: - tarinfo = self.tarfile.next() - elif self.index < len(self.tarfile.members): - tarinfo = self.tarfile.members[self.index] - elif not self.tarfile._loaded: - tarinfo = self.tarfile.next() - if not tarinfo: - self.tarfile._loaded = True - raise StopIteration - else: - raise StopIteration - self.index += 1 - return tarinfo #-------------------- # exported functions diff --git a/Misc/NEWS b/Misc/NEWS index b57dbf1..92572fa 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,9 @@ Core and Builtins Library ------- +- Issue #22227: The TarFile iterator is reimplemented using generator. + This implementation is simpler that using class. + - Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. Optimized ElementTree parsing; it is now 10% faster. -- cgit v0.12 From 22adf2ac022d6667a8689d6fac8af44c7663e3f2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Dec 2015 12:43:54 +0200 Subject: Issue #25873: Optimized iterating ElementTree. Iterating elements Element.iter() is now 40% faster, iterating text Element.itertext() is now up to 2.5 times faster. --- Misc/NEWS | 4 + Modules/_elementtree.c | 259 +++++++++++++++++++------------------------------ 2 files changed, 105 insertions(+), 158 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index ad37eb7..6dc02e3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,10 @@ Core and Builtins Library ------- +- Issue #25873: Optimized iterating ElementTree. Iterating elements + Element.iter() is now 40% faster, iterating text Element.itertext() + is now up to 2.5 times faster. + - Issue #25902: Fixed various refcount issues in ElementTree iteration. - Issue #22227: The TarFile iterator is reimplemented using generator. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 949e3a0..ad3e45c 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1986,22 +1986,22 @@ static PySequenceMethods element_as_sequence = { * pre-order traversal. To keep track of which sub-element should be returned * next, a stack of parents is maintained. This is a standard stack-based * iterative pre-order traversal of a tree. - * The stack is managed using a single-linked list starting at parent_stack. - * Each stack node contains the saved parent to which we should return after + * The stack is managed using a continuous array. + * Each stack item contains the saved parent to which we should return after * the current one is exhausted, and the next child to examine in that parent. */ typedef struct ParentLocator_t { ElementObject *parent; Py_ssize_t child_index; - struct ParentLocator_t *next; } ParentLocator; typedef struct { PyObject_HEAD ParentLocator *parent_stack; + Py_ssize_t parent_stack_used; + Py_ssize_t parent_stack_size; ElementObject *root_element; PyObject *sought_tag; - int root_done; int gettext; } ElementIterObject; @@ -2009,13 +2009,11 @@ typedef struct { static void elementiter_dealloc(ElementIterObject *it) { - ParentLocator *p = it->parent_stack; - while (p) { - ParentLocator *temp = p; - Py_XDECREF(p->parent); - p = p->next; - PyObject_Free(temp); - } + Py_ssize_t i = it->parent_stack_used; + it->parent_stack_used = 0; + while (i--) + Py_XDECREF(it->parent_stack[i].parent); + PyMem_Free(it->parent_stack); Py_XDECREF(it->sought_tag); Py_XDECREF(it->root_element); @@ -2027,11 +2025,9 @@ elementiter_dealloc(ElementIterObject *it) static int elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) { - ParentLocator *p = it->parent_stack; - while (p) { - Py_VISIT(p->parent); - p = p->next; - } + Py_ssize_t i = it->parent_stack_used; + while (i--) + Py_VISIT(it->parent_stack[i].parent); Py_VISIT(it->root_element); Py_VISIT(it->sought_tag); @@ -2040,17 +2036,25 @@ elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) /* Helper function for elementiter_next. Add a new parent to the parent stack. */ -static ParentLocator * -parent_stack_push_new(ParentLocator *stack, ElementObject *parent) +static int +parent_stack_push_new(ElementIterObject *it, ElementObject *parent) { - ParentLocator *new_node = PyObject_Malloc(sizeof(ParentLocator)); - if (new_node) { - new_node->parent = parent; - Py_INCREF(parent); - new_node->child_index = 0; - new_node->next = stack; + ParentLocator *item; + + if (it->parent_stack_used >= it->parent_stack_size) { + Py_ssize_t new_size = it->parent_stack_size * 2; /* never overflow */ + ParentLocator *parent_stack = it->parent_stack; + PyMem_Resize(parent_stack, ParentLocator, new_size); + if (parent_stack == NULL) + return -1; + it->parent_stack = parent_stack; + it->parent_stack_size = new_size; } - return new_node; + item = it->parent_stack + it->parent_stack_used++; + Py_INCREF(parent); + item->parent = parent; + item->child_index = 0; + return 0; } static PyObject * @@ -2067,151 +2071,91 @@ elementiter_next(ElementIterObject *it) * - itertext() also has to handle tail, after finishing with all the * children of a node. */ - ElementObject *cur_parent; - Py_ssize_t child_index; int rc; ElementObject *elem; + PyObject *text; while (1) { /* Handle the case reached in the beginning and end of iteration, where - * the parent stack is empty. The root_done flag gives us indication - * whether we've just started iterating (so root_done is 0), in which - * case the root is returned. If root_done is 1 and we're here, the + * the parent stack is empty. If root_element is NULL and we're here, the * iterator is exhausted. */ - if (!it->parent_stack->parent) { - if (it->root_done) { + if (!it->parent_stack_used) { + if (!it->root_element) { PyErr_SetNone(PyExc_StopIteration); return NULL; - } else { - elem = it->root_element; - it->parent_stack = parent_stack_push_new(it->parent_stack, - elem); - if (!it->parent_stack) { - PyErr_NoMemory(); - return NULL; - } + } - Py_INCREF(elem); - it->root_done = 1; - rc = (it->sought_tag == Py_None); - if (!rc) { - rc = PyObject_RichCompareBool(elem->tag, - it->sought_tag, Py_EQ); - if (rc < 0) { - Py_DECREF(elem); - return NULL; - } - } - if (rc) { - if (it->gettext) { - PyObject *text = element_get_text(elem); - if (!text) { - Py_DECREF(elem); - return NULL; - } - Py_INCREF(text); - Py_DECREF(elem); - rc = PyObject_IsTrue(text); - if (rc > 0) - return text; - Py_DECREF(text); - if (rc < 0) - return NULL; - } else { - return (PyObject *)elem; - } - } - else { - Py_DECREF(elem); + elem = it->root_element; /* steals a reference */ + it->root_element = NULL; + } + else { + /* See if there are children left to traverse in the current parent. If + * yes, visit the next child. If not, pop the stack and try again. + */ + ParentLocator *item = &it->parent_stack[it->parent_stack_used - 1]; + Py_ssize_t child_index = item->child_index; + ElementObjectExtra *extra; + elem = item->parent; + extra = elem->extra; + if (!extra || child_index >= extra->length) { + it->parent_stack_used--; + /* Note that extra condition on it->parent_stack_used here; + * this is because itertext() is supposed to only return *inner* + * text, not text following the element it began iteration with. + */ + if (it->gettext && it->parent_stack_used) { + text = element_get_tail(elem); + goto gettext; } + Py_DECREF(elem); + continue; } + + elem = (ElementObject *)extra->children[child_index]; + item->child_index++; + Py_INCREF(elem); } - /* See if there are children left to traverse in the current parent. If - * yes, visit the next child. If not, pop the stack and try again. - */ - cur_parent = it->parent_stack->parent; - child_index = it->parent_stack->child_index; - if (cur_parent->extra && child_index < cur_parent->extra->length) { - elem = (ElementObject *)cur_parent->extra->children[child_index]; - it->parent_stack->child_index++; - it->parent_stack = parent_stack_push_new(it->parent_stack, - elem); - if (!it->parent_stack) { - PyErr_NoMemory(); - return NULL; - } + if (parent_stack_push_new(it, elem) < 0) { + Py_DECREF(elem); + PyErr_NoMemory(); + return NULL; + } + if (it->gettext) { + text = element_get_text(elem); + goto gettext; + } - Py_INCREF(elem); - if (it->gettext) { - PyObject *text = element_get_text(elem); - if (!text) { - Py_DECREF(elem); - return NULL; - } - Py_INCREF(text); - Py_DECREF(elem); - rc = PyObject_IsTrue(text); - if (rc > 0) - return text; - Py_DECREF(text); - if (rc < 0) - return NULL; - } else { - rc = (it->sought_tag == Py_None); - if (!rc) { - rc = PyObject_RichCompareBool(elem->tag, - it->sought_tag, Py_EQ); - if (rc < 0) { - Py_DECREF(elem); - return NULL; - } - } - if (rc) { - return (PyObject *)elem; - } - Py_DECREF(elem); - } + if (it->sought_tag == Py_None) + return (PyObject *)elem; + + rc = PyObject_RichCompareBool(elem->tag, it->sought_tag, Py_EQ); + if (rc > 0) + return (PyObject *)elem; + + Py_DECREF(elem); + if (rc < 0) + return NULL; + continue; + +gettext: + if (!text) { + Py_DECREF(elem); + return NULL; + } + if (text == Py_None) { + Py_DECREF(elem); } else { - PyObject *tail; - ParentLocator *next; - if (it->gettext) { - Py_INCREF(cur_parent); - tail = element_get_tail(cur_parent); - if (!tail) { - Py_DECREF(cur_parent); - return NULL; - } - Py_INCREF(tail); - Py_DECREF(cur_parent); - } - else { - tail = Py_None; - Py_INCREF(tail); - } - next = it->parent_stack->next; - cur_parent = it->parent_stack->parent; - PyObject_Free(it->parent_stack); - it->parent_stack = next; - Py_XDECREF(cur_parent); - - /* Note that extra condition on it->parent_stack->parent here; - * this is because itertext() is supposed to only return *inner* - * text, not text following the element it began iteration with. - */ - if (it->parent_stack->parent) { - rc = PyObject_IsTrue(tail); - if (rc > 0) - return tail; - Py_DECREF(tail); - if (rc < 0) - return NULL; - } - else { - Py_DECREF(tail); - } + Py_INCREF(text); + Py_DECREF(elem); + rc = PyObject_IsTrue(text); + if (rc > 0) + return text; + Py_DECREF(text); + if (rc < 0) + return NULL; } } @@ -2263,6 +2207,7 @@ static PyTypeObject ElementIter_Type = { 0, /* tp_new */ }; +#define INIT_PARENT_STACK_SIZE 8 static PyObject * create_elementiter(ElementObject *self, PyObject *tag, int gettext) @@ -2275,22 +2220,20 @@ create_elementiter(ElementObject *self, PyObject *tag, int gettext) Py_INCREF(tag); it->sought_tag = tag; - it->root_done = 0; it->gettext = gettext; Py_INCREF(self); it->root_element = self; PyObject_GC_Track(it); - it->parent_stack = PyObject_Malloc(sizeof(ParentLocator)); + it->parent_stack = PyMem_New(ParentLocator, INIT_PARENT_STACK_SIZE); if (it->parent_stack == NULL) { Py_DECREF(it); PyErr_NoMemory(); return NULL; } - it->parent_stack->parent = NULL; - it->parent_stack->child_index = 0; - it->parent_stack->next = NULL; + it->parent_stack_used = 0; + it->parent_stack_size = INIT_PARENT_STACK_SIZE; return (PyObject *)it; } -- cgit v0.12 From 060ed718cecbfb7bb3292b76eef8312896ba7317 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Dec 2015 12:57:27 +0200 Subject: Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. --- Misc/NEWS | 2 ++ Modules/_elementtree.c | 78 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 6dc02e3..ff1beef 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,8 @@ Core and Builtins Library ------- +- Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. + - Issue #25873: Optimized iterating ElementTree. Iterating elements Element.iter() is now 40% faster, iterating text Element.itertext() is now up to 2.5 times faster. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index ad3e45c..5908c72 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -129,30 +129,6 @@ elementtree_free(void *m) /* helpers */ LOCAL(PyObject*) -deepcopy(PyObject* object, PyObject* memo) -{ - /* do a deep copy of the given object */ - PyObject* args; - PyObject* result; - elementtreestate *st = ET_STATE_GLOBAL; - - if (!st->deepcopy_obj) { - PyErr_SetString( - PyExc_RuntimeError, - "deepcopy helper not found" - ); - return NULL; - } - - args = PyTuple_Pack(2, object, memo); - if (!args) - return NULL; - result = PyObject_CallObject(st->deepcopy_obj, args); - Py_DECREF(args); - return result; -} - -LOCAL(PyObject*) list_join(PyObject* list) { /* join list elements (destroying the list in the process) */ @@ -748,6 +724,9 @@ _elementtree_Element___copy___impl(ElementObject *self) return (PyObject*) element; } +/* Helper for a deep copy. */ +LOCAL(PyObject *) deepcopy(PyObject *, PyObject *); + /*[clinic input] _elementtree.Element.__deepcopy__ @@ -838,6 +817,57 @@ _elementtree_Element___deepcopy__(ElementObject *self, PyObject *memo) return NULL; } +LOCAL(PyObject *) +deepcopy(PyObject *object, PyObject *memo) +{ + /* do a deep copy of the given object */ + PyObject *args; + PyObject *result; + elementtreestate *st; + + /* Fast paths */ + if (object == Py_None || PyUnicode_CheckExact(object)) { + Py_INCREF(object); + return object; + } + + if (Py_REFCNT(object) == 1) { + if (PyDict_CheckExact(object)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + int simple = 1; + while (PyDict_Next(object, &pos, &key, &value)) { + if (!PyUnicode_CheckExact(key) || !PyUnicode_CheckExact(value)) { + simple = 0; + break; + } + } + if (simple) + return PyDict_Copy(object); + /* Fall through to general case */ + } + else if (Element_CheckExact(object)) { + return _elementtree_Element___deepcopy__((ElementObject *)object, memo); + } + } + + /* General case */ + st = ET_STATE_GLOBAL; + if (!st->deepcopy_obj) { + PyErr_SetString(PyExc_RuntimeError, + "deepcopy helper not found"); + return NULL; + } + + args = PyTuple_Pack(2, object, memo); + if (!args) + return NULL; + result = PyObject_CallObject(st->deepcopy_obj, args); + Py_DECREF(args); + return result; +} + + /*[clinic input] _elementtree.Element.__sizeof__ -> Py_ssize_t -- cgit v0.12 From 2d06e8445587d9b4d0bf79bdb08ab4743b780249 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 25 Dec 2015 19:53:18 +0200 Subject: Issue #25923: Added the const qualifier to static constant arrays. --- Include/py_curses.h | 4 ++-- Include/pymath.h | 2 +- Modules/_csv.c | 8 ++++---- Modules/_ctypes/_ctypes.c | 14 +++++++------- Modules/_ctypes/callproc.c | 16 ++++++++-------- Modules/_curses_panel.c | 2 +- Modules/_datetimemodule.c | 20 ++++++++++---------- Modules/_dbmmodule.c | 8 ++++---- Modules/_gdbmmodule.c | 2 +- Modules/_io/textio.c | 4 ++-- Modules/_randommodule.c | 2 +- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/cursor.c | 4 ++-- Modules/_sqlite/module.c | 4 ++-- Modules/_sre.c | 2 +- Modules/_struct.c | 4 ++-- Modules/_testbuffer.c | 2 +- Modules/_testcapimodule.c | 4 ++-- Modules/arraymodule.c | 18 +++++++++--------- Modules/audioop.c | 20 +++++++++++--------- Modules/binascii.c | 14 +++++++------- Modules/getaddrinfo.c | 2 +- Modules/getpath.c | 4 ++-- Modules/main.c | 14 +++++++------- Modules/parsermodule.c | 4 ++-- Modules/posixmodule.c | 10 +++++----- Modules/selectmodule.c | 2 +- Modules/timemodule.c | 4 ++-- Modules/unicodedata.c | 4 ++-- Objects/object.c | 2 +- Objects/structseq.c | 6 +++--- Objects/typeobject.c | 2 +- Objects/unicodeobject.c | 12 ++++++------ Parser/parsetok.c | 4 ++-- Parser/pgen.c | 2 +- Python/ast.c | 4 ++-- Python/dtoa.c | 4 ++-- Python/formatter_unicode.c | 2 +- Python/import.c | 6 +++--- Python/importdl.c | 4 ++-- Python/mystrtoul.c | 6 +++--- Python/pystrtod.c | 9 +++++---- Python/pythonrun.c | 4 ++-- Python/sysmodule.c | 6 ++++-- 44 files changed, 139 insertions(+), 134 deletions(-) diff --git a/Include/py_curses.h b/Include/py_curses.h index f2c08f6..3c21697 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -103,8 +103,8 @@ static void **PyCurses_API; #endif /* general error messages */ -static char *catchall_ERR = "curses function returned ERR"; -static char *catchall_NULL = "curses function returned NULL"; +static const char catchall_ERR[] = "curses function returned ERR"; +static const char catchall_NULL[] = "curses function returned NULL"; /* Function Prototype Macros - They are ugly but very, very useful. ;-) diff --git a/Include/pymath.h b/Include/pymath.h index 1ea9ac1..ed76053 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -169,7 +169,7 @@ PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); #pragma float_control (pop) #define Py_NAN __icc_nan() #else /* ICC_NAN_RELAXED as default for Intel Compiler */ - static union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; + static const union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; #define Py_NAN (__nan_store.__icc_nan) #endif /* ICC_NAN_STRICT */ #endif /* __INTEL_COMPILER */ diff --git a/Modules/_csv.c b/Modules/_csv.c index fe85069..c0be739 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -60,10 +60,10 @@ typedef enum { typedef struct { QuoteStyle style; - char *name; + const char *name; } StyleDesc; -static StyleDesc quote_styles[] = { +static const StyleDesc quote_styles[] = { { QUOTE_MINIMAL, "QUOTE_MINIMAL" }, { QUOTE_ALL, "QUOTE_ALL" }, { QUOTE_NONNUMERIC, "QUOTE_NONNUMERIC" }, @@ -286,7 +286,7 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) static int dialect_check_quoting(int quoting) { - StyleDesc *qs; + const StyleDesc *qs; for (qs = quote_styles; qs->name; qs++) { if ((int)qs->style == quoting) @@ -1633,7 +1633,7 @@ PyMODINIT_FUNC PyInit__csv(void) { PyObject *module; - StyleDesc *style; + const StyleDesc *style; if (PyType_Ready(&Dialect_Type) < 0) return NULL; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 34a1099..fccc8ba 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -435,7 +435,7 @@ UnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return StructUnionType_new(type, args, kwds, 0); } -static char from_address_doc[] = +static const char from_address_doc[] = "C.from_address(integer) -> C instance\naccess a C instance at the specified address"; static PyObject * @@ -453,7 +453,7 @@ CDataType_from_address(PyObject *type, PyObject *value) return PyCData_AtAddress(type, buf); } -static char from_buffer_doc[] = +static const char from_buffer_doc[] = "C.from_buffer(object, offset=0) -> C instance\ncreate a C instance from a writeable buffer"; static int @@ -524,7 +524,7 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return result; } -static char from_buffer_copy_doc[] = +static const char from_buffer_copy_doc[] = "C.from_buffer_copy(object, offset=0) -> C instance\ncreate a C instance from a readable buffer"; static PyObject * @@ -566,7 +566,7 @@ CDataType_from_buffer_copy(PyObject *type, PyObject *args) return result; } -static char in_dll_doc[] = +static const char in_dll_doc[] = "C.in_dll(dll, name) -> C instance\naccess a C instance in a dll"; static PyObject * @@ -623,7 +623,7 @@ CDataType_in_dll(PyObject *type, PyObject *args) return PyCData_AtAddress(type, address); } -static char from_param_doc[] = +static const char from_param_doc[] = "Convert a Python object into a function call parameter."; static PyObject * @@ -1481,7 +1481,7 @@ _type_ attribute. */ -static char *SIMPLE_TYPE_CHARS = "cbBhHiIlLdfuzZqQPXOv?g"; +static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdfuzZqQPXOv?g"; static PyObject * c_wchar_p_from_param(PyObject *type, PyObject *value) @@ -5118,7 +5118,7 @@ static const char module_docs[] = #ifdef MS_WIN32 -static char comerror_doc[] = "Raised when a COM method call failed."; +static const char comerror_doc[] = "Raised when a COM method call failed."; int comerror_init(PyObject *self, PyObject *args, PyObject *kwds) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 03a911f..68981fe 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1201,7 +1201,7 @@ _parse_voidp(PyObject *obj, void **address) #ifdef MS_WIN32 -static char format_error_doc[] = +static const char format_error_doc[] = "FormatError([integer]) -> string\n\ \n\ Convert a win32 error code into a string. If the error code is not\n\ @@ -1225,7 +1225,7 @@ static PyObject *format_error(PyObject *self, PyObject *args) return result; } -static char load_library_doc[] = +static const char load_library_doc[] = "LoadLibrary(name) -> handle\n\ \n\ Load an executable (usually a DLL), and return a handle to it.\n\ @@ -1254,7 +1254,7 @@ static PyObject *load_library(PyObject *self, PyObject *args) #endif } -static char free_library_doc[] = +static const char free_library_doc[] = "FreeLibrary(handle) -> void\n\ \n\ Free the handle of an executable previously loaded by LoadLibrary.\n"; @@ -1269,7 +1269,7 @@ static PyObject *free_library(PyObject *self, PyObject *args) return Py_None; } -static char copy_com_pointer_doc[] = +static const char copy_com_pointer_doc[] = "CopyComPointer(src, dst) -> HRESULT value\n"; static PyObject * @@ -1439,7 +1439,7 @@ call_cdeclfunction(PyObject *self, PyObject *args) /***************************************************************** * functions */ -static char sizeof_doc[] = +static const char sizeof_doc[] = "sizeof(C type) -> integer\n" "sizeof(C instance) -> integer\n" "Return the size in bytes of a C instance"; @@ -1460,7 +1460,7 @@ sizeof_func(PyObject *self, PyObject *obj) return NULL; } -static char alignment_doc[] = +static const char alignment_doc[] = "alignment(C type) -> integer\n" "alignment(C instance) -> integer\n" "Return the alignment requirements of a C instance"; @@ -1483,7 +1483,7 @@ align_func(PyObject *self, PyObject *obj) return NULL; } -static char byref_doc[] = +static const char byref_doc[] = "byref(C instance[, offset=0]) -> byref-object\n" "Return a pointer lookalike to a C instance, only usable\n" "as function argument"; @@ -1527,7 +1527,7 @@ byref(PyObject *self, PyObject *args) return (PyObject *)parg; } -static char addressof_doc[] = +static const char addressof_doc[] = "addressof(C instance) -> integer\n" "Return the address of the C instance internal buffer"; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 759b731..a9c406f 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -6,7 +6,7 @@ /* Release Number */ -static char *PyCursesVersion = "2.1"; +static const char PyCursesVersion[] = "2.1"; /* Includes */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 55988c5..e8b7ae8 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -184,12 +184,12 @@ divide_nearest(PyObject *m, PyObject *n) * and the number of days before that month in the same year. These * are correct for non-leap years only. */ -static int _days_in_month[] = { +static const int _days_in_month[] = { 0, /* unused; this vector uses 1-based indexing */ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; -static int _days_before_month[] = { +static const int _days_before_month[] = { 0, /* unused; this vector uses 1-based indexing */ 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 }; @@ -1009,10 +1009,10 @@ append_keyword_tzinfo(PyObject *repr, PyObject *tzinfo) static PyObject * format_ctime(PyDateTime_Date *date, int hours, int minutes, int seconds) { - static const char *DayNames[] = { + static const char * const DayNames[] = { "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" }; - static const char *MonthNames[] = { + static const char * const MonthNames[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; @@ -2307,7 +2307,7 @@ static PyMethodDef delta_methods[] = { {NULL, NULL}, }; -static char delta_doc[] = +static const char delta_doc[] = PyDoc_STR("Difference between two datetime values."); static PyNumberMethods delta_as_number = { @@ -2886,7 +2886,7 @@ static PyMethodDef date_methods[] = { {NULL, NULL} }; -static char date_doc[] = +static const char date_doc[] = PyDoc_STR("date(year, month, day) --> date object"); static PyNumberMethods date_as_number = { @@ -3155,7 +3155,7 @@ static PyMethodDef tzinfo_methods[] = { {NULL, NULL} }; -static char tzinfo_doc[] = +static const char tzinfo_doc[] = PyDoc_STR("Abstract base class for time zone info objects."); static PyTypeObject PyDateTime_TZInfoType = { @@ -3387,7 +3387,7 @@ static PyMethodDef timezone_methods[] = { {NULL, NULL} }; -static char timezone_doc[] = +static const char timezone_doc[] = PyDoc_STR("Fixed offset from UTC implementation of tzinfo."); static PyTypeObject PyDateTime_TimeZoneType = { @@ -3877,7 +3877,7 @@ static PyMethodDef time_methods[] = { {NULL, NULL} }; -static char time_doc[] = +static const char time_doc[] = PyDoc_STR("time([hour[, minute[, second[, microsecond[, tzinfo]]]]]) --> a time object\n\ \n\ All arguments are optional. tzinfo may be None, or an instance of\n\ @@ -5065,7 +5065,7 @@ static PyMethodDef datetime_methods[] = { {NULL, NULL} }; -static char datetime_doc[] = +static const char datetime_doc[] = PyDoc_STR("datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])\n\ \n\ The year, month and day arguments are required. tzinfo may be None, or an\n\ diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 02899e4..5e7ec1a 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -14,16 +14,16 @@ */ #if defined(HAVE_NDBM_H) #include -static char *which_dbm = "GNU gdbm"; /* EMX port of GDBM */ +static const char which_dbm[] = "GNU gdbm"; /* EMX port of GDBM */ #elif defined(HAVE_GDBM_NDBM_H) #include -static char *which_dbm = "GNU gdbm"; +static const char which_dbm[] = "GNU gdbm"; #elif defined(HAVE_GDBM_DASH_NDBM_H) #include -static char *which_dbm = "GNU gdbm"; +static const char which_dbm[] = "GNU gdbm"; #elif defined(HAVE_BERKDB_H) #include -static char *which_dbm = "Berkeley DB"; +static const char which_dbm[] = "Berkeley DB"; #else #error "No ndbm.h available!" #endif diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index f070a14..bf7b036 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -615,7 +615,7 @@ dbmopen_impl(PyModuleDef *module, const char *name, const char *flags, return newdbmobject(name, iflags, mode); } -static char dbmmodule_open_flags[] = "rwcn" +static const char dbmmodule_open_flags[] = "rwcn" #ifdef GDBM_FAST "f" #endif diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index b232b02..d018623 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -772,7 +772,7 @@ typedef struct { encodefunc_t encodefunc; } encodefuncentry; -static encodefuncentry encodefuncs[] = { +static const encodefuncentry encodefuncs[] = { {"ascii", (encodefunc_t) ascii_encode}, {"iso8859-1", (encodefunc_t) latin1_encode}, {"utf-8", (encodefunc_t) utf8_encode}, @@ -1022,7 +1022,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, goto error; } else if (PyUnicode_Check(res)) { - encodefuncentry *e = encodefuncs; + const encodefuncentry *e = encodefuncs; while (e->name != NULL) { if (!PyUnicode_CompareWithASCIIString(res, e->name)) { self->encodefunc = e->encodefunc; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 95ad4a4..a85d905 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -99,7 +99,7 @@ static PY_UINT32_T genrand_int32(RandomObject *self) { PY_UINT32_T y; - static PY_UINT32_T mag01[2]={0x0U, MATRIX_A}; + static const PY_UINT32_T mag01[2] = {0x0U, MATRIX_A}; /* mag01[x] = x * MATRIX_A for x=0,1 */ PY_UINT32_T *mt; diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7018f9f..75ed1f8 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1622,7 +1622,7 @@ pysqlite_connection_exit(pysqlite_Connection* self, PyObject* args) Py_RETURN_FALSE; } -static char connection_doc[] = +static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); static PyGetSetDef connection_getset[] = { diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index d909738..1c240d6 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -27,7 +27,7 @@ PyObject* pysqlite_cursor_iternext(pysqlite_Cursor* self); -static char* errmsg_fetch_across_rollback = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from."; +static const char errmsg_fetch_across_rollback[] = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from."; static pysqlite_StatementKind detect_statement_type(const char* statement) { @@ -1050,7 +1050,7 @@ static struct PyMemberDef cursor_members[] = {NULL} }; -static char cursor_doc[] = +static const char cursor_doc[] = PyDoc_STR("SQLite database cursor class."); PyTypeObject pysqlite_CursorType = { diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 7a7e860..ff2e3a5 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -261,13 +261,13 @@ static PyMethodDef module_methods[] = { }; struct _IntConstantPair { - char* constant_name; + const char *constant_name; int constant_value; }; typedef struct _IntConstantPair IntConstantPair; -static IntConstantPair _int_constants[] = { +static const IntConstantPair _int_constants[] = { {"PARSE_DECLTYPES", PARSE_DECLTYPES}, {"PARSE_COLNAMES", PARSE_COLNAMES}, diff --git a/Modules/_sre.c b/Modules/_sre.c index 919a069..f597a70 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -35,7 +35,7 @@ * other compatibility work. */ -static char copyright[] = +static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #define PY_SSIZE_T_CLEAN diff --git a/Modules/_struct.c b/Modules/_struct.c index b61f9f6..b18c71d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -723,7 +723,7 @@ np_void_p(char *p, PyObject *v, const formatdef *f) return 0; } -static formatdef native_table[] = { +static const formatdef native_table[] = { {'x', sizeof(char), 0, NULL}, {'b', sizeof(char), 0, nu_byte, np_byte}, {'B', sizeof(char), 0, nu_ubyte, np_ubyte}, @@ -2280,7 +2280,7 @@ PyInit__struct(void) /* Check endian and swap in faster functions */ { - formatdef *native = native_table; + const formatdef *native = native_table; formatdef *other, *ptr; #if PY_LITTLE_ENDIAN other = lilendian_table; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 43db8a8..13d3ccc 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -13,7 +13,7 @@ PyObject *Struct = NULL; PyObject *calcsize = NULL; /* cache simple format string */ -static const char *simple_fmt = "B"; +static const char simple_fmt[] = "B"; PyObject *simple_format = NULL; #define SIMPLE_FORMAT(fmt) (fmt == NULL || strcmp(fmt, "B") == 0) #define FIX_FORMAT(fmt) (fmt == NULL ? "B" : fmt) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a896af0..cc3f52d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -887,7 +887,7 @@ static PyObject * getargs_keywords(PyObject *self, PyObject *args, PyObject *kwargs) { static char *keywords[] = {"arg1","arg2","arg3","arg4","arg5", NULL}; - static char *fmt="(ii)i|(i(ii))(iii)i"; + static const char fmt[] = "(ii)i|(i(ii))(iii)i"; int int_args[10]={-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, fmt, keywords, @@ -3769,7 +3769,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_LONGLONG", "T_ULONGLONG", #endif NULL}; - static char *fmt = "|bbBhHiIlknfds#" + static const char fmt[] = "|bbBhHiIlknfds#" #ifdef HAVE_LONG_LONG "LK" #endif diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 6af75a4..1b0a282 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -31,7 +31,7 @@ struct arraydescr { int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); - char *formats; + const char *formats; int is_integer_type; int is_signed; }; @@ -40,7 +40,7 @@ typedef struct arrayobject { PyObject_VAR_HEAD char *ob_item; Py_ssize_t allocated; - struct arraydescr *ob_descr; + const struct arraydescr *ob_descr; PyObject *weakreflist; /* List of weak references */ int ob_exports; /* Number of exported buffers */ } arrayobject; @@ -511,7 +511,7 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) * Don't forget to update typecode_to_mformat_code() if you add a new * typecode. */ -static struct arraydescr descriptors[] = { +static const struct arraydescr descriptors[] = { {'b', 1, b_getitem, b_setitem, "b", 1, 1}, {'B', 1, BB_getitem, BB_setitem, "B", 1, 0}, {'u', sizeof(Py_UNICODE), u_getitem, u_setitem, "u", 0, 0}, @@ -539,7 +539,7 @@ class array.array "arrayobject *" "&Arraytype" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=ad43d37e942a8854]*/ static PyObject * -newarrayobject(PyTypeObject *type, Py_ssize_t size, struct arraydescr *descr) +newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) { arrayobject *op; size_t nbytes; @@ -1946,7 +1946,7 @@ array__array_reconstructor_impl(PyModuleDef *module, PyTypeObject *arraytype, { PyObject *converted_items; PyObject *result; - struct arraydescr *descr; + const struct arraydescr *descr; if (!PyType_Check(arraytype)) { PyErr_Format(PyExc_TypeError, @@ -2084,7 +2084,7 @@ array__array_reconstructor_impl(PyModuleDef *module, PyTypeObject *arraytype, Py_ssize_t itemcount = Py_SIZE(items) / mf_descr.size; const unsigned char *memstr = (unsigned char *)PyBytes_AS_STRING(items); - struct arraydescr *descr; + const struct arraydescr *descr; /* If possible, try to pack array's items using a data type * that fits better. This may result in an array with narrower @@ -2554,7 +2554,7 @@ array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) view->format = NULL; view->internal = NULL; if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { - view->format = self->ob_descr->formats; + view->format = (char *)self->ob_descr->formats; #ifdef Py_UNICODE_WIDE if (self->ob_descr->typecode == 'u') { view->format = "w"; @@ -2595,7 +2595,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { int c; PyObject *initial = NULL, *it = NULL; - struct arraydescr *descr; + const struct arraydescr *descr; if (type == &Arraytype && !_PyArg_NoKeywords("array.array()", kwds)) return NULL; @@ -2987,7 +2987,7 @@ array_modexec(PyObject *m) char buffer[Py_ARRAY_LENGTH(descriptors)], *p; PyObject *typecodes; Py_ssize_t size = 0; - struct arraydescr *descr; + const struct arraydescr *descr; if (PyType_Ready(&Arraytype) < 0) return -1; diff --git a/Modules/audioop.c b/Modules/audioop.c index 3b05aec..d97a369 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -51,13 +51,15 @@ fbound(double val, double minval, double maxval) #define SEG_SHIFT (4) /* Left shift for segment number. */ #define SEG_MASK (0x70) /* Segment field mask. */ -static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF, - 0x1FF, 0x3FF, 0x7FF, 0xFFF}; -static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF, - 0x3FF, 0x7FF, 0xFFF, 0x1FFF}; +static const PyInt16 seg_aend[8] = { + 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF +}; +static const PyInt16 seg_uend[8] = { + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF +}; static PyInt16 -search(PyInt16 val, PyInt16 *table, int size) +search(PyInt16 val, const PyInt16 *table, int size) { int i; @@ -70,7 +72,7 @@ search(PyInt16 val, PyInt16 *table, int size) #define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc]) #define st_alaw2linear16(uc) (_st_alaw2linear16[uc]) -static PyInt16 _st_ulaw2linear16[256] = { +static const PyInt16 _st_ulaw2linear16[256] = { -32124, -31100, -30076, -29052, -28028, -27004, -25980, -24956, -23932, -22908, -21884, -20860, -19836, -18812, -17788, -16764, -15996, -15484, -14972, -14460, -13948, @@ -176,7 +178,7 @@ st_14linear2ulaw(PyInt16 pcm_val) /* 2's complement (14-bit range) */ } -static PyInt16 _st_alaw2linear16[256] = { +static const PyInt16 _st_alaw2linear16[256] = { -5504, -5248, -6016, -5760, -4480, -4224, -4992, -4736, -7552, -7296, -8064, -7808, -6528, -6272, -7040, -6784, -2752, -2624, -3008, -2880, -2240, @@ -270,12 +272,12 @@ st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */ /* End of code taken from sox */ /* Intel ADPCM step variation table */ -static int indexTable[16] = { +static const int indexTable[16] = { -1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8, }; -static int stepsizeTable[89] = { +static const int stepsizeTable[89] = { 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, diff --git a/Modules/binascii.c b/Modules/binascii.c index ccd81fa..9df48da 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -74,7 +74,7 @@ static PyObject *Incomplete; #define SKIP 0x7E #define FAIL 0x7D -static unsigned char table_a2b_hqx[256] = { +static const unsigned char table_a2b_hqx[256] = { /* ^@ ^A ^B ^C ^D ^E ^F ^G */ /* 0*/ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, /* \b \t \n ^K ^L \r ^N ^O */ @@ -125,10 +125,10 @@ static unsigned char table_a2b_hqx[256] = { FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, }; -static unsigned char table_b2a_hqx[] = +static const unsigned char table_b2a_hqx[] = "!\"#$%&'()*+,-012345689@ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr"; -static char table_a2b_base64[] = { +static const char table_a2b_base64[] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, @@ -144,12 +144,12 @@ static char table_a2b_base64[] = { /* Max binary chunk size; limited only by available memory */ #define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) -static unsigned char table_b2a_base64[] = +static const unsigned char table_b2a_base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static unsigned short crctab_hqx[256] = { +static const unsigned short crctab_hqx[256] = { 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, @@ -977,7 +977,7 @@ binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) using byte-swap instructions. ********************************************************************/ -static unsigned int crc_32_tab[256] = { +static const unsigned int crc_32_tab[256] = { 0x00000000U, 0x77073096U, 0xee0e612cU, 0x990951baU, 0x076dc419U, 0x706af48fU, 0xe963a535U, 0x9e6495a3U, 0x0edb8832U, 0x79dcb8a4U, 0xe0d5e91eU, 0x97d2d988U, 0x09b64c2bU, 0x7eb17cbdU, 0xe7b82d07U, @@ -1201,7 +1201,7 @@ binascii_unhexlify_impl(PyModuleDef *module, Py_buffer *hexstr) return binascii_a2b_hex_impl(module, hexstr); } -static int table_hex[128] = { +static const int table_hex[128] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index e2a2edf..33d7078 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -136,7 +136,7 @@ static int get_addr(const char *, int, struct addrinfo **, struct addrinfo *, int); static int str_isnumber(const char *); -static char *ai_errlist[] = { +static const char * const ai_errlist[] = { "success.", "address family for hostname not supported.", /* EAI_ADDRFAMILY */ "temporary failure in name resolution.", /* EAI_AGAIN */ diff --git a/Modules/getpath.c b/Modules/getpath.c index 03d292c..30a0e99 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -477,8 +477,8 @@ calculate_path(void) { extern wchar_t *Py_GetProgramName(void); - static wchar_t delimiter[2] = {DELIM, '\0'}; - static wchar_t separator[2] = {SEP, '\0'}; + static const wchar_t delimiter[2] = {DELIM, '\0'}; + static const wchar_t separator[2] = {SEP, '\0'}; char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */ wchar_t *rtpypath = NULL; wchar_t *home = Py_GetPythonHome(); diff --git a/Modules/main.c b/Modules/main.c index 0fbdb69..4358cc8 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -42,11 +42,11 @@ static int orig_argc; #define PROGRAM_OPTS BASE_OPTS /* Short usage message (with %s for argv0) */ -static char *usage_line = +static const char usage_line[] = "usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; /* Long usage message, split into parts < 512 bytes */ -static char *usage_1 = "\ +static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ @@ -56,7 +56,7 @@ Options and arguments (and corresponding environment variables):\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ -h : print this help message and exit (also --help)\n\ "; -static char *usage_2 = "\ +static const char usage_2[] = "\ -i : inspect interactively after running script; forces a prompt even\n\ if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ -I : isolate Python from the user's environment (implies -E and -s)\n\ @@ -67,7 +67,7 @@ static char *usage_2 = "\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ "; -static char *usage_3 = "\ +static const char usage_3[] = "\ -u : unbuffered binary stdout and stderr, stdin always buffered;\n\ also PYTHONUNBUFFERED=x\n\ see man page for details on internal buffering relating to '-u'\n\ @@ -79,7 +79,7 @@ static char *usage_3 = "\ -x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ -X opt : set implementation-specific option\n\ "; -static char *usage_4 = "\ +static const char usage_4[] = "\ file : program read from script file\n\ - : program read from stdin (default; interactive mode if a tty)\n\ arg ...: arguments passed to program in sys.argv[1:]\n\n\ @@ -88,14 +88,14 @@ PYTHONSTARTUP: file executed on interactive startup (no default)\n\ PYTHONPATH : '%c'-separated list of directories prefixed to the\n\ default module search path. The result is sys.path.\n\ "; -static char *usage_5 = +static const char usage_5[] = "PYTHONHOME : alternate directory (or %c).\n" " The default module search path uses %s.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" "PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n\ "; -static char *usage_6 = "\ +static const char usage_6[] = "\ PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n\ to seed the hashes of str, bytes and datetime objects. It can also be\n\ set to an integer in the range [0,4294967295] to get hash values with a\n\ diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 6471b8e..4c5f2ef 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -53,7 +53,7 @@ extern grammar _PyParser_Grammar; /* From graminit.c */ /* String constants used to initialize module attributes. * */ -static char parser_copyright_string[] = +static const char parser_copyright_string[] = "Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\ University, Blacksburg, Virginia, USA, and Fred L. Drake, Jr., Reston,\n\ Virginia, USA. Portions copyright 1991-1995 by Stichting Mathematisch\n\ @@ -63,7 +63,7 @@ Centrum, Amsterdam, The Netherlands."; PyDoc_STRVAR(parser_doc_string, "This is an interface to Python's internal parser."); -static char parser_version_string[] = "0.5"; +static const char parser_version_string[] = "0.5"; typedef PyObject* (*SeqMaker) (Py_ssize_t length); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7ec1f47..7a2b661 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9479,7 +9479,7 @@ os__getdiskusage_impl(PyModuleDef *module, Py_UNICODE *path) * sufficiently pervasive that it's not worth the loss of readability. */ struct constdef { - char *name; + const char *name; int value; }; @@ -10822,7 +10822,7 @@ os_getxattr_impl(PyModuleDef *module, path_t *path, path_t *attribute, for (i = 0; ; i++) { void *ptr; ssize_t result; - static Py_ssize_t buffer_sizes[] = {128, XATTR_SIZE_MAX, 0}; + static const Py_ssize_t buffer_sizes[] = {128, XATTR_SIZE_MAX, 0}; Py_ssize_t buffer_size = buffer_sizes[i]; if (!buffer_size) { path_error(path); @@ -10988,7 +10988,7 @@ os_listxattr_impl(PyModuleDef *module, path_t *path, int follow_symlinks) for (i = 0; ; i++) { char *start, *trace, *end; ssize_t length; - static Py_ssize_t buffer_sizes[] = { 256, XATTR_LIST_MAX, 0 }; + static const Py_ssize_t buffer_sizes[] = { 256, XATTR_LIST_MAX, 0 }; Py_ssize_t buffer_size = buffer_sizes[i]; if (!buffer_size) { /* ERANGE */ @@ -12821,7 +12821,7 @@ static struct PyModuleDef posixmodule = { }; -static char *have_functions[] = { +static const char * const have_functions[] = { #ifdef HAVE_FACCESSAT "HAVE_FACCESSAT", @@ -12956,7 +12956,7 @@ INITFUNC(void) { PyObject *m, *v; PyObject *list; - char **trace; + const char * const *trace; #if defined(HAVE_SYMLINK) && defined(MS_WINDOWS) win32_can_symlink = enable_symlink(); diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index b3ac807..70f2db0 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1842,7 +1842,7 @@ kqueue_event_init(kqueue_event_Object *self, PyObject *args, PyObject *kwds) PyObject *pfd; static char *kwlist[] = {"ident", "filter", "flags", "fflags", "data", "udata", NULL}; - static char *fmt = "O|hHI" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; + static const char fmt[] = "O|hHI" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; EV_SET(&(self->e), 0, EVFILT_READ, EV_ADD, 0, 0, 0); /* defaults */ diff --git a/Modules/timemodule.c b/Modules/timemodule.c index d2caacd..31f0ce5 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -732,10 +732,10 @@ _asctime(struct tm *timeptr) { /* Inspired by Open Group reference implementation available at * http://pubs.opengroup.org/onlinepubs/009695399/functions/asctime.html */ - static char wday_name[7][4] = { + static const char wday_name[7][4] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; - static char mon_name[12][4] = { + static const char mon_name[12][4] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index fe4e908..7d518fa 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -884,7 +884,7 @@ _gethash(const char *s, int len, int scale) return h; } -static char *hangul_syllables[][3] = { +static const char * const hangul_syllables[][3] = { { "G", "A", "" }, { "GG", "AE", "G" }, { "N", "YA", "GG" }, @@ -1057,7 +1057,7 @@ find_syllable(const char *str, int *len, int *pos, int count, int column) int i, len1; *len = -1; for (i = 0; i < count; i++) { - char *s = hangul_syllables[i][column]; + const char *s = hangul_syllables[i][column]; len1 = Py_SAFE_DOWNCAST(strlen(s), size_t, int); if (len1 <= *len) continue; diff --git a/Objects/object.c b/Objects/object.c index e1718ea..417a97d 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -644,7 +644,7 @@ PyObject_Bytes(PyObject *v) /* Map rich comparison operators to their swapped version, e.g. LT <--> GT */ int _Py_SwappedOp[] = {Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE}; -static char *opstrings[] = {"<", "<=", "==", "!=", ">", ">="}; +static const char * const opstrings[] = {"<", "<=", "==", "!=", ">", ">="}; /* Perform a rich comparison, raising TypeError when the requested comparison operator is not supported. */ diff --git a/Objects/structseq.c b/Objects/structseq.c index 7209738..e315cba 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -4,9 +4,9 @@ #include "Python.h" #include "structmember.h" -static char visible_length_key[] = "n_sequence_fields"; -static char real_length_key[] = "n_fields"; -static char unnamed_fields_key[] = "n_unnamed_fields"; +static const char visible_length_key[] = "n_sequence_fields"; +static const char real_length_key[] = "n_fields"; +static const char unnamed_fields_key[] = "n_unnamed_fields"; /* Fields with this name have only a field index, not a field name. They are only allowed for indices < n_visible_fields. */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 341e18c..b4d23b2 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2690,7 +2690,7 @@ error: return NULL; } -static short slotoffsets[] = { +static const short slotoffsets[] = { -1, /* invalid slot */ #include "typeslots.inc" }; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a985d6f..fef184a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -272,7 +272,7 @@ raise_encode_exception(PyObject **exceptionObject, const char *reason); /* Same for linebreaks */ -static unsigned char ascii_linebreak[] = { +static const unsigned char ascii_linebreak[] = { 0, 0, 0, 0, 0, 0, 0, 0, /* 0x000A, * LINE FEED */ /* 0x000B, * LINE TABULATION */ @@ -4135,7 +4135,7 @@ unicode_decode_call_errorhandler_wchar( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, PyObject **output, Py_ssize_t *outpos) { - static char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -4243,7 +4243,7 @@ unicode_decode_call_errorhandler_writer( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, _PyUnicodeWriter *writer /* PyObject **output, Py_ssize_t *outpos */) { - static char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -6560,7 +6560,7 @@ unicode_encode_call_errorhandler(const char *errors, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { - static char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; + static const char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; Py_ssize_t len; PyObject *restuple; PyObject *resunicode; @@ -8572,7 +8572,7 @@ unicode_translate_call_errorhandler(const char *errors, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { - static char *argparse = "O!n;translating error handler must return (str, int) tuple"; + static const char *argparse = "O!n;translating error handler must return (str, int) tuple"; Py_ssize_t i_newpos; PyObject *restuple; @@ -12156,7 +12156,7 @@ unicode_lower(PyObject *self) #define BOTHSTRIP 2 /* Arrays indexed by above */ -static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; +static const char * const stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; #define STRIPNAME(i) (stripformat[i]+3) diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 629dee5..ebe9495 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -161,10 +161,10 @@ PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD #if 0 -static char with_msg[] = +static const char with_msg[] = "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n"; -static char as_msg[] = +static const char as_msg[] = "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n"; static void diff --git a/Parser/pgen.c b/Parser/pgen.c index f3031ae..6ecb311 100644 --- a/Parser/pgen.c +++ b/Parser/pgen.c @@ -134,7 +134,7 @@ addnfa(nfagrammar *gr, char *name) #ifdef Py_DEBUG -static char REQNFMT[] = "metacompile: less than %d children\n"; +static const char REQNFMT[] = "metacompile: less than %d children\n"; #define REQN(i, count) do { \ if (i < count) { \ diff --git a/Python/ast.c b/Python/ast.c index 77ebc83..328ee5d 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -870,7 +870,7 @@ get_operator(const node *n) } } -static const char* FORBIDDEN[] = { +static const char * const FORBIDDEN[] = { "None", "True", "False", @@ -887,7 +887,7 @@ forbidden_name(struct compiling *c, identifier name, const node *n, return 1; } if (full_checks) { - const char **p; + const char * const *p; for (p = FORBIDDEN; *p; p++) { if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { ast_error(c, n, "assignment to keyword"); diff --git a/Python/dtoa.c b/Python/dtoa.c index 3da546e..3121cd6 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -747,7 +747,7 @@ pow5mult(Bigint *b, int k) { Bigint *b1, *p5, *p51; int i; - static int p05[3] = { 5, 25, 125 }; + static const int p05[3] = { 5, 25, 125 }; if ((i = k & 3)) { b = multadd(b, p05[i-1], 0); @@ -803,7 +803,7 @@ pow5mult(Bigint *b, int k) { Bigint *b1, *p5, *p51; int i; - static int p05[3] = { 5, 25, 125 }; + static const int p05[3] = { 5, 25, 125 }; if ((i = k & 3)) { b = multadd(b, p05[i-1], 0); diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index 056bb76..a428fbe 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -656,7 +656,7 @@ fill_number(_PyUnicodeWriter *writer, const NumberFieldWidths *spec, return 0; } -static char no_grouping[1] = {CHAR_MAX}; +static const char no_grouping[1] = {CHAR_MAX}; /* Find the decimal point character(s?), thousands_separator(s?), and grouping description, either for the current locale if type is diff --git a/Python/import.c b/Python/import.c index edf030d..8d7bfe9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -320,7 +320,7 @@ PyImport_GetModuleDict(void) /* List of names to clear in sys */ -static char* sys_deletes[] = { +static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", "path_hooks", "path_importer_cache", "meta_path", @@ -330,7 +330,7 @@ static char* sys_deletes[] = { NULL }; -static char* sys_files[] = { +static const char * const sys_files[] = { "stdin", "__stdin__", "stdout", "__stdout__", "stderr", "__stderr__", @@ -347,7 +347,7 @@ PyImport_Cleanup(void) PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; PyObject *weaklist = NULL; - char **p; + const char * const *p; if (modules == NULL) return; /* Already done */ diff --git a/Python/importdl.c b/Python/importdl.c index 1aa585d..ac03289 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -23,8 +23,8 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, const char *pathname, FILE *fp); #endif -static const char *ascii_only_prefix = "PyInit"; -static const char *nonascii_prefix = "PyInitU"; +static const char * const ascii_only_prefix = "PyInit"; +static const char * const nonascii_prefix = "PyInitU"; /* Get the variable part of a module's export symbol name. * Returns a bytes instance. For non-ASCII-named modules, the name is diff --git a/Python/mystrtoul.c b/Python/mystrtoul.c index 98429d4..a85790e 100644 --- a/Python/mystrtoul.c +++ b/Python/mystrtoul.c @@ -17,7 +17,7 @@ * smallmax[base] is the largest unsigned long i such that * i * base doesn't overflow unsigned long. */ -static unsigned long smallmax[] = { +static const unsigned long smallmax[] = { 0, /* bases 0 and 1 are invalid */ 0, ULONG_MAX / 2, @@ -62,14 +62,14 @@ static unsigned long smallmax[] = { * Note that this is pessimistic if sizeof(long) > 4. */ #if SIZEOF_LONG == 4 -static int digitlimit[] = { +static const int digitlimit[] = { 0, 0, 32, 20, 16, 13, 12, 11, 10, 10, /* 0 - 9 */ 9, 9, 8, 8, 8, 8, 8, 7, 7, 7, /* 10 - 19 */ 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, /* 20 - 29 */ 6, 6, 6, 6, 6, 6, 6}; /* 30 - 36 */ #elif SIZEOF_LONG == 8 /* [int(math.floor(math.log(2**64, i))) for i in range(2, 37)] */ -static int digitlimit[] = { +static const int digitlimit[] = { 0, 0, 64, 40, 32, 27, 24, 22, 21, 20, /* 0 - 9 */ 19, 18, 17, 17, 16, 16, 16, 15, 15, 15, /* 10 - 19 */ 14, 14, 14, 14, 13, 13, 13, 13, 13, 13, /* 20 - 29 */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 209c908..5f3af92 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -881,12 +881,12 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val, #define OFS_E 2 /* The lengths of these are known to the code below, so don't change them */ -static char *lc_float_strings[] = { +static const char * const lc_float_strings[] = { "inf", "nan", "e", }; -static char *uc_float_strings[] = { +static const char * const uc_float_strings[] = { "INF", "NAN", "E", @@ -925,7 +925,8 @@ static char * format_float_short(double d, char format_code, int mode, int precision, int always_add_sign, int add_dot_0_if_integer, - int use_alt_formatting, char **float_strings, int *type) + int use_alt_formatting, const char * const *float_strings, + int *type) { char *buf = NULL; char *p = NULL; @@ -1176,7 +1177,7 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val, int flags, int *type) { - char **float_strings = lc_float_strings; + const char * const *float_strings = lc_float_strings; int mode; /* Validate format_code, and map upper and lower case. Compute the diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 1a5dab5..cfe197b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -785,11 +785,11 @@ print_exception(PyObject *f, PyObject *value) PyErr_Clear(); } -static const char *cause_message = +static const char cause_message[] = "\nThe above exception was the direct cause " "of the following exception:\n\n"; -static const char *context_message = +static const char context_message[] = "\nDuring handling of the above exception, " "another exception occurred:\n\n"; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e0aa233..f784f75 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -346,8 +346,10 @@ static PyObject *whatstrings[7] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL}; static int trace_init(void) { - static char *whatnames[7] = {"call", "exception", "line", "return", - "c_call", "c_exception", "c_return"}; + static const char * const whatnames[7] = { + "call", "exception", "line", "return", + "c_call", "c_exception", "c_return" + }; PyObject *name; int i; for (i = 0; i < 7; ++i) { -- cgit v0.12 From ef1585eb9a488ae8ce3ff057f43a7048b941cc1c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 25 Dec 2015 20:01:53 +0200 Subject: Issue #25923: Added more const qualifiers to signatures of static and private functions. --- Include/bytes_methods.h | 6 ++--- Include/pythonrun.h | 8 +++---- Misc/coverity_model.c | 2 +- Modules/_ctypes/_ctypes.c | 4 ++-- Modules/_ctypes/callbacks.c | 2 +- Modules/_ctypes/callproc.c | 2 +- Modules/_ctypes/ctypes.h | 2 +- Modules/_curses_panel.c | 2 +- Modules/_datetimemodule.c | 2 +- Modules/_io/_iomodule.h | 2 +- Modules/_io/bufferedio.c | 2 +- Modules/_io/fileio.c | 4 ++-- Modules/_io/textio.c | 20 ++++++++-------- Modules/_json.c | 4 ++-- Modules/_localemodule.c | 2 +- Modules/_pickle.c | 2 +- Modules/_posixsubprocess.c | 2 +- Modules/_scproxy.c | 2 +- Modules/_sre.c | 2 +- Modules/_ssl.c | 4 ++-- Modules/_struct.c | 6 ++--- Modules/_testmultiphase.c | 2 +- Modules/_tkinter.c | 8 +++---- Modules/_winapi.c | 4 ++-- Modules/binascii.c | 44 +++++++++++++++++++++------------- Modules/gcmodule.c | 2 +- Modules/getaddrinfo.c | 2 +- Modules/main.c | 2 +- Modules/mathmodule.c | 4 ++-- Modules/parsermodule.c | 10 ++++---- Modules/posixmodule.c | 43 +++++++++++++++++++-------------- Modules/pyexpat.c | 6 ++--- Modules/socketmodule.c | 4 ++-- Modules/timemodule.c | 2 +- Modules/xxlimited.c | 2 +- Modules/xxmodule.c | 2 +- Modules/zipimport.c | 2 +- Modules/zlibmodule.c | 2 +- Objects/bytearrayobject.c | 8 +++---- Objects/bytes_methods.c | 6 ++--- Objects/bytesobject.c | 2 +- Objects/descrobject.c | 2 +- Objects/dictobject.c | 2 +- Objects/memoryobject.c | 4 ++-- Objects/typeobject.c | 10 ++++---- Objects/unicodeobject.c | 6 ++--- Objects/weakrefobject.c | 2 +- Parser/pgen.c | 4 ++-- Parser/pgenmain.c | 4 ++-- Parser/tokenizer.c | 6 ++--- Python/_warnings.c | 2 +- Python/ceval.c | 4 ++-- Python/dtoa.c | 2 +- Python/dynload_win.c | 2 +- Python/getargs.c | 58 ++++++++++++++++++++++----------------------- Python/marshal.c | 34 +++++++++++++------------- Python/modsupport.c | 4 ++-- Python/pylifecycle.c | 4 ++-- Python/pythonrun.c | 8 +++---- Python/symtable.c | 4 ++-- 60 files changed, 210 insertions(+), 191 deletions(-) diff --git a/Include/bytes_methods.h b/Include/bytes_methods.h index 11d5f42..dbc033c 100644 --- a/Include/bytes_methods.h +++ b/Include/bytes_methods.h @@ -17,9 +17,9 @@ extern PyObject* _Py_bytes_istitle(const char *cptr, Py_ssize_t len); /* These store their len sized answer in the given preallocated *result arg. */ extern void _Py_bytes_lower(char *result, const char *cptr, Py_ssize_t len); extern void _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len); -extern void _Py_bytes_title(char *result, char *s, Py_ssize_t len); -extern void _Py_bytes_capitalize(char *result, char *s, Py_ssize_t len); -extern void _Py_bytes_swapcase(char *result, char *s, Py_ssize_t len); +extern void _Py_bytes_title(char *result, const char *s, Py_ssize_t len); +extern void _Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len); +extern void _Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len); /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); diff --git a/Include/pythonrun.h b/Include/pythonrun.h index f92148d..3b93a80 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -66,8 +66,8 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile( const char *filename, /* decoded from the filesystem encoding */ const char* enc, int start, - char *ps1, - char *ps2, + const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena); @@ -76,8 +76,8 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject( PyObject *filename, const char* enc, int start, - char *ps1, - char *ps2, + const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena); diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c index 493e7c1..f776d76 100644 --- a/Misc/coverity_model.c +++ b/Misc/coverity_model.c @@ -94,7 +94,7 @@ wchar_t *Py_DecodeLocale(const char* arg, size_t *size) } /* Parser/pgenmain.c */ -grammar *getgrammar(char *filename) +grammar *getgrammar(const char *filename) { grammar *g; __coverity_tainted_data_sink__(filename); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index fccc8ba..ac5e5fa 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3195,7 +3195,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) } static int -_get_name(PyObject *obj, char **pname) +_get_name(PyObject *obj, const char **pname) { #ifdef MS_WIN32 if (PyLong_Check(obj)) { @@ -3223,7 +3223,7 @@ _get_name(PyObject *obj, char **pname) static PyObject * PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) { - char *name; + const char *name; int (* address)(void); PyObject *ftuple; PyObject *dll; diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 7cd6164..00e8e66 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -77,7 +77,7 @@ PyTypeObject PyCThunk_Type = { /**************************************************************/ static void -PrintError(char *msg, ...) +PrintError(const char *msg, ...) { char buf[512]; PyObject *f = PySys_GetObject("stderr"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 68981fe..870a0d4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -928,7 +928,7 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) * Raise a new exception 'exc_class', adding additional text to the original * exception string. */ -void _ctypes_extend_error(PyObject *exc_class, char *fmt, ...) +void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) { va_list vargs; PyObject *tp, *v, *tb, *s, *cls_str, *msg_str; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 0d3f724..b06ba8a 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -327,7 +327,7 @@ extern int PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr); -extern void _ctypes_extend_error(PyObject *exc_class, char *fmt, ...); +extern void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...); struct basespec { CDataObject *base; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index a9c406f..26bf094 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -56,7 +56,7 @@ static struct PyModuleDef _curses_panelmodule; */ static PyObject * -PyCursesCheckERR(int code, char *fname) +PyCursesCheckERR(int code, const char *fname) { if (code != ERR) { Py_INCREF(Py_None); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index e8b7ae8..3e5e195 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -873,7 +873,7 @@ get_tzinfo_member(PyObject *self) * this returns NULL. Else result is returned. */ static PyObject * -call_tzinfo_method(PyObject *tzinfo, char *name, PyObject *tzinfoarg) +call_tzinfo_method(PyObject *tzinfo, const char *name, PyObject *tzinfoarg) { PyObject *offset; diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 0c6eae2..3c48ff3 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -60,7 +60,7 @@ extern PyObject *_PyIncrementalNewlineDecoder_decode( * Otherwise, the line ending is specified by readnl, a str object */ extern Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - int kind, char *start, char *end, Py_ssize_t *consumed); + int kind, const char *start, const char *end, Py_ssize_t *consumed); /* Return 1 if an EnvironmentError with errno == EINTR is set (and then clears the error indicator), 0 otherwise. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 6bb2200..16f8cdf 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -659,7 +659,7 @@ _bufferedreader_raw_read(buffered *self, char *start, Py_ssize_t len); /* Sets the current error to BlockingIOError */ static void -_set_BlockingIOError(char *msg, Py_ssize_t written) +_set_BlockingIOError(const char *msg, Py_ssize_t written) { PyObject *err; PyErr_Clear(); diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index dbd604a..8bf3922 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -540,7 +540,7 @@ err_closed(void) } static PyObject * -err_mode(char *action) +err_mode(const char *action) { _PyIO_State *state = IO_STATE(); if (state != NULL) @@ -1043,7 +1043,7 @@ _io_FileIO_truncate_impl(fileio *self, PyObject *posobj) } #endif /* HAVE_FTRUNCATE */ -static char * +static const char * mode_string(fileio *self) { if (self->created) { diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index d018623..140da10 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1648,8 +1648,8 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) /* NOTE: `end` must point to the real end of the Py_UCS4 storage, that is to the NUL character. Otherwise the function will produce incorrect results. */ -static char * -find_control_char(int kind, char *s, char *end, Py_UCS4 ch) +static const char * +find_control_char(int kind, const char *s, const char *end, Py_UCS4 ch) { if (kind == PyUnicode_1BYTE_KIND) { assert(ch < 256); @@ -1669,13 +1669,13 @@ find_control_char(int kind, char *s, char *end, Py_UCS4 ch) Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - int kind, char *start, char *end, Py_ssize_t *consumed) + int kind, const char *start, const char *end, Py_ssize_t *consumed) { Py_ssize_t len = ((char*)end - (char*)start)/kind; if (translated) { /* Newlines are already translated, only search for \n */ - char *pos = find_control_char(kind, start, end, '\n'); + const char *pos = find_control_char(kind, start, end, '\n'); if (pos != NULL) return (pos - start)/kind + 1; else { @@ -1687,7 +1687,7 @@ _PyIO_find_line_ending( /* Universal newline search. Find any of \r, \r\n, \n * The decoder ensures that \r\n are not split in two pieces */ - char *s = start; + const char *s = start; for (;;) { Py_UCS4 ch; /* Fast path for non-control chars. The loop always ends @@ -1717,21 +1717,21 @@ _PyIO_find_line_ending( /* Assume that readnl is an ASCII character. */ assert(PyUnicode_KIND(readnl) == PyUnicode_1BYTE_KIND); if (readnl_len == 1) { - char *pos = find_control_char(kind, start, end, nl[0]); + const char *pos = find_control_char(kind, start, end, nl[0]); if (pos != NULL) return (pos - start)/kind + 1; *consumed = len; return -1; } else { - char *s = start; - char *e = end - (readnl_len - 1)*kind; - char *pos; + const char *s = start; + const char *e = end - (readnl_len - 1)*kind; + const char *pos; if (e < s) e = s; while (s < e) { Py_ssize_t i; - char *pos = find_control_char(kind, s, end, nl[0]); + const char *pos = find_control_char(kind, s, end, nl[0]); if (pos == NULL || pos >= e) break; for (i = 1; i < readnl_len; i++) { diff --git a/Modules/_json.c b/Modules/_json.c index f63d758..3c857ae 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -112,7 +112,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, PyObject *dct, Py_ssiz static PyObject * _encoded_const(PyObject *obj); static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); +raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end); static PyObject * encoder_encode_string(PyEncoderObject *s, PyObject *obj); static PyObject * @@ -323,7 +323,7 @@ escape_unicode(PyObject *pystr) } static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) +raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end) { /* Use JSONDecodeError exception to raise a nice looking ValueError subclass */ static PyObject *JSONDecodeError = NULL; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index b1d6add..a92fb10 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -49,7 +49,7 @@ PyDoc_STRVAR(setlocale__doc__, /* the grouping is terminated by either 0 or CHAR_MAX */ static PyObject* -copy_grouping(char* s) +copy_grouping(const char* s) { int i; PyObject *result, *val = NULL; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 487b533..9d36346 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2187,7 +2187,7 @@ error: } static int -write_utf8(PicklerObject *self, char *data, Py_ssize_t size) +write_utf8(PicklerObject *self, const char *data, Py_ssize_t size) { char header[9]; Py_ssize_t len; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 8bedab5..a0109fb 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -72,7 +72,7 @@ _enable_gc(int need_to_reenable_gc, PyObject *gc_module) /* Convert ASCII to a positive int, no libc call. no overflow. -1 on error. */ static int -_pos_int_from_ascii(char *name) +_pos_int_from_ascii(const char *name) { int num = 0; while (*name >= '0' && *name <= '9') { diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 66b6e34..68be458 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -130,7 +130,7 @@ error: } static int -set_proxy(PyObject* proxies, char* proto, CFDictionaryRef proxyDict, +set_proxy(PyObject* proxies, const char* proto, CFDictionaryRef proxyDict, CFStringRef enabledKey, CFStringRef hostKey, CFStringRef portKey) { diff --git a/Modules/_sre.c b/Modules/_sre.c index f597a70..fb0ab03 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -714,7 +714,7 @@ _sre_SRE_Pattern_search_impl(PatternObject *self, PyObject *string, } static PyObject* -call(char* module, char* function, PyObject* args) +call(const char* module, const char* function, PyObject* args) { PyObject* name; PyObject* mod; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8818d26..5968ed5 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -378,7 +378,7 @@ fail: } static PyObject * -PySSL_SetError(PySSLSocket *obj, int ret, char *filename, int lineno) +PySSL_SetError(PySSLSocket *obj, int ret, const char *filename, int lineno) { PyObject *type = PySSLErrorObject; char *errstr = NULL; @@ -460,7 +460,7 @@ PySSL_SetError(PySSLSocket *obj, int ret, char *filename, int lineno) } static PyObject * -_setSSLError (char *errstr, int errcode, char *filename, int lineno) { +_setSSLError (const char *errstr, int errcode, const char *filename, int lineno) { if (errstr == NULL) errcode = ERR_peek_last_error(); diff --git a/Modules/_struct.c b/Modules/_struct.c index b18c71d..820e004 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1189,7 +1189,7 @@ static formatdef lilendian_table[] = { static const formatdef * -whichtable(char **pfmt) +whichtable(const char **pfmt) { const char *fmt = (*pfmt)++; /* May be backed out of later */ switch (*fmt) { @@ -1268,7 +1268,7 @@ prepare_s(PyStructObject *self) fmt = PyBytes_AS_STRING(self->s_format); - f = whichtable((char **)&fmt); + f = whichtable(&fmt); s = fmt; size = 0; @@ -1457,7 +1457,7 @@ s_dealloc(PyStructObject *s) } static PyObject * -s_unpack_internal(PyStructObject *soself, char *startfrom) { +s_unpack_internal(PyStructObject *soself, const char *startfrom) { formatcode *code; Py_ssize_t i = 0; PyObject *result = PyTuple_New(soself->s_len); diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 2005205..03eda27 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -61,7 +61,7 @@ Example_getattro(ExampleObject *self, PyObject *name) } static int -Example_setattr(ExampleObject *self, char *name, PyObject *v) +Example_setattr(ExampleObject *self, const char *name, PyObject *v) { if (self->x_attr == NULL) { self->x_attr = PyDict_New(); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 41ad5f9..768df81 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -841,7 +841,7 @@ PyTclObject_dealloc(PyTclObject *self) Py_DECREF(tp); } -static char* +static const char * PyTclObject_TclString(PyObject *self) { return Tcl_GetString(((PyTclObject*)self)->value); @@ -1726,7 +1726,7 @@ static int varname_converter(PyObject *in, void *_out) { char *s; - char **out = (char**)_out; + const char **out = (const char**)_out; if (PyBytes_Check(in)) { if (PyBytes_Size(in) > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "bytes object is too long"); @@ -1846,7 +1846,7 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) static PyObject * SetVar(PyObject *self, PyObject *args, int flags) { - char *name1, *name2; + const char *name1, *name2; PyObject *newValue; PyObject *res = NULL; Tcl_Obj *newval, *ok; @@ -1915,7 +1915,7 @@ Tkapp_GlobalSetVar(PyObject *self, PyObject *args) static PyObject * GetVar(PyObject *self, PyObject *args, int flags) { - char *name1, *name2=NULL; + const char *name1, *name2=NULL; PyObject *res = NULL; Tcl_Obj *tres; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 3e7f187..c4d4264 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -675,7 +675,7 @@ _winapi_CreatePipe_impl(PyModuleDef *module, PyObject *pipe_attrs, /* helpers for createprocess */ static unsigned long -getulong(PyObject* obj, char* name) +getulong(PyObject* obj, const char* name) { PyObject* value; unsigned long ret; @@ -691,7 +691,7 @@ getulong(PyObject* obj, char* name) } static HANDLE -gethandle(PyObject* obj, char* name) +gethandle(PyObject* obj, const char* name) { PyObject* value; HANDLE ret; diff --git a/Modules/binascii.c b/Modules/binascii.c index 9df48da..a306acd 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -256,7 +256,8 @@ static PyObject * binascii_a2b_uu_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=5779f39b0b48459f input=7cafeaf73df63d1c]*/ { - unsigned char *ascii_data, *bin_data; + const unsigned char *ascii_data; + unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -342,7 +343,8 @@ static PyObject * binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=181021b69bb9a414 input=00fdf458ce8b465b]*/ { - unsigned char *ascii_data, *bin_data; + unsigned char *ascii_data; + const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -389,7 +391,7 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) static int -binascii_find_valid(unsigned char *s, Py_ssize_t slen, int num) +binascii_find_valid(const unsigned char *s, Py_ssize_t slen, int num) { /* Finds & returns the (num+1)th ** valid character for base64, or -1 if none. @@ -426,7 +428,8 @@ static PyObject * binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=3e351b702bed56d2 input=5872acf6e1cac243]*/ { - unsigned char *ascii_data, *bin_data; + const unsigned char *ascii_data; + unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -522,7 +525,8 @@ static PyObject * binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) /*[clinic end generated code: output=19e1dd719a890b50 input=7b2ea6fa38d8924c]*/ { - unsigned char *ascii_data, *bin_data; + unsigned char *ascii_data; + const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -589,7 +593,8 @@ static PyObject * binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=60bcdbbd28b105cd input=0d914c680e0eed55]*/ { - unsigned char *ascii_data, *bin_data; + const unsigned char *ascii_data; + unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -667,7 +672,8 @@ static PyObject * binascii_rlecode_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=0905da344dbf0648 input=e1f1712447a82b09]*/ { - unsigned char *in_data, *out_data; + const unsigned char *in_data; + unsigned char *out_data; unsigned char ch; Py_ssize_t in, inend, len; _PyBytesWriter writer; @@ -728,7 +734,8 @@ static PyObject * binascii_b2a_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=5a987810d5e3cdbb input=9596ebe019fe12ba]*/ { - unsigned char *ascii_data, *bin_data; + unsigned char *ascii_data; + const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -782,7 +789,8 @@ static PyObject * binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=f7afd89b789946ab input=54cdd49fc014402c]*/ { - unsigned char *in_data, *out_data; + const unsigned char *in_data; + unsigned char *out_data; unsigned char in_byte, in_repeat; Py_ssize_t in_len; _PyBytesWriter writer; @@ -899,7 +907,7 @@ static unsigned int binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) /*[clinic end generated code: output=167c2dac62625717 input=add8c53712ccceda]*/ { - unsigned char *bin_data; + const unsigned char *bin_data; Py_ssize_t len; crc &= 0xffff; @@ -1050,7 +1058,7 @@ binascii_crc32_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) #ifdef USE_ZLIB_CRC32 /* This was taken from zlibmodule.c PyZlib_crc32 (but is PY_SSIZE_T_CLEAN) */ { - Byte *buf; + const Byte *buf; Py_ssize_t len; int signed_val; @@ -1061,7 +1069,7 @@ binascii_crc32_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) } #else /* USE_ZLIB_CRC32 */ { /* By Jim Ahlstrom; All rights transferred to CNRI */ - unsigned char *bin_data; + const unsigned char *bin_data; Py_ssize_t len; unsigned int result; @@ -1144,7 +1152,7 @@ static PyObject * binascii_a2b_hex_impl(PyModuleDef *module, Py_buffer *hexstr) /*[clinic end generated code: output=d61da452b5c6d290 input=9e1e7f2f94db24fd]*/ { - char* argbuf; + const char* argbuf; Py_ssize_t arglen; PyObject *retval; char* retbuf; @@ -1232,7 +1240,8 @@ binascii_a2b_qp_impl(PyModuleDef *module, Py_buffer *data, int header) { Py_ssize_t in, out; char ch; - unsigned char *ascii_data, *odata; + const unsigned char *ascii_data; + unsigned char *odata; Py_ssize_t datalen = 0; PyObject *rv; @@ -1338,13 +1347,14 @@ binascii_b2a_qp_impl(PyModuleDef *module, Py_buffer *data, int quotetabs, /*[clinic end generated code: output=a87ca9ccb94e2a9f input=7f2a9aaa008e92b2]*/ { Py_ssize_t in, out; - unsigned char *databuf, *odata; + const unsigned char *databuf; + unsigned char *odata; Py_ssize_t datalen = 0, odatalen = 0; PyObject *rv; unsigned int linelen = 0; unsigned char ch; int crlf = 0; - unsigned char *p; + const unsigned char *p; databuf = data->buf; datalen = data->len; @@ -1353,7 +1363,7 @@ binascii_b2a_qp_impl(PyModuleDef *module, Py_buffer *data, int quotetabs, /* XXX: this function has the side effect of converting all of * the end of lines to be the same depending on this detection * here */ - p = (unsigned char *) memchr(databuf, '\n', datalen); + p = (const unsigned char *) memchr(databuf, '\n', datalen); if ((p != NULL) && (p > databuf) && (*(p-1) == '\r')) crlf = 1; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index cb7222d..0c6f444 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -738,7 +738,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) } static void -debug_cycle(char *msg, PyObject *op) +debug_cycle(const char *msg, PyObject *op) { PySys_FormatStderr("gc: %s <%s %p>\n", msg, Py_TYPE(op)->tp_name, op); diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index 33d7078..13a9e40 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -198,7 +198,7 @@ if (pai->ai_flags & AI_CANONNAME) {\ #define ERR(err) { error = (err); goto bad; } -char * +const char * gai_strerror(int ecode) { if (ecode < 0 || ecode > EAI_MAX) diff --git a/Modules/main.c b/Modules/main.c index 4358cc8..35d07fd 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -103,7 +103,7 @@ PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n\ "; static int -usage(int exitcode, wchar_t* program) +usage(int exitcode, const wchar_t* program) { FILE *f = exitcode ? stderr : stdout; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 9359eb2..6b3e139 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -876,7 +876,7 @@ math_1_to_int(PyObject *arg, double (*func) (double), int can_overflow) } static PyObject * -math_2(PyObject *args, double (*func) (double, double), char *funcname) +math_2(PyObject *args, double (*func) (double, double), const char *funcname) { PyObject *ox, *oy; double x, y, r; @@ -1673,7 +1673,7 @@ PyDoc_STRVAR(math_modf_doc, in that int is larger than PY_SSIZE_T_MAX. */ static PyObject* -loghelper(PyObject* arg, double (*func)(double), char *funcname) +loghelper(PyObject* arg, double (*func)(double), const char *funcname) { /* If it is int, do it ourselves. */ if (PyLong_Check(arg)) { diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 4c5f2ef..deae049 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -578,13 +578,13 @@ parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw) } -/* err_string(char* message) +/* err_string(const char* message) * * Sets the error string for an exception of type ParserError. * */ static void -err_string(char *message) +err_string(const char *message) { PyErr_SetString(parser_error, message); } @@ -597,7 +597,7 @@ err_string(char *message) * */ static PyObject* -parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) +parser_do_parse(PyObject *args, PyObject *kw, const char *argspec, int type) { char* string = 0; PyObject* res = 0; @@ -984,7 +984,7 @@ build_node_tree(PyObject *tuple) /* * Validation routines used within the validation section: */ -static int validate_terminal(node *terminal, int type, char *string); +static int validate_terminal(node *terminal, int type, const char *string); #define validate_ampersand(ch) validate_terminal(ch, AMPER, "&") #define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^") @@ -1082,7 +1082,7 @@ validate_numnodes(node *n, int num, const char *const name) static int -validate_terminal(node *terminal, int type, char *string) +validate_terminal(node *terminal, int type, const char *string) { int res = (validate_ntype(terminal, type) && ((string == 0) || (strcmp(string, STR(terminal)) == 0))); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7a2b661..367e4f2 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -949,7 +949,8 @@ path_converter(PyObject *o, void *p) { } static void -argument_unavailable_error(char *function_name, char *argument_name) { +argument_unavailable_error(const char *function_name, const char *argument_name) +{ PyErr_Format(PyExc_NotImplementedError, "%s%s%s unavailable on this platform", (function_name != NULL) ? function_name : "", @@ -972,7 +973,8 @@ dir_fd_unavailable(PyObject *o, void *p) } static int -fd_specified(char *function_name, int fd) { +fd_specified(const char *function_name, int fd) +{ if (fd == -1) return 0; @@ -981,7 +983,8 @@ fd_specified(char *function_name, int fd) { } static int -follow_symlinks_specified(char *function_name, int follow_symlinks) { +follow_symlinks_specified(const char *function_name, int follow_symlinks) +{ if (follow_symlinks) return 0; @@ -990,7 +993,8 @@ follow_symlinks_specified(char *function_name, int follow_symlinks) { } static int -path_and_dir_fd_invalid(char *function_name, path_t *path, int dir_fd) { +path_and_dir_fd_invalid(const char *function_name, path_t *path, int dir_fd) +{ if (!path->narrow && !path->wide && (dir_fd != DEFAULT_DIR_FD)) { PyErr_Format(PyExc_ValueError, "%s: can't specify dir_fd without matching path", @@ -1001,7 +1005,8 @@ path_and_dir_fd_invalid(char *function_name, path_t *path, int dir_fd) { } static int -dir_fd_and_fd_invalid(char *function_name, int dir_fd, int fd) { +dir_fd_and_fd_invalid(const char *function_name, int dir_fd, int fd) +{ if ((dir_fd != DEFAULT_DIR_FD) && (fd != -1)) { PyErr_Format(PyExc_ValueError, "%s: can't specify both dir_fd and fd", @@ -1012,8 +1017,9 @@ dir_fd_and_fd_invalid(char *function_name, int dir_fd, int fd) { } static int -fd_and_follow_symlinks_invalid(char *function_name, int fd, - int follow_symlinks) { +fd_and_follow_symlinks_invalid(const char *function_name, int fd, + int follow_symlinks) +{ if ((fd > 0) && (!follow_symlinks)) { PyErr_Format(PyExc_ValueError, "%s: cannot use fd and follow_symlinks together", @@ -1024,8 +1030,9 @@ fd_and_follow_symlinks_invalid(char *function_name, int fd, } static int -dir_fd_and_follow_symlinks_invalid(char *function_name, int dir_fd, - int follow_symlinks) { +dir_fd_and_follow_symlinks_invalid(const char *function_name, int dir_fd, + int follow_symlinks) +{ if ((dir_fd != DEFAULT_DIR_FD) && (!follow_symlinks)) { PyErr_Format(PyExc_ValueError, "%s: cannot use dir_fd and follow_symlinks together", @@ -1220,7 +1227,7 @@ posix_error(void) #ifdef MS_WINDOWS static PyObject * -win32_error(char* function, const char* filename) +win32_error(const char* function, const char* filename) { /* XXX We should pass the function name along in the future. (winreg.c also wants to pass the function name.) @@ -1235,7 +1242,7 @@ win32_error(char* function, const char* filename) } static PyObject * -win32_error_object(char* function, PyObject* filename) +win32_error_object(const char* function, PyObject* filename) { /* XXX - see win32_error for comments on 'function' */ errno = GetLastError(); @@ -2100,7 +2107,7 @@ _pystat_fromstructstat(STRUCT_STAT *st) static PyObject * -posix_do_stat(char *function_name, path_t *path, +posix_do_stat(const char *function_name, path_t *path, int dir_fd, int follow_symlinks) { STRUCT_STAT st; @@ -4561,7 +4568,7 @@ typedef struct { #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) static int -utime_dir_fd(utime_t *ut, int dir_fd, char *path, int follow_symlinks) +utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) { #ifdef HAVE_UTIMENSAT int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; @@ -4610,7 +4617,7 @@ utime_fd(utime_t *ut, int fd) #ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS static int -utime_nofollow_symlinks(utime_t *ut, char *path) +utime_nofollow_symlinks(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT UTIME_TO_TIMESPEC; @@ -4626,7 +4633,7 @@ utime_nofollow_symlinks(utime_t *ut, char *path) #ifndef MS_WINDOWS static int -utime_default(utime_t *ut, char *path) +utime_default(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT UTIME_TO_TIMESPEC; @@ -7323,7 +7330,7 @@ _check_dirW(WCHAR *src, WCHAR *dest) /* Return True if the path at src relative to dest is a directory */ static int -_check_dirA(char *src, char *dest) +_check_dirA(const char *src, char *dest) { WIN32_FILE_ATTRIBUTE_DATA src_info; char dest_parent[MAX_PATH]; @@ -11835,7 +11842,7 @@ error: #else /* POSIX */ static char * -join_path_filename(char *path_narrow, char* filename, Py_ssize_t filename_len) +join_path_filename(const char *path_narrow, const char* filename, Py_ssize_t filename_len) { Py_ssize_t path_len; Py_ssize_t size; @@ -11867,7 +11874,7 @@ join_path_filename(char *path_narrow, char* filename, Py_ssize_t filename_len) } static PyObject * -DirEntry_from_posix_info(path_t *path, char *name, Py_ssize_t name_len, +DirEntry_from_posix_info(path_t *path, const char *name, Py_ssize_t name_len, ino_t d_ino #ifdef HAVE_DIRENT_D_TYPE , unsigned char d_type diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b45e3da..9267c69 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -91,7 +91,7 @@ static struct HandlerInfo handler_info[64]; * false on an exception. */ static int -set_error_attr(PyObject *err, char *name, int value) +set_error_attr(PyObject *err, const char *name, int value) { PyObject *v = PyLong_FromLong(value); @@ -218,7 +218,7 @@ flag_error(xmlparseobject *self) } static PyObject* -call_with_frame(char *funcname, int lineno, PyObject* func, PyObject* args, +call_with_frame(const char *funcname, int lineno, PyObject* func, PyObject* args, xmlparseobject *self) { PyObject *res; @@ -766,7 +766,7 @@ readinst(char *buf, int buf_size, PyObject *meth) { PyObject *str; Py_ssize_t len; - char *ptr; + const char *ptr; str = PyObject_CallFunction(meth, "n", buf_size); if (str == NULL) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bae9634..7ab534e 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -904,7 +904,7 @@ static PyThread_type_lock netdb_lock; an error occurred; then an exception is raised. */ static int -setipaddr(char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) +setipaddr(const char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) { struct addrinfo hints, *res; int error; @@ -1085,7 +1085,7 @@ makeipaddr(struct sockaddr *addr, int addrlen) an error occurred. */ static int -setbdaddr(char *name, bdaddr_t *bdaddr) +setbdaddr(const char *name, bdaddr_t *bdaddr) { unsigned int b0, b1, b2, b3, b4, b5; char ch; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 31f0ce5..0b6d461 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -311,7 +311,7 @@ tmtotuple(struct tm *p) Returns non-zero on success (parallels PyArg_ParseTuple). */ static int -parse_time_t_args(PyObject *args, char *format, time_t *pwhen) +parse_time_t_args(PyObject *args, const char *format, time_t *pwhen) { PyObject *ot = NULL; time_t whent; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 40c1760..c1a9be9 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -89,7 +89,7 @@ Xxo_getattro(XxoObject *self, PyObject *name) } static int -Xxo_setattr(XxoObject *self, char *name, PyObject *v) +Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { if (self->x_attr == NULL) { self->x_attr = PyDict_New(); diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 85230d9..0764407 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -76,7 +76,7 @@ Xxo_getattro(XxoObject *self, PyObject *name) } static int -Xxo_setattr(XxoObject *self, char *name, PyObject *v) +Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { if (self->x_attr == NULL) { self->x_attr = PyDict_New(); diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 7220faf..87c8cfc 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -815,7 +815,7 @@ static PyTypeObject ZipImporter_Type = { 4 bytes, encoded as little endian. This partially reimplements marshal.c:r_long() */ static long -get_long(unsigned char *buf) { +get_long(const unsigned char *buf) { long x; x = buf[0]; x |= (long)buf[1] << 8; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index a15fdb2..7d2f55a 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -53,7 +53,7 @@ typedef struct } compobject; static void -zlib_error(z_stream zst, int err, char *msg) +zlib_error(z_stream zst, int err, const char *msg) { const char *zmsg = Z_NULL; /* In case of a version mismatch, zst.msg won't be initialized. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 96ab57d..9e8ba39 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2576,8 +2576,8 @@ bytearray_remove_impl(PyByteArrayObject *self, int value) /* XXX These two helpers could be optimized if argsize == 1 */ static Py_ssize_t -lstrip_helper(char *myptr, Py_ssize_t mysize, - void *argptr, Py_ssize_t argsize) +lstrip_helper(const char *myptr, Py_ssize_t mysize, + const void *argptr, Py_ssize_t argsize) { Py_ssize_t i = 0; while (i < mysize && memchr(argptr, (unsigned char) myptr[i], argsize)) @@ -2586,8 +2586,8 @@ lstrip_helper(char *myptr, Py_ssize_t mysize, } static Py_ssize_t -rstrip_helper(char *myptr, Py_ssize_t mysize, - void *argptr, Py_ssize_t argsize) +rstrip_helper(const char *myptr, Py_ssize_t mysize, + const void *argptr, Py_ssize_t argsize) { Py_ssize_t i = mysize - 1; while (i >= 0 && memchr(argptr, (unsigned char) myptr[i], argsize)) diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index a299915..d025351 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -277,7 +277,7 @@ Return a titlecased version of B, i.e. ASCII words start with uppercase\n\ characters, all remaining cased characters have lowercase."); void -_Py_bytes_title(char *result, char *s, Py_ssize_t len) +_Py_bytes_title(char *result, const char *s, Py_ssize_t len) { Py_ssize_t i; int previous_is_cased = 0; @@ -306,7 +306,7 @@ Return a copy of B with only its first character capitalized (ASCII)\n\ and the rest lower-cased."); void -_Py_bytes_capitalize(char *result, char *s, Py_ssize_t len) +_Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len) { Py_ssize_t i; @@ -336,7 +336,7 @@ Return a copy of B with uppercase ASCII characters converted\n\ to lowercase ASCII and vice versa."); void -_Py_bytes_swapcase(char *result, char *s, Py_ssize_t len) +_Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len) { Py_ssize_t i; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index f980516..602dea6 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -308,7 +308,7 @@ PyBytes_FromFormatV(const char *format, va_list vargs) { Py_ssize_t i; - p = va_arg(vargs, char*); + p = va_arg(vargs, const char*); i = strlen(p); if (prec > 0 && i > prec) i = prec; diff --git a/Objects/descrobject.c b/Objects/descrobject.c index da11f6b..2002d21 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -22,7 +22,7 @@ descr_name(PyDescrObject *descr) } static PyObject * -descr_repr(PyDescrObject *descr, char *format) +descr_repr(PyDescrObject *descr, const char *format) { PyObject *name = NULL; if (descr->d_name != NULL && PyUnicode_Check(descr->d_name)) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 6f57db0..f1f31ed 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1925,7 +1925,7 @@ dict_fromkeys_impl(PyTypeObject *type, PyObject *iterable, PyObject *value) } static int -dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, char *methname) +dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, const char *methname) { PyObject *arg = NULL; int result = 0; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 10162cb..e355a83 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1133,7 +1133,7 @@ get_native_fmtchar(char *result, const char *fmt) return -1; } -Py_LOCAL_INLINE(char *) +Py_LOCAL_INLINE(const char *) get_native_fmtstr(const char *fmt) { int at = 0; @@ -1221,7 +1221,7 @@ cast_to_1D(PyMemoryViewObject *mv, PyObject *format) goto out; } - view->format = get_native_fmtstr(PyBytes_AS_STRING(asciifmt)); + view->format = (char *)get_native_fmtstr(PyBytes_AS_STRING(asciifmt)); if (view->format == NULL) { /* NOT_REACHED: get_native_fmtchar() already validates the format. */ PyErr_SetString(PyExc_RuntimeError, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b4d23b2..2e75ec3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -564,7 +564,7 @@ type_get_bases(PyTypeObject *type, void *context) static PyTypeObject *best_base(PyObject *); static int mro_internal(PyTypeObject *, PyObject **); Py_LOCAL_INLINE(int) type_is_subtype_base_chain(PyTypeObject *, PyTypeObject *); -static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, char *); +static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, const char *); static int add_subclass(PyTypeObject*, PyTypeObject*); static int add_all_subclasses(PyTypeObject *type, PyObject *bases); static void remove_subclass(PyTypeObject *, PyTypeObject *); @@ -1435,7 +1435,7 @@ _PyObject_LookupSpecial(PyObject *self, _Py_Identifier *attrid) as lookup_method to cache the interned name string object. */ static PyObject * -call_method(PyObject *o, _Py_Identifier *nameid, char *format, ...) +call_method(PyObject *o, _Py_Identifier *nameid, const char *format, ...) { va_list va; PyObject *args, *func = 0, *retval; @@ -1471,7 +1471,7 @@ call_method(PyObject *o, _Py_Identifier *nameid, char *format, ...) /* Clone of call_method() that returns NotImplemented when the lookup fails. */ static PyObject * -call_maybe(PyObject *o, _Py_Identifier *nameid, char *format, ...) +call_maybe(PyObject *o, _Py_Identifier *nameid, const char *format, ...) { va_list va; PyObject *args, *func = 0, *retval; @@ -3609,7 +3609,7 @@ same_slots_added(PyTypeObject *a, PyTypeObject *b) } static int -compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, char* attr) +compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* attr) { PyTypeObject *newbase, *oldbase; @@ -5348,7 +5348,7 @@ wrap_delitem(PyObject *self, PyObject *args, void *wrapped) /* Helper to check for object.__setattr__ or __delattr__ applied to a type. This is called the Carlo Verre hack after its discoverer. */ static int -hackcheck(PyObject *self, setattrofunc func, char *what) +hackcheck(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); while (type && type->tp_flags & Py_TPFLAGS_HEAPTYPE) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index fef184a..bda2469 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6986,7 +6986,7 @@ PyUnicode_AsASCIIString(PyObject *unicode) # define WC_ERR_INVALID_CHARS 0x0080 #endif -static char* +static const char* code_page_name(UINT code_page, PyObject **obj) { *obj = NULL; @@ -7094,7 +7094,7 @@ decode_code_page_errors(UINT code_page, PyObject *errorHandler = NULL; PyObject *exc = NULL; PyObject *encoding_obj = NULL; - char *encoding; + const char *encoding; DWORD err; int ret = -1; @@ -7438,7 +7438,7 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, PyObject *errorHandler = NULL; PyObject *exc = NULL; PyObject *encoding_obj = NULL; - char *encoding; + const char *encoding; Py_ssize_t newpos, newoutsize; PyObject *rep; int ret = -1; diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index d4d52e6..f42fe3d 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -265,7 +265,7 @@ insert_head(PyWeakReference *newref, PyWeakReference **list) } static int -parse_weakref_init_args(char *funcname, PyObject *args, PyObject *kwargs, +parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, PyObject **obp, PyObject **callbackp) { /* XXX Should check that kwargs == NULL or is empty. */ diff --git a/Parser/pgen.c b/Parser/pgen.c index 6ecb311..be35e02 100644 --- a/Parser/pgen.c +++ b/Parser/pgen.c @@ -379,7 +379,7 @@ typedef struct _ss_dfa { /* Forward */ static void printssdfa(int xx_nstates, ss_state *xx_state, int nbits, - labellist *ll, char *msg); + labellist *ll, const char *msg); static void simplify(int xx_nstates, ss_state *xx_state); static void convert(dfa *d, int xx_nstates, ss_state *xx_state); @@ -494,7 +494,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d) static void printssdfa(int xx_nstates, ss_state *xx_state, int nbits, - labellist *ll, char *msg) + labellist *ll, const char *msg) { int i, ibit, iarc; ss_state *yy; diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 0f055d6..3ca4afe 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -27,7 +27,7 @@ int Py_VerboseFlag; int Py_IgnoreEnvironmentFlag; /* Forward */ -grammar *getgrammar(char *filename); +grammar *getgrammar(const char *filename); void Py_Exit(int) _Py_NO_RETURN; @@ -76,7 +76,7 @@ main(int argc, char **argv) } grammar * -getgrammar(char *filename) +getgrammar(const char *filename) { FILE *fp; node *n; diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 9ca3cb4..be7cf49 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -202,8 +202,8 @@ error_ret(struct tok_state *tok) /* XXX */ } -static char * -get_normal_name(char *s) /* for utf-8 and latin-1 */ +static const char * +get_normal_name(const char *s) /* for utf-8 and latin-1 */ { char buf[13]; int i; @@ -264,7 +264,7 @@ get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *t if (begin < t) { char* r = new_string(begin, t - begin, tok); - char* q; + const char* q; if (!r) return 0; q = get_normal_name(r); diff --git a/Python/_warnings.c b/Python/_warnings.c index 978bad1..daa1355 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -921,7 +921,7 @@ PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) #undef PyErr_Warn PyAPI_FUNC(int) -PyErr_Warn(PyObject *category, char *text) +PyErr_Warn(PyObject *category, const char *text) { return PyErr_WarnEx(category, text, 1); } diff --git a/Python/ceval.c b/Python/ceval.c index dd9360c..6f74e86 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -125,7 +125,7 @@ static PyObject * load_args(PyObject ***, int); #ifdef LLTRACE static int lltrace; -static int prtrace(PyObject *, char *); +static int prtrace(PyObject *, const char *); #endif static int call_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, @@ -4308,7 +4308,7 @@ Error: #ifdef LLTRACE static int -prtrace(PyObject *v, char *str) +prtrace(PyObject *v, const char *str) { printf("%s ", str); if (PyObject_Print(v, stdout, 0) != 0) diff --git a/Python/dtoa.c b/Python/dtoa.c index 3121cd6..e0665b6 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -2315,7 +2315,7 @@ rv_alloc(int i) } static char * -nrv_alloc(char *s, char **rve, int n) +nrv_alloc(const char *s, char **rve, int n) { char *rv, *t; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 9fb0525..05050cf 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -41,7 +41,7 @@ const char *_PyImport_DynLoadFiletab[] = { /* Case insensitive string compare, to avoid any dependencies on particular C RTL implementations */ -static int strcasecmp (char *string1, char *string2) +static int strcasecmp (const char *string1, const char *string2) { int first, second; diff --git a/Python/getargs.c b/Python/getargs.c index c365fc2..7d45785 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -20,12 +20,12 @@ int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, #ifdef HAVE_DECLSPEC_DLL /* Export functions */ -PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, char *, ...); -PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, char *, ...); +PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); +PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, ...); PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); -PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, char *, va_list); +PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, const char *, va_list); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, va_list); #endif @@ -56,18 +56,18 @@ typedef struct { /* Forward */ static int vgetargs1(PyObject *, const char *, va_list *, int); static void seterror(Py_ssize_t, const char *, int *, const char *, const char *); -static char *convertitem(PyObject *, const char **, va_list *, int, int *, - char *, size_t, freelist_t *); -static char *converttuple(PyObject *, const char **, va_list *, int, - int *, char *, size_t, int, freelist_t *); -static char *convertsimple(PyObject *, const char **, va_list *, int, char *, - size_t, freelist_t *); -static Py_ssize_t convertbuffer(PyObject *, void **p, char **); -static int getbuffer(PyObject *, Py_buffer *, char**); +static const char *convertitem(PyObject *, const char **, va_list *, int, int *, + char *, size_t, freelist_t *); +static const char *converttuple(PyObject *, const char **, va_list *, int, + int *, char *, size_t, int, freelist_t *); +static const char *convertsimple(PyObject *, const char **, va_list *, int, + char *, size_t, freelist_t *); +static Py_ssize_t convertbuffer(PyObject *, void **p, const char **); +static int getbuffer(PyObject *, Py_buffer *, const char**); static int vgetargskeywords(PyObject *, PyObject *, const char *, char **, va_list *, int); -static char *skipitem(const char **, va_list *, int); +static const char *skipitem(const char **, va_list *, int); int PyArg_Parse(PyObject *args, const char *format, ...) @@ -82,7 +82,7 @@ PyArg_Parse(PyObject *args, const char *format, ...) } int -_PyArg_Parse_SizeT(PyObject *args, char *format, ...) +_PyArg_Parse_SizeT(PyObject *args, const char *format, ...) { int retval; va_list va; @@ -107,7 +107,7 @@ PyArg_ParseTuple(PyObject *args, const char *format, ...) } int -_PyArg_ParseTuple_SizeT(PyObject *args, char *format, ...) +_PyArg_ParseTuple_SizeT(PyObject *args, const char *format, ...) { int retval; va_list va; @@ -130,7 +130,7 @@ PyArg_VaParse(PyObject *args, const char *format, va_list va) } int -_PyArg_VaParse_SizeT(PyObject *args, char *format, va_list va) +_PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) { va_list lva; @@ -208,7 +208,7 @@ vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags) int endfmt = 0; const char *formatsave = format; Py_ssize_t i, len; - char *msg; + const char *msg; int compat = flags & FLAG_COMPAT; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; @@ -416,7 +416,7 @@ seterror(Py_ssize_t iarg, const char *msg, int *levels, const char *fname, and msgbuf is returned. */ -static char * +static const char * converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, int *levels, char *msgbuf, size_t bufsize, int toplevel, freelist_t *freelist) @@ -474,7 +474,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, format = *p_format; for (i = 0; i < n; i++) { - char *msg; + const char *msg; PyObject *item; item = PySequence_GetItem(arg, i); if (item == NULL) { @@ -501,11 +501,11 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* Convert a single item. */ -static char * +static const char * convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, int *levels, char *msgbuf, size_t bufsize, freelist_t *freelist) { - char *msg; + const char *msg; const char *format = *p_format; if (*format == '(' /* ')' */) { @@ -530,7 +530,7 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* Format an error message generated by convertsimple(). */ -static char * +static const char * converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) { assert(expected != NULL); @@ -566,7 +566,7 @@ float_argument_error(PyObject *arg) When you add new format codes, please don't forget poor skipitem() below. */ -static char * +static const char * convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *msgbuf, size_t bufsize, freelist_t *freelist) { @@ -851,7 +851,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'y': {/* any bytes-like object */ void **p = (void **)va_arg(*p_va, char **); - char *buf; + const char *buf; Py_ssize_t count; if (*format == '*') { if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) @@ -898,7 +898,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, PyBuffer_FillInfo(p, arg, sarg, len, 1, 0); } else { /* any bytes-like object */ - char *buf; + const char *buf; if (getbuffer(arg, p, &buf) < 0) return converterr(buf, arg, msgbuf, bufsize); } @@ -928,7 +928,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { /* read-only bytes-like object */ /* XXX Really? */ - char *buf; + const char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) return converterr(buf, arg, msgbuf, bufsize); @@ -1275,7 +1275,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } static Py_ssize_t -convertbuffer(PyObject *arg, void **p, char **errmsg) +convertbuffer(PyObject *arg, void **p, const char **errmsg) { PyBufferProcs *pb = Py_TYPE(arg)->tp_as_buffer; Py_ssize_t count; @@ -1297,7 +1297,7 @@ convertbuffer(PyObject *arg, void **p, char **errmsg) } static int -getbuffer(PyObject *arg, Py_buffer *view, char **errmsg) +getbuffer(PyObject *arg, Py_buffer *view, const char **errmsg) { if (PyObject_GetBuffer(arg, view, PyBUF_SIMPLE) != 0) { *errmsg = "bytes-like object"; @@ -1629,7 +1629,7 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, } -static char * +static const char * skipitem(const char **p_format, va_list *p_va, int flags) { const char *format = *p_format; @@ -1722,7 +1722,7 @@ skipitem(const char **p_format, va_list *p_va, int flags) case '(': /* bypass tuple, not handled at all previously */ { - char *msg; + const char *msg; for (;;) { if (*format==')') break; diff --git a/Python/marshal.c b/Python/marshal.c index 5b8de99..589eb80 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -643,7 +643,7 @@ typedef struct { PyObject *refs; /* a list */ } RFILE; -static char * +static const char * r_string(Py_ssize_t n, RFILE *p) { Py_ssize_t read = -1; @@ -729,7 +729,7 @@ r_byte(RFILE *p) c = getc(p->fp); } else { - char *ptr = r_string(1, p); + const char *ptr = r_string(1, p); if (ptr != NULL) c = *(unsigned char *) ptr; } @@ -740,9 +740,9 @@ static int r_short(RFILE *p) { short x = -1; - unsigned char *buffer; + const unsigned char *buffer; - buffer = (unsigned char *) r_string(2, p); + buffer = (const unsigned char *) r_string(2, p); if (buffer != NULL) { x = buffer[0]; x |= buffer[1] << 8; @@ -756,9 +756,9 @@ static long r_long(RFILE *p) { long x = -1; - unsigned char *buffer; + const unsigned char *buffer; - buffer = (unsigned char *) r_string(4, p); + buffer = (const unsigned char *) r_string(4, p); if (buffer != NULL) { x = buffer[0]; x |= (long)buffer[1] << 8; @@ -978,7 +978,8 @@ r_object(RFILE *p) case TYPE_FLOAT: { - char buf[256], *ptr; + char buf[256]; + const char *ptr; double dx; n = r_byte(p); if (n == EOF) { @@ -1001,9 +1002,9 @@ r_object(RFILE *p) case TYPE_BINARY_FLOAT: { - unsigned char *buf; + const unsigned char *buf; double x; - buf = (unsigned char *) r_string(8, p); + buf = (const unsigned char *) r_string(8, p); if (buf == NULL) break; x = _PyFloat_Unpack8(buf, 1); @@ -1016,7 +1017,8 @@ r_object(RFILE *p) case TYPE_COMPLEX: { - char buf[256], *ptr; + char buf[256]; + const char *ptr; Py_complex c; n = r_byte(p); if (n == EOF) { @@ -1053,15 +1055,15 @@ r_object(RFILE *p) case TYPE_BINARY_COMPLEX: { - unsigned char *buf; + const unsigned char *buf; Py_complex c; - buf = (unsigned char *) r_string(8, p); + buf = (const unsigned char *) r_string(8, p); if (buf == NULL) break; c.real = _PyFloat_Unpack8(buf, 1); if (c.real == -1.0 && PyErr_Occurred()) break; - buf = (unsigned char *) r_string(8, p); + buf = (const unsigned char *) r_string(8, p); if (buf == NULL) break; c.imag = _PyFloat_Unpack8(buf, 1); @@ -1074,7 +1076,7 @@ r_object(RFILE *p) case TYPE_STRING: { - char *ptr; + const char *ptr; n = r_long(p); if (PyErr_Occurred()) break; @@ -1119,7 +1121,7 @@ r_object(RFILE *p) } _read_ascii: { - char *ptr; + const char *ptr; ptr = r_string(n, p); if (ptr == NULL) break; @@ -1137,7 +1139,7 @@ r_object(RFILE *p) is_interned = 1; case TYPE_UNICODE: { - char *buffer; + const char *buffer; n = r_long(p); if (PyErr_Occurred()) diff --git a/Python/modsupport.c b/Python/modsupport.c index 6c938dd..7e6a65b 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -301,7 +301,7 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) case 'U': /* XXX deprecated alias */ { PyObject *v; - char *str = va_arg(*p_va, char *); + const char *str = va_arg(*p_va, const char *); Py_ssize_t n; if (**p_format == '#') { ++*p_format; @@ -334,7 +334,7 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) case 'y': { PyObject *v; - char *str = va_arg(*p_va, char *); + const char *str = va_arg(*p_va, const char *); Py_ssize_t n; if (**p_format == '#') { ++*p_format; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b7f6ec8..c84c46a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1004,8 +1004,8 @@ is_valid_fd(int fd) /* returns Py_None if the fd is not valid */ static PyObject* create_stdio(PyObject* io, - int fd, int write_mode, char* name, - char* encoding, char* errors) + int fd, int write_mode, const char* name, + const char* encoding, const char* errors) { PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res; const char* mode; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index cfe197b..8829699 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1138,8 +1138,8 @@ PyParser_ASTFromString(const char *s, const char *filename_str, int start, mod_ty PyParser_ASTFromFileObject(FILE *fp, PyObject *filename, const char* enc, - int start, char *ps1, - char *ps2, PyCompilerFlags *flags, int *errcode, + int start, const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { mod_ty mod; @@ -1171,8 +1171,8 @@ PyParser_ASTFromFileObject(FILE *fp, PyObject *filename, const char* enc, mod_ty PyParser_ASTFromFile(FILE *fp, const char *filename_str, const char* enc, - int start, char *ps1, - char *ps2, PyCompilerFlags *flags, int *errcode, + int start, const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { mod_ty mod; diff --git a/Python/symtable.c b/Python/symtable.c index 8431d51..806364a 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -160,7 +160,7 @@ PyTypeObject PySTEntry_Type = { }; static int symtable_analyze(struct symtable *st); -static int symtable_warn(struct symtable *st, char *msg, int lineno); +static int symtable_warn(struct symtable *st, const char *msg, int lineno); static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, int lineno, int col_offset); @@ -903,7 +903,7 @@ symtable_analyze(struct symtable *st) static int -symtable_warn(struct symtable *st, char *msg, int lineno) +symtable_warn(struct symtable *st, const char *msg, int lineno) { PyObject *message = PyUnicode_FromString(msg); if (message == NULL) -- cgit v0.12 From aab9f46c6d5508383709f8722a9328fa76dd8c1b Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Sat, 26 Dec 2015 12:35:47 +0000 Subject: Closes #25789: Improved buffering behaviour in launcher. --- PC/launcher.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PC/launcher.c b/PC/launcher.c index f7a0917..02cd9b9 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -98,7 +98,7 @@ error(int rc, wchar_t * format, ... ) MessageBox(NULL, message, TEXT("Python Launcher is sorry to say ..."), MB_OK); #endif - ExitProcess(rc); + exit(rc); } /* @@ -652,7 +652,7 @@ run_child(wchar_t * cmdline) if (!ok) error(RC_CREATE_PROCESS, L"Failed to get exit code of process"); debug(L"child process exit code: %d\n", rc); - ExitProcess(rc); + exit(rc); } static void @@ -1357,6 +1357,7 @@ process(int argc, wchar_t ** argv) wchar_t * av[2]; #endif + setvbuf(stderr, (char *)NULL, _IONBF, 0); wp = get_env(L"PYLAUNCH_DEBUG"); if ((wp != NULL) && (*wp != L'\0')) log_fp = stderr; -- cgit v0.12 From 1ed017ae92b32b27186d5793f6e58c526f350a2b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Dec 2015 15:51:32 +0200 Subject: Issue #20440: Cleaning up the code by using Py_SETREF and Py_CLEAR. Old code is correct, but with Py_SETREF and Py_CLEAR it can be cleaner. This patch doesn't fix bugs and hence there is no need to backport it. --- Modules/_collectionsmodule.c | 5 +---- Modules/_ctypes/_ctypes.c | 10 ++++------ Modules/_elementtree.c | 6 +----- Modules/_io/bytesio.c | 6 ++---- Modules/itertoolsmodule.c | 22 ++++++---------------- Modules/pyexpat.c | 17 ++++------------- Objects/listobject.c | 10 ++-------- Objects/tupleobject.c | 5 +---- Objects/typeobject.c | 13 ++----------- 9 files changed, 23 insertions(+), 71 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e3d1910..6a53584 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1222,7 +1222,6 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) static int deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) { - PyObject *old_value; block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; @@ -1249,9 +1248,7 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) b = b->leftlink; } Py_INCREF(v); - old_value = b->data[i]; - b->data[i] = v; - Py_DECREF(old_value); + Py_SETREF(b->data[i], v); return 0; } diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 5db9f90..dddeba0 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5123,23 +5123,22 @@ int comerror_init(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *hresult, *text, *details; - PyBaseExceptionObject *bself; PyObject *a; int status; if (!_PyArg_NoKeywords(Py_TYPE(self)->tp_name, kwds)) - return -1; + return -1; if (!PyArg_ParseTuple(args, "OOO:COMError", &hresult, &text, &details)) return -1; a = PySequence_GetSlice(args, 1, PySequence_Size(args)); if (!a) - return -1; + return -1; status = PyObject_SetAttrString(self, "args", a); Py_DECREF(a); if (status < 0) - return -1; + return -1; if (PyObject_SetAttrString(self, "hresult", hresult) < 0) return -1; @@ -5150,9 +5149,8 @@ comerror_init(PyObject *self, PyObject *args, PyObject *kwds) if (PyObject_SetAttrString(self, "details", details) < 0) return -1; - bself = (PyBaseExceptionObject *)self; Py_INCREF(args); - Py_SETREF(bself->args, args); + Py_SETREF((PyBaseExceptionObject *)self->args, args); return 0; } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f16d48f..580c53a 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2338,13 +2338,9 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self, PyObject *element_factory) /*[clinic end generated code: output=91cfa7558970ee96 input=1b424eeefc35249c]*/ { - PyObject *tmp; - if (element_factory) { Py_INCREF(element_factory); - tmp = self->element_factory; - self->element_factory = element_factory; - Py_XDECREF(tmp); + Py_SETREF(self->element_factory, element_factory); } return 0; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 99e71bc..6333e46 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -87,7 +87,7 @@ scan_eol(bytesio *self, Py_ssize_t len) static int unshare_buffer(bytesio *self, size_t size) { - PyObject *new_buf, *old_buf; + PyObject *new_buf; assert(SHARED_BUF(self)); assert(self->exports == 0); assert(size >= (size_t)self->string_size); @@ -96,9 +96,7 @@ unshare_buffer(bytesio *self, size_t size) return -1; memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf), self->string_size); - old_buf = self->buf; - self->buf = new_buf; - Py_DECREF(old_buf); + Py_SETREF(self->buf, new_buf); return 0; } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 2bd0594..a4ffa06 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -77,7 +77,7 @@ groupby_traverse(groupbyobject *gbo, visitproc visit, void *arg) static PyObject * groupby_next(groupbyobject *gbo) { - PyObject *newvalue, *newkey, *r, *grouper, *tmp; + PyObject *newvalue, *newkey, *r, *grouper; /* skip to next iteration group */ for (;;) { @@ -110,19 +110,12 @@ groupby_next(groupbyobject *gbo) } } - tmp = gbo->currkey; - gbo->currkey = newkey; - Py_XDECREF(tmp); - - tmp = gbo->currvalue; - gbo->currvalue = newvalue; - Py_XDECREF(tmp); + Py_SETREF(gbo->currkey, newkey); + Py_SETREF(gbo->currvalue, newvalue); } Py_INCREF(gbo->currkey); - tmp = gbo->tgtkey; - gbo->tgtkey = gbo->currkey; - Py_XDECREF(tmp); + Py_SETREF(gbo->tgtkey, gbo->currkey); grouper = _grouper_create(gbo, gbo->tgtkey); if (grouper == NULL) @@ -3445,7 +3438,7 @@ accumulate_traverse(accumulateobject *lz, visitproc visit, void *arg) static PyObject * accumulate_next(accumulateobject *lz) { - PyObject *val, *oldtotal, *newtotal; + PyObject *val, *newtotal; val = (*Py_TYPE(lz->it)->tp_iternext)(lz->it); if (val == NULL) @@ -3465,11 +3458,8 @@ accumulate_next(accumulateobject *lz) if (newtotal == NULL) return NULL; - oldtotal = lz->total; - lz->total = newtotal; - Py_DECREF(oldtotal); - Py_INCREF(newtotal); + Py_SETREF(lz->total, newtotal); return newtotal; } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9267c69..c6d3515 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1226,12 +1226,8 @@ xmlparse_dealloc(xmlparseobject *self) self->itself = NULL; if (self->handlers != NULL) { - PyObject *temp; - for (i = 0; handler_info[i].name != NULL; i++) { - temp = self->handlers[i]; - self->handlers[i] = NULL; - Py_XDECREF(temp); - } + for (i = 0; handler_info[i].name != NULL; i++) + Py_CLEAR(self->handlers[i]); PyMem_Free(self->handlers); self->handlers = NULL; } @@ -1345,7 +1341,6 @@ sethandler(xmlparseobject *self, PyObject *name, PyObject* v) int handlernum = handlername2int(name); if (handlernum >= 0) { xmlhandler c_handler = NULL; - PyObject *temp = self->handlers[handlernum]; if (v == Py_None) { /* If this is the character data handler, and a character @@ -1367,8 +1362,7 @@ sethandler(xmlparseobject *self, PyObject *name, PyObject* v) Py_INCREF(v); c_handler = handler_info[handlernum].handler; } - self->handlers[handlernum] = v; - Py_XDECREF(temp); + Py_SETREF(self->handlers[handlernum], v); handler_info[handlernum].setter(self->itself, c_handler); return 1; } @@ -1898,15 +1892,12 @@ static void clear_handlers(xmlparseobject *self, int initial) { int i = 0; - PyObject *temp; for (; handler_info[i].name != NULL; i++) { if (initial) self->handlers[i] = NULL; else { - temp = self->handlers[i]; - self->handlers[i] = NULL; - Py_XDECREF(temp); + Py_CLEAR(self->handlers[i]); handler_info[i].setter(self->itself, NULL); } } diff --git a/Objects/listobject.c b/Objects/listobject.c index eee7c68..fcc21cb 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -216,7 +216,6 @@ int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) { - PyObject *olditem; PyObject **p; if (!PyList_Check(op)) { Py_XDECREF(newitem); @@ -230,9 +229,7 @@ PyList_SetItem(PyObject *op, Py_ssize_t i, return -1; } p = ((PyListObject *)op) -> ob_item + i; - olditem = *p; - *p = newitem; - Py_XDECREF(olditem); + Py_SETREF(*p, newitem); return 0; } @@ -730,7 +727,6 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) static int list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) { - PyObject *old_value; if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); @@ -739,9 +735,7 @@ list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) if (v == NULL) return list_ass_slice(a, i, i+1, v); Py_INCREF(v); - old_value = a->ob_item[i]; - a->ob_item[i] = v; - Py_DECREF(old_value); + Py_SETREF(a->ob_item[i], v); return 0; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 7efa1a6..8e1b00b 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -149,7 +149,6 @@ PyTuple_GetItem(PyObject *op, Py_ssize_t i) int PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) { - PyObject *olditem; PyObject **p; if (!PyTuple_Check(op) || op->ob_refcnt != 1) { Py_XDECREF(newitem); @@ -163,9 +162,7 @@ PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) return -1; } p = ((PyTupleObject *)op) -> ob_item + i; - olditem = *p; - *p = newitem; - Py_XDECREF(olditem); + Py_SETREF(*p, newitem); return 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4fac9e8..78ef6ac 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -401,7 +401,6 @@ type_qualname(PyTypeObject *type, void *context) static int type_set_name(PyTypeObject *type, PyObject *value, void *context) { - PyHeapTypeObject* et; char *tp_name; PyObject *tmp; @@ -430,17 +429,9 @@ type_set_name(PyTypeObject *type, PyObject *value, void *context) if (tp_name == NULL) return -1; - et = (PyHeapTypeObject*)type; - - Py_INCREF(value); - - /* Wait until et is a sane state before Py_DECREF'ing the old et->ht_name - value. (Bug #16447.) */ - tmp = et->ht_name; - et->ht_name = value; - type->tp_name = tp_name; - Py_DECREF(tmp); + Py_INCREF(value); + Py_SETREF(((PyHeapTypeObject*)type)->ht_name, value); return 0; } -- cgit v0.12 From 1e3c3e906c9f1fcc463cfb3641d078eec773666f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 27 Dec 2015 13:17:04 -0800 Subject: Issue #25768: Make compileall functions return booleans and document the return values as well as test them. Thanks to Nicholas Chammas for the bug report and initial patch. --- Doc/library/compileall.rst | 12 ++++++++---- Doc/whatsnew/3.6.rst | 5 +++++ Lib/compileall.py | 16 ++++++++-------- Lib/test/test_compileall.py | 26 ++++++++++++++++++++++++-- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index c5736f2..679c2b4 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -103,7 +103,8 @@ Public functions .. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` - files along the way. + files along the way. Return a true value if all the files compiled successfully, + and a false value otherwise. The *maxlevels* parameter is used to limit the depth of the recursion; it defaults to ``10``. @@ -155,7 +156,8 @@ Public functions .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1) - Compile the file with path *fullname*. + Compile the file with path *fullname*. Return a true value if the file + compiled successfully, and a false value otherwise. If *ddir* is given, it is prepended to the path to the file being compiled for use in compilation time tracebacks, and is also compiled in to the @@ -191,8 +193,10 @@ Public functions .. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1) - Byte-compile all the :file:`.py` files found along ``sys.path``. If - *skip_curdir* is true (the default), the current directory is not included + Byte-compile all the :file:`.py` files found along ``sys.path``. Return a + true value if all the files compiled successfully, and a false value otherwise. + + If *skip_curdir* is true (the default), the current directory is not included in the search. All other parameters are passed to the :func:`compile_dir` function. Note that unlike the other compile functions, ``maxlevels`` defaults to ``0``. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index f97c70f..c660933 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -230,6 +230,11 @@ that may require changes to your code. Changes in the Python API ------------------------- +* The functions in the :mod:`compileall` module now return booleans instead + of ``1`` or ``0`` to represent success or failure, respectively. Thanks to + booleans being a subclass of integers, this should only be an issue if you + were doing identity checks for ``1`` or ``0``. See :issue:`25768`. + * Reading the :attr:`~urllib.parse.SplitResult.port` attribute of :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results now raises :exc:`ValueError` for out-of-range values, rather than diff --git a/Lib/compileall.py b/Lib/compileall.py index 0cc0c1d..67c5f5a 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -68,7 +68,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, """ files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, ddir=ddir) - success = 1 + success = True if workers is not None and workers != 1 and ProcessPoolExecutor is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') @@ -81,12 +81,12 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, legacy=legacy, optimize=optimize), files) - success = min(results, default=1) + success = min(results, default=True) else: for file in files: if not compile_file(file, ddir, force, rx, quiet, legacy, optimize): - success = 0 + success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, @@ -104,7 +104,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter """ - success = 1 + success = True name = os.path.basename(fullname) if ddir is not None: dfile = os.path.join(ddir, name) @@ -144,7 +144,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, ok = py_compile.compile(fullname, cfile, dfile, True, optimize=optimize) except py_compile.PyCompileError as err: - success = 0 + success = False if quiet >= 2: return success elif quiet: @@ -157,7 +157,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, msg = msg.decode(sys.stdout.encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: - success = 0 + success = False if quiet >= 2: return success elif quiet: @@ -167,7 +167,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, print(e.__class__.__name__ + ':', e) else: if ok == 0: - success = 0 + success = False return success def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, @@ -183,7 +183,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, legacy: as for compile_dir() (default False) optimize: as for compile_dir() (default -1) """ - success = 1 + success = True for dir in sys.path: if (not dir or dir == os.curdir) and skip_curdir: if quiet < 2: diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 2ce8a61..439c125 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -1,6 +1,7 @@ import sys import compileall import importlib.util +import test.test_importlib.util import os import pathlib import py_compile @@ -40,6 +41,11 @@ class CompileallTests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.directory) + def add_bad_source_file(self): + self.bad_source_path = os.path.join(self.directory, '_test_bad.py') + with open(self.bad_source_path, 'w') as file: + file.write('x (\n') + def data(self): with open(self.bc_path, 'rb') as file: data = file.read(8) @@ -78,15 +84,31 @@ class CompileallTests(unittest.TestCase): os.unlink(fn) except: pass - compileall.compile_file(self.source_path, force=False, quiet=True) + self.assertTrue(compileall.compile_file(self.source_path, + force=False, quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and not os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) - compileall.compile_dir(self.directory, force=False, quiet=True) + self.assertTrue(compileall.compile_dir(self.directory, force=False, + quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) os.unlink(self.bc_path2) + # Test against bad files + self.add_bad_source_file() + self.assertFalse(compileall.compile_file(self.bad_source_path, + force=False, quiet=2)) + self.assertFalse(compileall.compile_dir(self.directory, + force=False, quiet=2)) + + def test_compile_path(self): + self.assertTrue(compileall.compile_path(quiet=2)) + + with test.test_importlib.util.import_state(path=[self.directory]): + self.add_bad_source_file() + self.assertFalse(compileall.compile_path(skip_curdir=False, + force=True, quiet=2)) def test_no_pycache_in_non_package(self): # Bug 8563 reported that __pycache__ directories got created by diff --git a/Misc/ACKS b/Misc/ACKS index 09ad15c..c1bf5d1 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -235,6 +235,7 @@ Octavian Cerna Michael Cetrulo Dave Chambers Pascal Chambon +Nicholas Chammas John Chandler Hye-Shik Chang Jeffrey Chang diff --git a/Misc/NEWS b/Misc/NEWS index f0b83b1..e795234 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -123,6 +123,9 @@ Core and Builtins Library ------- +- Issue #25768: Have the functions in compileall return booleans instead of + ints and add proper documentation and tests for the return values. + - Issue #24103: Fixed possible use after free in ElementTree.XMLPullParser. - Issue #25860: os.fwalk() no longer skips remaining directories when error -- cgit v0.12 From ac1e7f6983ef4f0feef6389d458544bf924f6965 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 28 Dec 2015 23:50:19 +0200 Subject: Remove duplicate method in test_pathlib. Initial patch by Navneet Suman. --- Lib/test/test_pathlib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 1c53ab7..80cb97e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1437,14 +1437,14 @@ class _BasePathTest(object): self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) self.assertEqual(set(p.glob("../xyzzy")), set()) - def _check_resolve_relative(self, p, expected): - q = p.resolve() - self.assertEqual(q, expected) - def _check_resolve_absolute(self, p, expected): + def _check_resolve(self, p, expected): q = p.resolve() self.assertEqual(q, expected) + # this can be used to check both relative and absolute resolutions + _check_resolve_relative = _check_resolve_absolute = _check_resolve + @with_symlinks def test_resolve_common(self): P = self.cls -- cgit v0.12 From 53f2e0ad45e41c007f714e077ccf11643651ef28 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 28 Dec 2015 23:02:02 +0100 Subject: Issue #25928: Add Decimal.as_integer_ratio(). Python parts and docs by Mark Dickinson. --- Doc/library/decimal.rst | 13 +++++ Lib/_pydecimal.py | 52 +++++++++++++++++++ Lib/test/test_decimal.py | 33 ++++++++++++ Misc/NEWS | 2 + Modules/_decimal/_decimal.c | 101 +++++++++++++++++++++++++++++++++++++ Modules/_decimal/docstrings.h | 9 ++++ Modules/_decimal/tests/deccheck.py | 6 +-- 7 files changed, 213 insertions(+), 3 deletions(-) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 2de0ea0..14356aa 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -448,6 +448,19 @@ Decimal objects ``Decimal('321e+5').adjusted()`` returns seven. Used for determining the position of the most significant digit with respect to the decimal point. + .. method:: as_integer_ratio() + + Return a pair ``(n, d)`` of integers that represent the given + :class:`Decimal` instance as a fraction, in lowest terms and + with a positive denominator:: + + >>> Decimal('-3.14').as_integer_ratio() + (-157, 50) + + The conversion is exact. Raise OverflowError on infinities and ValueError + on NaNs. + + .. versionadded:: 3.6 .. method:: as_tuple() diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 05ba4ee..eb7bba8 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1010,6 +1010,58 @@ class Decimal(object): """ return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + def as_integer_ratio(self): + """Express a finite Decimal instance in the form n / d. + + Returns a pair (n, d) of integers. When called on an infinity + or NaN, raises OverflowError or ValueError respectively. + + >>> Decimal('3.14').as_integer_ratio() + (157, 50) + >>> Decimal('-123e5').as_integer_ratio() + (-12300000, 1) + >>> Decimal('0.00').as_integer_ratio() + (0, 1) + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("Cannot pass NaN " + "to decimal.as_integer_ratio.") + else: + raise OverflowError("Cannot pass infinity " + "to decimal.as_integer_ratio.") + + if not self: + return 0, 1 + + # Find n, d in lowest terms such that abs(self) == n / d; + # we'll deal with the sign later. + n = int(self._int) + if self._exp >= 0: + # self is an integer. + n, d = n * 10**self._exp, 1 + else: + # Find d2, d5 such that abs(self) = n / (2**d2 * 5**d5). + d5 = -self._exp + while d5 > 0 and n % 5 == 0: + n //= 5 + d5 -= 1 + + # (n & -n).bit_length() - 1 counts trailing zeros in binary + # representation of n (provided n is nonzero). + d2 = -self._exp + shift2 = min((n & -n).bit_length() - 1, d2) + if shift2: + n >>= shift2 + d2 -= shift2 + + d = 5**d5 << d2 + + if self._sign: + n = -n + return n, d + def __repr__(self): """Represents the number as an instance of Decimal.""" # Invariant: eval(repr(d)) == d diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 137aaa5..f6d58ff 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -2047,6 +2047,39 @@ class UsabilityTest(unittest.TestCase): d = Decimal( (1, (0, 2, 7, 1), 'F') ) self.assertEqual(d.as_tuple(), (1, (0,), 'F')) + def test_as_integer_ratio(self): + Decimal = self.decimal.Decimal + + # exceptional cases + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('inf')) + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('-inf')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('-nan')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('snan123')) + + for exp in range(-4, 2): + for coeff in range(1000): + for sign in '+', '-': + d = Decimal('%s%dE%d' % (sign, coeff, exp)) + pq = d.as_integer_ratio() + p, q = pq + + # check return type + self.assertIsInstance(pq, tuple) + self.assertIsInstance(p, int) + self.assertIsInstance(q, int) + + # check normalization: q should be positive; + # p should be relatively prime to q. + self.assertGreater(q, 0) + self.assertEqual(math.gcd(p, q), 1) + + # check that p/q actually gives the correct value + self.assertEqual(Decimal(p) / Decimal(q), d) + def test_subclassing(self): # Different behaviours when subclassing Decimal Decimal = self.decimal.Decimal diff --git a/Misc/NEWS b/Misc/NEWS index e795234..821904f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -123,6 +123,8 @@ Core and Builtins Library ------- +- Issue #25928: Add Decimal.as_integer_ratio(). + - Issue #25768: Have the functions in compileall return booleans instead of ints and add proper documentation and tests for the return values. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 112b44f..3c2ad85 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -3380,6 +3380,106 @@ dec_as_long(PyObject *dec, PyObject *context, int round) return (PyObject *) pylong; } +/* Convert a Decimal to its exact integer ratio representation. */ +static PyObject * +dec_as_integer_ratio(PyObject *self, PyObject *args UNUSED) +{ + PyObject *numerator = NULL; + PyObject *denominator = NULL; + PyObject *exponent = NULL; + PyObject *result = NULL; + PyObject *tmp; + mpd_ssize_t exp; + PyObject *context; + uint32_t status = 0; + PyNumberMethods *long_methods = PyLong_Type.tp_as_number; + + if (mpd_isspecial(MPD(self))) { + if (mpd_isnan(MPD(self))) { + PyErr_SetString(PyExc_ValueError, + "cannot convert NaN to integer ratio"); + } + else { + PyErr_SetString(PyExc_OverflowError, + "cannot convert Infinity to integer ratio"); + } + return NULL; + } + + CURRENT_CONTEXT(context); + + tmp = dec_alloc(); + if (tmp == NULL) { + return NULL; + } + + if (!mpd_qcopy(MPD(tmp), MPD(self), &status)) { + Py_DECREF(tmp); + PyErr_NoMemory(); + return NULL; + } + + exp = mpd_iszero(MPD(tmp)) ? 0 : MPD(tmp)->exp; + MPD(tmp)->exp = 0; + + /* context and rounding are unused here: the conversion is exact */ + numerator = dec_as_long(tmp, context, MPD_ROUND_FLOOR); + Py_DECREF(tmp); + if (numerator == NULL) { + goto error; + } + + exponent = PyLong_FromSsize_t(exp < 0 ? -exp : exp); + if (exponent == NULL) { + goto error; + } + + tmp = PyLong_FromLong(10); + if (tmp == NULL) { + goto error; + } + + Py_SETREF(exponent, long_methods->nb_power(tmp, exponent, Py_None)); + Py_DECREF(tmp); + if (exponent == NULL) { + goto error; + } + + if (exp >= 0) { + Py_SETREF(numerator, long_methods->nb_multiply(numerator, exponent)); + if (numerator == NULL) { + goto error; + } + denominator = PyLong_FromLong(1); + if (denominator == NULL) { + goto error; + } + } + else { + denominator = exponent; + exponent = NULL; + tmp = _PyLong_GCD(numerator, denominator); + if (tmp == NULL) { + goto error; + } + Py_SETREF(numerator, long_methods->nb_floor_divide(numerator, tmp)); + Py_SETREF(denominator, long_methods->nb_floor_divide(denominator, tmp)); + Py_DECREF(tmp); + if (numerator == NULL || denominator == NULL) { + goto error; + } + } + + result = PyTuple_Pack(2, numerator, denominator); + + +error: + Py_XDECREF(exponent); + Py_XDECREF(denominator); + Py_XDECREF(numerator); + return result; +} + static PyObject * PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) { @@ -4688,6 +4788,7 @@ static PyMethodDef dec_methods [] = /* Miscellaneous */ { "from_float", dec_from_float, METH_O|METH_CLASS, doc_from_float }, { "as_tuple", PyDec_AsTuple, METH_NOARGS, doc_as_tuple }, + { "as_integer_ratio", dec_as_integer_ratio, METH_NOARGS, doc_as_integer_ratio }, /* Special methods */ { "__copy__", dec_copy, METH_NOARGS, NULL }, diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index 71029a9..f7fd6e7 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -70,6 +70,15 @@ PyDoc_STRVAR(doc_as_tuple, Return a tuple representation of the number.\n\ \n"); +PyDoc_STRVAR(doc_as_integer_ratio, +"as_integer_ratio($self, /)\n--\n\n\ +Decimal.as_integer_ratio() -> (int, int)\n\ +\n\ +Return a pair of integers, whose ratio is exactly equal to the original\n\ +Decimal and with a positive denominator. The ratio is in lowest terms.\n\ +Raise OverflowError on infinities and a ValueError on NaNs.\n\ +\n"); + PyDoc_STRVAR(doc_canonical, "canonical($self, /)\n--\n\n\ Return the canonical encoding of the argument. Currently, the encoding\n\ diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py index ab7d5bd..f907531 100644 --- a/Modules/_decimal/tests/deccheck.py +++ b/Modules/_decimal/tests/deccheck.py @@ -50,8 +50,8 @@ Functions = { '__abs__', '__bool__', '__ceil__', '__complex__', '__copy__', '__floor__', '__float__', '__hash__', '__int__', '__neg__', '__pos__', '__reduce__', '__repr__', '__str__', '__trunc__', - 'adjusted', 'as_tuple', 'canonical', 'conjugate', 'copy_abs', - 'copy_negate', 'is_canonical', 'is_finite', 'is_infinite', + 'adjusted', 'as_integer_ratio', 'as_tuple', 'canonical', 'conjugate', + 'copy_abs', 'copy_negate', 'is_canonical', 'is_finite', 'is_infinite', 'is_nan', 'is_qnan', 'is_signed', 'is_snan', 'is_zero', 'radix' ), # Unary with optional context: @@ -128,7 +128,7 @@ ContextFunctions = { # Functions that require a restricted exponent range for reasonable runtimes. UnaryRestricted = [ '__ceil__', '__floor__', '__int__', '__trunc__', - 'to_integral', 'to_integral_value' + 'as_integer_ratio', 'to_integral', 'to_integral_value' ] BinaryRestricted = ['__round__'] -- cgit v0.12 From 6f5470880eb13cb28d189953375a79b61756ba78 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 28 Dec 2015 23:17:05 +0100 Subject: Fix typo. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7ee0355..b042090 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -126,7 +126,7 @@ Library - Issue #25928: Add Decimal.as_integer_ratio(). - Issue #25447: Copying the lru_cache() wrapper object now always works, - independedly from the type of the wrapped object (by returning the original + independently from the type of the wrapped object (by returning the original object unchanged). - Issue #25768: Have the functions in compileall return booleans instead of -- cgit v0.12 From 2335f44d8e3b71814b26583f4942ee16738b7486 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 29 Dec 2015 00:11:36 +0100 Subject: Issue #25928: Temporarily disable some tests in test_statistics in order to sort out its assumptions about the as_integer_ratio() interface. --- Lib/test/test_statistics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 0089ae8..9a54fe1 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -699,6 +699,7 @@ class ExactRatioTest(unittest.TestCase): num, den = statistics._exact_ratio(x) self.assertEqual(x, num/den) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal(self): D = Decimal _exact_ratio = statistics._exact_ratio @@ -730,6 +731,7 @@ class ExactRatioTest(unittest.TestCase): self.assertIs(ratio[1], None) self.assertEqual(type(ratio[0]), type(nan)) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_nan(self): NAN = Decimal("NAN") sNAN = Decimal("sNAN") @@ -1258,6 +1260,7 @@ class SumSpecialValues(NumericTestCase): with decimal.localcontext(decimal.BasicContext): self.assertRaises(decimal.InvalidOperation, statistics._sum, data) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_snan_raises(self): # Adding sNAN should raise InvalidOperation. sNAN = Decimal('sNAN') -- cgit v0.12 From 150660eefd90e8004beb034ce885befed44df309 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 29 Dec 2015 02:15:35 +0100 Subject: Issue #25940: Make buildbots usable again until a solution is found. --- Lib/test/test_ssl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 548feeb..d1a9088 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1342,6 +1342,7 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) +@unittest.skipIf(True, "temporarily disabled: see #25940") class NetworkedTests(unittest.TestCase): def test_connect(self): @@ -1711,6 +1712,7 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret + @unittest.skipIf(True, "temporarily disabled: see #25940") def test_handshake(self): with support.transient_internet("svn.python.org"): sock = socket.socket(socket.AF_INET) -- cgit v0.12 From 3c61a448f106c7cafb050ff7be5c5c421273e68f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:17:58 -0800 Subject: Issue #24725: Skip the test_socket.testFDPassEmpty on OS X. In OS X 10.11, the test fails consistently due to a platform change since 10.10. Thanks to Jeff Ramnani for the patch. --- Lib/test/test_socket.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 1e355ea..ad1efb2 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2809,6 +2809,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #24725") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. -- cgit v0.12 From 3bbad12b5bd41235bfb6e675606b046d30296eed Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:21:44 -0800 Subject: Tweak skipping message --- Lib/test/test_socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index ad1efb2..99707ce 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2809,7 +2809,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #24725") + @unittest.skipIf(sys.platform == "darwin", "see issue #24725") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. -- cgit v0.12 From 838629a1fea6ec30342522dc0b087fc17872f1f0 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:23:35 -0800 Subject: Issue #25234: Skip test_eintr.test_os_open under OS X. Test inconsistently hangs. --- Lib/test/eintrdata/eintr_tester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 531d576..e1ed3da 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -345,6 +345,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = os.open(path, os.O_WRONLY) os.close(fd) + @unittest.skipIf(sys.platform == "darwin", "hangs under OS X; see issue #25234") def test_os_open(self): self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", self.os_open) -- cgit v0.12 From 050391774a4d1233c392f4e113e51c5a44b3e0ba Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:28:19 -0800 Subject: Issue #25930: Document that os.unlink and os.remove are *semantically* identical. Saying that the functions were identical confused some users who were upset when the functions were no longer simply the same function under different names. Thanks to Anthony Sottile for the bug report and Swati Jaiswal for the initial patch. --- Doc/library/os.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index c1193ad..373cafc 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1789,7 +1789,7 @@ features: be raised; on Unix, the directory entry is removed but the storage allocated to the file is not made available until the original file is no longer in use. - This function is identical to :func:`unlink`. + This function is semantically identical to :func:`unlink`. .. versionadded:: 3.3 The *dir_fd* argument. @@ -2452,10 +2452,10 @@ features: .. function:: unlink(path, *, dir_fd=None) - Remove (delete) the file *path*. This function is identical to - :func:`remove`; the ``unlink`` name is its traditional Unix - name. Please see the documentation for :func:`remove` for - further information. + Remove (delete) the file *path*. This function is semantically + identical to :func:`remove`; the ``unlink`` name is its + traditional Unix name. Please see the documentation for + :func:`remove` for further information. .. versionadded:: 3.3 The *dir_fd* parameter. -- cgit v0.12 From eae30790417d7113599e21639815c1e00798b9ed Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:55:27 -0800 Subject: Issue #25802: Deprecate load_module() on importlib.machinery.SourceFileLoader and SourcelessFileLoader. They were the only remaining implementations of load_module() not documented as deprecated. --- Doc/library/importlib.rst | 8 + Doc/whatsnew/3.6.rst | 7 +- Lib/importlib/_bootstrap_external.py | 1 + Misc/NEWS | 3 + Python/importlib_external.h | 3033 +++++++++++++++++----------------- 5 files changed, 1535 insertions(+), 1517 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 02b0a11..d800835 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -921,6 +921,10 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.Loader.load_module` where specifying the name of the module to load is optional. + .. deprecated:: 3.6 + + Use :meth:`importlib.abc.Loader.exec_module` instead. + .. class:: SourcelessFileLoader(fullname, path) @@ -960,6 +964,10 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.Loader.load_module` where specifying the name of the module to load is optional. + .. deprecated:: 3.6 + + Use :meth:`importlib.abc.Loader.exec_module` instead. + .. class:: ExtensionFileLoader(fullname, path) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c660933..bf5161d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -189,7 +189,12 @@ become proper keywords in Python 3.7. Deprecated Python modules, functions and methods ------------------------------------------------ -* None yet. +* :meth:`importlib.machinery.SourceFileLoader` and + :meth:`importlib.machinery.SourcelessFileLoader` are now deprecated. They + were the only remaining implementations of + :meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not + been deprecated in previous versions of Python in favour of + :meth:`importlib.abc.Loader.exec_module`. Deprecated functions and types of the C API diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index c58a62a..f274501 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -655,6 +655,7 @@ class _LoaderBasics: _bootstrap._call_with_frames_removed(exec, code, module.__dict__) def load_module(self, fullname): + """This module is deprecated.""" return _bootstrap._load_module_shim(self, fullname) diff --git a/Misc/NEWS b/Misc/NEWS index b042090..bf3ee24 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -123,6 +123,9 @@ Core and Builtins Library ------- +- Issue #25802: Document as deprecated the remaining implementations of + importlib.abc.Loader.load_module(). + - Issue #25928: Add Decimal.as_integer_ratio(). - Issue #25447: Copying the lru_cache() wrapper object now always works, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index dd322a9..5828ceb 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1039,581 +1039,304 @@ const unsigned char _Py_M__importlib_external[] = { 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, 0,0,3,0,0,0,67,0,0,0,115,16,0,0,0,116, 0,0,106,1,0,124,0,0,124,1,0,131,2,0,83,41, - 1,78,41,2,114,114,0,0,0,218,17,95,108,111,97,100, - 95,109,111,100,117,108,101,95,115,104,105,109,41,2,114,100, - 0,0,0,114,119,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,11,108,111,97,100,95,109,111, - 100,117,108,101,145,2,0,0,115,2,0,0,0,0,1,122, - 25,95,76,111,97,100,101,114,66,97,115,105,99,115,46,108, - 111,97,100,95,109,111,100,117,108,101,78,41,8,114,105,0, - 0,0,114,104,0,0,0,114,106,0,0,0,114,107,0,0, - 0,114,153,0,0,0,114,180,0,0,0,114,185,0,0,0, - 114,187,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,178,0,0,0,121,2, - 0,0,115,10,0,0,0,12,3,6,2,12,8,12,3,12, - 8,114,178,0,0,0,99,0,0,0,0,0,0,0,0,0, - 0,0,0,4,0,0,0,64,0,0,0,115,106,0,0,0, - 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,100, - 2,0,132,0,0,90,3,0,100,3,0,100,4,0,132,0, - 0,90,4,0,100,5,0,100,6,0,132,0,0,90,5,0, - 100,7,0,100,8,0,132,0,0,90,6,0,100,9,0,100, - 10,0,132,0,0,90,7,0,100,11,0,100,18,0,100,13, - 0,100,14,0,132,0,1,90,8,0,100,15,0,100,16,0, - 132,0,0,90,9,0,100,17,0,83,41,19,218,12,83,111, - 117,114,99,101,76,111,97,100,101,114,99,2,0,0,0,0, - 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, - 10,0,0,0,116,0,0,130,1,0,100,1,0,83,41,2, - 122,178,79,112,116,105,111,110,97,108,32,109,101,116,104,111, - 100,32,116,104,97,116,32,114,101,116,117,114,110,115,32,116, - 104,101,32,109,111,100,105,102,105,99,97,116,105,111,110,32, - 116,105,109,101,32,40,97,110,32,105,110,116,41,32,102,111, - 114,32,116,104,101,10,32,32,32,32,32,32,32,32,115,112, - 101,99,105,102,105,101,100,32,112,97,116,104,44,32,119,104, - 101,114,101,32,112,97,116,104,32,105,115,32,97,32,115,116, - 114,46,10,10,32,32,32,32,32,32,32,32,82,97,105,115, + 1,122,26,84,104,105,115,32,109,111,100,117,108,101,32,105, + 115,32,100,101,112,114,101,99,97,116,101,100,46,41,2,114, + 114,0,0,0,218,17,95,108,111,97,100,95,109,111,100,117, + 108,101,95,115,104,105,109,41,2,114,100,0,0,0,114,119, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,11,108,111,97,100,95,109,111,100,117,108,101,145, + 2,0,0,115,2,0,0,0,0,2,122,25,95,76,111,97, + 100,101,114,66,97,115,105,99,115,46,108,111,97,100,95,109, + 111,100,117,108,101,78,41,8,114,105,0,0,0,114,104,0, + 0,0,114,106,0,0,0,114,107,0,0,0,114,153,0,0, + 0,114,180,0,0,0,114,185,0,0,0,114,187,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,178,0,0,0,121,2,0,0,115,10,0, + 0,0,12,3,6,2,12,8,12,3,12,8,114,178,0,0, + 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, + 0,0,64,0,0,0,115,106,0,0,0,101,0,0,90,1, + 0,100,0,0,90,2,0,100,1,0,100,2,0,132,0,0, + 90,3,0,100,3,0,100,4,0,132,0,0,90,4,0,100, + 5,0,100,6,0,132,0,0,90,5,0,100,7,0,100,8, + 0,132,0,0,90,6,0,100,9,0,100,10,0,132,0,0, + 90,7,0,100,11,0,100,18,0,100,13,0,100,14,0,132, + 0,1,90,8,0,100,15,0,100,16,0,132,0,0,90,9, + 0,100,17,0,83,41,19,218,12,83,111,117,114,99,101,76, + 111,97,100,101,114,99,2,0,0,0,0,0,0,0,2,0, + 0,0,1,0,0,0,67,0,0,0,115,10,0,0,0,116, + 0,0,130,1,0,100,1,0,83,41,2,122,178,79,112,116, + 105,111,110,97,108,32,109,101,116,104,111,100,32,116,104,97, + 116,32,114,101,116,117,114,110,115,32,116,104,101,32,109,111, + 100,105,102,105,99,97,116,105,111,110,32,116,105,109,101,32, + 40,97,110,32,105,110,116,41,32,102,111,114,32,116,104,101, + 10,32,32,32,32,32,32,32,32,115,112,101,99,105,102,105, + 101,100,32,112,97,116,104,44,32,119,104,101,114,101,32,112, + 97,116,104,32,105,115,32,97,32,115,116,114,46,10,10,32, + 32,32,32,32,32,32,32,82,97,105,115,101,115,32,73,79, + 69,114,114,111,114,32,119,104,101,110,32,116,104,101,32,112, + 97,116,104,32,99,97,110,110,111,116,32,98,101,32,104,97, + 110,100,108,101,100,46,10,32,32,32,32,32,32,32,32,78, + 41,1,218,7,73,79,69,114,114,111,114,41,2,114,100,0, + 0,0,114,35,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,10,112,97,116,104,95,109,116,105, + 109,101,152,2,0,0,115,2,0,0,0,0,6,122,23,83, + 111,117,114,99,101,76,111,97,100,101,114,46,112,97,116,104, + 95,109,116,105,109,101,99,2,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,67,0,0,0,115,19,0,0,0, + 100,1,0,124,0,0,106,0,0,124,1,0,131,1,0,105, + 1,0,83,41,2,97,170,1,0,0,79,112,116,105,111,110, + 97,108,32,109,101,116,104,111,100,32,114,101,116,117,114,110, + 105,110,103,32,97,32,109,101,116,97,100,97,116,97,32,100, + 105,99,116,32,102,111,114,32,116,104,101,32,115,112,101,99, + 105,102,105,101,100,32,112,97,116,104,10,32,32,32,32,32, + 32,32,32,116,111,32,98,121,32,116,104,101,32,112,97,116, + 104,32,40,115,116,114,41,46,10,32,32,32,32,32,32,32, + 32,80,111,115,115,105,98,108,101,32,107,101,121,115,58,10, + 32,32,32,32,32,32,32,32,45,32,39,109,116,105,109,101, + 39,32,40,109,97,110,100,97,116,111,114,121,41,32,105,115, + 32,116,104,101,32,110,117,109,101,114,105,99,32,116,105,109, + 101,115,116,97,109,112,32,111,102,32,108,97,115,116,32,115, + 111,117,114,99,101,10,32,32,32,32,32,32,32,32,32,32, + 99,111,100,101,32,109,111,100,105,102,105,99,97,116,105,111, + 110,59,10,32,32,32,32,32,32,32,32,45,32,39,115,105, + 122,101,39,32,40,111,112,116,105,111,110,97,108,41,32,105, + 115,32,116,104,101,32,115,105,122,101,32,105,110,32,98,121, + 116,101,115,32,111,102,32,116,104,101,32,115,111,117,114,99, + 101,32,99,111,100,101,46,10,10,32,32,32,32,32,32,32, + 32,73,109,112,108,101,109,101,110,116,105,110,103,32,116,104, + 105,115,32,109,101,116,104,111,100,32,97,108,108,111,119,115, + 32,116,104,101,32,108,111,97,100,101,114,32,116,111,32,114, + 101,97,100,32,98,121,116,101,99,111,100,101,32,102,105,108, + 101,115,46,10,32,32,32,32,32,32,32,32,82,97,105,115, 101,115,32,73,79,69,114,114,111,114,32,119,104,101,110,32, 116,104,101,32,112,97,116,104,32,99,97,110,110,111,116,32, 98,101,32,104,97,110,100,108,101,100,46,10,32,32,32,32, - 32,32,32,32,78,41,1,218,7,73,79,69,114,114,111,114, + 32,32,32,32,114,126,0,0,0,41,1,114,190,0,0,0, 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, - 104,95,109,116,105,109,101,151,2,0,0,115,2,0,0,0, - 0,6,122,23,83,111,117,114,99,101,76,111,97,100,101,114, - 46,112,97,116,104,95,109,116,105,109,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, - 115,19,0,0,0,100,1,0,124,0,0,106,0,0,124,1, - 0,131,1,0,105,1,0,83,41,2,97,170,1,0,0,79, - 112,116,105,111,110,97,108,32,109,101,116,104,111,100,32,114, - 101,116,117,114,110,105,110,103,32,97,32,109,101,116,97,100, - 97,116,97,32,100,105,99,116,32,102,111,114,32,116,104,101, - 32,115,112,101,99,105,102,105,101,100,32,112,97,116,104,10, - 32,32,32,32,32,32,32,32,116,111,32,98,121,32,116,104, - 101,32,112,97,116,104,32,40,115,116,114,41,46,10,32,32, - 32,32,32,32,32,32,80,111,115,115,105,98,108,101,32,107, - 101,121,115,58,10,32,32,32,32,32,32,32,32,45,32,39, - 109,116,105,109,101,39,32,40,109,97,110,100,97,116,111,114, - 121,41,32,105,115,32,116,104,101,32,110,117,109,101,114,105, - 99,32,116,105,109,101,115,116,97,109,112,32,111,102,32,108, - 97,115,116,32,115,111,117,114,99,101,10,32,32,32,32,32, - 32,32,32,32,32,99,111,100,101,32,109,111,100,105,102,105, - 99,97,116,105,111,110,59,10,32,32,32,32,32,32,32,32, - 45,32,39,115,105,122,101,39,32,40,111,112,116,105,111,110, - 97,108,41,32,105,115,32,116,104,101,32,115,105,122,101,32, - 105,110,32,98,121,116,101,115,32,111,102,32,116,104,101,32, - 115,111,117,114,99,101,32,99,111,100,101,46,10,10,32,32, - 32,32,32,32,32,32,73,109,112,108,101,109,101,110,116,105, - 110,103,32,116,104,105,115,32,109,101,116,104,111,100,32,97, - 108,108,111,119,115,32,116,104,101,32,108,111,97,100,101,114, - 32,116,111,32,114,101,97,100,32,98,121,116,101,99,111,100, - 101,32,102,105,108,101,115,46,10,32,32,32,32,32,32,32, - 32,82,97,105,115,101,115,32,73,79,69,114,114,111,114,32, - 119,104,101,110,32,116,104,101,32,112,97,116,104,32,99,97, - 110,110,111,116,32,98,101,32,104,97,110,100,108,101,100,46, - 10,32,32,32,32,32,32,32,32,114,126,0,0,0,41,1, - 114,190,0,0,0,41,2,114,100,0,0,0,114,35,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,10,112,97,116,104,95,115,116,97,116,115,159,2,0,0, - 115,2,0,0,0,0,11,122,23,83,111,117,114,99,101,76, - 111,97,100,101,114,46,112,97,116,104,95,115,116,97,116,115, - 99,4,0,0,0,0,0,0,0,4,0,0,0,3,0,0, - 0,67,0,0,0,115,16,0,0,0,124,0,0,106,0,0, - 124,2,0,124,3,0,131,2,0,83,41,1,122,228,79,112, - 116,105,111,110,97,108,32,109,101,116,104,111,100,32,119,104, - 105,99,104,32,119,114,105,116,101,115,32,100,97,116,97,32, - 40,98,121,116,101,115,41,32,116,111,32,97,32,102,105,108, - 101,32,112,97,116,104,32,40,97,32,115,116,114,41,46,10, - 10,32,32,32,32,32,32,32,32,73,109,112,108,101,109,101, - 110,116,105,110,103,32,116,104,105,115,32,109,101,116,104,111, - 100,32,97,108,108,111,119,115,32,102,111,114,32,116,104,101, - 32,119,114,105,116,105,110,103,32,111,102,32,98,121,116,101, - 99,111,100,101,32,102,105,108,101,115,46,10,10,32,32,32, - 32,32,32,32,32,84,104,101,32,115,111,117,114,99,101,32, - 112,97,116,104,32,105,115,32,110,101,101,100,101,100,32,105, - 110,32,111,114,100,101,114,32,116,111,32,99,111,114,114,101, - 99,116,108,121,32,116,114,97,110,115,102,101,114,32,112,101, - 114,109,105,115,115,105,111,110,115,10,32,32,32,32,32,32, - 32,32,41,1,218,8,115,101,116,95,100,97,116,97,41,4, - 114,100,0,0,0,114,90,0,0,0,90,10,99,97,99,104, - 101,95,112,97,116,104,114,53,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,15,95,99,97,99, - 104,101,95,98,121,116,101,99,111,100,101,172,2,0,0,115, - 2,0,0,0,0,8,122,28,83,111,117,114,99,101,76,111, - 97,100,101,114,46,95,99,97,99,104,101,95,98,121,116,101, - 99,111,100,101,99,3,0,0,0,0,0,0,0,3,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,122,150,79,112,116,105,111,110,97,108,32,109, - 101,116,104,111,100,32,119,104,105,99,104,32,119,114,105,116, - 101,115,32,100,97,116,97,32,40,98,121,116,101,115,41,32, - 116,111,32,97,32,102,105,108,101,32,112,97,116,104,32,40, - 97,32,115,116,114,41,46,10,10,32,32,32,32,32,32,32, - 32,73,109,112,108,101,109,101,110,116,105,110,103,32,116,104, - 105,115,32,109,101,116,104,111,100,32,97,108,108,111,119,115, - 32,102,111,114,32,116,104,101,32,119,114,105,116,105,110,103, - 32,111,102,32,98,121,116,101,99,111,100,101,32,102,105,108, - 101,115,46,10,32,32,32,32,32,32,32,32,78,114,4,0, - 0,0,41,3,114,100,0,0,0,114,35,0,0,0,114,53, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,192,0,0,0,182,2,0,0,115,0,0,0,0, - 122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,115, - 101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0, - 5,0,0,0,16,0,0,0,67,0,0,0,115,105,0,0, - 0,124,0,0,106,0,0,124,1,0,131,1,0,125,2,0, - 121,19,0,124,0,0,106,1,0,124,2,0,131,1,0,125, - 3,0,87,110,58,0,4,116,2,0,107,10,0,114,94,0, - 1,125,4,0,1,122,26,0,116,3,0,100,1,0,100,2, - 0,124,1,0,131,1,1,124,4,0,130,2,0,87,89,100, - 3,0,100,3,0,125,4,0,126,4,0,88,110,1,0,88, - 116,4,0,124,3,0,131,1,0,83,41,4,122,52,67,111, - 110,99,114,101,116,101,32,105,109,112,108,101,109,101,110,116, - 97,116,105,111,110,32,111,102,32,73,110,115,112,101,99,116, - 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, - 101,46,122,39,115,111,117,114,99,101,32,110,111,116,32,97, - 118,97,105,108,97,98,108,101,32,116,104,114,111,117,103,104, - 32,103,101,116,95,100,97,116,97,40,41,114,98,0,0,0, - 78,41,5,114,151,0,0,0,218,8,103,101,116,95,100,97, - 116,97,114,40,0,0,0,114,99,0,0,0,114,149,0,0, - 0,41,5,114,100,0,0,0,114,119,0,0,0,114,35,0, - 0,0,114,147,0,0,0,218,3,101,120,99,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,10,103,101,116, - 95,115,111,117,114,99,101,189,2,0,0,115,14,0,0,0, - 0,2,15,1,3,1,19,1,18,1,9,1,31,1,122,23, - 83,111,117,114,99,101,76,111,97,100,101,114,46,103,101,116, - 95,115,111,117,114,99,101,218,9,95,111,112,116,105,109,105, - 122,101,114,29,0,0,0,99,3,0,0,0,1,0,0,0, - 4,0,0,0,9,0,0,0,67,0,0,0,115,34,0,0, - 0,116,0,0,106,1,0,116,2,0,124,1,0,124,2,0, - 100,1,0,100,2,0,100,3,0,100,4,0,124,3,0,131, - 4,2,83,41,5,122,130,82,101,116,117,114,110,32,116,104, - 101,32,99,111,100,101,32,111,98,106,101,99,116,32,99,111, - 109,112,105,108,101,100,32,102,114,111,109,32,115,111,117,114, - 99,101,46,10,10,32,32,32,32,32,32,32,32,84,104,101, - 32,39,100,97,116,97,39,32,97,114,103,117,109,101,110,116, - 32,99,97,110,32,98,101,32,97,110,121,32,111,98,106,101, - 99,116,32,116,121,112,101,32,116,104,97,116,32,99,111,109, - 112,105,108,101,40,41,32,115,117,112,112,111,114,116,115,46, - 10,32,32,32,32,32,32,32,32,114,183,0,0,0,218,12, - 100,111,110,116,95,105,110,104,101,114,105,116,84,114,68,0, - 0,0,41,3,114,114,0,0,0,114,182,0,0,0,218,7, - 99,111,109,112,105,108,101,41,4,114,100,0,0,0,114,53, - 0,0,0,114,35,0,0,0,114,197,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,14,115,111, - 117,114,99,101,95,116,111,95,99,111,100,101,199,2,0,0, - 115,4,0,0,0,0,5,21,1,122,27,83,111,117,114,99, - 101,76,111,97,100,101,114,46,115,111,117,114,99,101,95,116, - 111,95,99,111,100,101,99,2,0,0,0,0,0,0,0,10, - 0,0,0,43,0,0,0,67,0,0,0,115,183,1,0,0, - 124,0,0,106,0,0,124,1,0,131,1,0,125,2,0,100, - 1,0,125,3,0,121,16,0,116,1,0,124,2,0,131,1, - 0,125,4,0,87,110,24,0,4,116,2,0,107,10,0,114, - 63,0,1,1,1,100,1,0,125,4,0,89,110,205,0,88, - 121,19,0,124,0,0,106,3,0,124,2,0,131,1,0,125, - 5,0,87,110,18,0,4,116,4,0,107,10,0,114,103,0, - 1,1,1,89,110,165,0,88,116,5,0,124,5,0,100,2, - 0,25,131,1,0,125,3,0,121,19,0,124,0,0,106,6, - 0,124,4,0,131,1,0,125,6,0,87,110,18,0,4,116, - 7,0,107,10,0,114,159,0,1,1,1,89,110,109,0,88, - 121,34,0,116,8,0,124,6,0,100,3,0,124,5,0,100, - 4,0,124,1,0,100,5,0,124,4,0,131,1,3,125,7, - 0,87,110,24,0,4,116,9,0,116,10,0,102,2,0,107, - 10,0,114,220,0,1,1,1,89,110,48,0,88,116,11,0, - 106,12,0,100,6,0,124,4,0,124,2,0,131,3,0,1, - 116,13,0,124,7,0,100,4,0,124,1,0,100,7,0,124, - 4,0,100,8,0,124,2,0,131,1,3,83,124,0,0,106, - 6,0,124,2,0,131,1,0,125,8,0,124,0,0,106,14, - 0,124,8,0,124,2,0,131,2,0,125,9,0,116,11,0, - 106,12,0,100,9,0,124,2,0,131,2,0,1,116,15,0, - 106,16,0,12,114,179,1,124,4,0,100,1,0,107,9,0, - 114,179,1,124,3,0,100,1,0,107,9,0,114,179,1,116, - 17,0,124,9,0,124,3,0,116,18,0,124,8,0,131,1, - 0,131,3,0,125,6,0,121,39,0,124,0,0,106,19,0, - 124,2,0,124,4,0,124,6,0,131,3,0,1,116,11,0, - 106,12,0,100,10,0,124,4,0,131,2,0,1,87,110,18, - 0,4,116,2,0,107,10,0,114,178,1,1,1,1,89,110, - 1,0,88,124,9,0,83,41,11,122,190,67,111,110,99,114, - 101,116,101,32,105,109,112,108,101,109,101,110,116,97,116,105, - 111,110,32,111,102,32,73,110,115,112,101,99,116,76,111,97, - 100,101,114,46,103,101,116,95,99,111,100,101,46,10,10,32, - 32,32,32,32,32,32,32,82,101,97,100,105,110,103,32,111, - 102,32,98,121,116,101,99,111,100,101,32,114,101,113,117,105, - 114,101,115,32,112,97,116,104,95,115,116,97,116,115,32,116, - 111,32,98,101,32,105,109,112,108,101,109,101,110,116,101,100, - 46,32,84,111,32,119,114,105,116,101,10,32,32,32,32,32, - 32,32,32,98,121,116,101,99,111,100,101,44,32,115,101,116, - 95,100,97,116,97,32,109,117,115,116,32,97,108,115,111,32, - 98,101,32,105,109,112,108,101,109,101,110,116,101,100,46,10, - 10,32,32,32,32,32,32,32,32,78,114,126,0,0,0,114, - 132,0,0,0,114,98,0,0,0,114,35,0,0,0,122,13, - 123,125,32,109,97,116,99,104,101,115,32,123,125,114,89,0, - 0,0,114,90,0,0,0,122,19,99,111,100,101,32,111,98, - 106,101,99,116,32,102,114,111,109,32,123,125,122,10,119,114, - 111,116,101,32,123,33,114,125,41,20,114,151,0,0,0,114, - 79,0,0,0,114,66,0,0,0,114,191,0,0,0,114,189, - 0,0,0,114,14,0,0,0,114,194,0,0,0,114,40,0, - 0,0,114,135,0,0,0,114,99,0,0,0,114,130,0,0, - 0,114,114,0,0,0,114,129,0,0,0,114,141,0,0,0, - 114,200,0,0,0,114,7,0,0,0,218,19,100,111,110,116, - 95,119,114,105,116,101,95,98,121,116,101,99,111,100,101,114, - 144,0,0,0,114,31,0,0,0,114,193,0,0,0,41,10, - 114,100,0,0,0,114,119,0,0,0,114,90,0,0,0,114, - 133,0,0,0,114,89,0,0,0,218,2,115,116,114,53,0, - 0,0,218,10,98,121,116,101,115,95,100,97,116,97,114,147, - 0,0,0,90,11,99,111,100,101,95,111,98,106,101,99,116, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 181,0,0,0,207,2,0,0,115,78,0,0,0,0,7,15, - 1,6,1,3,1,16,1,13,1,11,2,3,1,19,1,13, - 1,5,2,16,1,3,1,19,1,13,1,5,2,3,1,9, - 1,12,1,13,1,19,1,5,2,12,1,7,1,15,1,6, - 1,7,1,15,1,18,1,16,1,22,1,12,1,9,1,15, - 1,3,1,19,1,20,1,13,1,5,1,122,21,83,111,117, - 114,99,101,76,111,97,100,101,114,46,103,101,116,95,99,111, - 100,101,78,114,87,0,0,0,41,10,114,105,0,0,0,114, - 104,0,0,0,114,106,0,0,0,114,190,0,0,0,114,191, - 0,0,0,114,193,0,0,0,114,192,0,0,0,114,196,0, - 0,0,114,200,0,0,0,114,181,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,188,0,0,0,149,2,0,0,115,14,0,0,0,12,2, - 12,8,12,13,12,10,12,7,12,10,18,8,114,188,0,0, - 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, - 0,0,0,0,0,0,115,112,0,0,0,101,0,0,90,1, - 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, - 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, - 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, - 0,101,7,0,135,0,0,102,1,0,100,8,0,100,9,0, - 134,0,0,131,1,0,90,8,0,101,7,0,100,10,0,100, - 11,0,132,0,0,131,1,0,90,9,0,100,12,0,100,13, - 0,132,0,0,90,10,0,135,0,0,83,41,14,218,10,70, - 105,108,101,76,111,97,100,101,114,122,103,66,97,115,101,32, - 102,105,108,101,32,108,111,97,100,101,114,32,99,108,97,115, - 115,32,119,104,105,99,104,32,105,109,112,108,101,109,101,110, - 116,115,32,116,104,101,32,108,111,97,100,101,114,32,112,114, - 111,116,111,99,111,108,32,109,101,116,104,111,100,115,32,116, - 104,97,116,10,32,32,32,32,114,101,113,117,105,114,101,32, - 102,105,108,101,32,115,121,115,116,101,109,32,117,115,97,103, - 101,46,99,3,0,0,0,0,0,0,0,3,0,0,0,2, - 0,0,0,67,0,0,0,115,22,0,0,0,124,1,0,124, - 0,0,95,0,0,124,2,0,124,0,0,95,1,0,100,1, - 0,83,41,2,122,75,67,97,99,104,101,32,116,104,101,32, - 109,111,100,117,108,101,32,110,97,109,101,32,97,110,100,32, - 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, - 102,105,108,101,32,102,111,117,110,100,32,98,121,32,116,104, - 101,10,32,32,32,32,32,32,32,32,102,105,110,100,101,114, - 46,78,41,2,114,98,0,0,0,114,35,0,0,0,41,3, - 114,100,0,0,0,114,119,0,0,0,114,35,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,179, - 0,0,0,8,3,0,0,115,4,0,0,0,0,3,9,1, - 122,19,70,105,108,101,76,111,97,100,101,114,46,95,95,105, - 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, - 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, - 0,0,106,0,0,124,1,0,106,0,0,107,2,0,111,33, - 0,124,0,0,106,1,0,124,1,0,106,1,0,107,2,0, - 83,41,1,78,41,2,218,9,95,95,99,108,97,115,115,95, - 95,114,111,0,0,0,41,2,114,100,0,0,0,218,5,111, - 116,104,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,95,101,113,95,95,14,3,0,0,115, - 4,0,0,0,0,1,18,1,122,17,70,105,108,101,76,111, - 97,100,101,114,46,95,95,101,113,95,95,99,1,0,0,0, - 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, - 115,26,0,0,0,116,0,0,124,0,0,106,1,0,131,1, - 0,116,0,0,124,0,0,106,2,0,131,1,0,65,83,41, - 1,78,41,3,218,4,104,97,115,104,114,98,0,0,0,114, - 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,8,95,95,104,97, - 115,104,95,95,18,3,0,0,115,2,0,0,0,0,1,122, - 19,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, - 115,104,95,95,99,2,0,0,0,0,0,0,0,2,0,0, - 0,3,0,0,0,3,0,0,0,115,22,0,0,0,116,0, - 0,116,1,0,124,0,0,131,2,0,106,2,0,124,1,0, - 131,1,0,83,41,1,122,100,76,111,97,100,32,97,32,109, - 111,100,117,108,101,32,102,114,111,109,32,97,32,102,105,108, - 101,46,10,10,32,32,32,32,32,32,32,32,84,104,105,115, - 32,109,101,116,104,111,100,32,105,115,32,100,101,112,114,101, - 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, - 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, - 100,46,10,10,32,32,32,32,32,32,32,32,41,3,218,5, - 115,117,112,101,114,114,204,0,0,0,114,187,0,0,0,41, - 2,114,100,0,0,0,114,119,0,0,0,41,1,114,205,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,187,0,0, - 0,21,3,0,0,115,2,0,0,0,0,10,122,22,70,105, - 108,101,76,111,97,100,101,114,46,108,111,97,100,95,109,111, - 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,7,0,0,0,124,0, - 0,106,0,0,83,41,1,122,58,82,101,116,117,114,110,32, - 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, - 115,111,117,114,99,101,32,102,105,108,101,32,97,115,32,102, - 111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100, - 101,114,46,41,1,114,35,0,0,0,41,2,114,100,0,0, - 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,151,0,0,0,33,3,0,0,115,2, - 0,0,0,0,3,122,23,70,105,108,101,76,111,97,100,101, - 114,46,103,101,116,95,102,105,108,101,110,97,109,101,99,2, - 0,0,0,0,0,0,0,3,0,0,0,9,0,0,0,67, - 0,0,0,115,42,0,0,0,116,0,0,106,1,0,124,1, - 0,100,1,0,131,2,0,143,17,0,125,2,0,124,2,0, - 106,2,0,131,0,0,83,87,100,2,0,81,82,88,100,2, - 0,83,41,3,122,39,82,101,116,117,114,110,32,116,104,101, - 32,100,97,116,97,32,102,114,111,109,32,112,97,116,104,32, - 97,115,32,114,97,119,32,98,121,116,101,115,46,218,1,114, - 78,41,3,114,49,0,0,0,114,50,0,0,0,90,4,114, - 101,97,100,41,3,114,100,0,0,0,114,35,0,0,0,114, - 54,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,194,0,0,0,38,3,0,0,115,4,0,0, - 0,0,2,21,1,122,19,70,105,108,101,76,111,97,100,101, - 114,46,103,101,116,95,100,97,116,97,41,11,114,105,0,0, - 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, - 114,179,0,0,0,114,207,0,0,0,114,209,0,0,0,114, - 116,0,0,0,114,187,0,0,0,114,151,0,0,0,114,194, - 0,0,0,114,4,0,0,0,114,4,0,0,0,41,1,114, - 205,0,0,0,114,5,0,0,0,114,204,0,0,0,3,3, - 0,0,115,14,0,0,0,12,3,6,2,12,6,12,4,12, - 3,24,12,18,5,114,204,0,0,0,99,0,0,0,0,0, - 0,0,0,0,0,0,0,4,0,0,0,64,0,0,0,115, - 64,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, - 100,1,0,90,3,0,100,2,0,100,3,0,132,0,0,90, - 4,0,100,4,0,100,5,0,132,0,0,90,5,0,100,6, - 0,100,7,0,100,8,0,100,9,0,132,0,1,90,6,0, - 100,10,0,83,41,11,218,16,83,111,117,114,99,101,70,105, - 108,101,76,111,97,100,101,114,122,62,67,111,110,99,114,101, - 116,101,32,105,109,112,108,101,109,101,110,116,97,116,105,111, - 110,32,111,102,32,83,111,117,114,99,101,76,111,97,100,101, - 114,32,117,115,105,110,103,32,116,104,101,32,102,105,108,101, - 32,115,121,115,116,101,109,46,99,2,0,0,0,0,0,0, - 0,3,0,0,0,4,0,0,0,67,0,0,0,115,34,0, - 0,0,116,0,0,124,1,0,131,1,0,125,2,0,100,1, - 0,124,2,0,106,1,0,100,2,0,124,2,0,106,2,0, - 105,2,0,83,41,3,122,33,82,101,116,117,114,110,32,116, - 104,101,32,109,101,116,97,100,97,116,97,32,102,111,114,32, - 116,104,101,32,112,97,116,104,46,114,126,0,0,0,114,127, - 0,0,0,41,3,114,39,0,0,0,218,8,115,116,95,109, - 116,105,109,101,90,7,115,116,95,115,105,122,101,41,3,114, - 100,0,0,0,114,35,0,0,0,114,202,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,191,0, - 0,0,48,3,0,0,115,4,0,0,0,0,2,12,1,122, - 27,83,111,117,114,99,101,70,105,108,101,76,111,97,100,101, - 114,46,112,97,116,104,95,115,116,97,116,115,99,4,0,0, - 0,0,0,0,0,5,0,0,0,5,0,0,0,67,0,0, - 0,115,34,0,0,0,116,0,0,124,1,0,131,1,0,125, - 4,0,124,0,0,106,1,0,124,2,0,124,3,0,100,1, - 0,124,4,0,131,2,1,83,41,2,78,218,5,95,109,111, - 100,101,41,2,114,97,0,0,0,114,192,0,0,0,41,5, - 114,100,0,0,0,114,90,0,0,0,114,89,0,0,0,114, - 53,0,0,0,114,42,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,193,0,0,0,53,3,0, - 0,115,4,0,0,0,0,2,12,1,122,32,83,111,117,114, - 99,101,70,105,108,101,76,111,97,100,101,114,46,95,99,97, - 99,104,101,95,98,121,116,101,99,111,100,101,114,214,0,0, - 0,105,182,1,0,0,99,3,0,0,0,1,0,0,0,9, - 0,0,0,17,0,0,0,67,0,0,0,115,62,1,0,0, - 116,0,0,124,1,0,131,1,0,92,2,0,125,4,0,125, - 5,0,103,0,0,125,6,0,120,54,0,124,4,0,114,80, - 0,116,1,0,124,4,0,131,1,0,12,114,80,0,116,0, - 0,124,4,0,131,1,0,92,2,0,125,4,0,125,7,0, - 124,6,0,106,2,0,124,7,0,131,1,0,1,113,27,0, - 87,120,135,0,116,3,0,124,6,0,131,1,0,68,93,121, - 0,125,7,0,116,4,0,124,4,0,124,7,0,131,2,0, - 125,4,0,121,17,0,116,5,0,106,6,0,124,4,0,131, - 1,0,1,87,113,94,0,4,116,7,0,107,10,0,114,155, - 0,1,1,1,119,94,0,89,113,94,0,4,116,8,0,107, - 10,0,114,214,0,1,125,8,0,1,122,28,0,116,9,0, - 106,10,0,100,1,0,124,4,0,124,8,0,131,3,0,1, - 100,2,0,83,87,89,100,2,0,100,2,0,125,8,0,126, - 8,0,88,113,94,0,88,113,94,0,87,121,36,0,116,11, - 0,124,1,0,124,2,0,124,3,0,131,3,0,1,116,9, - 0,106,10,0,100,3,0,124,1,0,131,2,0,1,87,110, - 56,0,4,116,8,0,107,10,0,114,57,1,1,125,8,0, - 1,122,24,0,116,9,0,106,10,0,100,1,0,124,1,0, - 124,8,0,131,3,0,1,87,89,100,2,0,100,2,0,125, - 8,0,126,8,0,88,110,1,0,88,100,2,0,83,41,4, - 122,27,87,114,105,116,101,32,98,121,116,101,115,32,100,97, - 116,97,32,116,111,32,97,32,102,105,108,101,46,122,27,99, - 111,117,108,100,32,110,111,116,32,99,114,101,97,116,101,32, - 123,33,114,125,58,32,123,33,114,125,78,122,12,99,114,101, - 97,116,101,100,32,123,33,114,125,41,12,114,38,0,0,0, - 114,46,0,0,0,114,157,0,0,0,114,33,0,0,0,114, - 28,0,0,0,114,3,0,0,0,90,5,109,107,100,105,114, - 218,15,70,105,108,101,69,120,105,115,116,115,69,114,114,111, - 114,114,40,0,0,0,114,114,0,0,0,114,129,0,0,0, - 114,55,0,0,0,41,9,114,100,0,0,0,114,35,0,0, - 0,114,53,0,0,0,114,214,0,0,0,218,6,112,97,114, - 101,110,116,114,94,0,0,0,114,27,0,0,0,114,23,0, - 0,0,114,195,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,192,0,0,0,58,3,0,0,115, - 42,0,0,0,0,2,18,1,6,2,22,1,18,1,17,2, - 19,1,15,1,3,1,17,1,13,2,7,1,18,3,9,1, - 10,1,27,1,3,1,16,1,20,1,18,2,12,1,122,25, - 83,111,117,114,99,101,70,105,108,101,76,111,97,100,101,114, - 46,115,101,116,95,100,97,116,97,78,41,7,114,105,0,0, - 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, - 114,191,0,0,0,114,193,0,0,0,114,192,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,212,0,0,0,44,3,0,0,115,8,0,0, - 0,12,2,6,2,12,5,12,5,114,212,0,0,0,99,0, - 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, - 0,0,0,115,46,0,0,0,101,0,0,90,1,0,100,0, - 0,90,2,0,100,1,0,90,3,0,100,2,0,100,3,0, - 132,0,0,90,4,0,100,4,0,100,5,0,132,0,0,90, - 5,0,100,6,0,83,41,7,218,20,83,111,117,114,99,101, - 108,101,115,115,70,105,108,101,76,111,97,100,101,114,122,45, - 76,111,97,100,101,114,32,119,104,105,99,104,32,104,97,110, - 100,108,101,115,32,115,111,117,114,99,101,108,101,115,115,32, - 102,105,108,101,32,105,109,112,111,114,116,115,46,99,2,0, - 0,0,0,0,0,0,5,0,0,0,6,0,0,0,67,0, - 0,0,115,76,0,0,0,124,0,0,106,0,0,124,1,0, - 131,1,0,125,2,0,124,0,0,106,1,0,124,2,0,131, - 1,0,125,3,0,116,2,0,124,3,0,100,1,0,124,1, - 0,100,2,0,124,2,0,131,1,2,125,4,0,116,3,0, - 124,4,0,100,1,0,124,1,0,100,3,0,124,2,0,131, - 1,2,83,41,4,78,114,98,0,0,0,114,35,0,0,0, - 114,89,0,0,0,41,4,114,151,0,0,0,114,194,0,0, - 0,114,135,0,0,0,114,141,0,0,0,41,5,114,100,0, - 0,0,114,119,0,0,0,114,35,0,0,0,114,53,0,0, - 0,114,203,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,93,3,0,0,115,8, - 0,0,0,0,1,15,1,15,1,24,1,122,29,83,111,117, - 114,99,101,108,101,115,115,70,105,108,101,76,111,97,100,101, - 114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, - 4,0,0,0,100,1,0,83,41,2,122,39,82,101,116,117, - 114,110,32,78,111,110,101,32,97,115,32,116,104,101,114,101, - 32,105,115,32,110,111,32,115,111,117,114,99,101,32,99,111, - 100,101,46,78,114,4,0,0,0,41,2,114,100,0,0,0, - 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,99,3,0,0,115,2,0, - 0,0,0,2,122,31,83,111,117,114,99,101,108,101,115,115, - 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,115, - 111,117,114,99,101,78,41,6,114,105,0,0,0,114,104,0, - 0,0,114,106,0,0,0,114,107,0,0,0,114,181,0,0, - 0,114,196,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,89, - 3,0,0,115,6,0,0,0,12,2,6,2,12,6,114,217, - 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, - 3,0,0,0,64,0,0,0,115,136,0,0,0,101,0,0, - 90,1,0,100,0,0,90,2,0,100,1,0,90,3,0,100, - 2,0,100,3,0,132,0,0,90,4,0,100,4,0,100,5, - 0,132,0,0,90,5,0,100,6,0,100,7,0,132,0,0, - 90,6,0,100,8,0,100,9,0,132,0,0,90,7,0,100, - 10,0,100,11,0,132,0,0,90,8,0,100,12,0,100,13, - 0,132,0,0,90,9,0,100,14,0,100,15,0,132,0,0, - 90,10,0,100,16,0,100,17,0,132,0,0,90,11,0,101, - 12,0,100,18,0,100,19,0,132,0,0,131,1,0,90,13, - 0,100,20,0,83,41,21,218,19,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,122,93,76,111, - 97,100,101,114,32,102,111,114,32,101,120,116,101,110,115,105, - 111,110,32,109,111,100,117,108,101,115,46,10,10,32,32,32, - 32,84,104,101,32,99,111,110,115,116,114,117,99,116,111,114, - 32,105,115,32,100,101,115,105,103,110,101,100,32,116,111,32, - 119,111,114,107,32,119,105,116,104,32,70,105,108,101,70,105, - 110,100,101,114,46,10,10,32,32,32,32,99,3,0,0,0, - 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, - 115,22,0,0,0,124,1,0,124,0,0,95,0,0,124,2, - 0,124,0,0,95,1,0,100,0,0,83,41,1,78,41,2, - 114,98,0,0,0,114,35,0,0,0,41,3,114,100,0,0, - 0,114,98,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,116, - 3,0,0,115,4,0,0,0,0,1,9,1,122,28,69,120, - 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, - 114,46,95,95,105,110,105,116,95,95,99,2,0,0,0,0, - 0,0,0,2,0,0,0,2,0,0,0,67,0,0,0,115, - 34,0,0,0,124,0,0,106,0,0,124,1,0,106,0,0, - 107,2,0,111,33,0,124,0,0,106,1,0,124,1,0,106, - 1,0,107,2,0,83,41,1,78,41,2,114,205,0,0,0, - 114,111,0,0,0,41,2,114,100,0,0,0,114,206,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,207,0,0,0,120,3,0,0,115,4,0,0,0,0,1, - 18,1,122,26,69,120,116,101,110,115,105,111,110,70,105,108, - 101,76,111,97,100,101,114,46,95,95,101,113,95,95,99,1, - 0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,67, - 0,0,0,115,26,0,0,0,116,0,0,124,0,0,106,1, - 0,131,1,0,116,0,0,124,0,0,106,2,0,131,1,0, - 65,83,41,1,78,41,3,114,208,0,0,0,114,98,0,0, - 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,209,0,0, - 0,124,3,0,0,115,2,0,0,0,0,1,122,28,69,120, - 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, - 114,46,95,95,104,97,115,104,95,95,99,2,0,0,0,0, - 0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115, - 50,0,0,0,116,0,0,106,1,0,116,2,0,106,3,0, - 124,1,0,131,2,0,125,2,0,116,0,0,106,4,0,100, - 1,0,124,1,0,106,5,0,124,0,0,106,6,0,131,3, - 0,1,124,2,0,83,41,2,122,38,67,114,101,97,116,101, - 32,97,110,32,117,110,105,116,105,97,108,105,122,101,100,32, - 101,120,116,101,110,115,105,111,110,32,109,111,100,117,108,101, - 122,38,101,120,116,101,110,115,105,111,110,32,109,111,100,117, - 108,101,32,123,33,114,125,32,108,111,97,100,101,100,32,102, - 114,111,109,32,123,33,114,125,41,7,114,114,0,0,0,114, - 182,0,0,0,114,139,0,0,0,90,14,99,114,101,97,116, - 101,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, - 0,0,0,114,35,0,0,0,41,3,114,100,0,0,0,114, - 158,0,0,0,114,184,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,127,3,0, - 0,115,10,0,0,0,0,2,6,1,15,1,9,1,16,1, - 122,33,69,120,116,101,110,115,105,111,110,70,105,108,101,76, - 111,97,100,101,114,46,99,114,101,97,116,101,95,109,111,100, - 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, - 4,0,0,0,67,0,0,0,115,48,0,0,0,116,0,0, - 106,1,0,116,2,0,106,3,0,124,1,0,131,2,0,1, - 116,0,0,106,4,0,100,1,0,124,0,0,106,5,0,124, - 0,0,106,6,0,131,3,0,1,100,2,0,83,41,3,122, - 30,73,110,105,116,105,97,108,105,122,101,32,97,110,32,101, - 120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,122, - 40,101,120,116,101,110,115,105,111,110,32,109,111,100,117,108, - 101,32,123,33,114,125,32,101,120,101,99,117,116,101,100,32, - 102,114,111,109,32,123,33,114,125,78,41,7,114,114,0,0, - 0,114,182,0,0,0,114,139,0,0,0,90,12,101,120,101, - 99,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, - 0,0,0,114,35,0,0,0,41,2,114,100,0,0,0,114, - 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,185,0,0,0,135,3,0,0,115,6,0,0, - 0,0,2,19,1,9,1,122,31,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,46,101,120,101, - 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, - 0,2,0,0,0,4,0,0,0,3,0,0,0,115,48,0, - 0,0,116,0,0,124,0,0,106,1,0,131,1,0,100,1, - 0,25,137,0,0,116,2,0,135,0,0,102,1,0,100,2, - 0,100,3,0,134,0,0,116,3,0,68,131,1,0,131,1, - 0,83,41,4,122,49,82,101,116,117,114,110,32,84,114,117, - 101,32,105,102,32,116,104,101,32,101,120,116,101,110,115,105, - 111,110,32,109,111,100,117,108,101,32,105,115,32,97,32,112, - 97,99,107,97,103,101,46,114,29,0,0,0,99,1,0,0, - 0,0,0,0,0,2,0,0,0,4,0,0,0,51,0,0, - 0,115,31,0,0,0,124,0,0,93,21,0,125,1,0,136, - 0,0,100,0,0,124,1,0,23,107,2,0,86,1,113,3, - 0,100,1,0,83,41,2,114,179,0,0,0,78,114,4,0, - 0,0,41,2,114,22,0,0,0,218,6,115,117,102,102,105, - 120,41,1,218,9,102,105,108,101,95,110,97,109,101,114,4, - 0,0,0,114,5,0,0,0,250,9,60,103,101,110,101,120, - 112,114,62,144,3,0,0,115,2,0,0,0,6,1,122,49, - 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, - 100,101,114,46,105,115,95,112,97,99,107,97,103,101,46,60, - 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, - 62,41,4,114,38,0,0,0,114,35,0,0,0,218,3,97, - 110,121,218,18,69,88,84,69,78,83,73,79,78,95,83,85, - 70,70,73,88,69,83,41,2,114,100,0,0,0,114,119,0, - 0,0,114,4,0,0,0,41,1,114,220,0,0,0,114,5, - 0,0,0,114,153,0,0,0,141,3,0,0,115,6,0,0, - 0,0,2,19,1,18,1,122,30,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,46,105,115,95, - 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, - 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, - 0,100,1,0,83,41,2,122,63,82,101,116,117,114,110,32, - 78,111,110,101,32,97,115,32,97,110,32,101,120,116,101,110, - 115,105,111,110,32,109,111,100,117,108,101,32,99,97,110,110, - 111,116,32,99,114,101,97,116,101,32,97,32,99,111,100,101, - 32,111,98,106,101,99,116,46,78,114,4,0,0,0,41,2, - 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,181,0,0,0,147,3, - 0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,110, - 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,103, - 101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,0, - 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, - 0,100,1,0,83,41,2,122,53,82,101,116,117,114,110,32, - 78,111,110,101,32,97,115,32,101,120,116,101,110,115,105,111, - 110,32,109,111,100,117,108,101,115,32,104,97,118,101,32,110, - 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, - 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 196,0,0,0,151,3,0,0,115,2,0,0,0,0,2,122, - 30,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, - 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, + 104,95,115,116,97,116,115,160,2,0,0,115,2,0,0,0, + 0,11,122,23,83,111,117,114,99,101,76,111,97,100,101,114, + 46,112,97,116,104,95,115,116,97,116,115,99,4,0,0,0, + 0,0,0,0,4,0,0,0,3,0,0,0,67,0,0,0, + 115,16,0,0,0,124,0,0,106,0,0,124,2,0,124,3, + 0,131,2,0,83,41,1,122,228,79,112,116,105,111,110,97, + 108,32,109,101,116,104,111,100,32,119,104,105,99,104,32,119, + 114,105,116,101,115,32,100,97,116,97,32,40,98,121,116,101, + 115,41,32,116,111,32,97,32,102,105,108,101,32,112,97,116, + 104,32,40,97,32,115,116,114,41,46,10,10,32,32,32,32, + 32,32,32,32,73,109,112,108,101,109,101,110,116,105,110,103, + 32,116,104,105,115,32,109,101,116,104,111,100,32,97,108,108, + 111,119,115,32,102,111,114,32,116,104,101,32,119,114,105,116, + 105,110,103,32,111,102,32,98,121,116,101,99,111,100,101,32, + 102,105,108,101,115,46,10,10,32,32,32,32,32,32,32,32, + 84,104,101,32,115,111,117,114,99,101,32,112,97,116,104,32, + 105,115,32,110,101,101,100,101,100,32,105,110,32,111,114,100, + 101,114,32,116,111,32,99,111,114,114,101,99,116,108,121,32, + 116,114,97,110,115,102,101,114,32,112,101,114,109,105,115,115, + 105,111,110,115,10,32,32,32,32,32,32,32,32,41,1,218, + 8,115,101,116,95,100,97,116,97,41,4,114,100,0,0,0, + 114,90,0,0,0,90,10,99,97,99,104,101,95,112,97,116, + 104,114,53,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,15,95,99,97,99,104,101,95,98,121, + 116,101,99,111,100,101,173,2,0,0,115,2,0,0,0,0, + 8,122,28,83,111,117,114,99,101,76,111,97,100,101,114,46, + 95,99,97,99,104,101,95,98,121,116,101,99,111,100,101,99, + 3,0,0,0,0,0,0,0,3,0,0,0,1,0,0,0, + 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,122, + 150,79,112,116,105,111,110,97,108,32,109,101,116,104,111,100, + 32,119,104,105,99,104,32,119,114,105,116,101,115,32,100,97, + 116,97,32,40,98,121,116,101,115,41,32,116,111,32,97,32, + 102,105,108,101,32,112,97,116,104,32,40,97,32,115,116,114, + 41,46,10,10,32,32,32,32,32,32,32,32,73,109,112,108, + 101,109,101,110,116,105,110,103,32,116,104,105,115,32,109,101, + 116,104,111,100,32,97,108,108,111,119,115,32,102,111,114,32, + 116,104,101,32,119,114,105,116,105,110,103,32,111,102,32,98, + 121,116,101,99,111,100,101,32,102,105,108,101,115,46,10,32, + 32,32,32,32,32,32,32,78,114,4,0,0,0,41,3,114, + 100,0,0,0,114,35,0,0,0,114,53,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,192,0, + 0,0,183,2,0,0,115,0,0,0,0,122,21,83,111,117, + 114,99,101,76,111,97,100,101,114,46,115,101,116,95,100,97, + 116,97,99,2,0,0,0,0,0,0,0,5,0,0,0,16, + 0,0,0,67,0,0,0,115,105,0,0,0,124,0,0,106, + 0,0,124,1,0,131,1,0,125,2,0,121,19,0,124,0, + 0,106,1,0,124,2,0,131,1,0,125,3,0,87,110,58, + 0,4,116,2,0,107,10,0,114,94,0,1,125,4,0,1, + 122,26,0,116,3,0,100,1,0,100,2,0,124,1,0,131, + 1,1,124,4,0,130,2,0,87,89,100,3,0,100,3,0, + 125,4,0,126,4,0,88,110,1,0,88,116,4,0,124,3, + 0,131,1,0,83,41,4,122,52,67,111,110,99,114,101,116, + 101,32,105,109,112,108,101,109,101,110,116,97,116,105,111,110, + 32,111,102,32,73,110,115,112,101,99,116,76,111,97,100,101, + 114,46,103,101,116,95,115,111,117,114,99,101,46,122,39,115, + 111,117,114,99,101,32,110,111,116,32,97,118,97,105,108,97, + 98,108,101,32,116,104,114,111,117,103,104,32,103,101,116,95, + 100,97,116,97,40,41,114,98,0,0,0,78,41,5,114,151, + 0,0,0,218,8,103,101,116,95,100,97,116,97,114,40,0, + 0,0,114,99,0,0,0,114,149,0,0,0,41,5,114,100, + 0,0,0,114,119,0,0,0,114,35,0,0,0,114,147,0, + 0,0,218,3,101,120,99,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,10,103,101,116,95,115,111,117,114, + 99,101,190,2,0,0,115,14,0,0,0,0,2,15,1,3, + 1,19,1,18,1,9,1,31,1,122,23,83,111,117,114,99, + 101,76,111,97,100,101,114,46,103,101,116,95,115,111,117,114, + 99,101,218,9,95,111,112,116,105,109,105,122,101,114,29,0, + 0,0,99,3,0,0,0,1,0,0,0,4,0,0,0,9, + 0,0,0,67,0,0,0,115,34,0,0,0,116,0,0,106, + 1,0,116,2,0,124,1,0,124,2,0,100,1,0,100,2, + 0,100,3,0,100,4,0,124,3,0,131,4,2,83,41,5, + 122,130,82,101,116,117,114,110,32,116,104,101,32,99,111,100, + 101,32,111,98,106,101,99,116,32,99,111,109,112,105,108,101, + 100,32,102,114,111,109,32,115,111,117,114,99,101,46,10,10, + 32,32,32,32,32,32,32,32,84,104,101,32,39,100,97,116, + 97,39,32,97,114,103,117,109,101,110,116,32,99,97,110,32, + 98,101,32,97,110,121,32,111,98,106,101,99,116,32,116,121, + 112,101,32,116,104,97,116,32,99,111,109,112,105,108,101,40, + 41,32,115,117,112,112,111,114,116,115,46,10,32,32,32,32, + 32,32,32,32,114,183,0,0,0,218,12,100,111,110,116,95, + 105,110,104,101,114,105,116,84,114,68,0,0,0,41,3,114, + 114,0,0,0,114,182,0,0,0,218,7,99,111,109,112,105, + 108,101,41,4,114,100,0,0,0,114,53,0,0,0,114,35, + 0,0,0,114,197,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,14,115,111,117,114,99,101,95, + 116,111,95,99,111,100,101,200,2,0,0,115,4,0,0,0, + 0,5,21,1,122,27,83,111,117,114,99,101,76,111,97,100, + 101,114,46,115,111,117,114,99,101,95,116,111,95,99,111,100, + 101,99,2,0,0,0,0,0,0,0,10,0,0,0,43,0, + 0,0,67,0,0,0,115,183,1,0,0,124,0,0,106,0, + 0,124,1,0,131,1,0,125,2,0,100,1,0,125,3,0, + 121,16,0,116,1,0,124,2,0,131,1,0,125,4,0,87, + 110,24,0,4,116,2,0,107,10,0,114,63,0,1,1,1, + 100,1,0,125,4,0,89,110,205,0,88,121,19,0,124,0, + 0,106,3,0,124,2,0,131,1,0,125,5,0,87,110,18, + 0,4,116,4,0,107,10,0,114,103,0,1,1,1,89,110, + 165,0,88,116,5,0,124,5,0,100,2,0,25,131,1,0, + 125,3,0,121,19,0,124,0,0,106,6,0,124,4,0,131, + 1,0,125,6,0,87,110,18,0,4,116,7,0,107,10,0, + 114,159,0,1,1,1,89,110,109,0,88,121,34,0,116,8, + 0,124,6,0,100,3,0,124,5,0,100,4,0,124,1,0, + 100,5,0,124,4,0,131,1,3,125,7,0,87,110,24,0, + 4,116,9,0,116,10,0,102,2,0,107,10,0,114,220,0, + 1,1,1,89,110,48,0,88,116,11,0,106,12,0,100,6, + 0,124,4,0,124,2,0,131,3,0,1,116,13,0,124,7, + 0,100,4,0,124,1,0,100,7,0,124,4,0,100,8,0, + 124,2,0,131,1,3,83,124,0,0,106,6,0,124,2,0, + 131,1,0,125,8,0,124,0,0,106,14,0,124,8,0,124, + 2,0,131,2,0,125,9,0,116,11,0,106,12,0,100,9, + 0,124,2,0,131,2,0,1,116,15,0,106,16,0,12,114, + 179,1,124,4,0,100,1,0,107,9,0,114,179,1,124,3, + 0,100,1,0,107,9,0,114,179,1,116,17,0,124,9,0, + 124,3,0,116,18,0,124,8,0,131,1,0,131,3,0,125, + 6,0,121,39,0,124,0,0,106,19,0,124,2,0,124,4, + 0,124,6,0,131,3,0,1,116,11,0,106,12,0,100,10, + 0,124,4,0,131,2,0,1,87,110,18,0,4,116,2,0, + 107,10,0,114,178,1,1,1,1,89,110,1,0,88,124,9, + 0,83,41,11,122,190,67,111,110,99,114,101,116,101,32,105, + 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, + 32,73,110,115,112,101,99,116,76,111,97,100,101,114,46,103, + 101,116,95,99,111,100,101,46,10,10,32,32,32,32,32,32, + 32,32,82,101,97,100,105,110,103,32,111,102,32,98,121,116, + 101,99,111,100,101,32,114,101,113,117,105,114,101,115,32,112, + 97,116,104,95,115,116,97,116,115,32,116,111,32,98,101,32, + 105,109,112,108,101,109,101,110,116,101,100,46,32,84,111,32, + 119,114,105,116,101,10,32,32,32,32,32,32,32,32,98,121, + 116,101,99,111,100,101,44,32,115,101,116,95,100,97,116,97, + 32,109,117,115,116,32,97,108,115,111,32,98,101,32,105,109, + 112,108,101,109,101,110,116,101,100,46,10,10,32,32,32,32, + 32,32,32,32,78,114,126,0,0,0,114,132,0,0,0,114, + 98,0,0,0,114,35,0,0,0,122,13,123,125,32,109,97, + 116,99,104,101,115,32,123,125,114,89,0,0,0,114,90,0, + 0,0,122,19,99,111,100,101,32,111,98,106,101,99,116,32, + 102,114,111,109,32,123,125,122,10,119,114,111,116,101,32,123, + 33,114,125,41,20,114,151,0,0,0,114,79,0,0,0,114, + 66,0,0,0,114,191,0,0,0,114,189,0,0,0,114,14, + 0,0,0,114,194,0,0,0,114,40,0,0,0,114,135,0, + 0,0,114,99,0,0,0,114,130,0,0,0,114,114,0,0, + 0,114,129,0,0,0,114,141,0,0,0,114,200,0,0,0, + 114,7,0,0,0,218,19,100,111,110,116,95,119,114,105,116, + 101,95,98,121,116,101,99,111,100,101,114,144,0,0,0,114, + 31,0,0,0,114,193,0,0,0,41,10,114,100,0,0,0, + 114,119,0,0,0,114,90,0,0,0,114,133,0,0,0,114, + 89,0,0,0,218,2,115,116,114,53,0,0,0,218,10,98, + 121,116,101,115,95,100,97,116,97,114,147,0,0,0,90,11, + 99,111,100,101,95,111,98,106,101,99,116,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,181,0,0,0,208, + 2,0,0,115,78,0,0,0,0,7,15,1,6,1,3,1, + 16,1,13,1,11,2,3,1,19,1,13,1,5,2,16,1, + 3,1,19,1,13,1,5,2,3,1,9,1,12,1,13,1, + 19,1,5,2,12,1,7,1,15,1,6,1,7,1,15,1, + 18,1,16,1,22,1,12,1,9,1,15,1,3,1,19,1, + 20,1,13,1,5,1,122,21,83,111,117,114,99,101,76,111, + 97,100,101,114,46,103,101,116,95,99,111,100,101,78,114,87, + 0,0,0,41,10,114,105,0,0,0,114,104,0,0,0,114, + 106,0,0,0,114,190,0,0,0,114,191,0,0,0,114,193, + 0,0,0,114,192,0,0,0,114,196,0,0,0,114,200,0, + 0,0,114,181,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,188,0,0,0, + 150,2,0,0,115,14,0,0,0,12,2,12,8,12,13,12, + 10,12,7,12,10,18,8,114,188,0,0,0,99,0,0,0, + 0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0, + 0,115,112,0,0,0,101,0,0,90,1,0,100,0,0,90, + 2,0,100,1,0,90,3,0,100,2,0,100,3,0,132,0, + 0,90,4,0,100,4,0,100,5,0,132,0,0,90,5,0, + 100,6,0,100,7,0,132,0,0,90,6,0,101,7,0,135, + 0,0,102,1,0,100,8,0,100,9,0,134,0,0,131,1, + 0,90,8,0,101,7,0,100,10,0,100,11,0,132,0,0, + 131,1,0,90,9,0,100,12,0,100,13,0,132,0,0,90, + 10,0,135,0,0,83,41,14,218,10,70,105,108,101,76,111, + 97,100,101,114,122,103,66,97,115,101,32,102,105,108,101,32, + 108,111,97,100,101,114,32,99,108,97,115,115,32,119,104,105, + 99,104,32,105,109,112,108,101,109,101,110,116,115,32,116,104, + 101,32,108,111,97,100,101,114,32,112,114,111,116,111,99,111, + 108,32,109,101,116,104,111,100,115,32,116,104,97,116,10,32, + 32,32,32,114,101,113,117,105,114,101,32,102,105,108,101,32, + 115,121,115,116,101,109,32,117,115,97,103,101,46,99,3,0, + 0,0,0,0,0,0,3,0,0,0,2,0,0,0,67,0, + 0,0,115,22,0,0,0,124,1,0,124,0,0,95,0,0, + 124,2,0,124,0,0,95,1,0,100,1,0,83,41,2,122, + 75,67,97,99,104,101,32,116,104,101,32,109,111,100,117,108, + 101,32,110,97,109,101,32,97,110,100,32,116,104,101,32,112, + 97,116,104,32,116,111,32,116,104,101,32,102,105,108,101,32, + 102,111,117,110,100,32,98,121,32,116,104,101,10,32,32,32, + 32,32,32,32,32,102,105,110,100,101,114,46,78,41,2,114, + 98,0,0,0,114,35,0,0,0,41,3,114,100,0,0,0, + 114,119,0,0,0,114,35,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,179,0,0,0,9,3, + 0,0,115,4,0,0,0,0,3,9,1,122,19,70,105,108, + 101,76,111,97,100,101,114,46,95,95,105,110,105,116,95,95, + 99,2,0,0,0,0,0,0,0,2,0,0,0,2,0,0, + 0,67,0,0,0,115,34,0,0,0,124,0,0,106,0,0, + 124,1,0,106,0,0,107,2,0,111,33,0,124,0,0,106, + 1,0,124,1,0,106,1,0,107,2,0,83,41,1,78,41, + 2,218,9,95,95,99,108,97,115,115,95,95,114,111,0,0, + 0,41,2,114,100,0,0,0,218,5,111,116,104,101,114,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, + 95,95,101,113,95,95,15,3,0,0,115,4,0,0,0,0, + 1,18,1,122,17,70,105,108,101,76,111,97,100,101,114,46, + 95,95,101,113,95,95,99,1,0,0,0,0,0,0,0,1, + 0,0,0,3,0,0,0,67,0,0,0,115,26,0,0,0, + 116,0,0,124,0,0,106,1,0,131,1,0,116,0,0,124, + 0,0,106,2,0,131,1,0,65,83,41,1,78,41,3,218, + 4,104,97,115,104,114,98,0,0,0,114,35,0,0,0,41, + 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,8,95,95,104,97,115,104,95,95,19, + 3,0,0,115,2,0,0,0,0,1,122,19,70,105,108,101, + 76,111,97,100,101,114,46,95,95,104,97,115,104,95,95,99, + 2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, + 3,0,0,0,115,22,0,0,0,116,0,0,116,1,0,124, + 0,0,131,2,0,106,2,0,124,1,0,131,1,0,83,41, + 1,122,100,76,111,97,100,32,97,32,109,111,100,117,108,101, + 32,102,114,111,109,32,97,32,102,105,108,101,46,10,10,32, + 32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,104, + 111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,100, + 46,32,32,85,115,101,32,101,120,101,99,95,109,111,100,117, + 108,101,40,41,32,105,110,115,116,101,97,100,46,10,10,32, + 32,32,32,32,32,32,32,41,3,218,5,115,117,112,101,114, + 114,204,0,0,0,114,187,0,0,0,41,2,114,100,0,0, + 0,114,119,0,0,0,41,1,114,205,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,187,0,0,0,22,3,0,0, + 115,2,0,0,0,0,10,122,22,70,105,108,101,76,111,97, + 100,101,114,46,108,111,97,100,95,109,111,100,117,108,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, 67,0,0,0,115,7,0,0,0,124,0,0,106,0,0,83, 41,1,122,58,82,101,116,117,114,110,32,116,104,101,32,112, @@ -1622,959 +1345,1237 @@ const unsigned char _Py_M__importlib_external[] = { 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,151,0,0,0,155,3,0,0,115,2,0,0,0,0,3, - 122,32,69,120,116,101,110,115,105,111,110,70,105,108,101,76, - 111,97,100,101,114,46,103,101,116,95,102,105,108,101,110,97, - 109,101,78,41,14,114,105,0,0,0,114,104,0,0,0,114, - 106,0,0,0,114,107,0,0,0,114,179,0,0,0,114,207, - 0,0,0,114,209,0,0,0,114,180,0,0,0,114,185,0, - 0,0,114,153,0,0,0,114,181,0,0,0,114,196,0,0, - 0,114,116,0,0,0,114,151,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 218,0,0,0,108,3,0,0,115,20,0,0,0,12,6,6, - 2,12,4,12,4,12,3,12,8,12,6,12,6,12,4,12, - 4,114,218,0,0,0,99,0,0,0,0,0,0,0,0,0, - 0,0,0,2,0,0,0,64,0,0,0,115,130,0,0,0, - 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, - 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, - 0,100,5,0,132,0,0,90,5,0,100,6,0,100,7,0, - 132,0,0,90,6,0,100,8,0,100,9,0,132,0,0,90, - 7,0,100,10,0,100,11,0,132,0,0,90,8,0,100,12, - 0,100,13,0,132,0,0,90,9,0,100,14,0,100,15,0, - 132,0,0,90,10,0,100,16,0,100,17,0,132,0,0,90, - 11,0,100,18,0,100,19,0,132,0,0,90,12,0,100,20, - 0,83,41,21,218,14,95,78,97,109,101,115,112,97,99,101, - 80,97,116,104,97,38,1,0,0,82,101,112,114,101,115,101, - 110,116,115,32,97,32,110,97,109,101,115,112,97,99,101,32, - 112,97,99,107,97,103,101,39,115,32,112,97,116,104,46,32, - 32,73,116,32,117,115,101,115,32,116,104,101,32,109,111,100, - 117,108,101,32,110,97,109,101,10,32,32,32,32,116,111,32, - 102,105,110,100,32,105,116,115,32,112,97,114,101,110,116,32, - 109,111,100,117,108,101,44,32,97,110,100,32,102,114,111,109, - 32,116,104,101,114,101,32,105,116,32,108,111,111,107,115,32, - 117,112,32,116,104,101,32,112,97,114,101,110,116,39,115,10, - 32,32,32,32,95,95,112,97,116,104,95,95,46,32,32,87, - 104,101,110,32,116,104,105,115,32,99,104,97,110,103,101,115, - 44,32,116,104,101,32,109,111,100,117,108,101,39,115,32,111, - 119,110,32,112,97,116,104,32,105,115,32,114,101,99,111,109, - 112,117,116,101,100,44,10,32,32,32,32,117,115,105,110,103, - 32,112,97,116,104,95,102,105,110,100,101,114,46,32,32,70, - 111,114,32,116,111,112,45,108,101,118,101,108,32,109,111,100, - 117,108,101,115,44,32,116,104,101,32,112,97,114,101,110,116, - 32,109,111,100,117,108,101,39,115,32,112,97,116,104,10,32, - 32,32,32,105,115,32,115,121,115,46,112,97,116,104,46,99, - 4,0,0,0,0,0,0,0,4,0,0,0,2,0,0,0, - 67,0,0,0,115,52,0,0,0,124,1,0,124,0,0,95, - 0,0,124,2,0,124,0,0,95,1,0,116,2,0,124,0, - 0,106,3,0,131,0,0,131,1,0,124,0,0,95,4,0, - 124,3,0,124,0,0,95,5,0,100,0,0,83,41,1,78, - 41,6,218,5,95,110,97,109,101,218,5,95,112,97,116,104, - 114,93,0,0,0,218,16,95,103,101,116,95,112,97,114,101, - 110,116,95,112,97,116,104,218,17,95,108,97,115,116,95,112, - 97,114,101,110,116,95,112,97,116,104,218,12,95,112,97,116, - 104,95,102,105,110,100,101,114,41,4,114,100,0,0,0,114, - 98,0,0,0,114,35,0,0,0,218,11,112,97,116,104,95, - 102,105,110,100,101,114,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,179,0,0,0,168,3,0,0,115,8, - 0,0,0,0,1,9,1,9,1,21,1,122,23,95,78,97, - 109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,110, - 105,116,95,95,99,1,0,0,0,0,0,0,0,4,0,0, - 0,3,0,0,0,67,0,0,0,115,53,0,0,0,124,0, - 0,106,0,0,106,1,0,100,1,0,131,1,0,92,3,0, - 125,1,0,125,2,0,125,3,0,124,2,0,100,2,0,107, - 2,0,114,43,0,100,6,0,83,124,1,0,100,5,0,102, - 2,0,83,41,7,122,62,82,101,116,117,114,110,115,32,97, - 32,116,117,112,108,101,32,111,102,32,40,112,97,114,101,110, - 116,45,109,111,100,117,108,101,45,110,97,109,101,44,32,112, - 97,114,101,110,116,45,112,97,116,104,45,97,116,116,114,45, - 110,97,109,101,41,114,58,0,0,0,114,30,0,0,0,114, - 7,0,0,0,114,35,0,0,0,90,8,95,95,112,97,116, - 104,95,95,41,2,122,3,115,121,115,122,4,112,97,116,104, - 41,2,114,225,0,0,0,114,32,0,0,0,41,4,114,100, - 0,0,0,114,216,0,0,0,218,3,100,111,116,90,2,109, - 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,23,95,102,105,110,100,95,112,97,114,101,110,116,95,112, - 97,116,104,95,110,97,109,101,115,174,3,0,0,115,8,0, - 0,0,0,2,27,1,12,2,4,3,122,38,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,102,105,110,100, - 95,112,97,114,101,110,116,95,112,97,116,104,95,110,97,109, - 101,115,99,1,0,0,0,0,0,0,0,3,0,0,0,3, - 0,0,0,67,0,0,0,115,38,0,0,0,124,0,0,106, - 0,0,131,0,0,92,2,0,125,1,0,125,2,0,116,1, - 0,116,2,0,106,3,0,124,1,0,25,124,2,0,131,2, - 0,83,41,1,78,41,4,114,232,0,0,0,114,110,0,0, - 0,114,7,0,0,0,218,7,109,111,100,117,108,101,115,41, - 3,114,100,0,0,0,90,18,112,97,114,101,110,116,95,109, - 111,100,117,108,101,95,110,97,109,101,90,14,112,97,116,104, - 95,97,116,116,114,95,110,97,109,101,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,227,0,0,0,184,3, - 0,0,115,4,0,0,0,0,1,18,1,122,31,95,78,97, - 109,101,115,112,97,99,101,80,97,116,104,46,95,103,101,116, - 95,112,97,114,101,110,116,95,112,97,116,104,99,1,0,0, - 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, - 0,115,118,0,0,0,116,0,0,124,0,0,106,1,0,131, - 0,0,131,1,0,125,1,0,124,1,0,124,0,0,106,2, - 0,107,3,0,114,111,0,124,0,0,106,3,0,124,0,0, - 106,4,0,124,1,0,131,2,0,125,2,0,124,2,0,100, - 0,0,107,9,0,114,102,0,124,2,0,106,5,0,100,0, - 0,107,8,0,114,102,0,124,2,0,106,6,0,114,102,0, - 124,2,0,106,6,0,124,0,0,95,7,0,124,1,0,124, - 0,0,95,2,0,124,0,0,106,7,0,83,41,1,78,41, - 8,114,93,0,0,0,114,227,0,0,0,114,228,0,0,0, - 114,229,0,0,0,114,225,0,0,0,114,120,0,0,0,114, - 150,0,0,0,114,226,0,0,0,41,3,114,100,0,0,0, - 90,11,112,97,114,101,110,116,95,112,97,116,104,114,158,0, + 114,151,0,0,0,34,3,0,0,115,2,0,0,0,0,3, + 122,23,70,105,108,101,76,111,97,100,101,114,46,103,101,116, + 95,102,105,108,101,110,97,109,101,99,2,0,0,0,0,0, + 0,0,3,0,0,0,9,0,0,0,67,0,0,0,115,42, + 0,0,0,116,0,0,106,1,0,124,1,0,100,1,0,131, + 2,0,143,17,0,125,2,0,124,2,0,106,2,0,131,0, + 0,83,87,100,2,0,81,82,88,100,2,0,83,41,3,122, + 39,82,101,116,117,114,110,32,116,104,101,32,100,97,116,97, + 32,102,114,111,109,32,112,97,116,104,32,97,115,32,114,97, + 119,32,98,121,116,101,115,46,218,1,114,78,41,3,114,49, + 0,0,0,114,50,0,0,0,90,4,114,101,97,100,41,3, + 114,100,0,0,0,114,35,0,0,0,114,54,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,194, + 0,0,0,39,3,0,0,115,4,0,0,0,0,2,21,1, + 122,19,70,105,108,101,76,111,97,100,101,114,46,103,101,116, + 95,100,97,116,97,41,11,114,105,0,0,0,114,104,0,0, + 0,114,106,0,0,0,114,107,0,0,0,114,179,0,0,0, + 114,207,0,0,0,114,209,0,0,0,114,116,0,0,0,114, + 187,0,0,0,114,151,0,0,0,114,194,0,0,0,114,4, + 0,0,0,114,4,0,0,0,41,1,114,205,0,0,0,114, + 5,0,0,0,114,204,0,0,0,4,3,0,0,115,14,0, + 0,0,12,3,6,2,12,6,12,4,12,3,24,12,18,5, + 114,204,0,0,0,99,0,0,0,0,0,0,0,0,0,0, + 0,0,4,0,0,0,64,0,0,0,115,64,0,0,0,101, + 0,0,90,1,0,100,0,0,90,2,0,100,1,0,90,3, + 0,100,2,0,100,3,0,132,0,0,90,4,0,100,4,0, + 100,5,0,132,0,0,90,5,0,100,6,0,100,7,0,100, + 8,0,100,9,0,132,0,1,90,6,0,100,10,0,83,41, + 11,218,16,83,111,117,114,99,101,70,105,108,101,76,111,97, + 100,101,114,122,62,67,111,110,99,114,101,116,101,32,105,109, + 112,108,101,109,101,110,116,97,116,105,111,110,32,111,102,32, + 83,111,117,114,99,101,76,111,97,100,101,114,32,117,115,105, + 110,103,32,116,104,101,32,102,105,108,101,32,115,121,115,116, + 101,109,46,99,2,0,0,0,0,0,0,0,3,0,0,0, + 4,0,0,0,67,0,0,0,115,34,0,0,0,116,0,0, + 124,1,0,131,1,0,125,2,0,100,1,0,124,2,0,106, + 1,0,100,2,0,124,2,0,106,2,0,105,2,0,83,41, + 3,122,33,82,101,116,117,114,110,32,116,104,101,32,109,101, + 116,97,100,97,116,97,32,102,111,114,32,116,104,101,32,112, + 97,116,104,46,114,126,0,0,0,114,127,0,0,0,41,3, + 114,39,0,0,0,218,8,115,116,95,109,116,105,109,101,90, + 7,115,116,95,115,105,122,101,41,3,114,100,0,0,0,114, + 35,0,0,0,114,202,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,191,0,0,0,49,3,0, + 0,115,4,0,0,0,0,2,12,1,122,27,83,111,117,114, + 99,101,70,105,108,101,76,111,97,100,101,114,46,112,97,116, + 104,95,115,116,97,116,115,99,4,0,0,0,0,0,0,0, + 5,0,0,0,5,0,0,0,67,0,0,0,115,34,0,0, + 0,116,0,0,124,1,0,131,1,0,125,4,0,124,0,0, + 106,1,0,124,2,0,124,3,0,100,1,0,124,4,0,131, + 2,1,83,41,2,78,218,5,95,109,111,100,101,41,2,114, + 97,0,0,0,114,192,0,0,0,41,5,114,100,0,0,0, + 114,90,0,0,0,114,89,0,0,0,114,53,0,0,0,114, + 42,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,193,0,0,0,54,3,0,0,115,4,0,0, + 0,0,2,12,1,122,32,83,111,117,114,99,101,70,105,108, + 101,76,111,97,100,101,114,46,95,99,97,99,104,101,95,98, + 121,116,101,99,111,100,101,114,214,0,0,0,105,182,1,0, + 0,99,3,0,0,0,1,0,0,0,9,0,0,0,17,0, + 0,0,67,0,0,0,115,62,1,0,0,116,0,0,124,1, + 0,131,1,0,92,2,0,125,4,0,125,5,0,103,0,0, + 125,6,0,120,54,0,124,4,0,114,80,0,116,1,0,124, + 4,0,131,1,0,12,114,80,0,116,0,0,124,4,0,131, + 1,0,92,2,0,125,4,0,125,7,0,124,6,0,106,2, + 0,124,7,0,131,1,0,1,113,27,0,87,120,135,0,116, + 3,0,124,6,0,131,1,0,68,93,121,0,125,7,0,116, + 4,0,124,4,0,124,7,0,131,2,0,125,4,0,121,17, + 0,116,5,0,106,6,0,124,4,0,131,1,0,1,87,113, + 94,0,4,116,7,0,107,10,0,114,155,0,1,1,1,119, + 94,0,89,113,94,0,4,116,8,0,107,10,0,114,214,0, + 1,125,8,0,1,122,28,0,116,9,0,106,10,0,100,1, + 0,124,4,0,124,8,0,131,3,0,1,100,2,0,83,87, + 89,100,2,0,100,2,0,125,8,0,126,8,0,88,113,94, + 0,88,113,94,0,87,121,36,0,116,11,0,124,1,0,124, + 2,0,124,3,0,131,3,0,1,116,9,0,106,10,0,100, + 3,0,124,1,0,131,2,0,1,87,110,56,0,4,116,8, + 0,107,10,0,114,57,1,1,125,8,0,1,122,24,0,116, + 9,0,106,10,0,100,1,0,124,1,0,124,8,0,131,3, + 0,1,87,89,100,2,0,100,2,0,125,8,0,126,8,0, + 88,110,1,0,88,100,2,0,83,41,4,122,27,87,114,105, + 116,101,32,98,121,116,101,115,32,100,97,116,97,32,116,111, + 32,97,32,102,105,108,101,46,122,27,99,111,117,108,100,32, + 110,111,116,32,99,114,101,97,116,101,32,123,33,114,125,58, + 32,123,33,114,125,78,122,12,99,114,101,97,116,101,100,32, + 123,33,114,125,41,12,114,38,0,0,0,114,46,0,0,0, + 114,157,0,0,0,114,33,0,0,0,114,28,0,0,0,114, + 3,0,0,0,90,5,109,107,100,105,114,218,15,70,105,108, + 101,69,120,105,115,116,115,69,114,114,111,114,114,40,0,0, + 0,114,114,0,0,0,114,129,0,0,0,114,55,0,0,0, + 41,9,114,100,0,0,0,114,35,0,0,0,114,53,0,0, + 0,114,214,0,0,0,218,6,112,97,114,101,110,116,114,94, + 0,0,0,114,27,0,0,0,114,23,0,0,0,114,195,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,188, - 3,0,0,115,16,0,0,0,0,2,18,1,15,1,21,3, - 27,1,9,1,12,1,9,1,122,27,95,78,97,109,101,115, - 112,97,99,101,80,97,116,104,46,95,114,101,99,97,108,99, - 117,108,97,116,101,99,1,0,0,0,0,0,0,0,1,0, - 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,116, - 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, - 1,78,41,2,218,4,105,116,101,114,114,234,0,0,0,41, - 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,201, - 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,95,105,116,101, - 114,95,95,99,1,0,0,0,0,0,0,0,1,0,0,0, - 2,0,0,0,67,0,0,0,115,16,0,0,0,116,0,0, - 124,0,0,106,1,0,131,0,0,131,1,0,83,41,1,78, - 41,2,114,31,0,0,0,114,234,0,0,0,41,1,114,100, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,7,95,95,108,101,110,95,95,204,3,0,0,115, - 2,0,0,0,0,1,122,22,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,46,95,95,108,101,110,95,95,99,1, - 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, - 0,0,0,115,16,0,0,0,100,1,0,106,0,0,124,0, - 0,106,1,0,131,1,0,83,41,2,78,122,20,95,78,97, - 109,101,115,112,97,99,101,80,97,116,104,40,123,33,114,125, - 41,41,2,114,47,0,0,0,114,226,0,0,0,41,1,114, - 100,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,95,114,101,112,114,95,95,207,3,0, - 0,115,2,0,0,0,0,1,122,23,95,78,97,109,101,115, - 112,97,99,101,80,97,116,104,46,95,95,114,101,112,114,95, - 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, - 0,0,67,0,0,0,115,16,0,0,0,124,1,0,124,0, - 0,106,0,0,131,0,0,107,6,0,83,41,1,78,41,1, - 114,234,0,0,0,41,2,114,100,0,0,0,218,4,105,116, - 101,109,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,210, - 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,95,99,111,110, - 116,97,105,110,115,95,95,99,2,0,0,0,0,0,0,0, - 2,0,0,0,2,0,0,0,67,0,0,0,115,20,0,0, - 0,124,0,0,106,0,0,106,1,0,124,1,0,131,1,0, - 1,100,0,0,83,41,1,78,41,2,114,226,0,0,0,114, - 157,0,0,0,41,2,114,100,0,0,0,114,239,0,0,0, + 0,114,192,0,0,0,59,3,0,0,115,42,0,0,0,0, + 2,18,1,6,2,22,1,18,1,17,2,19,1,15,1,3, + 1,17,1,13,2,7,1,18,3,9,1,10,1,27,1,3, + 1,16,1,20,1,18,2,12,1,122,25,83,111,117,114,99, + 101,70,105,108,101,76,111,97,100,101,114,46,115,101,116,95, + 100,97,116,97,78,41,7,114,105,0,0,0,114,104,0,0, + 0,114,106,0,0,0,114,107,0,0,0,114,191,0,0,0, + 114,193,0,0,0,114,192,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,212, + 0,0,0,45,3,0,0,115,8,0,0,0,12,2,6,2, + 12,5,12,5,114,212,0,0,0,99,0,0,0,0,0,0, + 0,0,0,0,0,0,2,0,0,0,64,0,0,0,115,46, + 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, + 1,0,90,3,0,100,2,0,100,3,0,132,0,0,90,4, + 0,100,4,0,100,5,0,132,0,0,90,5,0,100,6,0, + 83,41,7,218,20,83,111,117,114,99,101,108,101,115,115,70, + 105,108,101,76,111,97,100,101,114,122,45,76,111,97,100,101, + 114,32,119,104,105,99,104,32,104,97,110,100,108,101,115,32, + 115,111,117,114,99,101,108,101,115,115,32,102,105,108,101,32, + 105,109,112,111,114,116,115,46,99,2,0,0,0,0,0,0, + 0,5,0,0,0,6,0,0,0,67,0,0,0,115,76,0, + 0,0,124,0,0,106,0,0,124,1,0,131,1,0,125,2, + 0,124,0,0,106,1,0,124,2,0,131,1,0,125,3,0, + 116,2,0,124,3,0,100,1,0,124,1,0,100,2,0,124, + 2,0,131,1,2,125,4,0,116,3,0,124,4,0,100,1, + 0,124,1,0,100,3,0,124,2,0,131,1,2,83,41,4, + 78,114,98,0,0,0,114,35,0,0,0,114,89,0,0,0, + 41,4,114,151,0,0,0,114,194,0,0,0,114,135,0,0, + 0,114,141,0,0,0,41,5,114,100,0,0,0,114,119,0, + 0,0,114,35,0,0,0,114,53,0,0,0,114,203,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,181,0,0,0,94,3,0,0,115,8,0,0,0,0,1, + 15,1,15,1,24,1,122,29,83,111,117,114,99,101,108,101, + 115,115,70,105,108,101,76,111,97,100,101,114,46,103,101,116, + 95,99,111,100,101,99,2,0,0,0,0,0,0,0,2,0, + 0,0,1,0,0,0,67,0,0,0,115,4,0,0,0,100, + 1,0,83,41,2,122,39,82,101,116,117,114,110,32,78,111, + 110,101,32,97,115,32,116,104,101,114,101,32,105,115,32,110, + 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, + 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 157,0,0,0,213,3,0,0,115,2,0,0,0,0,1,122, - 21,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, - 97,112,112,101,110,100,78,41,13,114,105,0,0,0,114,104, - 0,0,0,114,106,0,0,0,114,107,0,0,0,114,179,0, - 0,0,114,232,0,0,0,114,227,0,0,0,114,234,0,0, - 0,114,236,0,0,0,114,237,0,0,0,114,238,0,0,0, - 114,240,0,0,0,114,157,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,224, - 0,0,0,161,3,0,0,115,20,0,0,0,12,5,6,2, - 12,6,12,10,12,4,12,13,12,3,12,3,12,3,12,3, - 114,224,0,0,0,99,0,0,0,0,0,0,0,0,0,0, - 0,0,3,0,0,0,64,0,0,0,115,118,0,0,0,101, - 0,0,90,1,0,100,0,0,90,2,0,100,1,0,100,2, - 0,132,0,0,90,3,0,101,4,0,100,3,0,100,4,0, - 132,0,0,131,1,0,90,5,0,100,5,0,100,6,0,132, - 0,0,90,6,0,100,7,0,100,8,0,132,0,0,90,7, - 0,100,9,0,100,10,0,132,0,0,90,8,0,100,11,0, - 100,12,0,132,0,0,90,9,0,100,13,0,100,14,0,132, - 0,0,90,10,0,100,15,0,100,16,0,132,0,0,90,11, - 0,100,17,0,83,41,18,218,16,95,78,97,109,101,115,112, - 97,99,101,76,111,97,100,101,114,99,4,0,0,0,0,0, - 0,0,4,0,0,0,4,0,0,0,67,0,0,0,115,25, - 0,0,0,116,0,0,124,1,0,124,2,0,124,3,0,131, - 3,0,124,0,0,95,1,0,100,0,0,83,41,1,78,41, - 2,114,224,0,0,0,114,226,0,0,0,41,4,114,100,0, - 0,0,114,98,0,0,0,114,35,0,0,0,114,230,0,0, + 196,0,0,0,100,3,0,0,115,2,0,0,0,0,2,122, + 31,83,111,117,114,99,101,108,101,115,115,70,105,108,101,76, + 111,97,100,101,114,46,103,101,116,95,115,111,117,114,99,101, + 78,41,6,114,105,0,0,0,114,104,0,0,0,114,106,0, + 0,0,114,107,0,0,0,114,181,0,0,0,114,196,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,217,0,0,0,90,3,0,0,115,6, + 0,0,0,12,2,6,2,12,6,114,217,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, + 0,0,0,115,136,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,90,3,0,100,2,0,100,3,0, + 132,0,0,90,4,0,100,4,0,100,5,0,132,0,0,90, + 5,0,100,6,0,100,7,0,132,0,0,90,6,0,100,8, + 0,100,9,0,132,0,0,90,7,0,100,10,0,100,11,0, + 132,0,0,90,8,0,100,12,0,100,13,0,132,0,0,90, + 9,0,100,14,0,100,15,0,132,0,0,90,10,0,100,16, + 0,100,17,0,132,0,0,90,11,0,101,12,0,100,18,0, + 100,19,0,132,0,0,131,1,0,90,13,0,100,20,0,83, + 41,21,218,19,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,122,93,76,111,97,100,101,114,32, + 102,111,114,32,101,120,116,101,110,115,105,111,110,32,109,111, + 100,117,108,101,115,46,10,10,32,32,32,32,84,104,101,32, + 99,111,110,115,116,114,117,99,116,111,114,32,105,115,32,100, + 101,115,105,103,110,101,100,32,116,111,32,119,111,114,107,32, + 119,105,116,104,32,70,105,108,101,70,105,110,100,101,114,46, + 10,10,32,32,32,32,99,3,0,0,0,0,0,0,0,3, + 0,0,0,2,0,0,0,67,0,0,0,115,22,0,0,0, + 124,1,0,124,0,0,95,0,0,124,2,0,124,0,0,95, + 1,0,100,0,0,83,41,1,78,41,2,114,98,0,0,0, + 114,35,0,0,0,41,3,114,100,0,0,0,114,98,0,0, + 0,114,35,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,179,0,0,0,117,3,0,0,115,4, + 0,0,0,0,1,9,1,122,28,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,105, + 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, + 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, + 0,0,106,0,0,124,1,0,106,0,0,107,2,0,111,33, + 0,124,0,0,106,1,0,124,1,0,106,1,0,107,2,0, + 83,41,1,78,41,2,114,205,0,0,0,114,111,0,0,0, + 41,2,114,100,0,0,0,114,206,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,207,0,0,0, + 121,3,0,0,115,4,0,0,0,0,1,18,1,122,26,69, + 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, + 101,114,46,95,95,101,113,95,95,99,1,0,0,0,0,0, + 0,0,1,0,0,0,3,0,0,0,67,0,0,0,115,26, + 0,0,0,116,0,0,124,0,0,106,1,0,131,1,0,116, + 0,0,124,0,0,106,2,0,131,1,0,65,83,41,1,78, + 41,3,114,208,0,0,0,114,98,0,0,0,114,35,0,0, + 0,41,1,114,100,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,209,0,0,0,125,3,0,0, + 115,2,0,0,0,0,1,122,28,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,104, + 97,115,104,95,95,99,2,0,0,0,0,0,0,0,3,0, + 0,0,4,0,0,0,67,0,0,0,115,50,0,0,0,116, + 0,0,106,1,0,116,2,0,106,3,0,124,1,0,131,2, + 0,125,2,0,116,0,0,106,4,0,100,1,0,124,1,0, + 106,5,0,124,0,0,106,6,0,131,3,0,1,124,2,0, + 83,41,2,122,38,67,114,101,97,116,101,32,97,110,32,117, + 110,105,116,105,97,108,105,122,101,100,32,101,120,116,101,110, + 115,105,111,110,32,109,111,100,117,108,101,122,38,101,120,116, + 101,110,115,105,111,110,32,109,111,100,117,108,101,32,123,33, + 114,125,32,108,111,97,100,101,100,32,102,114,111,109,32,123, + 33,114,125,41,7,114,114,0,0,0,114,182,0,0,0,114, + 139,0,0,0,90,14,99,114,101,97,116,101,95,100,121,110, + 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, + 0,0,0,41,3,114,100,0,0,0,114,158,0,0,0,114, + 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,180,0,0,0,128,3,0,0,115,10,0,0, + 0,0,2,6,1,15,1,9,1,16,1,122,33,69,120,116, + 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, + 46,99,114,101,97,116,101,95,109,111,100,117,108,101,99,2, + 0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,67, + 0,0,0,115,48,0,0,0,116,0,0,106,1,0,116,2, + 0,106,3,0,124,1,0,131,2,0,1,116,0,0,106,4, + 0,100,1,0,124,0,0,106,5,0,124,0,0,106,6,0, + 131,3,0,1,100,2,0,83,41,3,122,30,73,110,105,116, + 105,97,108,105,122,101,32,97,110,32,101,120,116,101,110,115, + 105,111,110,32,109,111,100,117,108,101,122,40,101,120,116,101, + 110,115,105,111,110,32,109,111,100,117,108,101,32,123,33,114, + 125,32,101,120,101,99,117,116,101,100,32,102,114,111,109,32, + 123,33,114,125,78,41,7,114,114,0,0,0,114,182,0,0, + 0,114,139,0,0,0,90,12,101,120,101,99,95,100,121,110, + 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, + 0,0,0,41,2,114,100,0,0,0,114,184,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,185, + 0,0,0,136,3,0,0,115,6,0,0,0,0,2,19,1, + 9,1,122,31,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,46,101,120,101,99,95,109,111,100, + 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 4,0,0,0,3,0,0,0,115,48,0,0,0,116,0,0, + 124,0,0,106,1,0,131,1,0,100,1,0,25,137,0,0, + 116,2,0,135,0,0,102,1,0,100,2,0,100,3,0,134, + 0,0,116,3,0,68,131,1,0,131,1,0,83,41,4,122, + 49,82,101,116,117,114,110,32,84,114,117,101,32,105,102,32, + 116,104,101,32,101,120,116,101,110,115,105,111,110,32,109,111, + 100,117,108,101,32,105,115,32,97,32,112,97,99,107,97,103, + 101,46,114,29,0,0,0,99,1,0,0,0,0,0,0,0, + 2,0,0,0,4,0,0,0,51,0,0,0,115,31,0,0, + 0,124,0,0,93,21,0,125,1,0,136,0,0,100,0,0, + 124,1,0,23,107,2,0,86,1,113,3,0,100,1,0,83, + 41,2,114,179,0,0,0,78,114,4,0,0,0,41,2,114, + 22,0,0,0,218,6,115,117,102,102,105,120,41,1,218,9, + 102,105,108,101,95,110,97,109,101,114,4,0,0,0,114,5, + 0,0,0,250,9,60,103,101,110,101,120,112,114,62,145,3, + 0,0,115,2,0,0,0,6,1,122,49,69,120,116,101,110, + 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,105, + 115,95,112,97,99,107,97,103,101,46,60,108,111,99,97,108, + 115,62,46,60,103,101,110,101,120,112,114,62,41,4,114,38, + 0,0,0,114,35,0,0,0,218,3,97,110,121,218,18,69, + 88,84,69,78,83,73,79,78,95,83,85,70,70,73,88,69, + 83,41,2,114,100,0,0,0,114,119,0,0,0,114,4,0, + 0,0,41,1,114,220,0,0,0,114,5,0,0,0,114,153, + 0,0,0,142,3,0,0,115,6,0,0,0,0,2,19,1, + 18,1,122,30,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,46,105,115,95,112,97,99,107,97, + 103,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, + 0,0,0,67,0,0,0,115,4,0,0,0,100,1,0,83, + 41,2,122,63,82,101,116,117,114,110,32,78,111,110,101,32, + 97,115,32,97,110,32,101,120,116,101,110,115,105,111,110,32, + 109,111,100,117,108,101,32,99,97,110,110,111,116,32,99,114, + 101,97,116,101,32,97,32,99,111,100,101,32,111,98,106,101, + 99,116,46,78,114,4,0,0,0,41,2,114,100,0,0,0, + 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,181,0,0,0,148,3,0,0,115,2,0, + 0,0,0,2,122,28,69,120,116,101,110,115,105,111,110,70, + 105,108,101,76,111,97,100,101,114,46,103,101,116,95,99,111, + 100,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, + 0,0,0,67,0,0,0,115,4,0,0,0,100,1,0,83, + 41,2,122,53,82,101,116,117,114,110,32,78,111,110,101,32, + 97,115,32,101,120,116,101,110,115,105,111,110,32,109,111,100, + 117,108,101,115,32,104,97,118,101,32,110,111,32,115,111,117, + 114,99,101,32,99,111,100,101,46,78,114,4,0,0,0,41, + 2,114,100,0,0,0,114,119,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,196,0,0,0,152, + 3,0,0,115,2,0,0,0,0,2,122,30,69,120,116,101, + 110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,46, + 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, + 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, + 7,0,0,0,124,0,0,106,0,0,83,41,1,122,58,82, + 101,116,117,114,110,32,116,104,101,32,112,97,116,104,32,116, + 111,32,116,104,101,32,115,111,117,114,99,101,32,102,105,108, + 101,32,97,115,32,102,111,117,110,100,32,98,121,32,116,104, + 101,32,102,105,110,100,101,114,46,41,1,114,35,0,0,0, + 41,2,114,100,0,0,0,114,119,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,151,0,0,0, + 156,3,0,0,115,2,0,0,0,0,3,122,32,69,120,116, + 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, + 46,103,101,116,95,102,105,108,101,110,97,109,101,78,41,14, + 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, + 107,0,0,0,114,179,0,0,0,114,207,0,0,0,114,209, + 0,0,0,114,180,0,0,0,114,185,0,0,0,114,153,0, + 0,0,114,181,0,0,0,114,196,0,0,0,114,116,0,0, + 0,114,151,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,218,0,0,0,109, + 3,0,0,115,20,0,0,0,12,6,6,2,12,4,12,4, + 12,3,12,8,12,6,12,6,12,4,12,4,114,218,0,0, + 0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0, + 0,0,64,0,0,0,115,130,0,0,0,101,0,0,90,1, + 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, + 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, + 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, + 0,100,8,0,100,9,0,132,0,0,90,7,0,100,10,0, + 100,11,0,132,0,0,90,8,0,100,12,0,100,13,0,132, + 0,0,90,9,0,100,14,0,100,15,0,132,0,0,90,10, + 0,100,16,0,100,17,0,132,0,0,90,11,0,100,18,0, + 100,19,0,132,0,0,90,12,0,100,20,0,83,41,21,218, + 14,95,78,97,109,101,115,112,97,99,101,80,97,116,104,97, + 38,1,0,0,82,101,112,114,101,115,101,110,116,115,32,97, + 32,110,97,109,101,115,112,97,99,101,32,112,97,99,107,97, + 103,101,39,115,32,112,97,116,104,46,32,32,73,116,32,117, + 115,101,115,32,116,104,101,32,109,111,100,117,108,101,32,110, + 97,109,101,10,32,32,32,32,116,111,32,102,105,110,100,32, + 105,116,115,32,112,97,114,101,110,116,32,109,111,100,117,108, + 101,44,32,97,110,100,32,102,114,111,109,32,116,104,101,114, + 101,32,105,116,32,108,111,111,107,115,32,117,112,32,116,104, + 101,32,112,97,114,101,110,116,39,115,10,32,32,32,32,95, + 95,112,97,116,104,95,95,46,32,32,87,104,101,110,32,116, + 104,105,115,32,99,104,97,110,103,101,115,44,32,116,104,101, + 32,109,111,100,117,108,101,39,115,32,111,119,110,32,112,97, + 116,104,32,105,115,32,114,101,99,111,109,112,117,116,101,100, + 44,10,32,32,32,32,117,115,105,110,103,32,112,97,116,104, + 95,102,105,110,100,101,114,46,32,32,70,111,114,32,116,111, + 112,45,108,101,118,101,108,32,109,111,100,117,108,101,115,44, + 32,116,104,101,32,112,97,114,101,110,116,32,109,111,100,117, + 108,101,39,115,32,112,97,116,104,10,32,32,32,32,105,115, + 32,115,121,115,46,112,97,116,104,46,99,4,0,0,0,0, + 0,0,0,4,0,0,0,2,0,0,0,67,0,0,0,115, + 52,0,0,0,124,1,0,124,0,0,95,0,0,124,2,0, + 124,0,0,95,1,0,116,2,0,124,0,0,106,3,0,131, + 0,0,131,1,0,124,0,0,95,4,0,124,3,0,124,0, + 0,95,5,0,100,0,0,83,41,1,78,41,6,218,5,95, + 110,97,109,101,218,5,95,112,97,116,104,114,93,0,0,0, + 218,16,95,103,101,116,95,112,97,114,101,110,116,95,112,97, + 116,104,218,17,95,108,97,115,116,95,112,97,114,101,110,116, + 95,112,97,116,104,218,12,95,112,97,116,104,95,102,105,110, + 100,101,114,41,4,114,100,0,0,0,114,98,0,0,0,114, + 35,0,0,0,218,11,112,97,116,104,95,102,105,110,100,101, + 114,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,179,0,0,0,169,3,0,0,115,8,0,0,0,0,1, + 9,1,9,1,21,1,122,23,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,99, + 1,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, + 67,0,0,0,115,53,0,0,0,124,0,0,106,0,0,106, + 1,0,100,1,0,131,1,0,92,3,0,125,1,0,125,2, + 0,125,3,0,124,2,0,100,2,0,107,2,0,114,43,0, + 100,6,0,83,124,1,0,100,5,0,102,2,0,83,41,7, + 122,62,82,101,116,117,114,110,115,32,97,32,116,117,112,108, + 101,32,111,102,32,40,112,97,114,101,110,116,45,109,111,100, + 117,108,101,45,110,97,109,101,44,32,112,97,114,101,110,116, + 45,112,97,116,104,45,97,116,116,114,45,110,97,109,101,41, + 114,58,0,0,0,114,30,0,0,0,114,7,0,0,0,114, + 35,0,0,0,90,8,95,95,112,97,116,104,95,95,41,2, + 122,3,115,121,115,122,4,112,97,116,104,41,2,114,225,0, + 0,0,114,32,0,0,0,41,4,114,100,0,0,0,114,216, + 0,0,0,218,3,100,111,116,90,2,109,101,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,23,95,102,105, + 110,100,95,112,97,114,101,110,116,95,112,97,116,104,95,110, + 97,109,101,115,175,3,0,0,115,8,0,0,0,0,2,27, + 1,12,2,4,3,122,38,95,78,97,109,101,115,112,97,99, + 101,80,97,116,104,46,95,102,105,110,100,95,112,97,114,101, + 110,116,95,112,97,116,104,95,110,97,109,101,115,99,1,0, + 0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0, + 0,0,115,38,0,0,0,124,0,0,106,0,0,131,0,0, + 92,2,0,125,1,0,125,2,0,116,1,0,116,2,0,106, + 3,0,124,1,0,25,124,2,0,131,2,0,83,41,1,78, + 41,4,114,232,0,0,0,114,110,0,0,0,114,7,0,0, + 0,218,7,109,111,100,117,108,101,115,41,3,114,100,0,0, + 0,90,18,112,97,114,101,110,116,95,109,111,100,117,108,101, + 95,110,97,109,101,90,14,112,97,116,104,95,97,116,116,114, + 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,227,0,0,0,185,3,0,0,115,4,0, + 0,0,0,1,18,1,122,31,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,46,95,103,101,116,95,112,97,114,101, + 110,116,95,112,97,116,104,99,1,0,0,0,0,0,0,0, + 3,0,0,0,3,0,0,0,67,0,0,0,115,118,0,0, + 0,116,0,0,124,0,0,106,1,0,131,0,0,131,1,0, + 125,1,0,124,1,0,124,0,0,106,2,0,107,3,0,114, + 111,0,124,0,0,106,3,0,124,0,0,106,4,0,124,1, + 0,131,2,0,125,2,0,124,2,0,100,0,0,107,9,0, + 114,102,0,124,2,0,106,5,0,100,0,0,107,8,0,114, + 102,0,124,2,0,106,6,0,114,102,0,124,2,0,106,6, + 0,124,0,0,95,7,0,124,1,0,124,0,0,95,2,0, + 124,0,0,106,7,0,83,41,1,78,41,8,114,93,0,0, + 0,114,227,0,0,0,114,228,0,0,0,114,229,0,0,0, + 114,225,0,0,0,114,120,0,0,0,114,150,0,0,0,114, + 226,0,0,0,41,3,114,100,0,0,0,90,11,112,97,114, + 101,110,116,95,112,97,116,104,114,158,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,114, + 101,99,97,108,99,117,108,97,116,101,189,3,0,0,115,16, + 0,0,0,0,2,18,1,15,1,21,3,27,1,9,1,12, + 1,9,1,122,27,95,78,97,109,101,115,112,97,99,101,80, + 97,116,104,46,95,114,101,99,97,108,99,117,108,97,116,101, + 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, + 0,67,0,0,0,115,16,0,0,0,116,0,0,124,0,0, + 106,1,0,131,0,0,131,1,0,83,41,1,78,41,2,218, + 4,105,116,101,114,114,234,0,0,0,41,1,114,100,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,219,3,0,0,115,2,0,0,0,0,1, + 218,8,95,95,105,116,101,114,95,95,202,3,0,0,115,2, + 0,0,0,0,1,122,23,95,78,97,109,101,115,112,97,99, + 101,80,97,116,104,46,95,95,105,116,101,114,95,95,99,1, + 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, + 0,0,0,115,16,0,0,0,116,0,0,124,0,0,106,1, + 0,131,0,0,131,1,0,83,41,1,78,41,2,114,31,0, + 0,0,114,234,0,0,0,41,1,114,100,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,218,7,95, + 95,108,101,110,95,95,205,3,0,0,115,2,0,0,0,0, + 1,122,22,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,95,95,108,101,110,95,95,99,1,0,0,0,0,0, + 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, + 0,0,0,100,1,0,106,0,0,124,0,0,106,1,0,131, + 1,0,83,41,2,78,122,20,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,40,123,33,114,125,41,41,2,114,47, + 0,0,0,114,226,0,0,0,41,1,114,100,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, + 95,95,114,101,112,114,95,95,208,3,0,0,115,2,0,0, + 0,0,1,122,23,95,78,97,109,101,115,112,97,99,101,80, + 97,116,104,46,95,95,114,101,112,114,95,95,99,2,0,0, + 0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,0, + 0,115,16,0,0,0,124,1,0,124,0,0,106,0,0,131, + 0,0,107,6,0,83,41,1,78,41,1,114,234,0,0,0, + 41,2,114,100,0,0,0,218,4,105,116,101,109,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,95, + 99,111,110,116,97,105,110,115,95,95,211,3,0,0,115,2, + 0,0,0,0,1,122,27,95,78,97,109,101,115,112,97,99, + 101,80,97,116,104,46,95,95,99,111,110,116,97,105,110,115, + 95,95,99,2,0,0,0,0,0,0,0,2,0,0,0,2, + 0,0,0,67,0,0,0,115,20,0,0,0,124,0,0,106, + 0,0,106,1,0,124,1,0,131,1,0,1,100,0,0,83, + 41,1,78,41,2,114,226,0,0,0,114,157,0,0,0,41, + 2,114,100,0,0,0,114,239,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,214, + 3,0,0,115,2,0,0,0,0,1,122,21,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,97,112,112,101,110, + 100,78,41,13,114,105,0,0,0,114,104,0,0,0,114,106, + 0,0,0,114,107,0,0,0,114,179,0,0,0,114,232,0, + 0,0,114,227,0,0,0,114,234,0,0,0,114,236,0,0, + 0,114,237,0,0,0,114,238,0,0,0,114,240,0,0,0, + 114,157,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,224,0,0,0,162,3, + 0,0,115,20,0,0,0,12,5,6,2,12,6,12,10,12, + 4,12,13,12,3,12,3,12,3,12,3,114,224,0,0,0, + 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, + 0,64,0,0,0,115,118,0,0,0,101,0,0,90,1,0, + 100,0,0,90,2,0,100,1,0,100,2,0,132,0,0,90, + 3,0,101,4,0,100,3,0,100,4,0,132,0,0,131,1, + 0,90,5,0,100,5,0,100,6,0,132,0,0,90,6,0, + 100,7,0,100,8,0,132,0,0,90,7,0,100,9,0,100, + 10,0,132,0,0,90,8,0,100,11,0,100,12,0,132,0, + 0,90,9,0,100,13,0,100,14,0,132,0,0,90,10,0, + 100,15,0,100,16,0,132,0,0,90,11,0,100,17,0,83, + 41,18,218,16,95,78,97,109,101,115,112,97,99,101,76,111, + 97,100,101,114,99,4,0,0,0,0,0,0,0,4,0,0, + 0,4,0,0,0,67,0,0,0,115,25,0,0,0,116,0, + 0,124,1,0,124,2,0,124,3,0,131,3,0,124,0,0, + 95,1,0,100,0,0,83,41,1,78,41,2,114,224,0,0, + 0,114,226,0,0,0,41,4,114,100,0,0,0,114,98,0, + 0,0,114,35,0,0,0,114,230,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,179,0,0,0, + 220,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, + 109,101,115,112,97,99,101,76,111,97,100,101,114,46,95,95, + 105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,2, + 0,0,0,2,0,0,0,67,0,0,0,115,16,0,0,0, + 100,1,0,106,0,0,124,1,0,106,1,0,131,1,0,83, + 41,2,122,115,82,101,116,117,114,110,32,114,101,112,114,32, + 102,111,114,32,116,104,101,32,109,111,100,117,108,101,46,10, + 10,32,32,32,32,32,32,32,32,84,104,101,32,109,101,116, + 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, + 100,46,32,32,84,104,101,32,105,109,112,111,114,116,32,109, + 97,99,104,105,110,101,114,121,32,100,111,101,115,32,116,104, + 101,32,106,111,98,32,105,116,115,101,108,102,46,10,10,32, + 32,32,32,32,32,32,32,122,25,60,109,111,100,117,108,101, + 32,123,33,114,125,32,40,110,97,109,101,115,112,97,99,101, + 41,62,41,2,114,47,0,0,0,114,105,0,0,0,41,2, + 114,164,0,0,0,114,184,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,11,109,111,100,117,108, + 101,95,114,101,112,114,223,3,0,0,115,2,0,0,0,0, + 7,122,28,95,78,97,109,101,115,112,97,99,101,76,111,97, + 100,101,114,46,109,111,100,117,108,101,95,114,101,112,114,99, + 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, + 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,78, + 84,114,4,0,0,0,41,2,114,100,0,0,0,114,119,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,114,153,0,0,0,232,3,0,0,115,2,0,0,0,0, + 1,122,27,95,78,97,109,101,115,112,97,99,101,76,111,97, + 100,101,114,46,105,115,95,112,97,99,107,97,103,101,99,2, + 0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67, + 0,0,0,115,4,0,0,0,100,1,0,83,41,2,78,114, + 30,0,0,0,114,4,0,0,0,41,2,114,100,0,0,0, + 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,196,0,0,0,235,3,0,0,115,2,0, + 0,0,0,1,122,27,95,78,97,109,101,115,112,97,99,101, + 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, + 101,99,2,0,0,0,0,0,0,0,2,0,0,0,6,0, + 0,0,67,0,0,0,115,22,0,0,0,116,0,0,100,1, + 0,100,2,0,100,3,0,100,4,0,100,5,0,131,3,1, + 83,41,6,78,114,30,0,0,0,122,8,60,115,116,114,105, + 110,103,62,114,183,0,0,0,114,198,0,0,0,84,41,1, + 114,199,0,0,0,41,2,114,100,0,0,0,114,119,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,181,0,0,0,238,3,0,0,115,2,0,0,0,0,1, 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, - 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, - 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, - 115,16,0,0,0,100,1,0,106,0,0,124,1,0,106,1, - 0,131,1,0,83,41,2,122,115,82,101,116,117,114,110,32, - 114,101,112,114,32,102,111,114,32,116,104,101,32,109,111,100, - 117,108,101,46,10,10,32,32,32,32,32,32,32,32,84,104, - 101,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, - 101,99,97,116,101,100,46,32,32,84,104,101,32,105,109,112, - 111,114,116,32,109,97,99,104,105,110,101,114,121,32,100,111, - 101,115,32,116,104,101,32,106,111,98,32,105,116,115,101,108, - 102,46,10,10,32,32,32,32,32,32,32,32,122,25,60,109, - 111,100,117,108,101,32,123,33,114,125,32,40,110,97,109,101, - 115,112,97,99,101,41,62,41,2,114,47,0,0,0,114,105, - 0,0,0,41,2,114,164,0,0,0,114,184,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 109,111,100,117,108,101,95,114,101,112,114,222,3,0,0,115, - 2,0,0,0,0,7,122,28,95,78,97,109,101,115,112,97, - 99,101,76,111,97,100,101,114,46,109,111,100,117,108,101,95, - 114,101,112,114,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,78,84,114,4,0,0,0,41,2,114,100,0, - 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,153,0,0,0,231,3,0,0,115, - 2,0,0,0,0,1,122,27,95,78,97,109,101,115,112,97, - 99,101,76,111,97,100,101,114,46,105,115,95,112,97,99,107, - 97,103,101,99,2,0,0,0,0,0,0,0,2,0,0,0, - 1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,0, - 83,41,2,78,114,30,0,0,0,114,4,0,0,0,41,2, - 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,196,0,0,0,234,3, - 0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,101, - 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, - 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, - 0,0,0,6,0,0,0,67,0,0,0,115,22,0,0,0, - 116,0,0,100,1,0,100,2,0,100,3,0,100,4,0,100, - 5,0,131,3,1,83,41,6,78,114,30,0,0,0,122,8, - 60,115,116,114,105,110,103,62,114,183,0,0,0,114,198,0, - 0,0,84,41,1,114,199,0,0,0,41,2,114,100,0,0, - 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,237,3,0,0,115,2, - 0,0,0,0,1,122,25,95,78,97,109,101,115,112,97,99, - 101,76,111,97,100,101,114,46,103,101,116,95,99,111,100,101, - 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, - 0,67,0,0,0,115,4,0,0,0,100,1,0,83,41,2, - 122,42,85,115,101,32,100,101,102,97,117,108,116,32,115,101, - 109,97,110,116,105,99,115,32,102,111,114,32,109,111,100,117, - 108,101,32,99,114,101,97,116,105,111,110,46,78,114,4,0, - 0,0,41,2,114,100,0,0,0,114,158,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,180,0, - 0,0,240,3,0,0,115,0,0,0,0,122,30,95,78,97, - 109,101,115,112,97,99,101,76,111,97,100,101,114,46,99,114, - 101,97,116,101,95,109,111,100,117,108,101,99,2,0,0,0, + 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,0,0,83,41,1,78,114,4,0,0, - 0,41,2,114,100,0,0,0,114,184,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,185,0,0, - 0,243,3,0,0,115,2,0,0,0,0,1,122,28,95,78, - 97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,101, - 120,101,99,95,109,111,100,117,108,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, - 35,0,0,0,116,0,0,106,1,0,100,1,0,124,0,0, - 106,2,0,131,2,0,1,116,0,0,106,3,0,124,0,0, - 124,1,0,131,2,0,83,41,2,122,98,76,111,97,100,32, - 97,32,110,97,109,101,115,112,97,99,101,32,109,111,100,117, - 108,101,46,10,10,32,32,32,32,32,32,32,32,84,104,105, - 115,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, - 101,99,97,116,101,100,46,32,32,85,115,101,32,101,120,101, - 99,95,109,111,100,117,108,101,40,41,32,105,110,115,116,101, - 97,100,46,10,10,32,32,32,32,32,32,32,32,122,38,110, - 97,109,101,115,112,97,99,101,32,109,111,100,117,108,101,32, - 108,111,97,100,101,100,32,119,105,116,104,32,112,97,116,104, - 32,123,33,114,125,41,4,114,114,0,0,0,114,129,0,0, - 0,114,226,0,0,0,114,186,0,0,0,41,2,114,100,0, - 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,187,0,0,0,246,3,0,0,115, - 6,0,0,0,0,7,9,1,10,1,122,28,95,78,97,109, - 101,115,112,97,99,101,76,111,97,100,101,114,46,108,111,97, - 100,95,109,111,100,117,108,101,78,41,12,114,105,0,0,0, - 114,104,0,0,0,114,106,0,0,0,114,179,0,0,0,114, - 177,0,0,0,114,242,0,0,0,114,153,0,0,0,114,196, - 0,0,0,114,181,0,0,0,114,180,0,0,0,114,185,0, - 0,0,114,187,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,241,0,0,0, - 218,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, - 3,12,3,12,3,12,3,12,3,114,241,0,0,0,99,0, - 0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,64, - 0,0,0,115,160,0,0,0,101,0,0,90,1,0,100,0, - 0,90,2,0,100,1,0,90,3,0,101,4,0,100,2,0, - 100,3,0,132,0,0,131,1,0,90,5,0,101,4,0,100, - 4,0,100,5,0,132,0,0,131,1,0,90,6,0,101,4, - 0,100,6,0,100,7,0,132,0,0,131,1,0,90,7,0, - 101,4,0,100,8,0,100,9,0,132,0,0,131,1,0,90, - 8,0,101,4,0,100,10,0,100,11,0,100,12,0,132,1, - 0,131,1,0,90,9,0,101,4,0,100,10,0,100,10,0, - 100,13,0,100,14,0,132,2,0,131,1,0,90,10,0,101, - 4,0,100,10,0,100,15,0,100,16,0,132,1,0,131,1, - 0,90,11,0,100,10,0,83,41,17,218,10,80,97,116,104, - 70,105,110,100,101,114,122,62,77,101,116,97,32,112,97,116, - 104,32,102,105,110,100,101,114,32,102,111,114,32,115,121,115, - 46,112,97,116,104,32,97,110,100,32,112,97,99,107,97,103, - 101,32,95,95,112,97,116,104,95,95,32,97,116,116,114,105, - 98,117,116,101,115,46,99,1,0,0,0,0,0,0,0,2, - 0,0,0,4,0,0,0,67,0,0,0,115,55,0,0,0, - 120,48,0,116,0,0,106,1,0,106,2,0,131,0,0,68, - 93,31,0,125,1,0,116,3,0,124,1,0,100,1,0,131, - 2,0,114,16,0,124,1,0,106,4,0,131,0,0,1,113, - 16,0,87,100,2,0,83,41,3,122,125,67,97,108,108,32, - 116,104,101,32,105,110,118,97,108,105,100,97,116,101,95,99, - 97,99,104,101,115,40,41,32,109,101,116,104,111,100,32,111, - 110,32,97,108,108,32,112,97,116,104,32,101,110,116,114,121, - 32,102,105,110,100,101,114,115,10,32,32,32,32,32,32,32, - 32,115,116,111,114,101,100,32,105,110,32,115,121,115,46,112, - 97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99, - 104,101,115,32,40,119,104,101,114,101,32,105,109,112,108,101, - 109,101,110,116,101,100,41,46,218,17,105,110,118,97,108,105, - 100,97,116,101,95,99,97,99,104,101,115,78,41,5,114,7, - 0,0,0,218,19,112,97,116,104,95,105,109,112,111,114,116, - 101,114,95,99,97,99,104,101,218,6,118,97,108,117,101,115, - 114,108,0,0,0,114,244,0,0,0,41,2,114,164,0,0, - 0,218,6,102,105,110,100,101,114,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,244,0,0,0,8,4,0, - 0,115,6,0,0,0,0,4,22,1,15,1,122,28,80,97, - 116,104,70,105,110,100,101,114,46,105,110,118,97,108,105,100, - 97,116,101,95,99,97,99,104,101,115,99,2,0,0,0,0, - 0,0,0,3,0,0,0,12,0,0,0,67,0,0,0,115, - 107,0,0,0,116,0,0,106,1,0,100,1,0,107,9,0, - 114,41,0,116,0,0,106,1,0,12,114,41,0,116,2,0, - 106,3,0,100,2,0,116,4,0,131,2,0,1,120,59,0, - 116,0,0,106,1,0,68,93,44,0,125,2,0,121,14,0, - 124,2,0,124,1,0,131,1,0,83,87,113,51,0,4,116, - 5,0,107,10,0,114,94,0,1,1,1,119,51,0,89,113, - 51,0,88,113,51,0,87,100,1,0,83,100,1,0,83,41, - 3,122,113,83,101,97,114,99,104,32,115,101,113,117,101,110, - 99,101,32,111,102,32,104,111,111,107,115,32,102,111,114,32, - 97,32,102,105,110,100,101,114,32,102,111,114,32,39,112,97, - 116,104,39,46,10,10,32,32,32,32,32,32,32,32,73,102, - 32,39,104,111,111,107,115,39,32,105,115,32,102,97,108,115, - 101,32,116,104,101,110,32,117,115,101,32,115,121,115,46,112, - 97,116,104,95,104,111,111,107,115,46,10,10,32,32,32,32, - 32,32,32,32,78,122,23,115,121,115,46,112,97,116,104,95, - 104,111,111,107,115,32,105,115,32,101,109,112,116,121,41,6, - 114,7,0,0,0,218,10,112,97,116,104,95,104,111,111,107, - 115,114,60,0,0,0,114,61,0,0,0,114,118,0,0,0, - 114,99,0,0,0,41,3,114,164,0,0,0,114,35,0,0, - 0,90,4,104,111,111,107,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,11,95,112,97,116,104,95,104,111, - 111,107,115,16,4,0,0,115,16,0,0,0,0,7,25,1, - 16,1,16,1,3,1,14,1,13,1,12,2,122,22,80,97, - 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,104, - 111,111,107,115,99,2,0,0,0,0,0,0,0,3,0,0, - 0,19,0,0,0,67,0,0,0,115,123,0,0,0,124,1, - 0,100,1,0,107,2,0,114,53,0,121,16,0,116,0,0, - 106,1,0,131,0,0,125,1,0,87,110,22,0,4,116,2, - 0,107,10,0,114,52,0,1,1,1,100,2,0,83,89,110, - 1,0,88,121,17,0,116,3,0,106,4,0,124,1,0,25, - 125,2,0,87,110,46,0,4,116,5,0,107,10,0,114,118, - 0,1,1,1,124,0,0,106,6,0,124,1,0,131,1,0, - 125,2,0,124,2,0,116,3,0,106,4,0,124,1,0,60, - 89,110,1,0,88,124,2,0,83,41,3,122,210,71,101,116, - 32,116,104,101,32,102,105,110,100,101,114,32,102,111,114,32, - 116,104,101,32,112,97,116,104,32,101,110,116,114,121,32,102, - 114,111,109,32,115,121,115,46,112,97,116,104,95,105,109,112, - 111,114,116,101,114,95,99,97,99,104,101,46,10,10,32,32, - 32,32,32,32,32,32,73,102,32,116,104,101,32,112,97,116, - 104,32,101,110,116,114,121,32,105,115,32,110,111,116,32,105, - 110,32,116,104,101,32,99,97,99,104,101,44,32,102,105,110, - 100,32,116,104,101,32,97,112,112,114,111,112,114,105,97,116, - 101,32,102,105,110,100,101,114,10,32,32,32,32,32,32,32, - 32,97,110,100,32,99,97,99,104,101,32,105,116,46,32,73, - 102,32,110,111,32,102,105,110,100,101,114,32,105,115,32,97, - 118,97,105,108,97,98,108,101,44,32,115,116,111,114,101,32, - 78,111,110,101,46,10,10,32,32,32,32,32,32,32,32,114, - 30,0,0,0,78,41,7,114,3,0,0,0,114,45,0,0, - 0,218,17,70,105,108,101,78,111,116,70,111,117,110,100,69, - 114,114,111,114,114,7,0,0,0,114,245,0,0,0,114,131, - 0,0,0,114,249,0,0,0,41,3,114,164,0,0,0,114, - 35,0,0,0,114,247,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,218,20,95,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,33,4, - 0,0,115,22,0,0,0,0,8,12,1,3,1,16,1,13, - 3,9,1,3,1,17,1,13,1,15,1,18,1,122,31,80, - 97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,99,3, - 0,0,0,0,0,0,0,6,0,0,0,3,0,0,0,67, - 0,0,0,115,119,0,0,0,116,0,0,124,2,0,100,1, - 0,131,2,0,114,39,0,124,2,0,106,1,0,124,1,0, - 131,1,0,92,2,0,125,3,0,125,4,0,110,21,0,124, - 2,0,106,2,0,124,1,0,131,1,0,125,3,0,103,0, - 0,125,4,0,124,3,0,100,0,0,107,9,0,114,88,0, - 116,3,0,106,4,0,124,1,0,124,3,0,131,2,0,83, - 116,3,0,106,5,0,124,1,0,100,0,0,131,2,0,125, - 5,0,124,4,0,124,5,0,95,6,0,124,5,0,83,41, - 2,78,114,117,0,0,0,41,7,114,108,0,0,0,114,117, - 0,0,0,114,176,0,0,0,114,114,0,0,0,114,173,0, - 0,0,114,154,0,0,0,114,150,0,0,0,41,6,114,164, - 0,0,0,114,119,0,0,0,114,247,0,0,0,114,120,0, - 0,0,114,121,0,0,0,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,16,95,108,101, - 103,97,99,121,95,103,101,116,95,115,112,101,99,55,4,0, - 0,115,18,0,0,0,0,4,15,1,24,2,15,1,6,1, - 12,1,16,1,18,1,9,1,122,27,80,97,116,104,70,105, - 110,100,101,114,46,95,108,101,103,97,99,121,95,103,101,116, - 95,115,112,101,99,78,99,4,0,0,0,0,0,0,0,9, - 0,0,0,5,0,0,0,67,0,0,0,115,243,0,0,0, - 103,0,0,125,4,0,120,230,0,124,2,0,68,93,191,0, - 125,5,0,116,0,0,124,5,0,116,1,0,116,2,0,102, - 2,0,131,2,0,115,43,0,113,13,0,124,0,0,106,3, - 0,124,5,0,131,1,0,125,6,0,124,6,0,100,1,0, - 107,9,0,114,13,0,116,4,0,124,6,0,100,2,0,131, - 2,0,114,106,0,124,6,0,106,5,0,124,1,0,124,3, - 0,131,2,0,125,7,0,110,18,0,124,0,0,106,6,0, - 124,1,0,124,6,0,131,2,0,125,7,0,124,7,0,100, - 1,0,107,8,0,114,139,0,113,13,0,124,7,0,106,7, - 0,100,1,0,107,9,0,114,158,0,124,7,0,83,124,7, - 0,106,8,0,125,8,0,124,8,0,100,1,0,107,8,0, - 114,191,0,116,9,0,100,3,0,131,1,0,130,1,0,124, - 4,0,106,10,0,124,8,0,131,1,0,1,113,13,0,87, - 116,11,0,106,12,0,124,1,0,100,1,0,131,2,0,125, - 7,0,124,4,0,124,7,0,95,8,0,124,7,0,83,100, - 1,0,83,41,4,122,63,70,105,110,100,32,116,104,101,32, - 108,111,97,100,101,114,32,111,114,32,110,97,109,101,115,112, - 97,99,101,95,112,97,116,104,32,102,111,114,32,116,104,105, - 115,32,109,111,100,117,108,101,47,112,97,99,107,97,103,101, - 32,110,97,109,101,46,78,114,175,0,0,0,122,19,115,112, - 101,99,32,109,105,115,115,105,110,103,32,108,111,97,100,101, - 114,41,13,114,137,0,0,0,114,69,0,0,0,218,5,98, - 121,116,101,115,114,251,0,0,0,114,108,0,0,0,114,175, - 0,0,0,114,252,0,0,0,114,120,0,0,0,114,150,0, - 0,0,114,99,0,0,0,114,143,0,0,0,114,114,0,0, - 0,114,154,0,0,0,41,9,114,164,0,0,0,114,119,0, - 0,0,114,35,0,0,0,114,174,0,0,0,218,14,110,97, - 109,101,115,112,97,99,101,95,112,97,116,104,90,5,101,110, - 116,114,121,114,247,0,0,0,114,158,0,0,0,114,121,0, + 115,4,0,0,0,100,1,0,83,41,2,122,42,85,115,101, + 32,100,101,102,97,117,108,116,32,115,101,109,97,110,116,105, + 99,115,32,102,111,114,32,109,111,100,117,108,101,32,99,114, + 101,97,116,105,111,110,46,78,114,4,0,0,0,41,2,114, + 100,0,0,0,114,158,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,180,0,0,0,241,3,0, + 0,115,0,0,0,0,122,30,95,78,97,109,101,115,112,97, + 99,101,76,111,97,100,101,114,46,99,114,101,97,116,101,95, + 109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,2, + 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, + 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,100, + 0,0,0,114,184,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,185,0,0,0,244,3,0,0, + 115,2,0,0,0,0,1,122,28,95,78,97,109,101,115,112, + 97,99,101,76,111,97,100,101,114,46,101,120,101,99,95,109, + 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, + 0,0,3,0,0,0,67,0,0,0,115,35,0,0,0,116, + 0,0,106,1,0,100,1,0,124,0,0,106,2,0,131,2, + 0,1,116,0,0,106,3,0,124,0,0,124,1,0,131,2, + 0,83,41,2,122,98,76,111,97,100,32,97,32,110,97,109, + 101,115,112,97,99,101,32,109,111,100,117,108,101,46,10,10, + 32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116, + 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, + 100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100, + 117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10, + 32,32,32,32,32,32,32,32,122,38,110,97,109,101,115,112, + 97,99,101,32,109,111,100,117,108,101,32,108,111,97,100,101, + 100,32,119,105,116,104,32,112,97,116,104,32,123,33,114,125, + 41,4,114,114,0,0,0,114,129,0,0,0,114,226,0,0, + 0,114,186,0,0,0,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,9,95,103,101,116,95,115,112,101,99,70,4,0,0, - 115,40,0,0,0,0,5,6,1,13,1,21,1,3,1,15, - 1,12,1,15,1,21,2,18,1,12,1,3,1,15,1,4, - 1,9,1,12,1,12,5,17,2,18,1,9,1,122,20,80, - 97,116,104,70,105,110,100,101,114,46,95,103,101,116,95,115, - 112,101,99,99,4,0,0,0,0,0,0,0,6,0,0,0, - 4,0,0,0,67,0,0,0,115,140,0,0,0,124,2,0, - 100,1,0,107,8,0,114,21,0,116,0,0,106,1,0,125, - 2,0,124,0,0,106,2,0,124,1,0,124,2,0,124,3, - 0,131,3,0,125,4,0,124,4,0,100,1,0,107,8,0, - 114,58,0,100,1,0,83,124,4,0,106,3,0,100,1,0, - 107,8,0,114,132,0,124,4,0,106,4,0,125,5,0,124, - 5,0,114,125,0,100,2,0,124,4,0,95,5,0,116,6, - 0,124,1,0,124,5,0,124,0,0,106,2,0,131,3,0, - 124,4,0,95,4,0,124,4,0,83,100,1,0,83,110,4, - 0,124,4,0,83,100,1,0,83,41,3,122,98,102,105,110, - 100,32,116,104,101,32,109,111,100,117,108,101,32,111,110,32, - 115,121,115,46,112,97,116,104,32,111,114,32,39,112,97,116, - 104,39,32,98,97,115,101,100,32,111,110,32,115,121,115,46, - 112,97,116,104,95,104,111,111,107,115,32,97,110,100,10,32, - 32,32,32,32,32,32,32,115,121,115,46,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,46,78, - 90,9,110,97,109,101,115,112,97,99,101,41,7,114,7,0, - 0,0,114,35,0,0,0,114,255,0,0,0,114,120,0,0, - 0,114,150,0,0,0,114,152,0,0,0,114,224,0,0,0, - 41,6,114,164,0,0,0,114,119,0,0,0,114,35,0,0, - 0,114,174,0,0,0,114,158,0,0,0,114,254,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 175,0,0,0,102,4,0,0,115,26,0,0,0,0,4,12, - 1,9,1,21,1,12,1,4,1,15,1,9,1,6,3,9, - 1,24,1,4,2,7,2,122,20,80,97,116,104,70,105,110, - 100,101,114,46,102,105,110,100,95,115,112,101,99,99,3,0, - 0,0,0,0,0,0,4,0,0,0,3,0,0,0,67,0, - 0,0,115,41,0,0,0,124,0,0,106,0,0,124,1,0, - 124,2,0,131,2,0,125,3,0,124,3,0,100,1,0,107, - 8,0,114,34,0,100,1,0,83,124,3,0,106,1,0,83, - 41,2,122,170,102,105,110,100,32,116,104,101,32,109,111,100, - 117,108,101,32,111,110,32,115,121,115,46,112,97,116,104,32, - 111,114,32,39,112,97,116,104,39,32,98,97,115,101,100,32, - 111,110,32,115,121,115,46,112,97,116,104,95,104,111,111,107, - 115,32,97,110,100,10,32,32,32,32,32,32,32,32,115,121, - 115,46,112,97,116,104,95,105,109,112,111,114,116,101,114,95, - 99,97,99,104,101,46,10,10,32,32,32,32,32,32,32,32, - 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, - 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, - 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, - 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, - 2,114,175,0,0,0,114,120,0,0,0,41,4,114,164,0, - 0,0,114,119,0,0,0,114,35,0,0,0,114,158,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,176,0,0,0,124,4,0,0,115,8,0,0,0,0,8, - 18,1,12,1,4,1,122,22,80,97,116,104,70,105,110,100, - 101,114,46,102,105,110,100,95,109,111,100,117,108,101,41,12, - 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, - 107,0,0,0,114,177,0,0,0,114,244,0,0,0,114,249, - 0,0,0,114,251,0,0,0,114,252,0,0,0,114,255,0, - 0,0,114,175,0,0,0,114,176,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,243,0,0,0,4,4,0,0,115,22,0,0,0,12,2, - 6,2,18,8,18,17,18,22,18,15,3,1,18,31,3,1, - 21,21,3,1,114,243,0,0,0,99,0,0,0,0,0,0, - 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,133, + 0,114,187,0,0,0,247,3,0,0,115,6,0,0,0,0, + 7,9,1,10,1,122,28,95,78,97,109,101,115,112,97,99, + 101,76,111,97,100,101,114,46,108,111,97,100,95,109,111,100, + 117,108,101,78,41,12,114,105,0,0,0,114,104,0,0,0, + 114,106,0,0,0,114,179,0,0,0,114,177,0,0,0,114, + 242,0,0,0,114,153,0,0,0,114,196,0,0,0,114,181, + 0,0,0,114,180,0,0,0,114,185,0,0,0,114,187,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,241,0,0,0,219,3,0,0,115, + 16,0,0,0,12,1,12,3,18,9,12,3,12,3,12,3, + 12,3,12,3,114,241,0,0,0,99,0,0,0,0,0,0, + 0,0,0,0,0,0,5,0,0,0,64,0,0,0,115,160, 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, - 1,0,90,3,0,100,2,0,100,3,0,132,0,0,90,4, - 0,100,4,0,100,5,0,132,0,0,90,5,0,101,6,0, - 90,7,0,100,6,0,100,7,0,132,0,0,90,8,0,100, - 8,0,100,9,0,132,0,0,90,9,0,100,10,0,100,11, - 0,100,12,0,132,1,0,90,10,0,100,13,0,100,14,0, - 132,0,0,90,11,0,101,12,0,100,15,0,100,16,0,132, - 0,0,131,1,0,90,13,0,100,17,0,100,18,0,132,0, - 0,90,14,0,100,10,0,83,41,19,218,10,70,105,108,101, - 70,105,110,100,101,114,122,172,70,105,108,101,45,98,97,115, - 101,100,32,102,105,110,100,101,114,46,10,10,32,32,32,32, - 73,110,116,101,114,97,99,116,105,111,110,115,32,119,105,116, - 104,32,116,104,101,32,102,105,108,101,32,115,121,115,116,101, - 109,32,97,114,101,32,99,97,99,104,101,100,32,102,111,114, - 32,112,101,114,102,111,114,109,97,110,99,101,44,32,98,101, - 105,110,103,10,32,32,32,32,114,101,102,114,101,115,104,101, - 100,32,119,104,101,110,32,116,104,101,32,100,105,114,101,99, - 116,111,114,121,32,116,104,101,32,102,105,110,100,101,114,32, - 105,115,32,104,97,110,100,108,105,110,103,32,104,97,115,32, - 98,101,101,110,32,109,111,100,105,102,105,101,100,46,10,10, - 32,32,32,32,99,2,0,0,0,0,0,0,0,5,0,0, - 0,5,0,0,0,7,0,0,0,115,122,0,0,0,103,0, - 0,125,3,0,120,52,0,124,2,0,68,93,44,0,92,2, - 0,137,0,0,125,4,0,124,3,0,106,0,0,135,0,0, - 102,1,0,100,1,0,100,2,0,134,0,0,124,4,0,68, - 131,1,0,131,1,0,1,113,13,0,87,124,3,0,124,0, - 0,95,1,0,124,1,0,112,79,0,100,3,0,124,0,0, - 95,2,0,100,6,0,124,0,0,95,3,0,116,4,0,131, - 0,0,124,0,0,95,5,0,116,4,0,131,0,0,124,0, - 0,95,6,0,100,5,0,83,41,7,122,154,73,110,105,116, - 105,97,108,105,122,101,32,119,105,116,104,32,116,104,101,32, - 112,97,116,104,32,116,111,32,115,101,97,114,99,104,32,111, - 110,32,97,110,100,32,97,32,118,97,114,105,97,98,108,101, - 32,110,117,109,98,101,114,32,111,102,10,32,32,32,32,32, - 32,32,32,50,45,116,117,112,108,101,115,32,99,111,110,116, - 97,105,110,105,110,103,32,116,104,101,32,108,111,97,100,101, - 114,32,97,110,100,32,116,104,101,32,102,105,108,101,32,115, - 117,102,102,105,120,101,115,32,116,104,101,32,108,111,97,100, - 101,114,10,32,32,32,32,32,32,32,32,114,101,99,111,103, - 110,105,122,101,115,46,99,1,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,51,0,0,0,115,27,0,0,0, - 124,0,0,93,17,0,125,1,0,124,1,0,136,0,0,102, - 2,0,86,1,113,3,0,100,0,0,83,41,1,78,114,4, - 0,0,0,41,2,114,22,0,0,0,114,219,0,0,0,41, - 1,114,120,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,153,4,0,0,115,2,0,0,0,6,0, - 122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105, - 110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60, - 103,101,110,101,120,112,114,62,114,58,0,0,0,114,29,0, - 0,0,78,114,87,0,0,0,41,7,114,143,0,0,0,218, - 8,95,108,111,97,100,101,114,115,114,35,0,0,0,218,11, - 95,112,97,116,104,95,109,116,105,109,101,218,3,115,101,116, - 218,11,95,112,97,116,104,95,99,97,99,104,101,218,19,95, - 114,101,108,97,120,101,100,95,112,97,116,104,95,99,97,99, - 104,101,41,5,114,100,0,0,0,114,35,0,0,0,218,14, - 108,111,97,100,101,114,95,100,101,116,97,105,108,115,90,7, - 108,111,97,100,101,114,115,114,160,0,0,0,114,4,0,0, - 0,41,1,114,120,0,0,0,114,5,0,0,0,114,179,0, - 0,0,147,4,0,0,115,16,0,0,0,0,4,6,1,19, - 1,36,1,9,2,15,1,9,1,12,1,122,19,70,105,108, + 1,0,90,3,0,101,4,0,100,2,0,100,3,0,132,0, + 0,131,1,0,90,5,0,101,4,0,100,4,0,100,5,0, + 132,0,0,131,1,0,90,6,0,101,4,0,100,6,0,100, + 7,0,132,0,0,131,1,0,90,7,0,101,4,0,100,8, + 0,100,9,0,132,0,0,131,1,0,90,8,0,101,4,0, + 100,10,0,100,11,0,100,12,0,132,1,0,131,1,0,90, + 9,0,101,4,0,100,10,0,100,10,0,100,13,0,100,14, + 0,132,2,0,131,1,0,90,10,0,101,4,0,100,10,0, + 100,15,0,100,16,0,132,1,0,131,1,0,90,11,0,100, + 10,0,83,41,17,218,10,80,97,116,104,70,105,110,100,101, + 114,122,62,77,101,116,97,32,112,97,116,104,32,102,105,110, + 100,101,114,32,102,111,114,32,115,121,115,46,112,97,116,104, + 32,97,110,100,32,112,97,99,107,97,103,101,32,95,95,112, + 97,116,104,95,95,32,97,116,116,114,105,98,117,116,101,115, + 46,99,1,0,0,0,0,0,0,0,2,0,0,0,4,0, + 0,0,67,0,0,0,115,55,0,0,0,120,48,0,116,0, + 0,106,1,0,106,2,0,131,0,0,68,93,31,0,125,1, + 0,116,3,0,124,1,0,100,1,0,131,2,0,114,16,0, + 124,1,0,106,4,0,131,0,0,1,113,16,0,87,100,2, + 0,83,41,3,122,125,67,97,108,108,32,116,104,101,32,105, + 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, + 40,41,32,109,101,116,104,111,100,32,111,110,32,97,108,108, + 32,112,97,116,104,32,101,110,116,114,121,32,102,105,110,100, + 101,114,115,10,32,32,32,32,32,32,32,32,115,116,111,114, + 101,100,32,105,110,32,115,121,115,46,112,97,116,104,95,105, + 109,112,111,114,116,101,114,95,99,97,99,104,101,115,32,40, + 119,104,101,114,101,32,105,109,112,108,101,109,101,110,116,101, + 100,41,46,218,17,105,110,118,97,108,105,100,97,116,101,95, + 99,97,99,104,101,115,78,41,5,114,7,0,0,0,218,19, + 112,97,116,104,95,105,109,112,111,114,116,101,114,95,99,97, + 99,104,101,218,6,118,97,108,117,101,115,114,108,0,0,0, + 114,244,0,0,0,41,2,114,164,0,0,0,218,6,102,105, + 110,100,101,114,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,244,0,0,0,9,4,0,0,115,6,0,0, + 0,0,4,22,1,15,1,122,28,80,97,116,104,70,105,110, + 100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99, + 97,99,104,101,115,99,2,0,0,0,0,0,0,0,3,0, + 0,0,12,0,0,0,67,0,0,0,115,107,0,0,0,116, + 0,0,106,1,0,100,1,0,107,9,0,114,41,0,116,0, + 0,106,1,0,12,114,41,0,116,2,0,106,3,0,100,2, + 0,116,4,0,131,2,0,1,120,59,0,116,0,0,106,1, + 0,68,93,44,0,125,2,0,121,14,0,124,2,0,124,1, + 0,131,1,0,83,87,113,51,0,4,116,5,0,107,10,0, + 114,94,0,1,1,1,119,51,0,89,113,51,0,88,113,51, + 0,87,100,1,0,83,100,1,0,83,41,3,122,113,83,101, + 97,114,99,104,32,115,101,113,117,101,110,99,101,32,111,102, + 32,104,111,111,107,115,32,102,111,114,32,97,32,102,105,110, + 100,101,114,32,102,111,114,32,39,112,97,116,104,39,46,10, + 10,32,32,32,32,32,32,32,32,73,102,32,39,104,111,111, + 107,115,39,32,105,115,32,102,97,108,115,101,32,116,104,101, + 110,32,117,115,101,32,115,121,115,46,112,97,116,104,95,104, + 111,111,107,115,46,10,10,32,32,32,32,32,32,32,32,78, + 122,23,115,121,115,46,112,97,116,104,95,104,111,111,107,115, + 32,105,115,32,101,109,112,116,121,41,6,114,7,0,0,0, + 218,10,112,97,116,104,95,104,111,111,107,115,114,60,0,0, + 0,114,61,0,0,0,114,118,0,0,0,114,99,0,0,0, + 41,3,114,164,0,0,0,114,35,0,0,0,90,4,104,111, + 111,107,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,11,95,112,97,116,104,95,104,111,111,107,115,17,4, + 0,0,115,16,0,0,0,0,7,25,1,16,1,16,1,3, + 1,14,1,13,1,12,2,122,22,80,97,116,104,70,105,110, + 100,101,114,46,95,112,97,116,104,95,104,111,111,107,115,99, + 2,0,0,0,0,0,0,0,3,0,0,0,19,0,0,0, + 67,0,0,0,115,123,0,0,0,124,1,0,100,1,0,107, + 2,0,114,53,0,121,16,0,116,0,0,106,1,0,131,0, + 0,125,1,0,87,110,22,0,4,116,2,0,107,10,0,114, + 52,0,1,1,1,100,2,0,83,89,110,1,0,88,121,17, + 0,116,3,0,106,4,0,124,1,0,25,125,2,0,87,110, + 46,0,4,116,5,0,107,10,0,114,118,0,1,1,1,124, + 0,0,106,6,0,124,1,0,131,1,0,125,2,0,124,2, + 0,116,3,0,106,4,0,124,1,0,60,89,110,1,0,88, + 124,2,0,83,41,3,122,210,71,101,116,32,116,104,101,32, + 102,105,110,100,101,114,32,102,111,114,32,116,104,101,32,112, + 97,116,104,32,101,110,116,114,121,32,102,114,111,109,32,115, + 121,115,46,112,97,116,104,95,105,109,112,111,114,116,101,114, + 95,99,97,99,104,101,46,10,10,32,32,32,32,32,32,32, + 32,73,102,32,116,104,101,32,112,97,116,104,32,101,110,116, + 114,121,32,105,115,32,110,111,116,32,105,110,32,116,104,101, + 32,99,97,99,104,101,44,32,102,105,110,100,32,116,104,101, + 32,97,112,112,114,111,112,114,105,97,116,101,32,102,105,110, + 100,101,114,10,32,32,32,32,32,32,32,32,97,110,100,32, + 99,97,99,104,101,32,105,116,46,32,73,102,32,110,111,32, + 102,105,110,100,101,114,32,105,115,32,97,118,97,105,108,97, + 98,108,101,44,32,115,116,111,114,101,32,78,111,110,101,46, + 10,10,32,32,32,32,32,32,32,32,114,30,0,0,0,78, + 41,7,114,3,0,0,0,114,45,0,0,0,218,17,70,105, + 108,101,78,111,116,70,111,117,110,100,69,114,114,111,114,114, + 7,0,0,0,114,245,0,0,0,114,131,0,0,0,114,249, + 0,0,0,41,3,114,164,0,0,0,114,35,0,0,0,114, + 247,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,20,95,112,97,116,104,95,105,109,112,111,114, + 116,101,114,95,99,97,99,104,101,34,4,0,0,115,22,0, + 0,0,0,8,12,1,3,1,16,1,13,3,9,1,3,1, + 17,1,13,1,15,1,18,1,122,31,80,97,116,104,70,105, + 110,100,101,114,46,95,112,97,116,104,95,105,109,112,111,114, + 116,101,114,95,99,97,99,104,101,99,3,0,0,0,0,0, + 0,0,6,0,0,0,3,0,0,0,67,0,0,0,115,119, + 0,0,0,116,0,0,124,2,0,100,1,0,131,2,0,114, + 39,0,124,2,0,106,1,0,124,1,0,131,1,0,92,2, + 0,125,3,0,125,4,0,110,21,0,124,2,0,106,2,0, + 124,1,0,131,1,0,125,3,0,103,0,0,125,4,0,124, + 3,0,100,0,0,107,9,0,114,88,0,116,3,0,106,4, + 0,124,1,0,124,3,0,131,2,0,83,116,3,0,106,5, + 0,124,1,0,100,0,0,131,2,0,125,5,0,124,4,0, + 124,5,0,95,6,0,124,5,0,83,41,2,78,114,117,0, + 0,0,41,7,114,108,0,0,0,114,117,0,0,0,114,176, + 0,0,0,114,114,0,0,0,114,173,0,0,0,114,154,0, + 0,0,114,150,0,0,0,41,6,114,164,0,0,0,114,119, + 0,0,0,114,247,0,0,0,114,120,0,0,0,114,121,0, + 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,16,95,108,101,103,97,99,121,95, + 103,101,116,95,115,112,101,99,56,4,0,0,115,18,0,0, + 0,0,4,15,1,24,2,15,1,6,1,12,1,16,1,18, + 1,9,1,122,27,80,97,116,104,70,105,110,100,101,114,46, + 95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,99, + 78,99,4,0,0,0,0,0,0,0,9,0,0,0,5,0, + 0,0,67,0,0,0,115,243,0,0,0,103,0,0,125,4, + 0,120,230,0,124,2,0,68,93,191,0,125,5,0,116,0, + 0,124,5,0,116,1,0,116,2,0,102,2,0,131,2,0, + 115,43,0,113,13,0,124,0,0,106,3,0,124,5,0,131, + 1,0,125,6,0,124,6,0,100,1,0,107,9,0,114,13, + 0,116,4,0,124,6,0,100,2,0,131,2,0,114,106,0, + 124,6,0,106,5,0,124,1,0,124,3,0,131,2,0,125, + 7,0,110,18,0,124,0,0,106,6,0,124,1,0,124,6, + 0,131,2,0,125,7,0,124,7,0,100,1,0,107,8,0, + 114,139,0,113,13,0,124,7,0,106,7,0,100,1,0,107, + 9,0,114,158,0,124,7,0,83,124,7,0,106,8,0,125, + 8,0,124,8,0,100,1,0,107,8,0,114,191,0,116,9, + 0,100,3,0,131,1,0,130,1,0,124,4,0,106,10,0, + 124,8,0,131,1,0,1,113,13,0,87,116,11,0,106,12, + 0,124,1,0,100,1,0,131,2,0,125,7,0,124,4,0, + 124,7,0,95,8,0,124,7,0,83,100,1,0,83,41,4, + 122,63,70,105,110,100,32,116,104,101,32,108,111,97,100,101, + 114,32,111,114,32,110,97,109,101,115,112,97,99,101,95,112, + 97,116,104,32,102,111,114,32,116,104,105,115,32,109,111,100, + 117,108,101,47,112,97,99,107,97,103,101,32,110,97,109,101, + 46,78,114,175,0,0,0,122,19,115,112,101,99,32,109,105, + 115,115,105,110,103,32,108,111,97,100,101,114,41,13,114,137, + 0,0,0,114,69,0,0,0,218,5,98,121,116,101,115,114, + 251,0,0,0,114,108,0,0,0,114,175,0,0,0,114,252, + 0,0,0,114,120,0,0,0,114,150,0,0,0,114,99,0, + 0,0,114,143,0,0,0,114,114,0,0,0,114,154,0,0, + 0,41,9,114,164,0,0,0,114,119,0,0,0,114,35,0, + 0,0,114,174,0,0,0,218,14,110,97,109,101,115,112,97, + 99,101,95,112,97,116,104,90,5,101,110,116,114,121,114,247, + 0,0,0,114,158,0,0,0,114,121,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,9,95,103, + 101,116,95,115,112,101,99,71,4,0,0,115,40,0,0,0, + 0,5,6,1,13,1,21,1,3,1,15,1,12,1,15,1, + 21,2,18,1,12,1,3,1,15,1,4,1,9,1,12,1, + 12,5,17,2,18,1,9,1,122,20,80,97,116,104,70,105, + 110,100,101,114,46,95,103,101,116,95,115,112,101,99,99,4, + 0,0,0,0,0,0,0,6,0,0,0,4,0,0,0,67, + 0,0,0,115,140,0,0,0,124,2,0,100,1,0,107,8, + 0,114,21,0,116,0,0,106,1,0,125,2,0,124,0,0, + 106,2,0,124,1,0,124,2,0,124,3,0,131,3,0,125, + 4,0,124,4,0,100,1,0,107,8,0,114,58,0,100,1, + 0,83,124,4,0,106,3,0,100,1,0,107,8,0,114,132, + 0,124,4,0,106,4,0,125,5,0,124,5,0,114,125,0, + 100,2,0,124,4,0,95,5,0,116,6,0,124,1,0,124, + 5,0,124,0,0,106,2,0,131,3,0,124,4,0,95,4, + 0,124,4,0,83,100,1,0,83,110,4,0,124,4,0,83, + 100,1,0,83,41,3,122,98,102,105,110,100,32,116,104,101, + 32,109,111,100,117,108,101,32,111,110,32,115,121,115,46,112, + 97,116,104,32,111,114,32,39,112,97,116,104,39,32,98,97, + 115,101,100,32,111,110,32,115,121,115,46,112,97,116,104,95, + 104,111,111,107,115,32,97,110,100,10,32,32,32,32,32,32, + 32,32,115,121,115,46,112,97,116,104,95,105,109,112,111,114, + 116,101,114,95,99,97,99,104,101,46,78,90,9,110,97,109, + 101,115,112,97,99,101,41,7,114,7,0,0,0,114,35,0, + 0,0,114,255,0,0,0,114,120,0,0,0,114,150,0,0, + 0,114,152,0,0,0,114,224,0,0,0,41,6,114,164,0, + 0,0,114,119,0,0,0,114,35,0,0,0,114,174,0,0, + 0,114,158,0,0,0,114,254,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,103, + 4,0,0,115,26,0,0,0,0,4,12,1,9,1,21,1, + 12,1,4,1,15,1,9,1,6,3,9,1,24,1,4,2, + 7,2,122,20,80,97,116,104,70,105,110,100,101,114,46,102, + 105,110,100,95,115,112,101,99,99,3,0,0,0,0,0,0, + 0,4,0,0,0,3,0,0,0,67,0,0,0,115,41,0, + 0,0,124,0,0,106,0,0,124,1,0,124,2,0,131,2, + 0,125,3,0,124,3,0,100,1,0,107,8,0,114,34,0, + 100,1,0,83,124,3,0,106,1,0,83,41,2,122,170,102, + 105,110,100,32,116,104,101,32,109,111,100,117,108,101,32,111, + 110,32,115,121,115,46,112,97,116,104,32,111,114,32,39,112, + 97,116,104,39,32,98,97,115,101,100,32,111,110,32,115,121, + 115,46,112,97,116,104,95,104,111,111,107,115,32,97,110,100, + 10,32,32,32,32,32,32,32,32,115,121,115,46,112,97,116, + 104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101, + 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, + 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, + 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, + 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, + 10,32,32,32,32,32,32,32,32,78,41,2,114,175,0,0, + 0,114,120,0,0,0,41,4,114,164,0,0,0,114,119,0, + 0,0,114,35,0,0,0,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,176,0,0,0, + 125,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, + 1,122,22,80,97,116,104,70,105,110,100,101,114,46,102,105, + 110,100,95,109,111,100,117,108,101,41,12,114,105,0,0,0, + 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, + 177,0,0,0,114,244,0,0,0,114,249,0,0,0,114,251, + 0,0,0,114,252,0,0,0,114,255,0,0,0,114,175,0, + 0,0,114,176,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,243,0,0,0, + 5,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, + 17,18,22,18,15,3,1,18,31,3,1,21,21,3,1,114, + 243,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, + 0,3,0,0,0,64,0,0,0,115,133,0,0,0,101,0, + 0,90,1,0,100,0,0,90,2,0,100,1,0,90,3,0, + 100,2,0,100,3,0,132,0,0,90,4,0,100,4,0,100, + 5,0,132,0,0,90,5,0,101,6,0,90,7,0,100,6, + 0,100,7,0,132,0,0,90,8,0,100,8,0,100,9,0, + 132,0,0,90,9,0,100,10,0,100,11,0,100,12,0,132, + 1,0,90,10,0,100,13,0,100,14,0,132,0,0,90,11, + 0,101,12,0,100,15,0,100,16,0,132,0,0,131,1,0, + 90,13,0,100,17,0,100,18,0,132,0,0,90,14,0,100, + 10,0,83,41,19,218,10,70,105,108,101,70,105,110,100,101, + 114,122,172,70,105,108,101,45,98,97,115,101,100,32,102,105, + 110,100,101,114,46,10,10,32,32,32,32,73,110,116,101,114, + 97,99,116,105,111,110,115,32,119,105,116,104,32,116,104,101, + 32,102,105,108,101,32,115,121,115,116,101,109,32,97,114,101, + 32,99,97,99,104,101,100,32,102,111,114,32,112,101,114,102, + 111,114,109,97,110,99,101,44,32,98,101,105,110,103,10,32, + 32,32,32,114,101,102,114,101,115,104,101,100,32,119,104,101, + 110,32,116,104,101,32,100,105,114,101,99,116,111,114,121,32, + 116,104,101,32,102,105,110,100,101,114,32,105,115,32,104,97, + 110,100,108,105,110,103,32,104,97,115,32,98,101,101,110,32, + 109,111,100,105,102,105,101,100,46,10,10,32,32,32,32,99, + 2,0,0,0,0,0,0,0,5,0,0,0,5,0,0,0, + 7,0,0,0,115,122,0,0,0,103,0,0,125,3,0,120, + 52,0,124,2,0,68,93,44,0,92,2,0,137,0,0,125, + 4,0,124,3,0,106,0,0,135,0,0,102,1,0,100,1, + 0,100,2,0,134,0,0,124,4,0,68,131,1,0,131,1, + 0,1,113,13,0,87,124,3,0,124,0,0,95,1,0,124, + 1,0,112,79,0,100,3,0,124,0,0,95,2,0,100,6, + 0,124,0,0,95,3,0,116,4,0,131,0,0,124,0,0, + 95,5,0,116,4,0,131,0,0,124,0,0,95,6,0,100, + 5,0,83,41,7,122,154,73,110,105,116,105,97,108,105,122, + 101,32,119,105,116,104,32,116,104,101,32,112,97,116,104,32, + 116,111,32,115,101,97,114,99,104,32,111,110,32,97,110,100, + 32,97,32,118,97,114,105,97,98,108,101,32,110,117,109,98, + 101,114,32,111,102,10,32,32,32,32,32,32,32,32,50,45, + 116,117,112,108,101,115,32,99,111,110,116,97,105,110,105,110, + 103,32,116,104,101,32,108,111,97,100,101,114,32,97,110,100, + 32,116,104,101,32,102,105,108,101,32,115,117,102,102,105,120, + 101,115,32,116,104,101,32,108,111,97,100,101,114,10,32,32, + 32,32,32,32,32,32,114,101,99,111,103,110,105,122,101,115, + 46,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, + 0,0,51,0,0,0,115,27,0,0,0,124,0,0,93,17, + 0,125,1,0,124,1,0,136,0,0,102,2,0,86,1,113, + 3,0,100,0,0,83,41,1,78,114,4,0,0,0,41,2, + 114,22,0,0,0,114,219,0,0,0,41,1,114,120,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, + 154,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, - 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, - 0,67,0,0,0,115,13,0,0,0,100,3,0,124,0,0, - 95,0,0,100,2,0,83,41,4,122,31,73,110,118,97,108, - 105,100,97,116,101,32,116,104,101,32,100,105,114,101,99,116, - 111,114,121,32,109,116,105,109,101,46,114,29,0,0,0,78, - 114,87,0,0,0,41,1,114,2,1,0,0,41,1,114,100, + 46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120, + 112,114,62,114,58,0,0,0,114,29,0,0,0,78,114,87, + 0,0,0,41,7,114,143,0,0,0,218,8,95,108,111,97, + 100,101,114,115,114,35,0,0,0,218,11,95,112,97,116,104, + 95,109,116,105,109,101,218,3,115,101,116,218,11,95,112,97, + 116,104,95,99,97,99,104,101,218,19,95,114,101,108,97,120, + 101,100,95,112,97,116,104,95,99,97,99,104,101,41,5,114, + 100,0,0,0,114,35,0,0,0,218,14,108,111,97,100,101, + 114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101, + 114,115,114,160,0,0,0,114,4,0,0,0,41,1,114,120, + 0,0,0,114,5,0,0,0,114,179,0,0,0,148,4,0, + 0,115,16,0,0,0,0,4,6,1,19,1,36,1,9,2, + 15,1,9,1,12,1,122,19,70,105,108,101,70,105,110,100, + 101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0, + 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, + 115,13,0,0,0,100,3,0,124,0,0,95,0,0,100,2, + 0,83,41,4,122,31,73,110,118,97,108,105,100,97,116,101, + 32,116,104,101,32,100,105,114,101,99,116,111,114,121,32,109, + 116,105,109,101,46,114,29,0,0,0,78,114,87,0,0,0, + 41,1,114,2,1,0,0,41,1,114,100,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,244,0, + 0,0,162,4,0,0,115,2,0,0,0,0,2,122,28,70, + 105,108,101,70,105,110,100,101,114,46,105,110,118,97,108,105, + 100,97,116,101,95,99,97,99,104,101,115,99,2,0,0,0, + 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, + 115,59,0,0,0,124,0,0,106,0,0,124,1,0,131,1, + 0,125,2,0,124,2,0,100,1,0,107,8,0,114,37,0, + 100,1,0,103,0,0,102,2,0,83,124,2,0,106,1,0, + 124,2,0,106,2,0,112,55,0,103,0,0,102,2,0,83, + 41,2,122,197,84,114,121,32,116,111,32,102,105,110,100,32, + 97,32,108,111,97,100,101,114,32,102,111,114,32,116,104,101, + 32,115,112,101,99,105,102,105,101,100,32,109,111,100,117,108, + 101,44,32,111,114,32,116,104,101,32,110,97,109,101,115,112, + 97,99,101,10,32,32,32,32,32,32,32,32,112,97,99,107, + 97,103,101,32,112,111,114,116,105,111,110,115,46,32,82,101, + 116,117,114,110,115,32,40,108,111,97,100,101,114,44,32,108, + 105,115,116,45,111,102,45,112,111,114,116,105,111,110,115,41, + 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, + 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, + 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, + 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, + 10,32,32,32,32,32,32,32,32,78,41,3,114,175,0,0, + 0,114,120,0,0,0,114,150,0,0,0,41,3,114,100,0, + 0,0,114,119,0,0,0,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,117,0,0,0, + 168,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, + 1,122,22,70,105,108,101,70,105,110,100,101,114,46,102,105, + 110,100,95,108,111,97,100,101,114,99,6,0,0,0,0,0, + 0,0,7,0,0,0,7,0,0,0,67,0,0,0,115,40, + 0,0,0,124,1,0,124,2,0,124,3,0,131,2,0,125, + 6,0,116,0,0,124,2,0,124,3,0,100,1,0,124,6, + 0,100,2,0,124,4,0,131,2,2,83,41,3,78,114,120, + 0,0,0,114,150,0,0,0,41,1,114,161,0,0,0,41, + 7,114,100,0,0,0,114,159,0,0,0,114,119,0,0,0, + 114,35,0,0,0,90,4,115,109,115,108,114,174,0,0,0, + 114,120,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,255,0,0,0,180,4,0,0,115,6,0, + 0,0,0,1,15,1,18,1,122,20,70,105,108,101,70,105, + 110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99, + 3,0,0,0,0,0,0,0,14,0,0,0,15,0,0,0, + 67,0,0,0,115,228,1,0,0,100,1,0,125,3,0,124, + 1,0,106,0,0,100,2,0,131,1,0,100,3,0,25,125, + 4,0,121,34,0,116,1,0,124,0,0,106,2,0,112,49, + 0,116,3,0,106,4,0,131,0,0,131,1,0,106,5,0, + 125,5,0,87,110,24,0,4,116,6,0,107,10,0,114,85, + 0,1,1,1,100,10,0,125,5,0,89,110,1,0,88,124, + 5,0,124,0,0,106,7,0,107,3,0,114,120,0,124,0, + 0,106,8,0,131,0,0,1,124,5,0,124,0,0,95,7, + 0,116,9,0,131,0,0,114,153,0,124,0,0,106,10,0, + 125,6,0,124,4,0,106,11,0,131,0,0,125,7,0,110, + 15,0,124,0,0,106,12,0,125,6,0,124,4,0,125,7, + 0,124,7,0,124,6,0,107,6,0,114,45,1,116,13,0, + 124,0,0,106,2,0,124,4,0,131,2,0,125,8,0,120, + 100,0,124,0,0,106,14,0,68,93,77,0,92,2,0,125, + 9,0,125,10,0,100,5,0,124,9,0,23,125,11,0,116, + 13,0,124,8,0,124,11,0,131,2,0,125,12,0,116,15, + 0,124,12,0,131,1,0,114,208,0,124,0,0,106,16,0, + 124,10,0,124,1,0,124,12,0,124,8,0,103,1,0,124, + 2,0,131,5,0,83,113,208,0,87,116,17,0,124,8,0, + 131,1,0,125,3,0,120,120,0,124,0,0,106,14,0,68, + 93,109,0,92,2,0,125,9,0,125,10,0,116,13,0,124, + 0,0,106,2,0,124,4,0,124,9,0,23,131,2,0,125, + 12,0,116,18,0,106,19,0,100,6,0,124,12,0,100,7, + 0,100,3,0,131,2,1,1,124,7,0,124,9,0,23,124, + 6,0,107,6,0,114,55,1,116,15,0,124,12,0,131,1, + 0,114,55,1,124,0,0,106,16,0,124,10,0,124,1,0, + 124,12,0,100,8,0,124,2,0,131,5,0,83,113,55,1, + 87,124,3,0,114,224,1,116,18,0,106,19,0,100,9,0, + 124,8,0,131,2,0,1,116,18,0,106,20,0,124,1,0, + 100,8,0,131,2,0,125,13,0,124,8,0,103,1,0,124, + 13,0,95,21,0,124,13,0,83,100,8,0,83,41,11,122, + 125,84,114,121,32,116,111,32,102,105,110,100,32,97,32,108, + 111,97,100,101,114,32,102,111,114,32,116,104,101,32,115,112, + 101,99,105,102,105,101,100,32,109,111,100,117,108,101,44,32, + 111,114,32,116,104,101,32,110,97,109,101,115,112,97,99,101, + 10,32,32,32,32,32,32,32,32,112,97,99,107,97,103,101, + 32,112,111,114,116,105,111,110,115,46,32,82,101,116,117,114, + 110,115,32,40,108,111,97,100,101,114,44,32,108,105,115,116, + 45,111,102,45,112,111,114,116,105,111,110,115,41,46,70,114, + 58,0,0,0,114,56,0,0,0,114,29,0,0,0,114,179, + 0,0,0,122,9,116,114,121,105,110,103,32,123,125,90,9, + 118,101,114,98,111,115,105,116,121,78,122,25,112,111,115,115, + 105,98,108,101,32,110,97,109,101,115,112,97,99,101,32,102, + 111,114,32,123,125,114,87,0,0,0,41,22,114,32,0,0, + 0,114,39,0,0,0,114,35,0,0,0,114,3,0,0,0, + 114,45,0,0,0,114,213,0,0,0,114,40,0,0,0,114, + 2,1,0,0,218,11,95,102,105,108,108,95,99,97,99,104, + 101,114,6,0,0,0,114,5,1,0,0,114,88,0,0,0, + 114,4,1,0,0,114,28,0,0,0,114,1,1,0,0,114, + 44,0,0,0,114,255,0,0,0,114,46,0,0,0,114,114, + 0,0,0,114,129,0,0,0,114,154,0,0,0,114,150,0, + 0,0,41,14,114,100,0,0,0,114,119,0,0,0,114,174, + 0,0,0,90,12,105,115,95,110,97,109,101,115,112,97,99, + 101,90,11,116,97,105,108,95,109,111,100,117,108,101,114,126, + 0,0,0,90,5,99,97,99,104,101,90,12,99,97,99,104, + 101,95,109,111,100,117,108,101,90,9,98,97,115,101,95,112, + 97,116,104,114,219,0,0,0,114,159,0,0,0,90,13,105, + 110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,117, + 108,108,95,112,97,116,104,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,175,0,0,0, + 185,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, + 1,34,1,13,1,11,1,15,1,10,1,9,2,9,1,9, + 1,15,2,9,1,6,2,12,1,18,1,22,1,10,1,15, + 1,12,1,32,4,12,2,22,1,22,1,22,1,16,1,12, + 1,15,1,14,1,6,1,16,1,18,1,12,1,4,1,122, + 20,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100, + 95,115,112,101,99,99,1,0,0,0,0,0,0,0,9,0, + 0,0,13,0,0,0,67,0,0,0,115,11,1,0,0,124, + 0,0,106,0,0,125,1,0,121,31,0,116,1,0,106,2, + 0,124,1,0,112,33,0,116,1,0,106,3,0,131,0,0, + 131,1,0,125,2,0,87,110,33,0,4,116,4,0,116,5, + 0,116,6,0,102,3,0,107,10,0,114,75,0,1,1,1, + 103,0,0,125,2,0,89,110,1,0,88,116,7,0,106,8, + 0,106,9,0,100,1,0,131,1,0,115,112,0,116,10,0, + 124,2,0,131,1,0,124,0,0,95,11,0,110,111,0,116, + 10,0,131,0,0,125,3,0,120,90,0,124,2,0,68,93, + 82,0,125,4,0,124,4,0,106,12,0,100,2,0,131,1, + 0,92,3,0,125,5,0,125,6,0,125,7,0,124,6,0, + 114,191,0,100,3,0,106,13,0,124,5,0,124,7,0,106, + 14,0,131,0,0,131,2,0,125,8,0,110,6,0,124,5, + 0,125,8,0,124,3,0,106,15,0,124,8,0,131,1,0, + 1,113,128,0,87,124,3,0,124,0,0,95,11,0,116,7, + 0,106,8,0,106,9,0,116,16,0,131,1,0,114,7,1, + 100,4,0,100,5,0,132,0,0,124,2,0,68,131,1,0, + 124,0,0,95,17,0,100,6,0,83,41,7,122,68,70,105, + 108,108,32,116,104,101,32,99,97,99,104,101,32,111,102,32, + 112,111,116,101,110,116,105,97,108,32,109,111,100,117,108,101, + 115,32,97,110,100,32,112,97,99,107,97,103,101,115,32,102, + 111,114,32,116,104,105,115,32,100,105,114,101,99,116,111,114, + 121,46,114,0,0,0,0,114,58,0,0,0,122,5,123,125, + 46,123,125,99,1,0,0,0,0,0,0,0,2,0,0,0, + 3,0,0,0,83,0,0,0,115,28,0,0,0,104,0,0, + 124,0,0,93,18,0,125,1,0,124,1,0,106,0,0,131, + 0,0,146,2,0,113,6,0,83,114,4,0,0,0,41,1, + 114,88,0,0,0,41,2,114,22,0,0,0,90,2,102,110, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,250, + 9,60,115,101,116,99,111,109,112,62,4,5,0,0,115,2, + 0,0,0,9,0,122,41,70,105,108,101,70,105,110,100,101, + 114,46,95,102,105,108,108,95,99,97,99,104,101,46,60,108, + 111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,62, + 78,41,18,114,35,0,0,0,114,3,0,0,0,90,7,108, + 105,115,116,100,105,114,114,45,0,0,0,114,250,0,0,0, + 218,15,80,101,114,109,105,115,115,105,111,110,69,114,114,111, + 114,218,18,78,111,116,65,68,105,114,101,99,116,111,114,121, + 69,114,114,111,114,114,7,0,0,0,114,8,0,0,0,114, + 9,0,0,0,114,3,1,0,0,114,4,1,0,0,114,83, + 0,0,0,114,47,0,0,0,114,88,0,0,0,218,3,97, + 100,100,114,10,0,0,0,114,5,1,0,0,41,9,114,100, + 0,0,0,114,35,0,0,0,90,8,99,111,110,116,101,110, + 116,115,90,21,108,111,119,101,114,95,115,117,102,102,105,120, + 95,99,111,110,116,101,110,116,115,114,239,0,0,0,114,98, + 0,0,0,114,231,0,0,0,114,219,0,0,0,90,8,110, + 101,119,95,110,97,109,101,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,7,1,0,0,231,4,0,0,115, + 34,0,0,0,0,2,9,1,3,1,31,1,22,3,11,3, + 18,1,18,7,9,1,13,1,24,1,6,1,27,2,6,1, + 17,1,9,1,18,1,122,22,70,105,108,101,70,105,110,100, + 101,114,46,95,102,105,108,108,95,99,97,99,104,101,99,1, + 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,7, + 0,0,0,115,25,0,0,0,135,0,0,135,1,0,102,2, + 0,100,1,0,100,2,0,134,0,0,125,2,0,124,2,0, + 83,41,3,97,20,1,0,0,65,32,99,108,97,115,115,32, + 109,101,116,104,111,100,32,119,104,105,99,104,32,114,101,116, + 117,114,110,115,32,97,32,99,108,111,115,117,114,101,32,116, + 111,32,117,115,101,32,111,110,32,115,121,115,46,112,97,116, + 104,95,104,111,111,107,10,32,32,32,32,32,32,32,32,119, + 104,105,99,104,32,119,105,108,108,32,114,101,116,117,114,110, + 32,97,110,32,105,110,115,116,97,110,99,101,32,117,115,105, + 110,103,32,116,104,101,32,115,112,101,99,105,102,105,101,100, + 32,108,111,97,100,101,114,115,32,97,110,100,32,116,104,101, + 32,112,97,116,104,10,32,32,32,32,32,32,32,32,99,97, + 108,108,101,100,32,111,110,32,116,104,101,32,99,108,111,115, + 117,114,101,46,10,10,32,32,32,32,32,32,32,32,73,102, + 32,116,104,101,32,112,97,116,104,32,99,97,108,108,101,100, + 32,111,110,32,116,104,101,32,99,108,111,115,117,114,101,32, + 105,115,32,110,111,116,32,97,32,100,105,114,101,99,116,111, + 114,121,44,32,73,109,112,111,114,116,69,114,114,111,114,32, + 105,115,10,32,32,32,32,32,32,32,32,114,97,105,115,101, + 100,46,10,10,32,32,32,32,32,32,32,32,99,1,0,0, + 0,0,0,0,0,1,0,0,0,4,0,0,0,19,0,0, + 0,115,43,0,0,0,116,0,0,124,0,0,131,1,0,115, + 30,0,116,1,0,100,1,0,100,2,0,124,0,0,131,1, + 1,130,1,0,136,0,0,124,0,0,136,1,0,140,1,0, + 83,41,3,122,45,80,97,116,104,32,104,111,111,107,32,102, + 111,114,32,105,109,112,111,114,116,108,105,98,46,109,97,99, + 104,105,110,101,114,121,46,70,105,108,101,70,105,110,100,101, + 114,46,122,30,111,110,108,121,32,100,105,114,101,99,116,111, + 114,105,101,115,32,97,114,101,32,115,117,112,112,111,114,116, + 101,100,114,35,0,0,0,41,2,114,46,0,0,0,114,99, + 0,0,0,41,1,114,35,0,0,0,41,2,114,164,0,0, + 0,114,6,1,0,0,114,4,0,0,0,114,5,0,0,0, + 218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95, + 70,105,108,101,70,105,110,100,101,114,16,5,0,0,115,6, + 0,0,0,0,2,12,1,18,1,122,54,70,105,108,101,70, + 105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46, + 60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111, + 111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101, + 114,114,4,0,0,0,41,3,114,164,0,0,0,114,6,1, + 0,0,114,12,1,0,0,114,4,0,0,0,41,2,114,164, + 0,0,0,114,6,1,0,0,114,5,0,0,0,218,9,112, + 97,116,104,95,104,111,111,107,6,5,0,0,115,4,0,0, + 0,0,10,21,6,122,20,70,105,108,101,70,105,110,100,101, + 114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,0, + 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, + 115,16,0,0,0,100,1,0,106,0,0,124,0,0,106,1, + 0,131,1,0,83,41,2,78,122,16,70,105,108,101,70,105, + 110,100,101,114,40,123,33,114,125,41,41,2,114,47,0,0, + 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,238,0,0, + 0,24,5,0,0,115,2,0,0,0,0,1,122,19,70,105, + 108,101,70,105,110,100,101,114,46,95,95,114,101,112,114,95, + 95,41,15,114,105,0,0,0,114,104,0,0,0,114,106,0, + 0,0,114,107,0,0,0,114,179,0,0,0,114,244,0,0, + 0,114,123,0,0,0,114,176,0,0,0,114,117,0,0,0, + 114,255,0,0,0,114,175,0,0,0,114,7,1,0,0,114, + 177,0,0,0,114,13,1,0,0,114,238,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,244,0,0,0,161,4,0,0,115,2,0,0,0, - 0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105, - 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, - 99,2,0,0,0,0,0,0,0,3,0,0,0,2,0,0, - 0,67,0,0,0,115,59,0,0,0,124,0,0,106,0,0, - 124,1,0,131,1,0,125,2,0,124,2,0,100,1,0,107, - 8,0,114,37,0,100,1,0,103,0,0,102,2,0,83,124, - 2,0,106,1,0,124,2,0,106,2,0,112,55,0,103,0, - 0,102,2,0,83,41,2,122,197,84,114,121,32,116,111,32, - 102,105,110,100,32,97,32,108,111,97,100,101,114,32,102,111, - 114,32,116,104,101,32,115,112,101,99,105,102,105,101,100,32, - 109,111,100,117,108,101,44,32,111,114,32,116,104,101,32,110, - 97,109,101,115,112,97,99,101,10,32,32,32,32,32,32,32, - 32,112,97,99,107,97,103,101,32,112,111,114,116,105,111,110, - 115,46,32,82,101,116,117,114,110,115,32,40,108,111,97,100, - 101,114,44,32,108,105,115,116,45,111,102,45,112,111,114,116, - 105,111,110,115,41,46,10,10,32,32,32,32,32,32,32,32, - 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, - 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, - 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, - 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, - 3,114,175,0,0,0,114,120,0,0,0,114,150,0,0,0, - 41,3,114,100,0,0,0,114,119,0,0,0,114,158,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,117,0,0,0,167,4,0,0,115,8,0,0,0,0,7, - 15,1,12,1,10,1,122,22,70,105,108,101,70,105,110,100, - 101,114,46,102,105,110,100,95,108,111,97,100,101,114,99,6, - 0,0,0,0,0,0,0,7,0,0,0,7,0,0,0,67, - 0,0,0,115,40,0,0,0,124,1,0,124,2,0,124,3, - 0,131,2,0,125,6,0,116,0,0,124,2,0,124,3,0, - 100,1,0,124,6,0,100,2,0,124,4,0,131,2,2,83, - 41,3,78,114,120,0,0,0,114,150,0,0,0,41,1,114, - 161,0,0,0,41,7,114,100,0,0,0,114,159,0,0,0, - 114,119,0,0,0,114,35,0,0,0,90,4,115,109,115,108, - 114,174,0,0,0,114,120,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,255,0,0,0,179,4, - 0,0,115,6,0,0,0,0,1,15,1,18,1,122,20,70, - 105,108,101,70,105,110,100,101,114,46,95,103,101,116,95,115, - 112,101,99,78,99,3,0,0,0,0,0,0,0,14,0,0, - 0,15,0,0,0,67,0,0,0,115,228,1,0,0,100,1, - 0,125,3,0,124,1,0,106,0,0,100,2,0,131,1,0, - 100,3,0,25,125,4,0,121,34,0,116,1,0,124,0,0, - 106,2,0,112,49,0,116,3,0,106,4,0,131,0,0,131, - 1,0,106,5,0,125,5,0,87,110,24,0,4,116,6,0, - 107,10,0,114,85,0,1,1,1,100,10,0,125,5,0,89, - 110,1,0,88,124,5,0,124,0,0,106,7,0,107,3,0, - 114,120,0,124,0,0,106,8,0,131,0,0,1,124,5,0, - 124,0,0,95,7,0,116,9,0,131,0,0,114,153,0,124, - 0,0,106,10,0,125,6,0,124,4,0,106,11,0,131,0, - 0,125,7,0,110,15,0,124,0,0,106,12,0,125,6,0, - 124,4,0,125,7,0,124,7,0,124,6,0,107,6,0,114, - 45,1,116,13,0,124,0,0,106,2,0,124,4,0,131,2, - 0,125,8,0,120,100,0,124,0,0,106,14,0,68,93,77, - 0,92,2,0,125,9,0,125,10,0,100,5,0,124,9,0, - 23,125,11,0,116,13,0,124,8,0,124,11,0,131,2,0, - 125,12,0,116,15,0,124,12,0,131,1,0,114,208,0,124, - 0,0,106,16,0,124,10,0,124,1,0,124,12,0,124,8, - 0,103,1,0,124,2,0,131,5,0,83,113,208,0,87,116, - 17,0,124,8,0,131,1,0,125,3,0,120,120,0,124,0, - 0,106,14,0,68,93,109,0,92,2,0,125,9,0,125,10, - 0,116,13,0,124,0,0,106,2,0,124,4,0,124,9,0, - 23,131,2,0,125,12,0,116,18,0,106,19,0,100,6,0, - 124,12,0,100,7,0,100,3,0,131,2,1,1,124,7,0, - 124,9,0,23,124,6,0,107,6,0,114,55,1,116,15,0, - 124,12,0,131,1,0,114,55,1,124,0,0,106,16,0,124, - 10,0,124,1,0,124,12,0,100,8,0,124,2,0,131,5, - 0,83,113,55,1,87,124,3,0,114,224,1,116,18,0,106, - 19,0,100,9,0,124,8,0,131,2,0,1,116,18,0,106, - 20,0,124,1,0,100,8,0,131,2,0,125,13,0,124,8, - 0,103,1,0,124,13,0,95,21,0,124,13,0,83,100,8, - 0,83,41,11,122,125,84,114,121,32,116,111,32,102,105,110, - 100,32,97,32,108,111,97,100,101,114,32,102,111,114,32,116, - 104,101,32,115,112,101,99,105,102,105,101,100,32,109,111,100, - 117,108,101,44,32,111,114,32,116,104,101,32,110,97,109,101, - 115,112,97,99,101,10,32,32,32,32,32,32,32,32,112,97, - 99,107,97,103,101,32,112,111,114,116,105,111,110,115,46,32, - 82,101,116,117,114,110,115,32,40,108,111,97,100,101,114,44, - 32,108,105,115,116,45,111,102,45,112,111,114,116,105,111,110, - 115,41,46,70,114,58,0,0,0,114,56,0,0,0,114,29, - 0,0,0,114,179,0,0,0,122,9,116,114,121,105,110,103, - 32,123,125,90,9,118,101,114,98,111,115,105,116,121,78,122, - 25,112,111,115,115,105,98,108,101,32,110,97,109,101,115,112, - 97,99,101,32,102,111,114,32,123,125,114,87,0,0,0,41, - 22,114,32,0,0,0,114,39,0,0,0,114,35,0,0,0, - 114,3,0,0,0,114,45,0,0,0,114,213,0,0,0,114, - 40,0,0,0,114,2,1,0,0,218,11,95,102,105,108,108, - 95,99,97,99,104,101,114,6,0,0,0,114,5,1,0,0, - 114,88,0,0,0,114,4,1,0,0,114,28,0,0,0,114, - 1,1,0,0,114,44,0,0,0,114,255,0,0,0,114,46, - 0,0,0,114,114,0,0,0,114,129,0,0,0,114,154,0, - 0,0,114,150,0,0,0,41,14,114,100,0,0,0,114,119, - 0,0,0,114,174,0,0,0,90,12,105,115,95,110,97,109, - 101,115,112,97,99,101,90,11,116,97,105,108,95,109,111,100, - 117,108,101,114,126,0,0,0,90,5,99,97,99,104,101,90, - 12,99,97,99,104,101,95,109,111,100,117,108,101,90,9,98, - 97,115,101,95,112,97,116,104,114,219,0,0,0,114,159,0, - 0,0,90,13,105,110,105,116,95,102,105,108,101,110,97,109, - 101,90,9,102,117,108,108,95,112,97,116,104,114,158,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,175,0,0,0,184,4,0,0,115,70,0,0,0,0,3, - 6,1,19,1,3,1,34,1,13,1,11,1,15,1,10,1, - 9,2,9,1,9,1,15,2,9,1,6,2,12,1,18,1, - 22,1,10,1,15,1,12,1,32,4,12,2,22,1,22,1, - 22,1,16,1,12,1,15,1,14,1,6,1,16,1,18,1, - 12,1,4,1,122,20,70,105,108,101,70,105,110,100,101,114, - 46,102,105,110,100,95,115,112,101,99,99,1,0,0,0,0, - 0,0,0,9,0,0,0,13,0,0,0,67,0,0,0,115, - 11,1,0,0,124,0,0,106,0,0,125,1,0,121,31,0, - 116,1,0,106,2,0,124,1,0,112,33,0,116,1,0,106, - 3,0,131,0,0,131,1,0,125,2,0,87,110,33,0,4, - 116,4,0,116,5,0,116,6,0,102,3,0,107,10,0,114, - 75,0,1,1,1,103,0,0,125,2,0,89,110,1,0,88, - 116,7,0,106,8,0,106,9,0,100,1,0,131,1,0,115, - 112,0,116,10,0,124,2,0,131,1,0,124,0,0,95,11, - 0,110,111,0,116,10,0,131,0,0,125,3,0,120,90,0, - 124,2,0,68,93,82,0,125,4,0,124,4,0,106,12,0, - 100,2,0,131,1,0,92,3,0,125,5,0,125,6,0,125, - 7,0,124,6,0,114,191,0,100,3,0,106,13,0,124,5, - 0,124,7,0,106,14,0,131,0,0,131,2,0,125,8,0, - 110,6,0,124,5,0,125,8,0,124,3,0,106,15,0,124, - 8,0,131,1,0,1,113,128,0,87,124,3,0,124,0,0, - 95,11,0,116,7,0,106,8,0,106,9,0,116,16,0,131, - 1,0,114,7,1,100,4,0,100,5,0,132,0,0,124,2, - 0,68,131,1,0,124,0,0,95,17,0,100,6,0,83,41, - 7,122,68,70,105,108,108,32,116,104,101,32,99,97,99,104, - 101,32,111,102,32,112,111,116,101,110,116,105,97,108,32,109, - 111,100,117,108,101,115,32,97,110,100,32,112,97,99,107,97, - 103,101,115,32,102,111,114,32,116,104,105,115,32,100,105,114, - 101,99,116,111,114,121,46,114,0,0,0,0,114,58,0,0, - 0,122,5,123,125,46,123,125,99,1,0,0,0,0,0,0, - 0,2,0,0,0,3,0,0,0,83,0,0,0,115,28,0, - 0,0,104,0,0,124,0,0,93,18,0,125,1,0,124,1, - 0,106,0,0,131,0,0,146,2,0,113,6,0,83,114,4, - 0,0,0,41,1,114,88,0,0,0,41,2,114,22,0,0, - 0,90,2,102,110,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,3, - 5,0,0,115,2,0,0,0,9,0,122,41,70,105,108,101, - 70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,99, - 104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,116, - 99,111,109,112,62,78,41,18,114,35,0,0,0,114,3,0, - 0,0,90,7,108,105,115,116,100,105,114,114,45,0,0,0, - 114,250,0,0,0,218,15,80,101,114,109,105,115,115,105,111, - 110,69,114,114,111,114,218,18,78,111,116,65,68,105,114,101, - 99,116,111,114,121,69,114,114,111,114,114,7,0,0,0,114, - 8,0,0,0,114,9,0,0,0,114,3,1,0,0,114,4, - 1,0,0,114,83,0,0,0,114,47,0,0,0,114,88,0, - 0,0,218,3,97,100,100,114,10,0,0,0,114,5,1,0, - 0,41,9,114,100,0,0,0,114,35,0,0,0,90,8,99, - 111,110,116,101,110,116,115,90,21,108,111,119,101,114,95,115, - 117,102,102,105,120,95,99,111,110,116,101,110,116,115,114,239, - 0,0,0,114,98,0,0,0,114,231,0,0,0,114,219,0, - 0,0,90,8,110,101,119,95,110,97,109,101,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,7,1,0,0, - 230,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, - 1,22,3,11,3,18,1,18,7,9,1,13,1,24,1,6, - 1,27,2,6,1,17,1,9,1,18,1,122,22,70,105,108, - 101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97, - 99,104,101,99,1,0,0,0,0,0,0,0,3,0,0,0, - 3,0,0,0,7,0,0,0,115,25,0,0,0,135,0,0, - 135,1,0,102,2,0,100,1,0,100,2,0,134,0,0,125, - 2,0,124,2,0,83,41,3,97,20,1,0,0,65,32,99, - 108,97,115,115,32,109,101,116,104,111,100,32,119,104,105,99, - 104,32,114,101,116,117,114,110,115,32,97,32,99,108,111,115, - 117,114,101,32,116,111,32,117,115,101,32,111,110,32,115,121, - 115,46,112,97,116,104,95,104,111,111,107,10,32,32,32,32, - 32,32,32,32,119,104,105,99,104,32,119,105,108,108,32,114, - 101,116,117,114,110,32,97,110,32,105,110,115,116,97,110,99, - 101,32,117,115,105,110,103,32,116,104,101,32,115,112,101,99, - 105,102,105,101,100,32,108,111,97,100,101,114,115,32,97,110, - 100,32,116,104,101,32,112,97,116,104,10,32,32,32,32,32, - 32,32,32,99,97,108,108,101,100,32,111,110,32,116,104,101, - 32,99,108,111,115,117,114,101,46,10,10,32,32,32,32,32, - 32,32,32,73,102,32,116,104,101,32,112,97,116,104,32,99, - 97,108,108,101,100,32,111,110,32,116,104,101,32,99,108,111, - 115,117,114,101,32,105,115,32,110,111,116,32,97,32,100,105, - 114,101,99,116,111,114,121,44,32,73,109,112,111,114,116,69, - 114,114,111,114,32,105,115,10,32,32,32,32,32,32,32,32, - 114,97,105,115,101,100,46,10,10,32,32,32,32,32,32,32, - 32,99,1,0,0,0,0,0,0,0,1,0,0,0,4,0, - 0,0,19,0,0,0,115,43,0,0,0,116,0,0,124,0, - 0,131,1,0,115,30,0,116,1,0,100,1,0,100,2,0, - 124,0,0,131,1,1,130,1,0,136,0,0,124,0,0,136, - 1,0,140,1,0,83,41,3,122,45,80,97,116,104,32,104, - 111,111,107,32,102,111,114,32,105,109,112,111,114,116,108,105, - 98,46,109,97,99,104,105,110,101,114,121,46,70,105,108,101, - 70,105,110,100,101,114,46,122,30,111,110,108,121,32,100,105, - 114,101,99,116,111,114,105,101,115,32,97,114,101,32,115,117, - 112,112,111,114,116,101,100,114,35,0,0,0,41,2,114,46, - 0,0,0,114,99,0,0,0,41,1,114,35,0,0,0,41, - 2,114,164,0,0,0,114,6,1,0,0,114,4,0,0,0, - 114,5,0,0,0,218,24,112,97,116,104,95,104,111,111,107, - 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,15, - 5,0,0,115,6,0,0,0,0,2,12,1,18,1,122,54, - 70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95, - 104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97, - 116,104,95,104,111,111,107,95,102,111,114,95,70,105,108,101, - 70,105,110,100,101,114,114,4,0,0,0,41,3,114,164,0, - 0,0,114,6,1,0,0,114,12,1,0,0,114,4,0,0, - 0,41,2,114,164,0,0,0,114,6,1,0,0,114,5,0, - 0,0,218,9,112,97,116,104,95,104,111,111,107,5,5,0, - 0,115,4,0,0,0,0,10,21,6,122,20,70,105,108,101, - 70,105,110,100,101,114,46,112,97,116,104,95,104,111,111,107, - 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, - 0,67,0,0,0,115,16,0,0,0,100,1,0,106,0,0, - 124,0,0,106,1,0,131,1,0,83,41,2,78,122,16,70, - 105,108,101,70,105,110,100,101,114,40,123,33,114,125,41,41, - 2,114,47,0,0,0,114,35,0,0,0,41,1,114,100,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,238,0,0,0,23,5,0,0,115,2,0,0,0,0, - 1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95, - 114,101,112,114,95,95,41,15,114,105,0,0,0,114,104,0, - 0,0,114,106,0,0,0,114,107,0,0,0,114,179,0,0, - 0,114,244,0,0,0,114,123,0,0,0,114,176,0,0,0, - 114,117,0,0,0,114,255,0,0,0,114,175,0,0,0,114, - 7,1,0,0,114,177,0,0,0,114,13,1,0,0,114,238, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,0,1,0,0,138,4,0,0, - 115,20,0,0,0,12,7,6,2,12,14,12,4,6,2,12, - 12,12,5,15,46,12,31,18,18,114,0,1,0,0,99,4, - 0,0,0,0,0,0,0,6,0,0,0,11,0,0,0,67, - 0,0,0,115,195,0,0,0,124,0,0,106,0,0,100,1, - 0,131,1,0,125,4,0,124,0,0,106,0,0,100,2,0, - 131,1,0,125,5,0,124,4,0,115,99,0,124,5,0,114, - 54,0,124,5,0,106,1,0,125,4,0,110,45,0,124,2, - 0,124,3,0,107,2,0,114,84,0,116,2,0,124,1,0, - 124,2,0,131,2,0,125,4,0,110,15,0,116,3,0,124, - 1,0,124,2,0,131,2,0,125,4,0,124,5,0,115,126, - 0,116,4,0,124,1,0,124,2,0,100,3,0,124,4,0, - 131,2,1,125,5,0,121,44,0,124,5,0,124,0,0,100, - 2,0,60,124,4,0,124,0,0,100,1,0,60,124,2,0, - 124,0,0,100,4,0,60,124,3,0,124,0,0,100,5,0, - 60,87,110,18,0,4,116,5,0,107,10,0,114,190,0,1, - 1,1,89,110,1,0,88,100,0,0,83,41,6,78,218,10, - 95,95,108,111,97,100,101,114,95,95,218,8,95,95,115,112, - 101,99,95,95,114,120,0,0,0,90,8,95,95,102,105,108, - 101,95,95,90,10,95,95,99,97,99,104,101,100,95,95,41, - 6,218,3,103,101,116,114,120,0,0,0,114,217,0,0,0, - 114,212,0,0,0,114,161,0,0,0,218,9,69,120,99,101, - 112,116,105,111,110,41,6,90,2,110,115,114,98,0,0,0, - 90,8,112,97,116,104,110,97,109,101,90,9,99,112,97,116, - 104,110,97,109,101,114,120,0,0,0,114,158,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,14, - 95,102,105,120,95,117,112,95,109,111,100,117,108,101,29,5, - 0,0,115,34,0,0,0,0,2,15,1,15,1,6,1,6, - 1,12,1,12,1,18,2,15,1,6,1,21,1,3,1,10, - 1,10,1,10,1,14,1,13,2,114,18,1,0,0,99,0, - 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,67, - 0,0,0,115,55,0,0,0,116,0,0,116,1,0,106,2, - 0,131,0,0,102,2,0,125,0,0,116,3,0,116,4,0, - 102,2,0,125,1,0,116,5,0,116,6,0,102,2,0,125, - 2,0,124,0,0,124,1,0,124,2,0,103,3,0,83,41, - 1,122,95,82,101,116,117,114,110,115,32,97,32,108,105,115, - 116,32,111,102,32,102,105,108,101,45,98,97,115,101,100,32, - 109,111,100,117,108,101,32,108,111,97,100,101,114,115,46,10, - 10,32,32,32,32,69,97,99,104,32,105,116,101,109,32,105, - 115,32,97,32,116,117,112,108,101,32,40,108,111,97,100,101, - 114,44,32,115,117,102,102,105,120,101,115,41,46,10,32,32, - 32,32,41,7,114,218,0,0,0,114,139,0,0,0,218,18, - 101,120,116,101,110,115,105,111,110,95,115,117,102,102,105,120, - 101,115,114,212,0,0,0,114,84,0,0,0,114,217,0,0, - 0,114,74,0,0,0,41,3,90,10,101,120,116,101,110,115, - 105,111,110,115,90,6,115,111,117,114,99,101,90,8,98,121, - 116,101,99,111,100,101,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,155,0,0,0,52,5,0,0,115,8, - 0,0,0,0,5,18,1,12,1,12,1,114,155,0,0,0, - 99,1,0,0,0,0,0,0,0,12,0,0,0,12,0,0, - 0,67,0,0,0,115,70,2,0,0,124,0,0,97,0,0, - 116,0,0,106,1,0,97,1,0,116,0,0,106,2,0,97, - 2,0,116,1,0,106,3,0,116,4,0,25,125,1,0,120, - 76,0,100,26,0,68,93,68,0,125,2,0,124,2,0,116, - 1,0,106,3,0,107,7,0,114,83,0,116,0,0,106,5, - 0,124,2,0,131,1,0,125,3,0,110,13,0,116,1,0, - 106,3,0,124,2,0,25,125,3,0,116,6,0,124,1,0, - 124,2,0,124,3,0,131,3,0,1,113,44,0,87,100,5, - 0,100,6,0,103,1,0,102,2,0,100,7,0,100,8,0, - 100,6,0,103,2,0,102,2,0,102,2,0,125,4,0,120, - 149,0,124,4,0,68,93,129,0,92,2,0,125,5,0,125, - 6,0,116,7,0,100,9,0,100,10,0,132,0,0,124,6, - 0,68,131,1,0,131,1,0,115,199,0,116,8,0,130,1, - 0,124,6,0,100,11,0,25,125,7,0,124,5,0,116,1, - 0,106,3,0,107,6,0,114,241,0,116,1,0,106,3,0, - 124,5,0,25,125,8,0,80,113,156,0,121,20,0,116,0, - 0,106,5,0,124,5,0,131,1,0,125,8,0,80,87,113, - 156,0,4,116,9,0,107,10,0,114,28,1,1,1,1,119, - 156,0,89,113,156,0,88,113,156,0,87,116,9,0,100,12, - 0,131,1,0,130,1,0,116,6,0,124,1,0,100,13,0, - 124,8,0,131,3,0,1,116,6,0,124,1,0,100,14,0, - 124,7,0,131,3,0,1,116,6,0,124,1,0,100,15,0, - 100,16,0,106,10,0,124,6,0,131,1,0,131,3,0,1, - 121,19,0,116,0,0,106,5,0,100,17,0,131,1,0,125, - 9,0,87,110,24,0,4,116,9,0,107,10,0,114,147,1, - 1,1,1,100,18,0,125,9,0,89,110,1,0,88,116,6, - 0,124,1,0,100,17,0,124,9,0,131,3,0,1,116,0, - 0,106,5,0,100,19,0,131,1,0,125,10,0,116,6,0, - 124,1,0,100,19,0,124,10,0,131,3,0,1,124,5,0, - 100,7,0,107,2,0,114,238,1,116,0,0,106,5,0,100, - 20,0,131,1,0,125,11,0,116,6,0,124,1,0,100,21, - 0,124,11,0,131,3,0,1,116,6,0,124,1,0,100,22, - 0,116,11,0,131,0,0,131,3,0,1,116,12,0,106,13, - 0,116,2,0,106,14,0,131,0,0,131,1,0,1,124,5, - 0,100,7,0,107,2,0,114,66,2,116,15,0,106,16,0, - 100,23,0,131,1,0,1,100,24,0,116,12,0,107,6,0, - 114,66,2,100,25,0,116,17,0,95,18,0,100,18,0,83, - 41,27,122,205,83,101,116,117,112,32,116,104,101,32,112,97, - 116,104,45,98,97,115,101,100,32,105,109,112,111,114,116,101, - 114,115,32,102,111,114,32,105,109,112,111,114,116,108,105,98, - 32,98,121,32,105,109,112,111,114,116,105,110,103,32,110,101, - 101,100,101,100,10,32,32,32,32,98,117,105,108,116,45,105, - 110,32,109,111,100,117,108,101,115,32,97,110,100,32,105,110, - 106,101,99,116,105,110,103,32,116,104,101,109,32,105,110,116, - 111,32,116,104,101,32,103,108,111,98,97,108,32,110,97,109, - 101,115,112,97,99,101,46,10,10,32,32,32,32,79,116,104, - 101,114,32,99,111,109,112,111,110,101,110,116,115,32,97,114, - 101,32,101,120,116,114,97,99,116,101,100,32,102,114,111,109, - 32,116,104,101,32,99,111,114,101,32,98,111,111,116,115,116, - 114,97,112,32,109,111,100,117,108,101,46,10,10,32,32,32, - 32,114,49,0,0,0,114,60,0,0,0,218,8,98,117,105, - 108,116,105,110,115,114,136,0,0,0,90,5,112,111,115,105, - 120,250,1,47,218,2,110,116,250,1,92,99,1,0,0,0, - 0,0,0,0,2,0,0,0,3,0,0,0,115,0,0,0, - 115,33,0,0,0,124,0,0,93,23,0,125,1,0,116,0, - 0,124,1,0,131,1,0,100,0,0,107,2,0,86,1,113, - 3,0,100,1,0,83,41,2,114,29,0,0,0,78,41,1, - 114,31,0,0,0,41,2,114,22,0,0,0,114,77,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,88,5,0,0,115,2,0,0,0,6,0, - 122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115, - 62,46,60,103,101,110,101,120,112,114,62,114,59,0,0,0, - 122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,117, - 105,114,101,115,32,112,111,115,105,120,32,111,114,32,110,116, - 114,3,0,0,0,114,25,0,0,0,114,21,0,0,0,114, - 30,0,0,0,90,7,95,116,104,114,101,97,100,78,90,8, - 95,119,101,97,107,114,101,102,90,6,119,105,110,114,101,103, - 114,163,0,0,0,114,6,0,0,0,122,4,46,112,121,119, - 122,6,95,100,46,112,121,100,84,41,4,122,3,95,105,111, - 122,9,95,119,97,114,110,105,110,103,115,122,8,98,117,105, - 108,116,105,110,115,122,7,109,97,114,115,104,97,108,41,19, - 114,114,0,0,0,114,7,0,0,0,114,139,0,0,0,114, - 233,0,0,0,114,105,0,0,0,90,18,95,98,117,105,108, - 116,105,110,95,102,114,111,109,95,110,97,109,101,114,109,0, - 0,0,218,3,97,108,108,218,14,65,115,115,101,114,116,105, - 111,110,69,114,114,111,114,114,99,0,0,0,114,26,0,0, - 0,114,11,0,0,0,114,223,0,0,0,114,143,0,0,0, - 114,19,1,0,0,114,84,0,0,0,114,157,0,0,0,114, - 162,0,0,0,114,167,0,0,0,41,12,218,17,95,98,111, - 111,116,115,116,114,97,112,95,109,111,100,117,108,101,90,11, - 115,101,108,102,95,109,111,100,117,108,101,90,12,98,117,105, - 108,116,105,110,95,110,97,109,101,90,14,98,117,105,108,116, - 105,110,95,109,111,100,117,108,101,90,10,111,115,95,100,101, - 116,97,105,108,115,90,10,98,117,105,108,116,105,110,95,111, - 115,114,21,0,0,0,114,25,0,0,0,90,9,111,115,95, - 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, - 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, - 111,100,117,108,101,90,13,119,105,110,114,101,103,95,109,111, - 100,117,108,101,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,115,101,116,117,112,63,5,0,0,115, - 82,0,0,0,0,8,6,1,9,1,9,3,13,1,13,1, - 15,1,18,2,13,1,20,3,33,1,19,2,31,1,10,1, - 15,1,13,1,4,2,3,1,15,1,5,1,13,1,12,2, - 12,1,16,1,16,1,25,3,3,1,19,1,13,2,11,1, - 16,3,15,1,16,3,12,1,15,1,16,3,19,1,19,1, - 12,1,13,1,12,1,114,27,1,0,0,99,1,0,0,0, - 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, - 115,116,0,0,0,116,0,0,124,0,0,131,1,0,1,116, - 1,0,131,0,0,125,1,0,116,2,0,106,3,0,106,4, - 0,116,5,0,106,6,0,124,1,0,140,0,0,103,1,0, - 131,1,0,1,116,7,0,106,8,0,100,1,0,107,2,0, - 114,78,0,116,2,0,106,9,0,106,10,0,116,11,0,131, - 1,0,1,116,2,0,106,9,0,106,10,0,116,12,0,131, - 1,0,1,116,5,0,124,0,0,95,5,0,116,13,0,124, - 0,0,95,13,0,100,2,0,83,41,3,122,41,73,110,115, - 116,97,108,108,32,116,104,101,32,112,97,116,104,45,98,97, - 115,101,100,32,105,109,112,111,114,116,32,99,111,109,112,111, - 110,101,110,116,115,46,114,22,1,0,0,78,41,14,114,27, - 1,0,0,114,155,0,0,0,114,7,0,0,0,114,248,0, - 0,0,114,143,0,0,0,114,0,1,0,0,114,13,1,0, - 0,114,3,0,0,0,114,105,0,0,0,218,9,109,101,116, - 97,95,112,97,116,104,114,157,0,0,0,114,162,0,0,0, - 114,243,0,0,0,114,212,0,0,0,41,2,114,26,1,0, - 0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,97, - 100,101,114,115,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,105,110,115,116,97,108,108,131,5,0, - 0,115,16,0,0,0,0,2,10,1,9,1,28,1,15,1, - 16,1,16,4,9,1,114,29,1,0,0,41,3,122,3,119, - 105,110,114,1,0,0,0,114,2,0,0,0,41,56,114,107, - 0,0,0,114,10,0,0,0,114,11,0,0,0,114,17,0, - 0,0,114,19,0,0,0,114,28,0,0,0,114,38,0,0, - 0,114,39,0,0,0,114,43,0,0,0,114,44,0,0,0, - 114,46,0,0,0,114,55,0,0,0,218,4,116,121,112,101, - 218,8,95,95,99,111,100,101,95,95,114,138,0,0,0,114, - 15,0,0,0,114,128,0,0,0,114,14,0,0,0,114,18, - 0,0,0,90,17,95,82,65,87,95,77,65,71,73,67,95, - 78,85,77,66,69,82,114,73,0,0,0,114,72,0,0,0, - 114,84,0,0,0,114,74,0,0,0,90,23,68,69,66,85, - 71,95,66,89,84,69,67,79,68,69,95,83,85,70,70,73, - 88,69,83,90,27,79,80,84,73,77,73,90,69,68,95,66, - 89,84,69,67,79,68,69,95,83,85,70,70,73,88,69,83, - 114,79,0,0,0,114,85,0,0,0,114,91,0,0,0,114, - 95,0,0,0,114,97,0,0,0,114,116,0,0,0,114,123, - 0,0,0,114,135,0,0,0,114,141,0,0,0,114,144,0, - 0,0,114,149,0,0,0,218,6,111,98,106,101,99,116,114, - 156,0,0,0,114,161,0,0,0,114,162,0,0,0,114,178, - 0,0,0,114,188,0,0,0,114,204,0,0,0,114,212,0, - 0,0,114,217,0,0,0,114,223,0,0,0,114,218,0,0, - 0,114,224,0,0,0,114,241,0,0,0,114,243,0,0,0, - 114,0,1,0,0,114,18,1,0,0,114,155,0,0,0,114, - 27,1,0,0,114,29,1,0,0,114,4,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,218,8,60, - 109,111,100,117,108,101,62,8,0,0,0,115,98,0,0,0, - 6,17,6,3,12,12,12,5,12,5,12,6,12,12,12,10, - 12,9,12,5,12,7,15,22,15,111,22,1,18,2,6,1, - 6,2,9,2,9,2,10,2,21,44,12,33,12,19,12,12, - 12,12,12,28,12,17,21,55,21,12,18,10,12,14,9,3, - 12,1,15,65,19,64,19,28,22,110,19,41,25,45,25,16, - 6,3,25,53,19,57,19,42,19,134,19,147,15,23,12,11, - 12,68, + 0,0,114,0,1,0,0,139,4,0,0,115,20,0,0,0, + 12,7,6,2,12,14,12,4,6,2,12,12,12,5,15,46, + 12,31,18,18,114,0,1,0,0,99,4,0,0,0,0,0, + 0,0,6,0,0,0,11,0,0,0,67,0,0,0,115,195, + 0,0,0,124,0,0,106,0,0,100,1,0,131,1,0,125, + 4,0,124,0,0,106,0,0,100,2,0,131,1,0,125,5, + 0,124,4,0,115,99,0,124,5,0,114,54,0,124,5,0, + 106,1,0,125,4,0,110,45,0,124,2,0,124,3,0,107, + 2,0,114,84,0,116,2,0,124,1,0,124,2,0,131,2, + 0,125,4,0,110,15,0,116,3,0,124,1,0,124,2,0, + 131,2,0,125,4,0,124,5,0,115,126,0,116,4,0,124, + 1,0,124,2,0,100,3,0,124,4,0,131,2,1,125,5, + 0,121,44,0,124,5,0,124,0,0,100,2,0,60,124,4, + 0,124,0,0,100,1,0,60,124,2,0,124,0,0,100,4, + 0,60,124,3,0,124,0,0,100,5,0,60,87,110,18,0, + 4,116,5,0,107,10,0,114,190,0,1,1,1,89,110,1, + 0,88,100,0,0,83,41,6,78,218,10,95,95,108,111,97, + 100,101,114,95,95,218,8,95,95,115,112,101,99,95,95,114, + 120,0,0,0,90,8,95,95,102,105,108,101,95,95,90,10, + 95,95,99,97,99,104,101,100,95,95,41,6,218,3,103,101, + 116,114,120,0,0,0,114,217,0,0,0,114,212,0,0,0, + 114,161,0,0,0,218,9,69,120,99,101,112,116,105,111,110, + 41,6,90,2,110,115,114,98,0,0,0,90,8,112,97,116, + 104,110,97,109,101,90,9,99,112,97,116,104,110,97,109,101, + 114,120,0,0,0,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,14,95,102,105,120,95, + 117,112,95,109,111,100,117,108,101,30,5,0,0,115,34,0, + 0,0,0,2,15,1,15,1,6,1,6,1,12,1,12,1, + 18,2,15,1,6,1,21,1,3,1,10,1,10,1,10,1, + 14,1,13,2,114,18,1,0,0,99,0,0,0,0,0,0, + 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,55, + 0,0,0,116,0,0,116,1,0,106,2,0,131,0,0,102, + 2,0,125,0,0,116,3,0,116,4,0,102,2,0,125,1, + 0,116,5,0,116,6,0,102,2,0,125,2,0,124,0,0, + 124,1,0,124,2,0,103,3,0,83,41,1,122,95,82,101, + 116,117,114,110,115,32,97,32,108,105,115,116,32,111,102,32, + 102,105,108,101,45,98,97,115,101,100,32,109,111,100,117,108, + 101,32,108,111,97,100,101,114,115,46,10,10,32,32,32,32, + 69,97,99,104,32,105,116,101,109,32,105,115,32,97,32,116, + 117,112,108,101,32,40,108,111,97,100,101,114,44,32,115,117, + 102,102,105,120,101,115,41,46,10,32,32,32,32,41,7,114, + 218,0,0,0,114,139,0,0,0,218,18,101,120,116,101,110, + 115,105,111,110,95,115,117,102,102,105,120,101,115,114,212,0, + 0,0,114,84,0,0,0,114,217,0,0,0,114,74,0,0, + 0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,90, + 6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,100, + 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,155,0,0,0,53,5,0,0,115,8,0,0,0,0,5, + 18,1,12,1,12,1,114,155,0,0,0,99,1,0,0,0, + 0,0,0,0,12,0,0,0,12,0,0,0,67,0,0,0, + 115,70,2,0,0,124,0,0,97,0,0,116,0,0,106,1, + 0,97,1,0,116,0,0,106,2,0,97,2,0,116,1,0, + 106,3,0,116,4,0,25,125,1,0,120,76,0,100,26,0, + 68,93,68,0,125,2,0,124,2,0,116,1,0,106,3,0, + 107,7,0,114,83,0,116,0,0,106,5,0,124,2,0,131, + 1,0,125,3,0,110,13,0,116,1,0,106,3,0,124,2, + 0,25,125,3,0,116,6,0,124,1,0,124,2,0,124,3, + 0,131,3,0,1,113,44,0,87,100,5,0,100,6,0,103, + 1,0,102,2,0,100,7,0,100,8,0,100,6,0,103,2, + 0,102,2,0,102,2,0,125,4,0,120,149,0,124,4,0, + 68,93,129,0,92,2,0,125,5,0,125,6,0,116,7,0, + 100,9,0,100,10,0,132,0,0,124,6,0,68,131,1,0, + 131,1,0,115,199,0,116,8,0,130,1,0,124,6,0,100, + 11,0,25,125,7,0,124,5,0,116,1,0,106,3,0,107, + 6,0,114,241,0,116,1,0,106,3,0,124,5,0,25,125, + 8,0,80,113,156,0,121,20,0,116,0,0,106,5,0,124, + 5,0,131,1,0,125,8,0,80,87,113,156,0,4,116,9, + 0,107,10,0,114,28,1,1,1,1,119,156,0,89,113,156, + 0,88,113,156,0,87,116,9,0,100,12,0,131,1,0,130, + 1,0,116,6,0,124,1,0,100,13,0,124,8,0,131,3, + 0,1,116,6,0,124,1,0,100,14,0,124,7,0,131,3, + 0,1,116,6,0,124,1,0,100,15,0,100,16,0,106,10, + 0,124,6,0,131,1,0,131,3,0,1,121,19,0,116,0, + 0,106,5,0,100,17,0,131,1,0,125,9,0,87,110,24, + 0,4,116,9,0,107,10,0,114,147,1,1,1,1,100,18, + 0,125,9,0,89,110,1,0,88,116,6,0,124,1,0,100, + 17,0,124,9,0,131,3,0,1,116,0,0,106,5,0,100, + 19,0,131,1,0,125,10,0,116,6,0,124,1,0,100,19, + 0,124,10,0,131,3,0,1,124,5,0,100,7,0,107,2, + 0,114,238,1,116,0,0,106,5,0,100,20,0,131,1,0, + 125,11,0,116,6,0,124,1,0,100,21,0,124,11,0,131, + 3,0,1,116,6,0,124,1,0,100,22,0,116,11,0,131, + 0,0,131,3,0,1,116,12,0,106,13,0,116,2,0,106, + 14,0,131,0,0,131,1,0,1,124,5,0,100,7,0,107, + 2,0,114,66,2,116,15,0,106,16,0,100,23,0,131,1, + 0,1,100,24,0,116,12,0,107,6,0,114,66,2,100,25, + 0,116,17,0,95,18,0,100,18,0,83,41,27,122,205,83, + 101,116,117,112,32,116,104,101,32,112,97,116,104,45,98,97, + 115,101,100,32,105,109,112,111,114,116,101,114,115,32,102,111, + 114,32,105,109,112,111,114,116,108,105,98,32,98,121,32,105, + 109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,10, + 32,32,32,32,98,117,105,108,116,45,105,110,32,109,111,100, + 117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,105, + 110,103,32,116,104,101,109,32,105,110,116,111,32,116,104,101, + 32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,99, + 101,46,10,10,32,32,32,32,79,116,104,101,114,32,99,111, + 109,112,111,110,101,110,116,115,32,97,114,101,32,101,120,116, + 114,97,99,116,101,100,32,102,114,111,109,32,116,104,101,32, + 99,111,114,101,32,98,111,111,116,115,116,114,97,112,32,109, + 111,100,117,108,101,46,10,10,32,32,32,32,114,49,0,0, + 0,114,60,0,0,0,218,8,98,117,105,108,116,105,110,115, + 114,136,0,0,0,90,5,112,111,115,105,120,250,1,47,218, + 2,110,116,250,1,92,99,1,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,115,0,0,0,115,33,0,0,0, + 124,0,0,93,23,0,125,1,0,116,0,0,124,1,0,131, + 1,0,100,0,0,107,2,0,86,1,113,3,0,100,1,0, + 83,41,2,114,29,0,0,0,78,41,1,114,31,0,0,0, + 41,2,114,22,0,0,0,114,77,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, + 89,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, + 116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,101, + 110,101,120,112,114,62,114,59,0,0,0,122,30,105,109,112, + 111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,32, + 112,111,115,105,120,32,111,114,32,110,116,114,3,0,0,0, + 114,25,0,0,0,114,21,0,0,0,114,30,0,0,0,90, + 7,95,116,104,114,101,97,100,78,90,8,95,119,101,97,107, + 114,101,102,90,6,119,105,110,114,101,103,114,163,0,0,0, + 114,6,0,0,0,122,4,46,112,121,119,122,6,95,100,46, + 112,121,100,84,41,4,122,3,95,105,111,122,9,95,119,97, + 114,110,105,110,103,115,122,8,98,117,105,108,116,105,110,115, + 122,7,109,97,114,115,104,97,108,41,19,114,114,0,0,0, + 114,7,0,0,0,114,139,0,0,0,114,233,0,0,0,114, + 105,0,0,0,90,18,95,98,117,105,108,116,105,110,95,102, + 114,111,109,95,110,97,109,101,114,109,0,0,0,218,3,97, + 108,108,218,14,65,115,115,101,114,116,105,111,110,69,114,114, + 111,114,114,99,0,0,0,114,26,0,0,0,114,11,0,0, + 0,114,223,0,0,0,114,143,0,0,0,114,19,1,0,0, + 114,84,0,0,0,114,157,0,0,0,114,162,0,0,0,114, + 167,0,0,0,41,12,218,17,95,98,111,111,116,115,116,114, + 97,112,95,109,111,100,117,108,101,90,11,115,101,108,102,95, + 109,111,100,117,108,101,90,12,98,117,105,108,116,105,110,95, + 110,97,109,101,90,14,98,117,105,108,116,105,110,95,109,111, + 100,117,108,101,90,10,111,115,95,100,101,116,97,105,108,115, + 90,10,98,117,105,108,116,105,110,95,111,115,114,21,0,0, + 0,114,25,0,0,0,90,9,111,115,95,109,111,100,117,108, + 101,90,13,116,104,114,101,97,100,95,109,111,100,117,108,101, + 90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,101, + 90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, + 95,115,101,116,117,112,64,5,0,0,115,82,0,0,0,0, + 8,6,1,9,1,9,3,13,1,13,1,15,1,18,2,13, + 1,20,3,33,1,19,2,31,1,10,1,15,1,13,1,4, + 2,3,1,15,1,5,1,13,1,12,2,12,1,16,1,16, + 1,25,3,3,1,19,1,13,2,11,1,16,3,15,1,16, + 3,12,1,15,1,16,3,19,1,19,1,12,1,13,1,12, + 1,114,27,1,0,0,99,1,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,67,0,0,0,115,116,0,0,0, + 116,0,0,124,0,0,131,1,0,1,116,1,0,131,0,0, + 125,1,0,116,2,0,106,3,0,106,4,0,116,5,0,106, + 6,0,124,1,0,140,0,0,103,1,0,131,1,0,1,116, + 7,0,106,8,0,100,1,0,107,2,0,114,78,0,116,2, + 0,106,9,0,106,10,0,116,11,0,131,1,0,1,116,2, + 0,106,9,0,106,10,0,116,12,0,131,1,0,1,116,5, + 0,124,0,0,95,5,0,116,13,0,124,0,0,95,13,0, + 100,2,0,83,41,3,122,41,73,110,115,116,97,108,108,32, + 116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,105, + 109,112,111,114,116,32,99,111,109,112,111,110,101,110,116,115, + 46,114,22,1,0,0,78,41,14,114,27,1,0,0,114,155, + 0,0,0,114,7,0,0,0,114,248,0,0,0,114,143,0, + 0,0,114,0,1,0,0,114,13,1,0,0,114,3,0,0, + 0,114,105,0,0,0,218,9,109,101,116,97,95,112,97,116, + 104,114,157,0,0,0,114,162,0,0,0,114,243,0,0,0, + 114,212,0,0,0,41,2,114,26,1,0,0,90,17,115,117, + 112,112,111,114,116,101,100,95,108,111,97,100,101,114,115,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, + 95,105,110,115,116,97,108,108,132,5,0,0,115,16,0,0, + 0,0,2,10,1,9,1,28,1,15,1,16,1,16,4,9, + 1,114,29,1,0,0,41,3,122,3,119,105,110,114,1,0, + 0,0,114,2,0,0,0,41,56,114,107,0,0,0,114,10, + 0,0,0,114,11,0,0,0,114,17,0,0,0,114,19,0, + 0,0,114,28,0,0,0,114,38,0,0,0,114,39,0,0, + 0,114,43,0,0,0,114,44,0,0,0,114,46,0,0,0, + 114,55,0,0,0,218,4,116,121,112,101,218,8,95,95,99, + 111,100,101,95,95,114,138,0,0,0,114,15,0,0,0,114, + 128,0,0,0,114,14,0,0,0,114,18,0,0,0,90,17, + 95,82,65,87,95,77,65,71,73,67,95,78,85,77,66,69, + 82,114,73,0,0,0,114,72,0,0,0,114,84,0,0,0, + 114,74,0,0,0,90,23,68,69,66,85,71,95,66,89,84, + 69,67,79,68,69,95,83,85,70,70,73,88,69,83,90,27, + 79,80,84,73,77,73,90,69,68,95,66,89,84,69,67,79, + 68,69,95,83,85,70,70,73,88,69,83,114,79,0,0,0, + 114,85,0,0,0,114,91,0,0,0,114,95,0,0,0,114, + 97,0,0,0,114,116,0,0,0,114,123,0,0,0,114,135, + 0,0,0,114,141,0,0,0,114,144,0,0,0,114,149,0, + 0,0,218,6,111,98,106,101,99,116,114,156,0,0,0,114, + 161,0,0,0,114,162,0,0,0,114,178,0,0,0,114,188, + 0,0,0,114,204,0,0,0,114,212,0,0,0,114,217,0, + 0,0,114,223,0,0,0,114,218,0,0,0,114,224,0,0, + 0,114,241,0,0,0,114,243,0,0,0,114,0,1,0,0, + 114,18,1,0,0,114,155,0,0,0,114,27,1,0,0,114, + 29,1,0,0,114,4,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,8,60,109,111,100,117,108, + 101,62,8,0,0,0,115,98,0,0,0,6,17,6,3,12, + 12,12,5,12,5,12,6,12,12,12,10,12,9,12,5,12, + 7,15,22,15,111,22,1,18,2,6,1,6,2,9,2,9, + 2,10,2,21,44,12,33,12,19,12,12,12,12,12,28,12, + 17,21,55,21,12,18,10,12,14,9,3,12,1,15,65,19, + 64,19,29,22,110,19,41,25,45,25,16,6,3,25,53,19, + 57,19,42,19,134,19,147,15,23,12,11,12,68, }; -- cgit v0.12 From bffa73e582c1a842d91d25ce93055f5e69074764 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 28 Dec 2015 21:51:02 -0800 Subject: Issue #25972, #20440: Fix compilation on Windows --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index dddeba0..c538ec7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5150,7 +5150,7 @@ comerror_init(PyObject *self, PyObject *args, PyObject *kwds) return -1; Py_INCREF(args); - Py_SETREF((PyBaseExceptionObject *)self->args, args); + Py_SETREF(((PyBaseExceptionObject *)self)->args, args); return 0; } -- cgit v0.12 From 0d250bc119489fa7d094d4a3fd2fd2fa0a508145 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Dec 2015 22:34:23 +0200 Subject: Issue #25971: Optimized creating Fractions from floats by 2 times and from Decimals by 3 times. Unified error messages in float.as_integer_ratio(), Decimal.as_integer_ratio(), and Fraction constructors. --- Lib/_pydecimal.py | 6 ++---- Lib/fractions.py | 32 ++++---------------------------- Lib/test/test_fractions.py | 14 +++++++------- Misc/NEWS | 3 +++ Objects/floatobject.c | 12 ++++++------ 5 files changed, 22 insertions(+), 45 deletions(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index eb7bba8..02365ca 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1026,11 +1026,9 @@ class Decimal(object): """ if self._is_special: if self.is_nan(): - raise ValueError("Cannot pass NaN " - "to decimal.as_integer_ratio.") + raise ValueError("cannot convert NaN to integer ratio") else: - raise OverflowError("Cannot pass infinity " - "to decimal.as_integer_ratio.") + raise OverflowError("cannot convert Infinity to integer ratio") if not self: return 0, 1 diff --git a/Lib/fractions.py b/Lib/fractions.py index 60b0728..64d746b 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -125,17 +125,9 @@ class Fraction(numbers.Rational): self._denominator = numerator.denominator return self - elif isinstance(numerator, float): - # Exact conversion from float - value = Fraction.from_float(numerator) - self._numerator = value._numerator - self._denominator = value._denominator - return self - - elif isinstance(numerator, Decimal): - value = Fraction.from_decimal(numerator) - self._numerator = value._numerator - self._denominator = value._denominator + elif isinstance(numerator, (float, Decimal)): + # Exact conversion + self._numerator, self._denominator = numerator.as_integer_ratio() return self elif isinstance(numerator, str): @@ -210,10 +202,6 @@ class Fraction(numbers.Rational): elif not isinstance(f, float): raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) - if math.isnan(f): - raise ValueError("Cannot convert %r to %s." % (f, cls.__name__)) - if math.isinf(f): - raise OverflowError("Cannot convert %r to %s." % (f, cls.__name__)) return cls(*f.as_integer_ratio()) @classmethod @@ -226,19 +214,7 @@ class Fraction(numbers.Rational): raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) - if dec.is_infinite(): - raise OverflowError( - "Cannot convert %s to %s." % (dec, cls.__name__)) - if dec.is_nan(): - raise ValueError("Cannot convert %s to %s." % (dec, cls.__name__)) - sign, digits, exp = dec.as_tuple() - digits = int(''.join(map(str, digits))) - if sign: - digits = -digits - if exp >= 0: - return cls(digits * 10 ** exp) - else: - return cls(digits, 10 ** -exp) + return cls(*dec.as_integer_ratio()) def limit_denominator(self, max_denominator=1000000): """Closest Fraction to self with denominator at most max_denominator. diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 1699852..73d2dd3 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -263,13 +263,13 @@ class FractionTest(unittest.TestCase): nan = inf - inf # bug 16469: error types should be consistent with float -> int self.assertRaisesMessage( - OverflowError, "Cannot convert inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, inf) self.assertRaisesMessage( - OverflowError, "Cannot convert -inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, -inf) self.assertRaisesMessage( - ValueError, "Cannot convert nan to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_float, nan) def testFromDecimal(self): @@ -284,16 +284,16 @@ class FractionTest(unittest.TestCase): # bug 16469: error types should be consistent with decimal -> int self.assertRaisesMessage( - OverflowError, "Cannot convert Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("inf")) self.assertRaisesMessage( - OverflowError, "Cannot convert -Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("-inf")) self.assertRaisesMessage( - ValueError, "Cannot convert NaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("nan")) self.assertRaisesMessage( - ValueError, "Cannot convert sNaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) def testLimitDenominator(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6ec585a..6333401 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -126,6 +126,9 @@ Core and Builtins Library ------- +- Issue #25971: Optimized creating Fractions from floats by 2 times and from + Decimals by 3 times. + - Issue #25802: Document as deprecated the remaining implementations of importlib.abc.Loader.load_module(). diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d92bec3..2949174 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1466,14 +1466,14 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) CONVERT_TO_DOUBLE(v, self); if (Py_IS_INFINITY(self)) { - PyErr_SetString(PyExc_OverflowError, - "Cannot pass infinity to float.as_integer_ratio."); - return NULL; + PyErr_SetString(PyExc_OverflowError, + "cannot convert Infinity to integer ratio"); + return NULL; } if (Py_IS_NAN(self)) { - PyErr_SetString(PyExc_ValueError, - "Cannot pass NaN to float.as_integer_ratio."); - return NULL; + PyErr_SetString(PyExc_ValueError, + "cannot convert NaN to integer ratio"); + return NULL; } PyFPE_START_PROTECT("as_integer_ratio", goto error); -- cgit v0.12 From 4e6aad1f7ac4e91a5e6c44e8ab3a2105acd65d74 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Dec 2015 22:55:48 +0200 Subject: Clean up float.as_integer_ratio(). --- Objects/floatobject.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 2949174..eb60659 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1451,18 +1451,12 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) int exponent; int i; - PyObject *prev; PyObject *py_exponent = NULL; PyObject *numerator = NULL; PyObject *denominator = NULL; PyObject *result_pair = NULL; PyNumberMethods *long_methods = PyLong_Type.tp_as_number; -#define INPLACE_UPDATE(obj, call) \ - prev = obj; \ - obj = call; \ - Py_DECREF(prev); \ - CONVERT_TO_DOUBLE(v, self); if (Py_IS_INFINITY(self)) { @@ -1489,29 +1483,31 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) to be truncated by PyLong_FromDouble(). */ numerator = PyLong_FromDouble(float_part); - if (numerator == NULL) goto error; + if (numerator == NULL) + goto error; + denominator = PyLong_FromLong(1); + if (denominator == NULL) + goto error; + py_exponent = PyLong_FromLong(Py_ABS(exponent)); + if (py_exponent == NULL) + goto error; /* fold in 2**exponent */ - denominator = PyLong_FromLong(1); - py_exponent = PyLong_FromLong(labs((long)exponent)); - if (py_exponent == NULL) goto error; - INPLACE_UPDATE(py_exponent, - long_methods->nb_lshift(denominator, py_exponent)); - if (py_exponent == NULL) goto error; if (exponent > 0) { - INPLACE_UPDATE(numerator, - long_methods->nb_multiply(numerator, py_exponent)); - if (numerator == NULL) goto error; + Py_SETREF(numerator, + long_methods->nb_lshift(numerator, py_exponent)); + if (numerator == NULL) + goto error; } else { - Py_DECREF(denominator); - denominator = py_exponent; - py_exponent = NULL; + Py_SETREF(denominator, + long_methods->nb_lshift(denominator, py_exponent)); + if (denominator == NULL) + goto error; } result_pair = PyTuple_Pack(2, numerator, denominator); -#undef INPLACE_UPDATE error: Py_XDECREF(py_exponent); Py_XDECREF(denominator); -- cgit v0.12 From 317f64f048b6da8a53870ce6a1d63ad458ece95f Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 2 Jan 2016 17:18:34 -0500 Subject: #21815: violate IMAP RFC to be compatible with, e.g., gmail and others, including imaplib's own behavior. I'm applying this only to 3.6 because there's a potential backward compatibility concern: if there are servers that include ] characters in the 'text' portion of their imap responses, this code change could introduce a new bug. Patch by Lita Cho, reviewed by Jessica McKellar, Berker Peksag, Maciej Szulik, silentghost, and me (I fleshed out the comments with the additional info/concerns.) --- Doc/library/imaplib.rst | 11 +++++++++++ Lib/imaplib.py | 10 +++++++++- Lib/test/test_imaplib.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 15b0932..fbb802c 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -500,6 +500,17 @@ An :class:`IMAP4` instance has the following methods: M.store(num, '+FLAGS', '\\Deleted') M.expunge() + .. note:: + + Creating flags containing ']' (for example: "[test]") violates + :rfc:`3501` (the IMAP protocol). However, imaplib has historically + allowed creation of such tags, and popular IMAP servers, such as Gmail, + accept and produce such flags. There are non-Python programs which also + create such tags. Although it is an RFC violation and IMAP clients and + servers are supposed to be strict, imaplib nontheless continues to allow + such tags to be created for backward compatibility reasons, and as of + python 3.5.2/3.6.0, handles them if they are sent from the server, since + this improves real-world compatibility. .. method:: IMAP4.subscribe(mailbox) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 4e8a4bb..a63ba8d 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -111,7 +111,15 @@ InternalDate = re.compile(br'.*INTERNALDATE "' # Literal is no longer used; kept for backward compatibility. Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) MapCRLF = re.compile(br'\r\n|\r|\n') -Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') +# We no longer exclude the ']' character from the data portion of the response +# code, even though it violates the RFC. Popular IMAP servers such as Gmail +# allow flags with ']', and there are programs (including imaplib!) that can +# produce them. The problem with this is if the 'text' portion of the response +# includes a ']' we'll parse the response wrong (which is the point of the RFC +# restriction). However, that seems less likely to be a problem in practice +# than being unable to correctly parse flags that include ']' chars, which +# was reported as a real-world problem in issue #21815. +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') # Untagged_status is no longer used; kept for backward compatibility Untagged_status = re.compile( diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 07157f5..8e4990b 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -243,6 +243,55 @@ class ThreadedNetworkedTests(unittest.TestCase): client.shutdown() @reap_threads + def test_bracket_flags(self): + + # This violates RFC 3501, which disallows ']' characters in tag names, + # but imaplib has allowed producing such tags forever, other programs + # also produce them (eg: OtherInbox's Organizer app as of 20140716), + # and Gmail, for example, accepts them and produces them. So we + # support them. See issue #21815. + + class BracketFlagHandler(SimpleIMAPHandler): + + def handle(self): + self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft'] + super().handle() + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + def cmd_SELECT(self, tag, args): + flag_msg = ' \\'.join(self.flags) + self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii')) + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.' + % flag_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_STORE(self, tag, args): + new_flags = args[2].strip('(').strip(')').split() + self.flags.extend(new_flags) + flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags) + msg = '* %s FETCH %s' % (args[0], flags_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', 'STORE completed.') + + with self.reaped_pair(BracketFlagHandler) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') + client.select('test') + typ, [data] = client.store(b'1', "+FLAGS", "[test]") + self.assertIn(b'[test]', data) + client.select('test') + typ, [data] = client.response('PERMANENTFLAGS') + self.assertIn(b'[test]', data) + + @reap_threads def test_issue5949(self): class EOFHandler(socketserver.StreamRequestHandler): diff --git a/Misc/NEWS b/Misc/NEWS index 83c07f5..7277838 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,10 @@ Core and Builtins Library ------- +- Issue #21815: Accept ] characters in the data portion of imap responses, + in order to handle the flags with square brackets accepted and produced + by servers such as gmail. + - Issue #25447: fileinput now uses sys.stdin as-is if it does not have a buffer attribute (restores backward compatibility). -- cgit v0.12 From 75f104a74535ddcbc390c4e8362fd7157bace185 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 2 Jan 2016 17:25:59 -0500 Subject: #21815: Make the doc change match what I actually did. --- Doc/library/imaplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index fbb802c..cd214ff 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -509,8 +509,8 @@ An :class:`IMAP4` instance has the following methods: create such tags. Although it is an RFC violation and IMAP clients and servers are supposed to be strict, imaplib nontheless continues to allow such tags to be created for backward compatibility reasons, and as of - python 3.5.2/3.6.0, handles them if they are sent from the server, since - this improves real-world compatibility. + python 3.6, handles them if they are sent from the server, since this + improves real-world compatibility. .. method:: IMAP4.subscribe(mailbox) -- cgit v0.12 From 576f132b986b5ee60e4b84d34a519a5edcd8c03e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 5 Jan 2016 21:27:54 +0200 Subject: Issue #20440: Cleaning up the code by using Py_SETREF. --- Modules/_datetimemodule.c | 24 ++++++------------------ Modules/_elementtree.c | 4 +--- Modules/_lsprof.c | 7 ++----- Modules/_pickle.c | 12 ++---------- Modules/readline.c | 4 +--- Objects/exceptions.c | 22 ++++++++-------------- Objects/frameobject.c | 6 +----- Objects/funcobject.c | 35 ++++++----------------------------- Objects/genobject.c | 12 ++---------- Objects/object.c | 8 +++----- Objects/typeobject.c | 6 ++---- Python/import.c | 10 +++------- Python/peephole.c | 4 +--- Python/sysmodule.c | 5 +---- 14 files changed, 39 insertions(+), 120 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3e5e195..4c8f3f6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1057,10 +1057,8 @@ format_utcoffset(char *buf, size_t buflen, const char *sep, } /* Offset is normalized, so it is negative if days < 0 */ if (GET_TD_DAYS(offset) < 0) { - PyObject *temp = offset; sign = '-'; - offset = delta_negative((PyDateTime_Delta *)offset); - Py_DECREF(temp); + Py_SETREF(offset, delta_negative((PyDateTime_Delta *)offset)); if (offset == NULL) return -1; } @@ -3047,10 +3045,8 @@ tzinfo_fromutc(PyDateTime_TZInfo *self, PyObject *dt) if (dst == Py_None) goto Inconsistent; if (delta_bool((PyDateTime_Delta *)dst) != 0) { - PyObject *temp = result; - result = add_datetime_timedelta((PyDateTime_DateTime *)result, - (PyDateTime_Delta *)dst, 1); - Py_DECREF(temp); + Py_SETREF(result, add_datetime_timedelta((PyDateTime_DateTime *)result, + (PyDateTime_Delta *)dst, 1)); if (result == NULL) goto Fail; } @@ -4157,10 +4153,7 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) tz); if (self != NULL && tz != Py_None) { /* Convert UTC to tzinfo's zone. */ - PyObject *temp = self; - - self = _PyObject_CallMethodId(tz, &PyId_fromutc, "O", self); - Py_DECREF(temp); + self = _PyObject_CallMethodId(tz, &PyId_fromutc, "N", self); } return self; } @@ -4195,10 +4188,7 @@ datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) tzinfo); if (self != NULL && tzinfo != Py_None) { /* Convert UTC to tzinfo's zone. */ - PyObject *temp = self; - - self = _PyObject_CallMethodId(tzinfo, &PyId_fromutc, "O", self); - Py_DECREF(temp); + self = _PyObject_CallMethodId(tzinfo, &PyId_fromutc, "N", self); } return self; } @@ -4421,9 +4411,7 @@ datetime_subtract(PyObject *left, PyObject *right) return NULL; if (offdiff != NULL) { - PyObject *temp = result; - result = delta_subtract(result, offdiff); - Py_DECREF(temp); + Py_SETREF(result, delta_subtract(result, offdiff)); Py_DECREF(offdiff); } } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 580c53a..0effc3f 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -396,10 +396,8 @@ element_init(PyObject *self, PyObject *args, PyObject *kwds) Py_XDECREF(attrib); /* Replace the objects already pointed to by tag, text and tail. */ - tmp = self_elem->tag; Py_INCREF(tag); - self_elem->tag = tag; - Py_DECREF(tmp); + Py_SETREF(self_elem->tag, tag); tmp = self_elem->text; Py_INCREF(Py_None); diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 66e534f..d3ed758 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -762,7 +762,6 @@ profiler_dealloc(ProfilerObject *op) static int profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) { - PyObject *o; PyObject *timer = NULL; double timeunit = 0.0; int subcalls = 1; @@ -777,11 +776,9 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) if (setSubcalls(pObj, subcalls) < 0 || setBuiltins(pObj, builtins) < 0) return -1; - o = pObj->externalTimer; - pObj->externalTimer = timer; - Py_XINCREF(timer); - Py_XDECREF(o); pObj->externalTimerUnit = timeunit; + Py_XINCREF(timer); + Py_SETREF(pObj->externalTimer, timer); return 0; } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index b42f4f6..bb3330a 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4494,8 +4494,6 @@ Pickler_get_persid(PicklerObject *self) static int Pickler_set_persid(PicklerObject *self, PyObject *value) { - PyObject *tmp; - if (value == NULL) { PyErr_SetString(PyExc_TypeError, "attribute deletion is not supported"); @@ -4507,10 +4505,8 @@ Pickler_set_persid(PicklerObject *self, PyObject *value) return -1; } - tmp = self->pers_func; Py_INCREF(value); - self->pers_func = value; - Py_XDECREF(tmp); /* self->pers_func can be NULL, so be careful. */ + Py_SETREF(self->pers_func, value); return 0; } @@ -6946,8 +6942,6 @@ Unpickler_get_persload(UnpicklerObject *self) static int Unpickler_set_persload(UnpicklerObject *self, PyObject *value) { - PyObject *tmp; - if (value == NULL) { PyErr_SetString(PyExc_TypeError, "attribute deletion is not supported"); @@ -6960,10 +6954,8 @@ Unpickler_set_persload(UnpicklerObject *self, PyObject *value) return -1; } - tmp = self->pers_func; Py_INCREF(value); - self->pers_func = value; - Py_XDECREF(tmp); /* self->pers_func can be NULL, so be careful. */ + Py_SETREF(self->pers_func, value); return 0; } diff --git a/Modules/readline.c b/Modules/readline.c index 939ff1a..6930415 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -321,10 +321,8 @@ set_hook(const char *funcname, PyObject **hook_var, PyObject *args) Py_CLEAR(*hook_var); } else if (PyCallable_Check(function)) { - PyObject *tmp = *hook_var; Py_INCREF(function); - *hook_var = function; - Py_XDECREF(tmp); + Py_SETREF(*hook_var, function); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 85f9472..7374368 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -59,15 +59,11 @@ BaseException_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int BaseException_init(PyBaseExceptionObject *self, PyObject *args, PyObject *kwds) { - PyObject *tmp; - if (!_PyArg_NoKeywords(Py_TYPE(self)->tp_name, kwds)) return -1; - tmp = self->args; - self->args = args; - Py_INCREF(self->args); - Py_XDECREF(tmp); + Py_INCREF(args); + Py_SETREF(self->args, args); return 0; } @@ -328,11 +324,10 @@ PyException_GetCause(PyObject *self) { /* Steals a reference to cause */ void -PyException_SetCause(PyObject *self, PyObject *cause) { - PyObject *old_cause = ((PyBaseExceptionObject *)self)->cause; - ((PyBaseExceptionObject *)self)->cause = cause; +PyException_SetCause(PyObject *self, PyObject *cause) +{ ((PyBaseExceptionObject *)self)->suppress_context = 1; - Py_XDECREF(old_cause); + Py_SETREF(((PyBaseExceptionObject *)self)->cause, cause); } PyObject * @@ -344,10 +339,9 @@ PyException_GetContext(PyObject *self) { /* Steals a reference to context */ void -PyException_SetContext(PyObject *self, PyObject *context) { - PyObject *old_context = ((PyBaseExceptionObject *)self)->context; - ((PyBaseExceptionObject *)self)->context = context; - Py_XDECREF(old_context); +PyException_SetContext(PyObject *self, PyObject *context) +{ + Py_SETREF(((PyBaseExceptionObject *)self)->context, context); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 37e626d..425a9ee 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -349,15 +349,11 @@ frame_gettrace(PyFrameObject *f, void *closure) static int frame_settrace(PyFrameObject *f, PyObject* v, void *closure) { - PyObject* old_value; - /* We rely on f_lineno being accurate when f_trace is set. */ f->f_lineno = PyFrame_GetLineNumber(f); - old_value = f->f_trace; Py_XINCREF(v); - f->f_trace = v; - Py_XDECREF(old_value); + Py_SETREF(f->f_trace, v); return 0; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 13daaba..2967634 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -249,7 +249,6 @@ func_get_code(PyFunctionObject *op) static int func_set_code(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; Py_ssize_t nfree, nclosure; /* Not legal to del f.func_code or to set it to anything @@ -270,10 +269,8 @@ func_set_code(PyFunctionObject *op, PyObject *value) nclosure, nfree); return -1; } - tmp = op->func_code; Py_INCREF(value); - op->func_code = value; - Py_DECREF(tmp); + Py_SETREF(op->func_code, value); return 0; } @@ -287,8 +284,6 @@ func_get_name(PyFunctionObject *op) static int func_set_name(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del f.func_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -296,10 +291,8 @@ func_set_name(PyFunctionObject *op, PyObject *value) "__name__ must be set to a string object"); return -1; } - tmp = op->func_name; Py_INCREF(value); - op->func_name = value; - Py_DECREF(tmp); + Py_SETREF(op->func_name, value); return 0; } @@ -313,8 +306,6 @@ func_get_qualname(PyFunctionObject *op) static int func_set_qualname(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del f.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -322,10 +313,8 @@ func_set_qualname(PyFunctionObject *op, PyObject *value) "__qualname__ must be set to a string object"); return -1; } - tmp = op->func_qualname; Py_INCREF(value); - op->func_qualname = value; - Py_DECREF(tmp); + Py_SETREF(op->func_qualname, value); return 0; } @@ -343,8 +332,6 @@ func_get_defaults(PyFunctionObject *op) static int func_set_defaults(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - /* Legal to del f.func_defaults. * Can only set func_defaults to NULL or a tuple. */ if (value == Py_None) @@ -354,10 +341,8 @@ func_set_defaults(PyFunctionObject *op, PyObject *value) "__defaults__ must be set to a tuple object"); return -1; } - tmp = op->func_defaults; Py_XINCREF(value); - op->func_defaults = value; - Py_XDECREF(tmp); + Py_SETREF(op->func_defaults, value); return 0; } @@ -375,8 +360,6 @@ func_get_kwdefaults(PyFunctionObject *op) static int func_set_kwdefaults(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - if (value == Py_None) value = NULL; /* Legal to del f.func_kwdefaults. @@ -386,10 +369,8 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value) "__kwdefaults__ must be set to a dict object"); return -1; } - tmp = op->func_kwdefaults; Py_XINCREF(value); - op->func_kwdefaults = value; - Py_XDECREF(tmp); + Py_SETREF(op->func_kwdefaults, value); return 0; } @@ -408,8 +389,6 @@ func_get_annotations(PyFunctionObject *op) static int func_set_annotations(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - if (value == Py_None) value = NULL; /* Legal to del f.func_annotations. @@ -420,10 +399,8 @@ func_set_annotations(PyFunctionObject *op, PyObject *value) "__annotations__ must be set to a dict object"); return -1; } - tmp = op->func_annotations; Py_XINCREF(value); - op->func_annotations = value; - Py_XDECREF(tmp); + Py_SETREF(op->func_annotations, value); return 0; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 00ebbf1..81a80b7 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -510,8 +510,6 @@ gen_get_name(PyGenObject *op) static int gen_set_name(PyGenObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del gen.gi_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -519,10 +517,8 @@ gen_set_name(PyGenObject *op, PyObject *value) "__name__ must be set to a string object"); return -1; } - tmp = op->gi_name; Py_INCREF(value); - op->gi_name = value; - Py_DECREF(tmp); + Py_SETREF(op->gi_name, value); return 0; } @@ -536,8 +532,6 @@ gen_get_qualname(PyGenObject *op) static int gen_set_qualname(PyGenObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del gen.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -545,10 +539,8 @@ gen_set_qualname(PyGenObject *op, PyObject *value) "__qualname__ must be set to a string object"); return -1; } - tmp = op->gi_qualname; Py_INCREF(value); - op->gi_qualname = value; - Py_DECREF(tmp); + Py_SETREF(op->gi_qualname, value); return 0; } diff --git a/Objects/object.c b/Objects/object.c index 417a97d..8072dbc 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1203,7 +1203,7 @@ PyObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) int PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { - PyObject *dict, **dictptr = _PyObject_GetDictPtr(obj); + PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { PyErr_SetString(PyExc_AttributeError, "This object has no __dict__"); @@ -1219,10 +1219,8 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - dict = *dictptr; - Py_XINCREF(value); - *dictptr = value; - Py_XDECREF(dict); + Py_INCREF(value); + Py_SETREF(*dictptr, value); return 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index c62255c..db15cf6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2092,7 +2092,7 @@ subtype_dict(PyObject *obj, void *context) static int subtype_setdict(PyObject *obj, PyObject *value, void *context) { - PyObject *dict, **dictptr; + PyObject **dictptr; PyTypeObject *base; base = get_builtin_base_with_dict(Py_TYPE(obj)); @@ -2123,10 +2123,8 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - dict = *dictptr; Py_XINCREF(value); - *dictptr = value; - Py_XDECREF(dict); + Py_SETREF(*dictptr, value); return 0; } diff --git a/Python/import.c b/Python/import.c index 8d7bfe9..325b936 100644 --- a/Python/import.c +++ b/Python/import.c @@ -879,10 +879,8 @@ update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) if (PyUnicode_Compare(co->co_filename, oldname)) return; - tmp = co->co_filename; - co->co_filename = newname; - Py_INCREF(co->co_filename); - Py_DECREF(tmp); + Py_INCREF(newname); + Py_SETREF(co->co_filename, newname); constants = co->co_consts; n = PyTuple_GET_SIZE(constants); @@ -1327,10 +1325,8 @@ remove_importlib_frames(void) (always_trim || PyUnicode_CompareWithASCIIString(code->co_name, remove_frames) == 0)) { - PyObject *tmp = *outer_link; - *outer_link = next; Py_XINCREF(next); - Py_DECREF(tmp); + Py_SETREF(*outer_link, next); prev_link = outer_link; } else { diff --git a/Python/peephole.c b/Python/peephole.c index 59ad3b7..4bf786e 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -118,9 +118,7 @@ tuple_of_constants(unsigned char *codestr, Py_ssize_t n, /* If it's a BUILD_SET, use the PyTuple we just built to create a PyFrozenSet, and use that as the constant instead: */ if (codestr[0] == BUILD_SET) { - PyObject *tuple = newconst; - newconst = PyFrozenSet_New(tuple); - Py_DECREF(tuple); + Py_SETREF(newconst, PyFrozenSet_New(newconst)); if (newconst == NULL) return 0; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f784f75..53dd4a1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -436,10 +436,7 @@ trace_trampoline(PyObject *self, PyFrameObject *frame, return -1; } if (result != Py_None) { - PyObject *temp = frame->f_trace; - frame->f_trace = NULL; - Py_XDECREF(temp); - frame->f_trace = result; + Py_SETREF(frame->f_trace, result); } else { Py_DECREF(result); -- cgit v0.12 From a85e927e39b0ef3637c0bb6efa851494b8d92662 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Jan 2016 14:33:09 -0800 Subject: Issue #25802: Add an examples section to importlib. Thanks to Berker Peksag for the patch review. --- Doc/library/imp.rst | 15 +++++--- Doc/library/importlib.rst | 91 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index 68a6b68..420031a 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -81,7 +81,9 @@ This module provides an interface to the mechanisms used to implement the .. deprecated:: 3.3 Use :func:`importlib.util.find_spec` instead unless Python 3.3 compatibility is required, in which case use - :func:`importlib.find_loader`. + :func:`importlib.find_loader`. For example usage of the former case, + see the :ref:`importlib-examples` section of the :mod:`importlib` + documentation. .. function:: load_module(name, file, pathname, description) @@ -108,9 +110,12 @@ This module provides an interface to the mechanisms used to implement the If previously used in conjunction with :func:`imp.find_module` then consider using :func:`importlib.import_module`, otherwise use the loader returned by the replacement you chose for :func:`imp.find_module`. If you - called :func:`imp.load_module` and related functions directly then use the - classes in :mod:`importlib.machinery`, e.g. - ``importlib.machinery.SourceFileLoader(name, path).load_module()``. + called :func:`imp.load_module` and related functions directly with file + path arguments then use a combination of + :func:`importlib.util.spec_from_file_location` and + :func:`importlib.util.module_from_spec`. See the :ref:`importlib-examples` + section of the :mod:`importlib` documentation for details of the various + approaches. .. function:: new_module(name) @@ -119,7 +124,7 @@ This module provides an interface to the mechanisms used to implement the in ``sys.modules``. .. deprecated:: 3.4 - Use :class:`types.ModuleType` instead. + Use :func:`importlib.util.module_from_spec` instead. .. function:: reload(module) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d800835..45766a7 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -256,7 +256,7 @@ ABC hierarchy:: module and *path* will be the value of :attr:`__path__` from the parent package. If a spec cannot be found, ``None`` is returned. When passed in, ``target`` is a module object that the finder may - use to make a more educated about what spec to return. + use to make a more educated guess about what spec to return. .. versionadded:: 3.4 @@ -306,7 +306,7 @@ ABC hierarchy:: within the :term:`path entry` to which it is assigned. If a spec cannot be found, ``None`` is returned. When passed in, ``target`` is a module object that the finder may use to make a more educated - about what spec to return. + guess about what spec to return. .. versionadded:: 3.4 @@ -1307,3 +1307,90 @@ an :term:`importer`. loader = importlib.machinery.SourceFileLoader lazy_loader = importlib.util.LazyLoader.factory(loader) finder = importlib.machinery.FileFinder(path, [(lazy_loader, suffixes)]) + +.. _importlib-examples: + +Examples +-------- + +To programmatically import a module, use :func:`importlib.import_module`. +:: + + import importlib + + itertools = importlib.import_module('itertools') + +If you need to find out if a module can be imported without actually doing the +import, then you should use :func:`importlib.util.find_spec`. +:: + + import importlib.util + import sys + + # For illustrative purposes. + name = 'itertools' + + spec = importlib.util.find_spec(name) + if spec is None: + print("can't find the itertools module") + else: + # If you chose to perform the actual import. + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # Adding the module to sys.modules is optional. + sys.modules[name] = module + +To import a Python source file directly, use the following recipe +(Python 3.4 and newer only):: + + import importlib.util + import sys + + # For illustrative purposes. + import tokenize + file_path = tokenize.__file__ + module_name = tokenize.__name__ + + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # Optional; only necessary if you want to be able to import the module + # by name later. + sys.modules[module_name] = module + +Import itself is implemented in Python code, making it possible to +expose most of the import machinery through importlib. The following +helps illustrate the various APIs that importlib exposes by providing an +approximate implementation of +:func:`importlib.import_module` (Python 3.4 and newer for importlib usage, +Python 3.6 and newer for other parts of the code). +:: + + import importlib.util + import sys + + def import_module(name, package=None): + """An approximate implementation of import.""" + absolute_name = importlib.util.resolve_name(name, package) + try: + return sys.modules[absolute_name] + except KeyError: + pass + + path = None + if '.' in absolute_name: + parent_name, _, child_name = absolute_name.rpartition('.') + parent_module = import_module(parent_name) + path = parent_module.spec.submodule_search_locations + for finder in sys.meta_path: + spec = finder.find_spec(absolute_name, path) + if spec is not None: + break + else: + raise ImportError(f'No module named {absolute_name!r}') + module = spec.loader.create_module(spec) + spec.loader.exec_module(module) + sys.modules[absolute_name] = module + if path is not None: + setattr(parent_module, child_name, module) + return module -- cgit v0.12 From 32d1e56bda420602fd4106dbc51ea1dc6f02ba14 Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Mon, 11 Jan 2016 14:53:01 +1100 Subject: Issue #26070: py.exe launcher fails to find in-place built binaries from earlier Python versions. --- PC/launcher.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/PC/launcher.c b/PC/launcher.c index b379a38..e6baae5 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -171,6 +171,9 @@ static wchar_t * location_checks[] = { L"\\", L"\\PCBuild\\win32\\", L"\\PCBuild\\amd64\\", + // To support early 32bit versions of Python that stuck the build binaries + // directly in PCBuild... + L"\\PCBuild\\", NULL }; -- cgit v0.12 From 96b531abd5bac9707e1b8f714c4d9259958726f9 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 11 Jan 2016 07:09:42 -0800 Subject: Issue #26069: Remove the deprecated apis in the trace module. --- Lib/test/test_trace.py | 48 +----------------------------------------------- Lib/trace.py | 43 ------------------------------------------- Misc/NEWS | 2 ++ 3 files changed, 3 insertions(+), 90 deletions(-) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 03dff84..55a3bfa 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,11 +1,10 @@ import os -import io import sys from test.support import TESTFN, rmtree, unlink, captured_stdout import unittest import trace -from trace import CoverageResults, Trace +from trace import Trace from test.tracedmodules import testmod @@ -366,50 +365,5 @@ class Test_Ignore(unittest.TestCase): self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz')) -class TestDeprecatedMethods(unittest.TestCase): - - def test_deprecated_usage(self): - sio = io.StringIO() - with self.assertWarns(DeprecationWarning): - trace.usage(sio) - self.assertIn('Usage:', sio.getvalue()) - - def test_deprecated_Ignore(self): - with self.assertWarns(DeprecationWarning): - trace.Ignore() - - def test_deprecated_modname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.modname("spam")) - - def test_deprecated_fullmodname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.fullmodname("spam")) - - def test_deprecated_find_lines_from_code(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines_from_code(foo.__code__, ["eggs"]) - - def test_deprecated_find_lines(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines(foo.__code__, ["eggs"]) - - def test_deprecated_find_strings(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_strings(fd.name) - - def test_deprecated_find_executable_linenos(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_executable_linenos(fd.name) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index f108266..b768829 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -58,7 +58,6 @@ import inspect import gc import dis import pickle -from warnings import warn as _warn from time import monotonic as _time try: @@ -810,47 +809,5 @@ def main(argv=None): if not no_report: results.write_results(missing, summary=summary, coverdir=coverdir) -# Deprecated API -def usage(outfile): - _warn("The trace.usage() function is deprecated", - DeprecationWarning, 2) - _usage(outfile) - -class Ignore(_Ignore): - def __init__(self, modules=None, dirs=None): - _warn("The class trace.Ignore is deprecated", - DeprecationWarning, 2) - _Ignore.__init__(self, modules, dirs) - -def modname(path): - _warn("The trace.modname() function is deprecated", - DeprecationWarning, 2) - return _modname(path) - -def fullmodname(path): - _warn("The trace.fullmodname() function is deprecated", - DeprecationWarning, 2) - return _fullmodname(path) - -def find_lines_from_code(code, strs): - _warn("The trace.find_lines_from_code() function is deprecated", - DeprecationWarning, 2) - return _find_lines_from_code(code, strs) - -def find_lines(code, strs): - _warn("The trace.find_lines() function is deprecated", - DeprecationWarning, 2) - return _find_lines(code, strs) - -def find_strings(filename, encoding=None): - _warn("The trace.find_strings() function is deprecated", - DeprecationWarning, 2) - return _find_strings(filename, encoding=None) - -def find_executable_linenos(filename): - _warn("The trace.find_executable_linenos() function is deprecated", - DeprecationWarning, 2) - return _find_executable_linenos(filename) - if __name__=='__main__': main() diff --git a/Misc/NEWS b/Misc/NEWS index 60799b6..d54c489 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,8 @@ Core and Builtins Library ------- +- Issue #26069: Remove the deprecated apis in the trace module. + - Issue #22138: Fix mock.patch behavior when patching descriptors. Restore original values after patching. Patch contributed by Sean McCully. -- cgit v0.12 From 37dc2b28834c95d24746c2958e9ea31d3e8d1968 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 11 Jan 2016 15:15:01 -0500 Subject: Issue #25486: Resurrect inspect.getargspec in 3.6. Backout a565aad5d6e1. The decision is that we shouldn't remove popular APIs (however long they are depreacted) from Python 3, while 2.7 is still around and supported. --- Doc/library/inspect.rst | 18 ++++++++++++++++++ Doc/whatsnew/3.6.rst | 3 --- Lib/inspect.py | 25 +++++++++++++++++++++++++ Lib/test/test_inspect.py | 40 +++++++++++++++++++++++++++++++++++----- Misc/NEWS | 3 +-- 5 files changed, 79 insertions(+), 10 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8045d85..8e8d725 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -796,6 +796,24 @@ Classes and functions classes using multiple inheritance and their descendants will appear multiple times. + +.. function:: getargspec(func) + + Get the names and default values of a Python function's arguments. A + :term:`named tuple` ``ArgSpec(args, varargs, keywords, defaults)`` is + returned. *args* is a list of the argument names. *varargs* and *keywords* + are the names of the ``*`` and ``**`` arguments or ``None``. *defaults* is a + tuple of default argument values or ``None`` if there are no default + arguments; if this tuple has *n* elements, they correspond to the last + *n* elements listed in *args*. + + .. deprecated:: 3.0 + Use :func:`signature` and + :ref:`Signature Object `, which provide a + better introspecting API for callables. This function will be removed + in Python 3.6. + + .. function:: getfullargspec(func) Get the names and default values of a Python function's arguments. A diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index bf5161d..f7dbbee 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -218,9 +218,6 @@ Removed API and Feature Removals ------------------------ -* ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). - :func:`inspect.getfullargspec` is an almost drop in replacement. - * ``inspect.getmoduleinfo()`` was removed (was deprecated since CPython 3.3). :func:`inspect.getmodulename` should be used for obtaining the module name for a given path. diff --git a/Lib/inspect.py b/Lib/inspect.py index 7615e52..74f3b3c 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1004,6 +1004,31 @@ def _getfullargs(co): varkw = co.co_varnames[nargs] return args, varargs, kwonlyargs, varkw + +ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') + +def getargspec(func): + """Get the names and default values of a function's arguments. + + A tuple of four things is returned: (args, varargs, keywords, defaults). + 'args' is a list of the argument names, including keyword-only argument names. + 'varargs' and 'keywords' are the names of the * and ** arguments or None. + 'defaults' is an n-tuple of the default values of the last n arguments. + + Use the getfullargspec() API for Python 3 code, as annotations + and keyword arguments are supported. getargspec() will raise ValueError + if the func has either annotations or keyword arguments. + """ + warnings.warn("inspect.getargspec() is deprecated, " + "use inspect.signature() instead", DeprecationWarning, + stacklevel=2) + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ + getfullargspec(func) + if kwonlyargs or ann: + raise ValueError("Function has keyword-only arguments or annotations" + ", use getfullargspec() API which can support them") + return ArgSpec(args, varargs, varkw, defaults) + FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index a88e7fd..283c922 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -628,6 +628,18 @@ class TestClassesAndFunctions(unittest.TestCase): got = inspect.getmro(D) self.assertEqual(expected, got) + def assertArgSpecEquals(self, routine, args_e, varargs_e=None, + varkw_e=None, defaults_e=None, formatted=None): + with self.assertWarns(DeprecationWarning): + args, varargs, varkw, defaults = inspect.getargspec(routine) + self.assertEqual(args, args_e) + self.assertEqual(varargs, varargs_e) + self.assertEqual(varkw, varkw_e) + self.assertEqual(defaults, defaults_e) + if formatted is not None: + self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults), + formatted) + def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, kwonlyargs_e=[], kwonlydefaults_e=None, @@ -646,6 +658,23 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs, kwonlydefaults, ann), formatted) + def test_getargspec(self): + self.assertArgSpecEquals(mod.eggs, ['x', 'y'], formatted='(x, y)') + + self.assertArgSpecEquals(mod.spam, + ['a', 'b', 'c', 'd', 'e', 'f'], + 'g', 'h', (3, 4, 5), + '(a, b, c, d=3, e=4, f=5, *g, **h)') + + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.keyworded, []) + + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.annotated, []) + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.keyword_only_arg, []) + + def test_getfullargspec(self): self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', kwonlyargs_e=['arg2'], @@ -659,19 +688,20 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs_e=['arg'], formatted='(*, arg)') - def test_fullargspec_api_ignores_wrapped(self): + def test_argspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) def ham(x, y): pass # Basic check + self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(functools.partial(ham), ['x', 'y'], formatted='(x, y)') # Other variants def check_method(f): - self.assertFullArgSpecEquals(f, ['self', 'x', 'y'], - formatted='(self, x, y)') + self.assertArgSpecEquals(f, ['self', 'x', 'y'], + formatted='(self, x, y)') class C: @functools.wraps(mod.spam) def ham(self, x, y): @@ -749,11 +779,11 @@ class TestClassesAndFunctions(unittest.TestCase): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) - def test_getfullargspec_method(self): + def test_getargspec_method(self): class A(object): def m(self): pass - self.assertFullArgSpecEquals(A.m, ['self']) + self.assertArgSpecEquals(A.m, ['self']) def test_classify_newstyle(self): class A(object): diff --git a/Misc/NEWS b/Misc/NEWS index 593ed24..089388e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -429,8 +429,7 @@ Library - Issue #23661: unittest.mock side_effects can now be exceptions again. This was a regression vs Python 3.4. Patch from Ignacio Rossi -- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo - functions. +- Issue #13248: Remove deprecated inspect.getmoduleinfo function. - Issue #25578: Fix (another) memory leak in SSLSocket.getpeercer(). -- cgit v0.12 From 121edbf7e26ad0ac887ac4a2bf46316deb05737e Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Tue, 12 Jan 2016 06:18:32 -0800 Subject: Issue25347 - Format the error message output of mock's assert_has_calls method. Patch contributed by Robert Zimmerman. --- Lib/unittest/mock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 976f663..21f49fa 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -820,7 +820,7 @@ class NonCallableMock(Base): if expected not in all_calls: raise AssertionError( 'Calls not found.\nExpected: %r\n' - 'Actual: %r' % (calls, self.mock_calls) + 'Actual: %r' % (_CallList(calls), self.mock_calls) ) from cause return -- cgit v0.12 From 436831dbe47e66afb0a4f5927859073015dbd4bb Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 13 Jan 2016 07:46:54 -0800 Subject: Issue22642 - Convert trace module's option handling mechanism from getopt to argparse. Patch contributed by SilentGhost. --- Lib/test/test_trace.py | 22 ++++ Lib/trace.py | 339 +++++++++++++++++++------------------------------ 2 files changed, 156 insertions(+), 205 deletions(-) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 55a3bfa..1d894aa 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,6 +1,7 @@ import os import sys from test.support import TESTFN, rmtree, unlink, captured_stdout +from test.support.script_helper import assert_python_ok, assert_python_failure import unittest import trace @@ -364,6 +365,27 @@ class Test_Ignore(unittest.TestCase): # Matched before. self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz')) +class TestCommandLine(unittest.TestCase): + + def test_failures(self): + _errors = ( + (b'filename is missing: required with the main options', '-l', '-T'), + (b'cannot specify both --listfuncs and (--trace or --count)', '-lc'), + (b'argument -R/--no-report: not allowed with argument -r/--report', '-rR'), + (b'must specify one of --trace, --count, --report, --listfuncs, or --trackcalls', '-g'), + (b'-r/--report requires -f/--file', '-r'), + (b'--summary can only be used with --count or --report', '-sT'), + (b'unrecognized arguments: -y', '-y')) + for message, *args in _errors: + *_, stderr = assert_python_failure('-m', 'trace', *args) + self.assertIn(message, stderr) + + def test_listfuncs_flag_success(self): + with open(TESTFN, 'w') as fd: + self.addCleanup(unlink, TESTFN) + fd.write("a = 1\n") + status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN) + self.assertIn(b'functions called:', stdout) if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index b768829..7afbe76 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -48,6 +48,7 @@ Sample use, programmatically r.write_results(show_missing=True, coverdir="/tmp") """ __all__ = ['Trace', 'CoverageResults'] +import argparse import linecache import os import re @@ -76,51 +77,6 @@ else: sys.settrace(None) threading.settrace(None) -def _usage(outfile): - outfile.write("""Usage: %s [OPTIONS] [ARGS] - -Meta-options: ---help Display this help then exit. ---version Output version information then exit. - -Otherwise, exactly one of the following three options must be given: --t, --trace Print each line to sys.stdout before it is executed. --c, --count Count the number of times each line is executed - and write the counts to .cover for each - module executed, in the module's directory. - See also `--coverdir', `--file', `--no-report' below. --l, --listfuncs Keep track of which functions are executed at least - once and write the results to sys.stdout after the - program exits. --T, --trackcalls Keep track of caller/called pairs and write the - results to sys.stdout after the program exits. --r, --report Generate a report from a counts file; do not execute - any code. `--file' must specify the results file to - read, which must have been created in a previous run - with `--count --file=FILE'. - -Modifiers: --f, --file= File to accumulate counts over several runs. --R, --no-report Do not generate the coverage report files. - Useful if you want to accumulate over several runs. --C, --coverdir= Directory where the report files. The coverage - report for . is written to file - //.cover. --m, --missing Annotate executable lines that were not executed - with '>>>>>> '. --s, --summary Write a brief summary on stdout for each file. - (Can only be used with --count or --report.) --g, --timing Prefix each line with the time since the program started. - Only used while tracing. - -Filters, may be repeated multiple times: ---ignore-module= Ignore the given module(s) and its submodules - (if it is a package). Accepts comma separated - list of module names ---ignore-dir= Ignore files in the given directory (multiple - directories can be joined by os.pathsep). -""" % sys.argv[0]) - PRAGMA_NOCOVER = "#pragma NO COVER" # Simple rx to find lines with no code. @@ -264,7 +220,13 @@ class CoverageResults: def write_results(self, show_missing=True, summary=False, coverdir=None): """ - @param coverdir + Write the coverage results. + + :param show_missing: Show lines that had no hits. + :param summary: Include coverage summary per module. + :param coverdir: If None, the results of each module are placed in it's + directory, otherwise it is included in the directory + specified. """ if self.calledfuncs: print() @@ -646,168 +608,135 @@ class Trace: calledfuncs=self._calledfuncs, callers=self._callers) -def _err_exit(msg): - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.exit(1) - -def main(argv=None): - import getopt - - if argv is None: - argv = sys.argv +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', version='trace 2.0') + + grp = parser.add_argument_group('Main options', + 'One of these (or --report) must be given') + + grp.add_argument('-c', '--count', action='store_true', + help='Count the number of times each line is executed and write ' + 'the counts to .cover for each module executed, in ' + 'the module\'s directory. See also --coverdir, --file, ' + '--no-report below.') + grp.add_argument('-t', '--trace', action='store_true', + help='Print each line to sys.stdout before it is executed') + grp.add_argument('-l', '--listfuncs', action='store_true', + help='Keep track of which functions are executed at least once ' + 'and write the results to sys.stdout after the program exits. ' + 'Cannot be specified alongside --trace or --count.') + grp.add_argument('-T', '--trackcalls', action='store_true', + help='Keep track of caller/called pairs and write the results to ' + 'sys.stdout after the program exits.') + + grp = parser.add_argument_group('Modifiers') + + _grp = grp.add_mutually_exclusive_group() + _grp.add_argument('-r', '--report', action='store_true', + help='Generate a report from a counts file; does not execute any ' + 'code. --file must specify the results file to read, which ' + 'must have been created in a previous run with --count ' + '--file=FILE') + _grp.add_argument('-R', '--no-report', action='store_true', + help='Do not generate the coverage report files. ' + 'Useful if you want to accumulate over several runs.') + + grp.add_argument('-f', '--file', + help='File to accumulate counts over several runs') + grp.add_argument('-C', '--coverdir', + help='Directory where the report files go. The coverage report ' + 'for . will be written to file ' + '//.cover') + grp.add_argument('-m', '--missing', action='store_true', + help='Annotate executable lines that were not executed with ' + '">>>>>> "') + grp.add_argument('-s', '--summary', action='store_true', + help='Write a brief summary for each file to sys.stdout. ' + 'Can only be used with --count or --report') + grp.add_argument('-g', '--timing', action='store_true', + help='Prefix each line with the time since the program started. ' + 'Only used while tracing') + + grp = parser.add_argument_group('Filters', + 'Can be specified multiple times') + grp.add_argument('--ignore-module', action='append', default=[], + help='Ignore the given module(s) and its submodules' + '(if it is a package). Accepts comma separated list of ' + 'module names.') + grp.add_argument('--ignore-dir', action='append', default=[], + help='Ignore files in the given directory ' + '(multiple directories can be joined by os.pathsep).') + + parser.add_argument('filename', nargs='?', + help='file to run as main program') + parser.add_argument('arguments', nargs=argparse.REMAINDER, + help='arguments to the program') + + opts = parser.parse_args() + + if opts.ignore_dir: + rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info) + _prefix = os.path.join(sys.base_prefix, *rel_path) + _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path) + + def parse_ignore_dir(s): + s = os.path.expanduser(os.path.expandvars(s)) + s = s.replace('$prefix', _prefix).replace('$exec_prefix', _exec_prefix) + return os.path.normpath(s) + + opts.ignore_module = [mod.strip() + for i in opts.ignore_module for mod in i.split(',')] + opts.ignore_dir = [parse_ignore_dir(s) + for i in opts.ignore_dir for s in i.split(os.pathsep)] + + if opts.report: + if not opts.file: + parser.error('-r/--report requires -f/--file') + results = CoverageResults(infile=opts.file, outfile=opts.file) + return results.write_results(opts.missing, opts.summary, opts.coverdir) + + if not any([opts.trace, opts.count, opts.listfuncs, opts.trackcalls]): + parser.error('must specify one of --trace, --count, --report, ' + '--listfuncs, or --trackcalls') + + if opts.listfuncs and (opts.count or opts.trace): + parser.error('cannot specify both --listfuncs and (--trace or --count)') + + if opts.summary and not opts.count: + parser.error('--summary can only be used with --count or --report') + + if opts.filename is None: + parser.error('filename is missing: required with the main options') + + sys.argv = opts.filename, *opts.arguments + sys.path[0] = os.path.dirname(opts.filename) + + t = Trace(opts.count, opts.trace, countfuncs=opts.listfuncs, + countcallers=opts.trackcalls, ignoremods=opts.ignore_module, + ignoredirs=opts.ignore_dir, infile=opts.file, + outfile=opts.file, timing=opts.timing) try: - opts, prog_argv = getopt.getopt(argv[1:], "tcrRf:d:msC:lTg", - ["help", "version", "trace", "count", - "report", "no-report", "summary", - "file=", "missing", - "ignore-module=", "ignore-dir=", - "coverdir=", "listfuncs", - "trackcalls", "timing"]) - - except getopt.error as msg: - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.stderr.write("Try `%s --help' for more information\n" - % sys.argv[0]) - sys.exit(1) - - trace = 0 - count = 0 - report = 0 - no_report = 0 - counts_file = None - missing = 0 - ignore_modules = [] - ignore_dirs = [] - coverdir = None - summary = 0 - listfuncs = False - countcallers = False - timing = False - - for opt, val in opts: - if opt == "--help": - _usage(sys.stdout) - sys.exit(0) - - if opt == "--version": - sys.stdout.write("trace 2.0\n") - sys.exit(0) - - if opt == "-T" or opt == "--trackcalls": - countcallers = True - continue - - if opt == "-l" or opt == "--listfuncs": - listfuncs = True - continue - - if opt == "-g" or opt == "--timing": - timing = True - continue - - if opt == "-t" or opt == "--trace": - trace = 1 - continue - - if opt == "-c" or opt == "--count": - count = 1 - continue - - if opt == "-r" or opt == "--report": - report = 1 - continue - - if opt == "-R" or opt == "--no-report": - no_report = 1 - continue - - if opt == "-f" or opt == "--file": - counts_file = val - continue - - if opt == "-m" or opt == "--missing": - missing = 1 - continue - - if opt == "-C" or opt == "--coverdir": - coverdir = val - continue - - if opt == "-s" or opt == "--summary": - summary = 1 - continue - - if opt == "--ignore-module": - for mod in val.split(","): - ignore_modules.append(mod.strip()) - continue - - if opt == "--ignore-dir": - for s in val.split(os.pathsep): - s = os.path.expandvars(s) - # should I also call expanduser? (after all, could use $HOME) - - s = s.replace("$prefix", - os.path.join(sys.base_prefix, "lib", - "python" + sys.version[:3])) - s = s.replace("$exec_prefix", - os.path.join(sys.base_exec_prefix, "lib", - "python" + sys.version[:3])) - s = os.path.normpath(s) - ignore_dirs.append(s) - continue - - assert 0, "Should never get here" - - if listfuncs and (count or trace): - _err_exit("cannot specify both --listfuncs and (--trace or --count)") - - if not (count or trace or report or listfuncs or countcallers): - _err_exit("must specify one of --trace, --count, --report, " - "--listfuncs, or --trackcalls") - - if report and no_report: - _err_exit("cannot specify both --report and --no-report") - - if report and not counts_file: - _err_exit("--report requires a --file") - - if no_report and len(prog_argv) == 0: - _err_exit("missing name of file to run") - - # everything is ready - if report: - results = CoverageResults(infile=counts_file, outfile=counts_file) - results.write_results(missing, summary=summary, coverdir=coverdir) - else: - sys.argv = prog_argv - progname = prog_argv[0] - sys.path[0] = os.path.split(progname)[0] - - t = Trace(count, trace, countfuncs=listfuncs, - countcallers=countcallers, ignoremods=ignore_modules, - ignoredirs=ignore_dirs, infile=counts_file, - outfile=counts_file, timing=timing) - try: - with open(progname) as fp: - code = compile(fp.read(), progname, 'exec') - # try to emulate __main__ namespace as much as possible - globs = { - '__file__': progname, - '__name__': '__main__', - '__package__': None, - '__cached__': None, - } - t.runctx(code, globs, globs) - except OSError as err: - _err_exit("Cannot run file %r because: %s" % (sys.argv[0], err)) - except SystemExit: - pass + with open(opts.filename) as fp: + code = compile(fp.read(), opts.filename, 'exec') + # try to emulate __main__ namespace as much as possible + globs = { + '__file__': opts.filename, + '__name__': '__main__', + '__package__': None, + '__cached__': None, + } + t.runctx(code, globs, globs) + except OSError as err: + sys.exit("Cannot run file %r because: %s" % (sys.argv[0], err)) + except SystemExit: + pass - results = t.results() + results = t.results() - if not no_report: - results.write_results(missing, summary=summary, coverdir=coverdir) + if not opts.no_report: + results.write_results(opts.missing, opts.summary, opts.coverdir) if __name__=='__main__': main() -- cgit v0.12 From 4e280a6f9fe9b052912de1aa11205346f91bb279 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 13 Jan 2016 07:48:57 -0800 Subject: Add a NEWS entry for Issue #22642. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 54c70fc..08f057d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,9 @@ Core and Builtins Library ------- +- Issue #22642: Convert trace module option parsing mechanism to argparse. + Patch contributed by SilentGhost. + - Issue #24705: Fix sysconfig._parse_makefile not expanding ${} vars appearing before $() vars. -- cgit v0.12 From 86f7109dadea2c5c97c15e1878b862e769199d1d Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 14 Jan 2016 00:11:39 -0800 Subject: Issue #25822: Add docstrings to the fields of urllib.parse results. Patch contributed by Swati Jaiswal. --- Lib/urllib/parse.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++-- Misc/NEWS | 3 +++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 5e2155c..cecd0b9 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -224,8 +224,71 @@ class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes): from collections import namedtuple _DefragResultBase = namedtuple('DefragResult', 'url fragment') -_SplitResultBase = namedtuple('SplitResult', 'scheme netloc path query fragment') -_ParseResultBase = namedtuple('ParseResult', 'scheme netloc path params query fragment') +_SplitResultBase = namedtuple( + 'SplitResult', 'scheme netloc path query fragment') +_ParseResultBase = namedtuple( + 'ParseResult', 'scheme netloc path params query fragment') + +_DefragResultBase.__doc__ = """ +DefragResult(url, fragment) + +A 2-tuple that contains the url without fragment identifier and the fragment +identifier as a separate argument. +""" + +_DefragResultBase.url.__doc__ = """The URL with no fragment identifier.""" + +_DefragResultBase.fragment.__doc__ = """ +Fragment identifier separated from URL, that allows indirect identification of a +secondary resource by reference to a primary resource and additional identifying +information. +""" + +_SplitResultBase.__doc__ = """ +SplitResult(scheme, netloc, path, query, fragment) + +A 5-tuple that contains the different components of a URL. Similar to +ParseResult, but does not split params. +""" + +_SplitResultBase.scheme.__doc__ = """Specifies URL scheme for the request.""" + +_SplitResultBase.netloc.__doc__ = """ +Network location where the request is made to. +""" + +_SplitResultBase.path.__doc__ = """ +The hierarchical path, such as the path to a file to download. +""" + +_SplitResultBase.query.__doc__ = """ +The query component, that contains non-hierarchical data, that along with data +in path component, identifies a resource in the scope of URI's scheme and +network location. +""" + +_SplitResultBase.fragment.__doc__ = """ +Fragment identifier, that allows indirect identification of a secondary resource +by reference to a primary resource and additional identifying information. +""" + +_ParseResultBase.__doc__ = """ +ParseResult(scheme, netloc, path, params, query, fragment) + +A 6-tuple that contains components of a parsed URL. +""" + +_ParseResultBase.scheme.__doc__ = _SplitResultBase.scheme.__doc__ +_ParseResultBase.netloc.__doc__ = _SplitResultBase.netloc.__doc__ +_ParseResultBase.path.__doc__ = _SplitResultBase.path.__doc__ +_ParseResultBase.params.__doc__ = """ +Parameters for last path element used to dereference the URI in order to provide +access to perform some operation on the resource. +""" + +_ParseResultBase.query.__doc__ = _SplitResultBase.query.__doc__ +_ParseResultBase.fragment.__doc__ = _SplitResultBase.fragment.__doc__ + # For backwards compatibility, alias _NetlocResultMixinStr # ResultBase is no longer part of the documented API, but it is diff --git a/Misc/NEWS b/Misc/NEWS index 08f057d..c3a2574 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,9 @@ Core and Builtins Library ------- +- Issue #25822: Add docstrings to the fields of urllib.parse results. + Patch contributed by Swati Jaiswal. + - Issue #22642: Convert trace module option parsing mechanism to argparse. Patch contributed by SilentGhost. -- cgit v0.12 From 8df98483222d480db087860656d3e369f2338663 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 14 Jan 2016 13:26:43 +0000 Subject: Issue #25940: test_ssl is working again --- Lib/test/test_ssl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index b3aee62..f95cfba 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1343,7 +1343,6 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) -@unittest.skipIf(True, "temporarily disabled: see #25940") class NetworkedTests(unittest.TestCase): def test_connect(self): @@ -1712,7 +1711,6 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret - @unittest.skipIf(True, "temporarily disabled: see #25940") def test_handshake(self): with support.transient_internet(REMOTE_HOST): sock = socket.socket(socket.AF_INET) -- cgit v0.12 From 63b8505281f9c26f7304b3e00d658b429b862d5b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 15 Jan 2016 13:33:03 -0800 Subject: Issue #25791: Raise an ImportWarning when __spec__ or __package__ are not defined for a relative import. This is the start of work to try and clean up import semantics to rely more on a module's spec than on the myriad attributes that get set on a module. Thanks to Rose Ames for the patch. --- Doc/reference/import.rst | 3 +- Doc/whatsnew/3.6.rst | 11 +- Lib/importlib/_bootstrap.py | 6 + .../test_importlib/import_/test___package__.py | 40 +- Misc/NEWS | 3 + Python/import.c | 64 ++- Python/importlib.h | 559 +++++++++++---------- 7 files changed, 376 insertions(+), 310 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 2144c1f..a162851 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -554,7 +554,8 @@ the module. details. This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. + relative imports for main modules -- as defined in :pep:`366` -- + when ``__spec__`` is not defined. .. attribute:: __spec__ diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index f7dbbee..a5b9be2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -209,7 +209,12 @@ Deprecated features * The ``pyvenv`` script has been deprecated in favour of ``python3 -m venv``. This prevents confusion as to what Python interpreter ``pyvenv`` is connected to and thus what Python interpreter will be used by the virtual - environment. See :issue:`25154`. + environment. (Contributed by Brett Cannon in :issue:`25154`.) + +* When performing a relative import, falling back on ``__name__`` and + ``__path__`` from the calling module when ``__spec__`` or + ``__package__`` are not defined now raises an :exc:`ImportWarning`. + (Contributed by Rose Ames in :issue:`25791`.) Removed @@ -251,6 +256,10 @@ Changes in the Python API :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. +* When performing a relative import, ``__spec__.parent`` is used + is ``__spec__`` is defined instead of ``__package__``. + (Contributed by Rose Ames in :issue:`25791`.) + Changes in the C API -------------------- diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6f62bb3..9adcf7b 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1032,8 +1032,14 @@ def _calc___package__(globals): to represent that its proper value is unknown. """ + spec = globals.get('__spec__') + if spec is not None: + return spec.parent package = globals.get('__package__') if package is None: + _warnings.warn("can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + ImportWarning, stacklevel=3) package = globals['__name__'] if '__path__' not in globals: package = package.rpartition('.')[0] diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index c7d3a2a..08e41c9 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -33,31 +33,39 @@ class Using__package__: """ - def test_using___package__(self): - # [__package__] + def import_module(self, globals_): with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: with util.import_state(meta_path=[importer]): self.__import__('pkg.fake') module = self.__import__('', - globals={'__package__': 'pkg.fake'}, + globals=globals_, fromlist=['attr'], level=2) + return module + + def test_using___package__(self): + # [__package__] + module = self.import_module({'__package__': 'pkg.fake'}) self.assertEqual(module.__name__, 'pkg') - def test_using___name__(self, package_as_None=False): + def test_using___name__(self): # [__name__] - globals_ = {'__name__': 'pkg.fake', '__path__': []} - if package_as_None: - globals_['__package__'] = None - with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: - with util.import_state(meta_path=[importer]): - self.__import__('pkg.fake') - module = self.__import__('', globals= globals_, - fromlist=['attr'], level=2) - self.assertEqual(module.__name__, 'pkg') + module = self.import_module({'__name__': 'pkg.fake', '__path__': []}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_using___name__(self): + with self.assertWarns(ImportWarning): + self.import_module({'__name__': 'pkg.fake', '__path__': []}) def test_None_as___package__(self): # [None] - self.test_using___name__(package_as_None=True) + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + self.assertEqual(module.__name__, 'pkg') + + def test_prefers___spec__(self): + globals = {'__spec__': FakeSpec()} + with self.assertRaises(SystemError): + self.__import__('', globals, {}, ['relimport'], 1) def test_bad__package__(self): globals = {'__package__': ''} @@ -70,6 +78,10 @@ class Using__package__: self.__import__('', globals, {}, ['relimport'], 1) +class FakeSpec: + parent = '' + + class Using__package__PEP302(Using__package__): mock_modules = util.mock_modules diff --git a/Misc/NEWS b/Misc/NEWS index da53b95..f87703f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #25791: Trying to resolve a relative import without __spec__ or + __package__ defined now raises an ImportWarning + - Issue #25961: Disallowed null characters in the type name. - Issue #25973: Fix segfault when an invalid nonlocal statement binds a name diff --git a/Python/import.c b/Python/import.c index 325b936..c1071c0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1359,6 +1359,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *final_mod = NULL; PyObject *mod = NULL; PyObject *package = NULL; + PyObject *spec = NULL; PyObject *globals = NULL; PyObject *fromlist = NULL; PyInterpreterState *interp = PyThreadState_GET()->interp; @@ -1414,39 +1415,60 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, goto error; } else if (level > 0) { - package = _PyDict_GetItemId(globals, &PyId___package__); + spec = _PyDict_GetItemId(globals, &PyId___spec__); + if (spec != NULL) { + package = PyObject_GetAttrString(spec, "parent"); + } if (package != NULL && package != Py_None) { - Py_INCREF(package); if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "package must be a string"); + PyErr_SetString(PyExc_TypeError, + "__spec__.parent must be a string"); goto error; } } else { - package = _PyDict_GetItemId(globals, &PyId___name__); - if (package == NULL) { - PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); - goto error; - } - else if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + package = _PyDict_GetItemId(globals, &PyId___package__); + if (package != NULL && package != Py_None) { + Py_INCREF(package); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "package must be a string"); + goto error; + } } - Py_INCREF(package); - - if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { - PyObject *partition = NULL; - PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); - if (borrowed_dot == NULL) { + else { + if (PyErr_WarnEx(PyExc_ImportWarning, + "can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", 1) < 0) { goto error; } - partition = PyUnicode_RPartition(package, borrowed_dot); - Py_DECREF(package); - if (partition == NULL) { + + package = _PyDict_GetItemId(globals, &PyId___name__); + if (package == NULL) { + PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); goto error; } - package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); - Py_DECREF(partition); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + goto error; + } + + if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *partition = NULL; + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL) { + goto error; + } + partition = PyUnicode_RPartition(package, borrowed_dot); + Py_DECREF(package); + if (partition == NULL) { + goto error; + } + package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); + Py_DECREF(partition); + } } } diff --git a/Python/importlib.h b/Python/importlib.h index a4daf62..709c5fd 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1057,7 +1057,7 @@ const unsigned char _Py_M__importlib[] = { 100,101,114,115,32,100,101,102,105,110,105,110,103,32,101,120, 101,99,95,109,111,100,117,108,101,40,41,32,109,117,115,116, 32,97,108,115,111,32,100,101,102,105,110,101,32,99,114,101, - 97,116,101,95,109,111,100,117,108,101,40,41,90,10,115,116, + 97,116,101,95,109,111,100,117,108,101,40,41,218,10,115,116, 97,99,107,108,101,118,101,108,233,2,0,0,0,41,9,114, 4,0,0,0,114,99,0,0,0,114,138,0,0,0,218,9, 95,119,97,114,110,105,110,103,115,218,4,119,97,114,110,218, @@ -1067,7 +1067,7 @@ const unsigned char _Py_M__importlib[] = { 0,0,0,114,10,0,0,0,114,11,0,0,0,218,16,109, 111,100,117,108,101,95,102,114,111,109,95,115,112,101,99,58, 2,0,0,115,20,0,0,0,0,3,6,1,18,3,21,1, - 18,1,9,2,13,1,12,1,15,1,13,1,114,144,0,0, + 18,1,9,2,13,1,12,1,15,1,13,1,114,145,0,0, 0,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, 0,0,67,0,0,0,115,149,0,0,0,124,0,0,106,0, 0,100,1,0,107,8,0,114,21,0,100,2,0,110,6,0, @@ -1149,7 +1149,7 @@ const unsigned char _Py_M__importlib[] = { 0,1,1,1,89,110,1,0,88,124,1,0,83,41,7,78, 114,91,0,0,0,114,134,0,0,0,114,131,0,0,0,114, 121,0,0,0,114,33,0,0,0,114,95,0,0,0,41,13, - 114,99,0,0,0,114,146,0,0,0,114,15,0,0,0,114, + 114,99,0,0,0,114,147,0,0,0,114,15,0,0,0,114, 14,0,0,0,114,21,0,0,0,114,6,0,0,0,114,91, 0,0,0,114,96,0,0,0,114,1,0,0,0,114,134,0, 0,0,114,4,0,0,0,114,122,0,0,0,114,95,0,0, @@ -1159,7 +1159,7 @@ const unsigned char _Py_M__importlib[] = { 112,97,116,105,98,108,101,118,2,0,0,115,40,0,0,0, 0,4,19,2,16,1,24,1,3,1,16,1,13,1,5,1, 24,1,3,4,12,1,15,1,29,1,13,1,5,1,24,1, - 3,1,13,1,13,1,5,1,114,148,0,0,0,99,1,0, + 3,1,13,1,13,1,5,1,114,149,0,0,0,99,1,0, 0,0,0,0,0,0,2,0,0,0,11,0,0,0,67,0, 0,0,115,159,0,0,0,124,0,0,106,0,0,100,0,0, 107,9,0,114,43,0,116,1,0,124,0,0,106,0,0,100, @@ -1174,14 +1174,14 @@ const unsigned char _Py_M__importlib[] = { 0,106,7,0,25,83,41,4,78,114,139,0,0,0,122,14, 109,105,115,115,105,110,103,32,108,111,97,100,101,114,114,15, 0,0,0,41,11,114,99,0,0,0,114,4,0,0,0,114, - 148,0,0,0,114,144,0,0,0,114,102,0,0,0,114,110, + 149,0,0,0,114,145,0,0,0,114,102,0,0,0,114,110, 0,0,0,114,77,0,0,0,114,15,0,0,0,114,139,0, 0,0,114,14,0,0,0,114,21,0,0,0,41,2,114,88, 0,0,0,114,89,0,0,0,114,10,0,0,0,114,10,0, 0,0,114,11,0,0,0,218,14,95,108,111,97,100,95,117, 110,108,111,99,107,101,100,147,2,0,0,115,20,0,0,0, 0,2,15,2,18,1,10,2,12,1,13,1,15,1,15,1, - 24,3,23,5,114,149,0,0,0,99,1,0,0,0,0,0, + 24,3,23,5,114,150,0,0,0,99,1,0,0,0,0,0, 0,0,1,0,0,0,9,0,0,0,67,0,0,0,115,47, 0,0,0,116,0,0,106,1,0,131,0,0,1,116,2,0, 124,0,0,106,3,0,131,1,0,143,15,0,1,116,4,0, @@ -1198,8 +1198,8 @@ const unsigned char _Py_M__importlib[] = { 100,117,108,101,115,44,32,116,104,97,116,32,101,120,105,115, 116,105,110,103,32,109,111,100,117,108,101,32,103,101,116,115, 10,32,32,32,32,99,108,111,98,98,101,114,101,100,46,10, - 10,32,32,32,32,78,41,5,114,57,0,0,0,114,145,0, - 0,0,114,54,0,0,0,114,15,0,0,0,114,149,0,0, + 10,32,32,32,32,78,41,5,114,57,0,0,0,114,146,0, + 0,0,114,54,0,0,0,114,15,0,0,0,114,150,0,0, 0,41,1,114,88,0,0,0,114,10,0,0,0,114,10,0, 0,0,114,11,0,0,0,114,87,0,0,0,170,2,0,0, 115,6,0,0,0,0,9,10,1,16,1,114,87,0,0,0, @@ -1274,8 +1274,8 @@ const unsigned char _Py_M__importlib[] = { 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,102, 105,110,100,95,115,112,101,99,40,41,32,105,110,115,116,101, 97,100,46,10,10,32,32,32,32,32,32,32,32,78,41,2, - 114,154,0,0,0,114,99,0,0,0,41,4,114,151,0,0, - 0,114,78,0,0,0,114,152,0,0,0,114,88,0,0,0, + 114,155,0,0,0,114,99,0,0,0,41,4,114,152,0,0, + 0,114,78,0,0,0,114,153,0,0,0,114,88,0,0,0, 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, 11,102,105,110,100,95,109,111,100,117,108,101,213,2,0,0, 115,4,0,0,0,0,9,18,1,122,27,66,117,105,108,116, @@ -1315,7 +1315,7 @@ const unsigned char _Py_M__importlib[] = { 32,97,115,32,98,117,105,108,116,45,105,110,32,109,111,100, 117,108,101,115,32,100,111,32,110,111,116,32,104,97,118,101, 32,99,111,100,101,32,111,98,106,101,99,116,115,46,78,114, - 10,0,0,0,41,2,114,151,0,0,0,114,78,0,0,0, + 10,0,0,0,41,2,114,152,0,0,0,114,78,0,0,0, 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, 8,103,101,116,95,99,111,100,101,238,2,0,0,115,2,0, 0,0,0,4,122,24,66,117,105,108,116,105,110,73,109,112, @@ -1326,7 +1326,7 @@ const unsigned char _Py_M__importlib[] = { 117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,32, 100,111,32,110,111,116,32,104,97,118,101,32,115,111,117,114, 99,101,32,99,111,100,101,46,78,114,10,0,0,0,41,2, - 114,151,0,0,0,114,78,0,0,0,114,10,0,0,0,114, + 114,152,0,0,0,114,78,0,0,0,114,10,0,0,0,114, 10,0,0,0,114,11,0,0,0,218,10,103,101,116,95,115, 111,117,114,99,101,244,2,0,0,115,2,0,0,0,0,4, 122,26,66,117,105,108,116,105,110,73,109,112,111,114,116,101, @@ -1336,7 +1336,7 @@ const unsigned char _Py_M__importlib[] = { 116,117,114,110,32,70,97,108,115,101,32,97,115,32,98,117, 105,108,116,45,105,110,32,109,111,100,117,108,101,115,32,97, 114,101,32,110,101,118,101,114,32,112,97,99,107,97,103,101, - 115,46,70,114,10,0,0,0,41,2,114,151,0,0,0,114, + 115,46,70,114,10,0,0,0,41,2,114,152,0,0,0,114, 78,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, 0,0,0,114,109,0,0,0,250,2,0,0,115,2,0,0, 0,0,4,122,26,66,117,105,108,116,105,110,73,109,112,111, @@ -1344,14 +1344,14 @@ const unsigned char _Py_M__importlib[] = { 17,114,1,0,0,0,114,0,0,0,0,114,2,0,0,0, 114,3,0,0,0,218,12,115,116,97,116,105,99,109,101,116, 104,111,100,114,92,0,0,0,218,11,99,108,97,115,115,109, - 101,116,104,111,100,114,154,0,0,0,114,155,0,0,0,114, - 138,0,0,0,114,139,0,0,0,114,81,0,0,0,114,156, - 0,0,0,114,157,0,0,0,114,109,0,0,0,114,90,0, - 0,0,114,146,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,150,0,0,0, + 101,116,104,111,100,114,155,0,0,0,114,156,0,0,0,114, + 138,0,0,0,114,139,0,0,0,114,81,0,0,0,114,157, + 0,0,0,114,158,0,0,0,114,109,0,0,0,114,90,0, + 0,0,114,147,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,151,0,0,0, 186,2,0,0,115,30,0,0,0,12,7,6,2,18,9,3, 1,21,8,3,1,18,11,18,8,18,5,3,1,21,5,3, - 1,21,5,3,1,21,5,114,150,0,0,0,99,0,0,0, + 1,21,5,3,1,21,5,114,151,0,0,0,99,0,0,0, 0,0,0,0,0,0,0,0,0,5,0,0,0,64,0,0, 0,115,211,0,0,0,101,0,0,90,1,0,100,0,0,90, 2,0,100,1,0,90,3,0,101,4,0,100,2,0,100,3, @@ -1399,9 +1399,9 @@ const unsigned char _Py_M__importlib[] = { 124,0,0,100,1,0,100,2,0,131,2,1,83,100,0,0, 83,100,0,0,83,41,3,78,114,107,0,0,0,90,6,102, 114,111,122,101,110,41,3,114,57,0,0,0,114,82,0,0, - 0,114,85,0,0,0,41,4,114,151,0,0,0,114,78,0, - 0,0,114,152,0,0,0,114,153,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,154,0,0,0, + 0,114,85,0,0,0,41,4,114,152,0,0,0,114,78,0, + 0,0,114,153,0,0,0,114,154,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,155,0,0,0, 21,3,0,0,115,6,0,0,0,0,2,15,1,19,2,122, 24,70,114,111,122,101,110,73,109,112,111,114,116,101,114,46, 102,105,110,100,95,115,112,101,99,99,3,0,0,0,0,0, @@ -1414,9 +1414,9 @@ const unsigned char _Py_M__importlib[] = { 101,99,97,116,101,100,46,32,32,85,115,101,32,102,105,110, 100,95,115,112,101,99,40,41,32,105,110,115,116,101,97,100, 46,10,10,32,32,32,32,32,32,32,32,78,41,2,114,57, - 0,0,0,114,82,0,0,0,41,3,114,151,0,0,0,114, - 78,0,0,0,114,152,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,155,0,0,0,28,3,0, + 0,0,0,114,82,0,0,0,41,3,114,152,0,0,0,114, + 78,0,0,0,114,153,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,114,156,0,0,0,28,3,0, 0,115,2,0,0,0,0,7,122,26,70,114,111,122,101,110, 73,109,112,111,114,116,101,114,46,102,105,110,100,95,109,111, 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, @@ -1424,7 +1424,7 @@ const unsigned char _Py_M__importlib[] = { 0,83,41,2,122,42,85,115,101,32,100,101,102,97,117,108, 116,32,115,101,109,97,110,116,105,99,115,32,102,111,114,32, 109,111,100,117,108,101,32,99,114,101,97,116,105,111,110,46, - 78,114,10,0,0,0,41,2,114,151,0,0,0,114,88,0, + 78,114,10,0,0,0,41,2,114,152,0,0,0,114,88,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, 0,114,138,0,0,0,37,3,0,0,115,0,0,0,0,122, 28,70,114,111,122,101,110,73,109,112,111,114,116,101,114,46, @@ -1457,8 +1457,8 @@ const unsigned char _Py_M__importlib[] = { 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, 100,46,10,10,32,32,32,32,32,32,32,32,41,1,114,90, - 0,0,0,41,2,114,151,0,0,0,114,78,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,146, + 0,0,0,41,2,114,152,0,0,0,114,78,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,147, 0,0,0,50,3,0,0,115,2,0,0,0,0,7,122,26, 70,114,111,122,101,110,73,109,112,111,114,116,101,114,46,108, 111,97,100,95,109,111,100,117,108,101,99,2,0,0,0,0, @@ -1467,9 +1467,9 @@ const unsigned char _Py_M__importlib[] = { 83,41,1,122,45,82,101,116,117,114,110,32,116,104,101,32, 99,111,100,101,32,111,98,106,101,99,116,32,102,111,114,32, 116,104,101,32,102,114,111,122,101,110,32,109,111,100,117,108, - 101,46,41,2,114,57,0,0,0,114,162,0,0,0,41,2, - 114,151,0,0,0,114,78,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,114,156,0,0,0,59,3, + 101,46,41,2,114,57,0,0,0,114,163,0,0,0,41,2, + 114,152,0,0,0,114,78,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,114,157,0,0,0,59,3, 0,0,115,2,0,0,0,0,4,122,23,70,114,111,122,101, 110,73,109,112,111,114,116,101,114,46,103,101,116,95,99,111, 100,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, @@ -1478,8 +1478,8 @@ const unsigned char _Py_M__importlib[] = { 97,115,32,102,114,111,122,101,110,32,109,111,100,117,108,101, 115,32,100,111,32,110,111,116,32,104,97,118,101,32,115,111, 117,114,99,101,32,99,111,100,101,46,78,114,10,0,0,0, - 41,2,114,151,0,0,0,114,78,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,157,0,0,0, + 41,2,114,152,0,0,0,114,78,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,158,0,0,0, 65,3,0,0,115,2,0,0,0,0,4,122,25,70,114,111, 122,101,110,73,109,112,111,114,116,101,114,46,103,101,116,95, 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, @@ -1489,21 +1489,21 @@ const unsigned char _Py_M__importlib[] = { 116,104,101,32,102,114,111,122,101,110,32,109,111,100,117,108, 101,32,105,115,32,97,32,112,97,99,107,97,103,101,46,41, 2,114,57,0,0,0,90,17,105,115,95,102,114,111,122,101, - 110,95,112,97,99,107,97,103,101,41,2,114,151,0,0,0, + 110,95,112,97,99,107,97,103,101,41,2,114,152,0,0,0, 114,78,0,0,0,114,10,0,0,0,114,10,0,0,0,114, 11,0,0,0,114,109,0,0,0,71,3,0,0,115,2,0, 0,0,0,4,122,25,70,114,111,122,101,110,73,109,112,111, 114,116,101,114,46,105,115,95,112,97,99,107,97,103,101,41, 16,114,1,0,0,0,114,0,0,0,0,114,2,0,0,0, - 114,3,0,0,0,114,158,0,0,0,114,92,0,0,0,114, - 159,0,0,0,114,154,0,0,0,114,155,0,0,0,114,138, - 0,0,0,114,139,0,0,0,114,146,0,0,0,114,84,0, - 0,0,114,156,0,0,0,114,157,0,0,0,114,109,0,0, + 114,3,0,0,0,114,159,0,0,0,114,92,0,0,0,114, + 160,0,0,0,114,155,0,0,0,114,156,0,0,0,114,138, + 0,0,0,114,139,0,0,0,114,147,0,0,0,114,84,0, + 0,0,114,157,0,0,0,114,158,0,0,0,114,109,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,160,0,0,0,3,3,0,0,115,30, + 114,11,0,0,0,114,161,0,0,0,3,3,0,0,115,30, 0,0,0,12,7,6,2,18,9,3,1,21,6,3,1,18, 8,18,4,18,9,18,9,3,1,21,5,3,1,21,5,3, - 1,114,160,0,0,0,99,0,0,0,0,0,0,0,0,0, + 1,114,161,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,2,0,0,0,64,0,0,0,115,46,0,0,0, 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, @@ -1516,7 +1516,7 @@ const unsigned char _Py_M__importlib[] = { 14,0,0,0,116,0,0,106,1,0,131,0,0,1,100,1, 0,83,41,2,122,24,65,99,113,117,105,114,101,32,116,104, 101,32,105,109,112,111,114,116,32,108,111,99,107,46,78,41, - 2,114,57,0,0,0,114,145,0,0,0,41,1,114,19,0, + 2,114,57,0,0,0,114,146,0,0,0,41,1,114,19,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, 0,114,23,0,0,0,84,3,0,0,115,2,0,0,0,0, 2,122,28,95,73,109,112,111,114,116,76,111,99,107,67,111, @@ -1538,8 +1538,8 @@ const unsigned char _Py_M__importlib[] = { 0,0,0,114,0,0,0,0,114,2,0,0,0,114,3,0, 0,0,114,23,0,0,0,114,30,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,165,0,0,0,80,3,0,0,115,6,0,0,0,12,2, - 6,2,12,4,114,165,0,0,0,99,3,0,0,0,0,0, + 114,166,0,0,0,80,3,0,0,115,6,0,0,0,12,2, + 6,2,12,4,114,166,0,0,0,99,3,0,0,0,0,0, 0,0,5,0,0,0,4,0,0,0,67,0,0,0,115,88, 0,0,0,124,1,0,106,0,0,100,1,0,124,2,0,100, 2,0,24,131,2,0,125,3,0,116,1,0,124,3,0,131, @@ -1562,17 +1562,17 @@ const unsigned char _Py_M__importlib[] = { 0,0,0,114,10,0,0,0,114,11,0,0,0,218,13,95, 114,101,115,111,108,118,101,95,110,97,109,101,93,3,0,0, 115,10,0,0,0,0,2,22,1,18,1,12,1,10,1,114, - 171,0,0,0,99,3,0,0,0,0,0,0,0,4,0,0, + 172,0,0,0,99,3,0,0,0,0,0,0,0,4,0,0, 0,3,0,0,0,67,0,0,0,115,47,0,0,0,124,0, 0,106,0,0,124,1,0,124,2,0,131,2,0,125,3,0, 124,3,0,100,0,0,107,8,0,114,34,0,100,0,0,83, 116,1,0,124,1,0,124,3,0,131,2,0,83,41,1,78, - 41,2,114,155,0,0,0,114,85,0,0,0,41,4,218,6, - 102,105,110,100,101,114,114,15,0,0,0,114,152,0,0,0, + 41,2,114,156,0,0,0,114,85,0,0,0,41,4,218,6, + 102,105,110,100,101,114,114,15,0,0,0,114,153,0,0,0, 114,99,0,0,0,114,10,0,0,0,114,10,0,0,0,114, 11,0,0,0,218,17,95,102,105,110,100,95,115,112,101,99, 95,108,101,103,97,99,121,102,3,0,0,115,8,0,0,0, - 0,3,18,1,12,1,4,1,114,173,0,0,0,99,3,0, + 0,3,18,1,12,1,4,1,114,174,0,0,0,99,3,0, 0,0,0,0,0,0,9,0,0,0,27,0,0,0,67,0, 0,0,115,42,1,0,0,116,0,0,106,1,0,100,1,0, 107,9,0,114,41,0,116,0,0,106,1,0,12,114,41,0, @@ -1597,19 +1597,19 @@ const unsigned char _Py_M__importlib[] = { 108,101,39,115,32,108,111,97,100,101,114,46,78,122,22,115, 121,115,46,109,101,116,97,95,112,97,116,104,32,105,115,32, 101,109,112,116,121,41,11,114,14,0,0,0,218,9,109,101, - 116,97,95,112,97,116,104,114,141,0,0,0,114,142,0,0, + 116,97,95,112,97,116,104,114,142,0,0,0,114,143,0,0, 0,218,13,73,109,112,111,114,116,87,97,114,110,105,110,103, - 114,21,0,0,0,114,165,0,0,0,114,154,0,0,0,114, - 96,0,0,0,114,173,0,0,0,114,95,0,0,0,41,9, - 114,15,0,0,0,114,152,0,0,0,114,153,0,0,0,90, - 9,105,115,95,114,101,108,111,97,100,114,172,0,0,0,114, - 154,0,0,0,114,88,0,0,0,114,89,0,0,0,114,95, + 114,21,0,0,0,114,166,0,0,0,114,155,0,0,0,114, + 96,0,0,0,114,174,0,0,0,114,95,0,0,0,41,9, + 114,15,0,0,0,114,153,0,0,0,114,154,0,0,0,90, + 9,105,115,95,114,101,108,111,97,100,114,173,0,0,0,114, + 155,0,0,0,114,88,0,0,0,114,89,0,0,0,114,95, 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, 0,0,218,10,95,102,105,110,100,95,115,112,101,99,111,3, 0,0,115,48,0,0,0,0,2,25,1,16,4,15,1,16, 1,10,1,3,1,13,1,13,1,18,1,12,1,8,2,25, 1,12,2,22,1,13,1,3,1,13,1,13,4,9,2,12, - 1,4,2,7,2,8,2,114,176,0,0,0,99,3,0,0, + 1,4,2,7,2,8,2,114,177,0,0,0,99,3,0,0, 0,0,0,0,0,4,0,0,0,4,0,0,0,67,0,0, 0,115,179,0,0,0,116,0,0,124,0,0,116,1,0,131, 2,0,115,42,0,116,2,0,100,1,0,106,3,0,116,4, @@ -1638,14 +1638,14 @@ const unsigned char _Py_M__importlib[] = { 101,32,110,97,109,101,78,41,9,218,10,105,115,105,110,115, 116,97,110,99,101,218,3,115,116,114,218,9,84,121,112,101, 69,114,114,111,114,114,50,0,0,0,114,13,0,0,0,114, - 168,0,0,0,114,14,0,0,0,114,21,0,0,0,218,11, + 169,0,0,0,114,14,0,0,0,114,21,0,0,0,218,11, 83,121,115,116,101,109,69,114,114,111,114,41,4,114,15,0, - 0,0,114,169,0,0,0,114,170,0,0,0,114,147,0,0, + 0,0,114,170,0,0,0,114,171,0,0,0,114,148,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, 218,13,95,115,97,110,105,116,121,95,99,104,101,99,107,151, 3,0,0,115,24,0,0,0,0,2,15,1,27,1,12,1, 12,1,6,1,15,1,15,1,15,1,6,2,21,1,19,1, - 114,181,0,0,0,122,16,78,111,32,109,111,100,117,108,101, + 114,182,0,0,0,122,16,78,111,32,109,111,100,117,108,101, 32,110,97,109,101,100,32,122,4,123,33,114,125,99,2,0, 0,0,0,0,0,0,8,0,0,0,12,0,0,0,67,0, 0,0,115,40,1,0,0,100,0,0,125,2,0,124,0,0, @@ -1669,21 +1669,21 @@ const unsigned char _Py_M__importlib[] = { 100,5,0,25,124,7,0,131,3,0,1,124,7,0,83,41, 6,78,114,121,0,0,0,114,33,0,0,0,122,23,59,32, 123,33,114,125,32,105,115,32,110,111,116,32,97,32,112,97, - 99,107,97,103,101,114,15,0,0,0,114,140,0,0,0,41, + 99,107,97,103,101,114,15,0,0,0,114,141,0,0,0,41, 12,114,122,0,0,0,114,14,0,0,0,114,21,0,0,0, 114,65,0,0,0,114,131,0,0,0,114,96,0,0,0,218, 8,95,69,82,82,95,77,83,71,114,50,0,0,0,114,77, - 0,0,0,114,176,0,0,0,114,149,0,0,0,114,5,0, + 0,0,0,114,177,0,0,0,114,150,0,0,0,114,5,0, 0,0,41,8,114,15,0,0,0,218,7,105,109,112,111,114, - 116,95,114,152,0,0,0,114,123,0,0,0,90,13,112,97, - 114,101,110,116,95,109,111,100,117,108,101,114,147,0,0,0, + 116,95,114,153,0,0,0,114,123,0,0,0,90,13,112,97, + 114,101,110,116,95,109,111,100,117,108,101,114,148,0,0,0, 114,88,0,0,0,114,89,0,0,0,114,10,0,0,0,114, 10,0,0,0,114,11,0,0,0,218,23,95,102,105,110,100, 95,97,110,100,95,108,111,97,100,95,117,110,108,111,99,107, 101,100,171,3,0,0,115,42,0,0,0,0,1,6,1,19, 1,6,1,15,1,13,2,15,1,11,1,13,1,3,1,13, 1,13,1,22,1,26,1,15,1,12,1,30,2,12,1,6, - 2,13,1,29,1,114,184,0,0,0,99,2,0,0,0,0, + 2,13,1,29,1,114,185,0,0,0,99,2,0,0,0,0, 0,0,0,2,0,0,0,10,0,0,0,67,0,0,0,115, 37,0,0,0,116,0,0,124,0,0,131,1,0,143,18,0, 1,116,1,0,124,0,0,124,1,0,131,2,0,83,87,100, @@ -1691,11 +1691,11 @@ const unsigned char _Py_M__importlib[] = { 100,32,97,110,100,32,108,111,97,100,32,116,104,101,32,109, 111,100,117,108,101,44,32,97,110,100,32,114,101,108,101,97, 115,101,32,116,104,101,32,105,109,112,111,114,116,32,108,111, - 99,107,46,78,41,2,114,54,0,0,0,114,184,0,0,0, - 41,2,114,15,0,0,0,114,183,0,0,0,114,10,0,0, + 99,107,46,78,41,2,114,54,0,0,0,114,185,0,0,0, + 41,2,114,15,0,0,0,114,184,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,11,0,0,0,218,14,95,102,105, 110,100,95,97,110,100,95,108,111,97,100,198,3,0,0,115, - 4,0,0,0,0,2,13,1,114,185,0,0,0,114,33,0, + 4,0,0,0,0,2,13,1,114,186,0,0,0,114,33,0, 0,0,99,3,0,0,0,0,0,0,0,5,0,0,0,4, 0,0,0,67,0,0,0,115,166,0,0,0,116,0,0,124, 0,0,124,1,0,124,2,0,131,3,0,1,124,2,0,100, @@ -1731,16 +1731,16 @@ const unsigned char _Py_M__importlib[] = { 78,122,40,105,109,112,111,114,116,32,111,102,32,123,125,32, 104,97,108,116,101,100,59,32,78,111,110,101,32,105,110,32, 115,121,115,46,109,111,100,117,108,101,115,114,15,0,0,0, - 41,12,114,181,0,0,0,114,171,0,0,0,114,57,0,0, - 0,114,145,0,0,0,114,14,0,0,0,114,21,0,0,0, - 114,185,0,0,0,218,11,95,103,99,100,95,105,109,112,111, + 41,12,114,182,0,0,0,114,172,0,0,0,114,57,0,0, + 0,114,146,0,0,0,114,14,0,0,0,114,21,0,0,0, + 114,186,0,0,0,218,11,95,103,99,100,95,105,109,112,111, 114,116,114,58,0,0,0,114,50,0,0,0,114,77,0,0, - 0,114,63,0,0,0,41,5,114,15,0,0,0,114,169,0, - 0,0,114,170,0,0,0,114,89,0,0,0,114,74,0,0, + 0,114,63,0,0,0,41,5,114,15,0,0,0,114,170,0, + 0,0,114,171,0,0,0,114,89,0,0,0,114,74,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,186,0,0,0,204,3,0,0,115,28,0,0,0,0,9, + 114,187,0,0,0,204,3,0,0,115,28,0,0,0,0,9, 16,1,12,1,18,1,10,1,15,1,13,1,13,1,12,1, - 10,1,6,1,9,1,18,1,10,1,114,186,0,0,0,99, + 10,1,6,1,9,1,18,1,10,1,114,187,0,0,0,99, 3,0,0,0,0,0,0,0,6,0,0,0,17,0,0,0, 67,0,0,0,115,239,0,0,0,116,0,0,124,0,0,100, 1,0,131,2,0,114,235,0,100,2,0,124,1,0,107,6, @@ -1776,213 +1776,226 @@ const unsigned char _Py_M__importlib[] = { 1,42,218,7,95,95,97,108,108,95,95,122,5,123,125,46, 123,125,78,41,13,114,4,0,0,0,114,130,0,0,0,218, 6,114,101,109,111,118,101,218,6,101,120,116,101,110,100,114, - 188,0,0,0,114,50,0,0,0,114,1,0,0,0,114,65, - 0,0,0,114,77,0,0,0,114,178,0,0,0,114,71,0, + 189,0,0,0,114,50,0,0,0,114,1,0,0,0,114,65, + 0,0,0,114,77,0,0,0,114,179,0,0,0,114,71,0, 0,0,218,15,95,69,82,82,95,77,83,71,95,80,82,69, 70,73,88,114,15,0,0,0,41,6,114,89,0,0,0,218, - 8,102,114,111,109,108,105,115,116,114,183,0,0,0,218,1, + 8,102,114,111,109,108,105,115,116,114,184,0,0,0,218,1, 120,90,9,102,114,111,109,95,110,97,109,101,90,3,101,120, 99,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, 218,16,95,104,97,110,100,108,101,95,102,114,111,109,108,105, 115,116,228,3,0,0,115,34,0,0,0,0,10,15,1,12, 1,12,1,13,1,15,1,16,1,13,1,15,1,21,1,3, - 1,17,1,18,4,21,1,15,1,3,1,26,1,114,194,0, - 0,0,99,1,0,0,0,0,0,0,0,2,0,0,0,2, - 0,0,0,67,0,0,0,115,72,0,0,0,124,0,0,106, + 1,17,1,18,4,21,1,15,1,3,1,26,1,114,195,0, + 0,0,99,1,0,0,0,0,0,0,0,3,0,0,0,5, + 0,0,0,67,0,0,0,115,128,0,0,0,124,0,0,106, 0,0,100,1,0,131,1,0,125,1,0,124,1,0,100,2, - 0,107,8,0,114,68,0,124,0,0,100,3,0,25,125,1, - 0,100,4,0,124,0,0,107,7,0,114,68,0,124,1,0, - 106,1,0,100,5,0,131,1,0,100,6,0,25,125,1,0, - 124,1,0,83,41,7,122,167,67,97,108,99,117,108,97,116, - 101,32,119,104,97,116,32,95,95,112,97,99,107,97,103,101, - 95,95,32,115,104,111,117,108,100,32,98,101,46,10,10,32, - 32,32,32,95,95,112,97,99,107,97,103,101,95,95,32,105, - 115,32,110,111,116,32,103,117,97,114,97,110,116,101,101,100, - 32,116,111,32,98,101,32,100,101,102,105,110,101,100,32,111, - 114,32,99,111,117,108,100,32,98,101,32,115,101,116,32,116, - 111,32,78,111,110,101,10,32,32,32,32,116,111,32,114,101, - 112,114,101,115,101,110,116,32,116,104,97,116,32,105,116,115, - 32,112,114,111,112,101,114,32,118,97,108,117,101,32,105,115, - 32,117,110,107,110,111,119,110,46,10,10,32,32,32,32,114, - 134,0,0,0,78,114,1,0,0,0,114,131,0,0,0,114, - 121,0,0,0,114,33,0,0,0,41,2,114,42,0,0,0, - 114,122,0,0,0,41,2,218,7,103,108,111,98,97,108,115, - 114,169,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,17,95,99,97,108,99,95,95,95,112,97, - 99,107,97,103,101,95,95,4,4,0,0,115,12,0,0,0, - 0,7,15,1,12,1,10,1,12,1,19,1,114,196,0,0, - 0,99,5,0,0,0,0,0,0,0,9,0,0,0,5,0, - 0,0,67,0,0,0,115,227,0,0,0,124,4,0,100,1, - 0,107,2,0,114,27,0,116,0,0,124,0,0,131,1,0, - 125,5,0,110,54,0,124,1,0,100,2,0,107,9,0,114, - 45,0,124,1,0,110,3,0,105,0,0,125,6,0,116,1, - 0,124,6,0,131,1,0,125,7,0,116,0,0,124,0,0, - 124,7,0,124,4,0,131,3,0,125,5,0,124,3,0,115, - 207,0,124,4,0,100,1,0,107,2,0,114,122,0,116,0, - 0,124,0,0,106,2,0,100,3,0,131,1,0,100,1,0, - 25,131,1,0,83,124,0,0,115,132,0,124,5,0,83,116, - 3,0,124,0,0,131,1,0,116,3,0,124,0,0,106,2, - 0,100,3,0,131,1,0,100,1,0,25,131,1,0,24,125, - 8,0,116,4,0,106,5,0,124,5,0,106,6,0,100,2, - 0,116,3,0,124,5,0,106,6,0,131,1,0,124,8,0, - 24,133,2,0,25,25,83,110,16,0,116,7,0,124,5,0, - 124,3,0,116,0,0,131,3,0,83,100,2,0,83,41,4, - 97,214,1,0,0,73,109,112,111,114,116,32,97,32,109,111, - 100,117,108,101,46,10,10,32,32,32,32,84,104,101,32,39, - 103,108,111,98,97,108,115,39,32,97,114,103,117,109,101,110, - 116,32,105,115,32,117,115,101,100,32,116,111,32,105,110,102, - 101,114,32,119,104,101,114,101,32,116,104,101,32,105,109,112, - 111,114,116,32,105,115,32,111,99,99,117,114,105,110,103,32, - 102,114,111,109,10,32,32,32,32,116,111,32,104,97,110,100, - 108,101,32,114,101,108,97,116,105,118,101,32,105,109,112,111, - 114,116,115,46,32,84,104,101,32,39,108,111,99,97,108,115, - 39,32,97,114,103,117,109,101,110,116,32,105,115,32,105,103, - 110,111,114,101,100,46,32,84,104,101,10,32,32,32,32,39, - 102,114,111,109,108,105,115,116,39,32,97,114,103,117,109,101, - 110,116,32,115,112,101,99,105,102,105,101,115,32,119,104,97, - 116,32,115,104,111,117,108,100,32,101,120,105,115,116,32,97, - 115,32,97,116,116,114,105,98,117,116,101,115,32,111,110,32, - 116,104,101,32,109,111,100,117,108,101,10,32,32,32,32,98, - 101,105,110,103,32,105,109,112,111,114,116,101,100,32,40,101, - 46,103,46,32,96,96,102,114,111,109,32,109,111,100,117,108, - 101,32,105,109,112,111,114,116,32,60,102,114,111,109,108,105, - 115,116,62,96,96,41,46,32,32,84,104,101,32,39,108,101, - 118,101,108,39,10,32,32,32,32,97,114,103,117,109,101,110, - 116,32,114,101,112,114,101,115,101,110,116,115,32,116,104,101, - 32,112,97,99,107,97,103,101,32,108,111,99,97,116,105,111, - 110,32,116,111,32,105,109,112,111,114,116,32,102,114,111,109, - 32,105,110,32,97,32,114,101,108,97,116,105,118,101,10,32, - 32,32,32,105,109,112,111,114,116,32,40,101,46,103,46,32, - 96,96,102,114,111,109,32,46,46,112,107,103,32,105,109,112, - 111,114,116,32,109,111,100,96,96,32,119,111,117,108,100,32, - 104,97,118,101,32,97,32,39,108,101,118,101,108,39,32,111, - 102,32,50,41,46,10,10,32,32,32,32,114,33,0,0,0, - 78,114,121,0,0,0,41,8,114,186,0,0,0,114,196,0, - 0,0,218,9,112,97,114,116,105,116,105,111,110,114,167,0, - 0,0,114,14,0,0,0,114,21,0,0,0,114,1,0,0, - 0,114,194,0,0,0,41,9,114,15,0,0,0,114,195,0, - 0,0,218,6,108,111,99,97,108,115,114,192,0,0,0,114, - 170,0,0,0,114,89,0,0,0,90,8,103,108,111,98,97, - 108,115,95,114,169,0,0,0,90,7,99,117,116,95,111,102, - 102,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,10,95,95,105,109,112,111,114,116,95,95,19,4,0,0, - 115,26,0,0,0,0,11,12,1,15,2,24,1,12,1,18, - 1,6,3,12,1,23,1,6,1,4,4,35,3,40,2,114, - 199,0,0,0,99,1,0,0,0,0,0,0,0,2,0,0, - 0,3,0,0,0,67,0,0,0,115,53,0,0,0,116,0, - 0,106,1,0,124,0,0,131,1,0,125,1,0,124,1,0, - 100,0,0,107,8,0,114,43,0,116,2,0,100,1,0,124, - 0,0,23,131,1,0,130,1,0,116,3,0,124,1,0,131, - 1,0,83,41,2,78,122,25,110,111,32,98,117,105,108,116, - 45,105,110,32,109,111,100,117,108,101,32,110,97,109,101,100, - 32,41,4,114,150,0,0,0,114,154,0,0,0,114,77,0, - 0,0,114,149,0,0,0,41,2,114,15,0,0,0,114,88, - 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,218,18,95,98,117,105,108,116,105,110,95,102,114,111, - 109,95,110,97,109,101,54,4,0,0,115,8,0,0,0,0, - 1,15,1,12,1,16,1,114,200,0,0,0,99,2,0,0, - 0,0,0,0,0,12,0,0,0,12,0,0,0,67,0,0, - 0,115,74,1,0,0,124,1,0,97,0,0,124,0,0,97, - 1,0,116,2,0,116,1,0,131,1,0,125,2,0,120,123, - 0,116,1,0,106,3,0,106,4,0,131,0,0,68,93,106, - 0,92,2,0,125,3,0,125,4,0,116,5,0,124,4,0, - 124,2,0,131,2,0,114,40,0,124,3,0,116,1,0,106, - 6,0,107,6,0,114,91,0,116,7,0,125,5,0,110,27, - 0,116,0,0,106,8,0,124,3,0,131,1,0,114,40,0, - 116,9,0,125,5,0,110,3,0,113,40,0,116,10,0,124, - 4,0,124,5,0,131,2,0,125,6,0,116,11,0,124,6, - 0,124,4,0,131,2,0,1,113,40,0,87,116,1,0,106, - 3,0,116,12,0,25,125,7,0,120,73,0,100,5,0,68, - 93,65,0,125,8,0,124,8,0,116,1,0,106,3,0,107, - 7,0,114,206,0,116,13,0,124,8,0,131,1,0,125,9, - 0,110,13,0,116,1,0,106,3,0,124,8,0,25,125,9, - 0,116,14,0,124,7,0,124,8,0,124,9,0,131,3,0, - 1,113,170,0,87,121,16,0,116,13,0,100,2,0,131,1, - 0,125,10,0,87,110,24,0,4,116,15,0,107,10,0,114, - 25,1,1,1,1,100,3,0,125,10,0,89,110,1,0,88, - 116,14,0,124,7,0,100,2,0,124,10,0,131,3,0,1, - 116,13,0,100,4,0,131,1,0,125,11,0,116,14,0,124, - 7,0,100,4,0,124,11,0,131,3,0,1,100,3,0,83, - 41,6,122,250,83,101,116,117,112,32,105,109,112,111,114,116, - 108,105,98,32,98,121,32,105,109,112,111,114,116,105,110,103, - 32,110,101,101,100,101,100,32,98,117,105,108,116,45,105,110, - 32,109,111,100,117,108,101,115,32,97,110,100,32,105,110,106, - 101,99,116,105,110,103,32,116,104,101,109,10,32,32,32,32, - 105,110,116,111,32,116,104,101,32,103,108,111,98,97,108,32, - 110,97,109,101,115,112,97,99,101,46,10,10,32,32,32,32, - 65,115,32,115,121,115,32,105,115,32,110,101,101,100,101,100, - 32,102,111,114,32,115,121,115,46,109,111,100,117,108,101,115, - 32,97,99,99,101,115,115,32,97,110,100,32,95,105,109,112, - 32,105,115,32,110,101,101,100,101,100,32,116,111,32,108,111, - 97,100,32,98,117,105,108,116,45,105,110,10,32,32,32,32, - 109,111,100,117,108,101,115,44,32,116,104,111,115,101,32,116, - 119,111,32,109,111,100,117,108,101,115,32,109,117,115,116,32, - 98,101,32,101,120,112,108,105,99,105,116,108,121,32,112,97, - 115,115,101,100,32,105,110,46,10,10,32,32,32,32,114,141, - 0,0,0,114,34,0,0,0,78,114,62,0,0,0,41,1, - 122,9,95,119,97,114,110,105,110,103,115,41,16,114,57,0, - 0,0,114,14,0,0,0,114,13,0,0,0,114,21,0,0, - 0,218,5,105,116,101,109,115,114,177,0,0,0,114,76,0, - 0,0,114,150,0,0,0,114,82,0,0,0,114,160,0,0, - 0,114,132,0,0,0,114,137,0,0,0,114,1,0,0,0, - 114,200,0,0,0,114,5,0,0,0,114,77,0,0,0,41, - 12,218,10,115,121,115,95,109,111,100,117,108,101,218,11,95, - 105,109,112,95,109,111,100,117,108,101,90,11,109,111,100,117, - 108,101,95,116,121,112,101,114,15,0,0,0,114,89,0,0, - 0,114,99,0,0,0,114,88,0,0,0,90,11,115,101,108, - 102,95,109,111,100,117,108,101,90,12,98,117,105,108,116,105, - 110,95,110,97,109,101,90,14,98,117,105,108,116,105,110,95, - 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, - 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, - 111,100,117,108,101,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,6,95,115,101,116,117,112,61,4,0,0, - 115,50,0,0,0,0,9,6,1,6,3,12,1,28,1,15, - 1,15,1,9,1,15,1,9,2,3,1,15,1,17,3,13, - 1,13,1,15,1,15,2,13,1,20,3,3,1,16,1,13, - 2,11,1,16,3,12,1,114,204,0,0,0,99,2,0,0, - 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, - 0,115,87,0,0,0,116,0,0,124,0,0,124,1,0,131, - 2,0,1,116,1,0,106,2,0,106,3,0,116,4,0,131, - 1,0,1,116,1,0,106,2,0,106,3,0,116,5,0,131, - 1,0,1,100,1,0,100,2,0,108,6,0,125,2,0,124, - 2,0,97,7,0,124,2,0,106,8,0,116,1,0,106,9, - 0,116,10,0,25,131,1,0,1,100,2,0,83,41,3,122, - 50,73,110,115,116,97,108,108,32,105,109,112,111,114,116,108, - 105,98,32,97,115,32,116,104,101,32,105,109,112,108,101,109, - 101,110,116,97,116,105,111,110,32,111,102,32,105,109,112,111, - 114,116,46,114,33,0,0,0,78,41,11,114,204,0,0,0, - 114,14,0,0,0,114,174,0,0,0,114,113,0,0,0,114, - 150,0,0,0,114,160,0,0,0,218,26,95,102,114,111,122, - 101,110,95,105,109,112,111,114,116,108,105,98,95,101,120,116, - 101,114,110,97,108,114,119,0,0,0,218,8,95,105,110,115, - 116,97,108,108,114,21,0,0,0,114,1,0,0,0,41,3, - 114,202,0,0,0,114,203,0,0,0,114,205,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,206, - 0,0,0,108,4,0,0,115,12,0,0,0,0,2,13,2, - 16,1,16,3,12,1,6,1,114,206,0,0,0,41,51,114, - 3,0,0,0,114,119,0,0,0,114,12,0,0,0,114,16, - 0,0,0,114,17,0,0,0,114,59,0,0,0,114,41,0, - 0,0,114,48,0,0,0,114,31,0,0,0,114,32,0,0, - 0,114,53,0,0,0,114,54,0,0,0,114,56,0,0,0, - 114,63,0,0,0,114,65,0,0,0,114,75,0,0,0,114, - 81,0,0,0,114,84,0,0,0,114,90,0,0,0,114,101, - 0,0,0,114,102,0,0,0,114,106,0,0,0,114,85,0, - 0,0,218,6,111,98,106,101,99,116,90,9,95,80,79,80, - 85,76,65,84,69,114,132,0,0,0,114,137,0,0,0,114, - 144,0,0,0,114,97,0,0,0,114,86,0,0,0,114,148, - 0,0,0,114,149,0,0,0,114,87,0,0,0,114,150,0, - 0,0,114,160,0,0,0,114,165,0,0,0,114,171,0,0, - 0,114,173,0,0,0,114,176,0,0,0,114,181,0,0,0, - 114,191,0,0,0,114,182,0,0,0,114,184,0,0,0,114, - 185,0,0,0,114,186,0,0,0,114,194,0,0,0,114,196, - 0,0,0,114,199,0,0,0,114,200,0,0,0,114,204,0, - 0,0,114,206,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,218,8,60,109,111, - 100,117,108,101,62,8,0,0,0,115,96,0,0,0,6,17, - 6,2,12,8,12,4,19,20,6,2,6,3,22,4,19,68, - 19,21,19,19,12,19,12,19,12,11,18,8,12,11,12,12, - 12,16,12,36,19,27,19,101,24,26,9,3,18,45,18,60, - 12,18,12,17,12,25,12,29,12,23,12,16,19,73,19,77, - 19,13,12,9,12,9,15,40,12,17,6,1,10,2,12,27, - 12,6,18,24,12,32,12,15,24,35,12,7,12,47, + 0,107,9,0,114,34,0,124,1,0,106,1,0,83,124,0, + 0,106,0,0,100,3,0,131,1,0,125,2,0,124,2,0, + 100,2,0,107,8,0,114,124,0,116,2,0,106,3,0,100, + 4,0,116,4,0,100,5,0,100,6,0,131,2,1,1,124, + 0,0,100,7,0,25,125,2,0,100,8,0,124,0,0,107, + 7,0,114,124,0,124,2,0,106,5,0,100,9,0,131,1, + 0,100,10,0,25,125,2,0,124,2,0,83,41,11,122,167, + 67,97,108,99,117,108,97,116,101,32,119,104,97,116,32,95, + 95,112,97,99,107,97,103,101,95,95,32,115,104,111,117,108, + 100,32,98,101,46,10,10,32,32,32,32,95,95,112,97,99, + 107,97,103,101,95,95,32,105,115,32,110,111,116,32,103,117, + 97,114,97,110,116,101,101,100,32,116,111,32,98,101,32,100, + 101,102,105,110,101,100,32,111,114,32,99,111,117,108,100,32, + 98,101,32,115,101,116,32,116,111,32,78,111,110,101,10,32, + 32,32,32,116,111,32,114,101,112,114,101,115,101,110,116,32, + 116,104,97,116,32,105,116,115,32,112,114,111,112,101,114,32, + 118,97,108,117,101,32,105,115,32,117,110,107,110,111,119,110, + 46,10,10,32,32,32,32,114,95,0,0,0,78,114,134,0, + 0,0,122,89,99,97,110,39,116,32,114,101,115,111,108,118, + 101,32,112,97,99,107,97,103,101,32,102,114,111,109,32,95, + 95,115,112,101,99,95,95,32,111,114,32,95,95,112,97,99, + 107,97,103,101,95,95,44,32,102,97,108,108,105,110,103,32, + 98,97,99,107,32,111,110,32,95,95,110,97,109,101,95,95, + 32,97,110,100,32,95,95,112,97,116,104,95,95,114,140,0, + 0,0,233,3,0,0,0,114,1,0,0,0,114,131,0,0, + 0,114,121,0,0,0,114,33,0,0,0,41,6,114,42,0, + 0,0,114,123,0,0,0,114,142,0,0,0,114,143,0,0, + 0,114,176,0,0,0,114,122,0,0,0,41,3,218,7,103, + 108,111,98,97,108,115,114,88,0,0,0,114,170,0,0,0, + 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, + 17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,101, + 95,95,4,4,0,0,115,22,0,0,0,0,7,15,1,12, + 1,7,1,15,1,12,1,9,2,13,1,10,1,12,1,19, + 1,114,198,0,0,0,99,5,0,0,0,0,0,0,0,9, + 0,0,0,5,0,0,0,67,0,0,0,115,227,0,0,0, + 124,4,0,100,1,0,107,2,0,114,27,0,116,0,0,124, + 0,0,131,1,0,125,5,0,110,54,0,124,1,0,100,2, + 0,107,9,0,114,45,0,124,1,0,110,3,0,105,0,0, + 125,6,0,116,1,0,124,6,0,131,1,0,125,7,0,116, + 0,0,124,0,0,124,7,0,124,4,0,131,3,0,125,5, + 0,124,3,0,115,207,0,124,4,0,100,1,0,107,2,0, + 114,122,0,116,0,0,124,0,0,106,2,0,100,3,0,131, + 1,0,100,1,0,25,131,1,0,83,124,0,0,115,132,0, + 124,5,0,83,116,3,0,124,0,0,131,1,0,116,3,0, + 124,0,0,106,2,0,100,3,0,131,1,0,100,1,0,25, + 131,1,0,24,125,8,0,116,4,0,106,5,0,124,5,0, + 106,6,0,100,2,0,116,3,0,124,5,0,106,6,0,131, + 1,0,124,8,0,24,133,2,0,25,25,83,110,16,0,116, + 7,0,124,5,0,124,3,0,116,0,0,131,3,0,83,100, + 2,0,83,41,4,97,214,1,0,0,73,109,112,111,114,116, + 32,97,32,109,111,100,117,108,101,46,10,10,32,32,32,32, + 84,104,101,32,39,103,108,111,98,97,108,115,39,32,97,114, + 103,117,109,101,110,116,32,105,115,32,117,115,101,100,32,116, + 111,32,105,110,102,101,114,32,119,104,101,114,101,32,116,104, + 101,32,105,109,112,111,114,116,32,105,115,32,111,99,99,117, + 114,105,110,103,32,102,114,111,109,10,32,32,32,32,116,111, + 32,104,97,110,100,108,101,32,114,101,108,97,116,105,118,101, + 32,105,109,112,111,114,116,115,46,32,84,104,101,32,39,108, + 111,99,97,108,115,39,32,97,114,103,117,109,101,110,116,32, + 105,115,32,105,103,110,111,114,101,100,46,32,84,104,101,10, + 32,32,32,32,39,102,114,111,109,108,105,115,116,39,32,97, + 114,103,117,109,101,110,116,32,115,112,101,99,105,102,105,101, + 115,32,119,104,97,116,32,115,104,111,117,108,100,32,101,120, + 105,115,116,32,97,115,32,97,116,116,114,105,98,117,116,101, + 115,32,111,110,32,116,104,101,32,109,111,100,117,108,101,10, + 32,32,32,32,98,101,105,110,103,32,105,109,112,111,114,116, + 101,100,32,40,101,46,103,46,32,96,96,102,114,111,109,32, + 109,111,100,117,108,101,32,105,109,112,111,114,116,32,60,102, + 114,111,109,108,105,115,116,62,96,96,41,46,32,32,84,104, + 101,32,39,108,101,118,101,108,39,10,32,32,32,32,97,114, + 103,117,109,101,110,116,32,114,101,112,114,101,115,101,110,116, + 115,32,116,104,101,32,112,97,99,107,97,103,101,32,108,111, + 99,97,116,105,111,110,32,116,111,32,105,109,112,111,114,116, + 32,102,114,111,109,32,105,110,32,97,32,114,101,108,97,116, + 105,118,101,10,32,32,32,32,105,109,112,111,114,116,32,40, + 101,46,103,46,32,96,96,102,114,111,109,32,46,46,112,107, + 103,32,105,109,112,111,114,116,32,109,111,100,96,96,32,119, + 111,117,108,100,32,104,97,118,101,32,97,32,39,108,101,118, + 101,108,39,32,111,102,32,50,41,46,10,10,32,32,32,32, + 114,33,0,0,0,78,114,121,0,0,0,41,8,114,187,0, + 0,0,114,198,0,0,0,218,9,112,97,114,116,105,116,105, + 111,110,114,168,0,0,0,114,14,0,0,0,114,21,0,0, + 0,114,1,0,0,0,114,195,0,0,0,41,9,114,15,0, + 0,0,114,197,0,0,0,218,6,108,111,99,97,108,115,114, + 193,0,0,0,114,171,0,0,0,114,89,0,0,0,90,8, + 103,108,111,98,97,108,115,95,114,170,0,0,0,90,7,99, + 117,116,95,111,102,102,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,10,95,95,105,109,112,111,114,116,95, + 95,25,4,0,0,115,26,0,0,0,0,11,12,1,15,2, + 24,1,12,1,18,1,6,3,12,1,23,1,6,1,4,4, + 35,3,40,2,114,201,0,0,0,99,1,0,0,0,0,0, + 0,0,2,0,0,0,3,0,0,0,67,0,0,0,115,53, + 0,0,0,116,0,0,106,1,0,124,0,0,131,1,0,125, + 1,0,124,1,0,100,0,0,107,8,0,114,43,0,116,2, + 0,100,1,0,124,0,0,23,131,1,0,130,1,0,116,3, + 0,124,1,0,131,1,0,83,41,2,78,122,25,110,111,32, + 98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,32, + 110,97,109,101,100,32,41,4,114,151,0,0,0,114,155,0, + 0,0,114,77,0,0,0,114,150,0,0,0,41,2,114,15, + 0,0,0,114,88,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,18,95,98,117,105,108,116,105, + 110,95,102,114,111,109,95,110,97,109,101,60,4,0,0,115, + 8,0,0,0,0,1,15,1,12,1,16,1,114,202,0,0, + 0,99,2,0,0,0,0,0,0,0,12,0,0,0,12,0, + 0,0,67,0,0,0,115,74,1,0,0,124,1,0,97,0, + 0,124,0,0,97,1,0,116,2,0,116,1,0,131,1,0, + 125,2,0,120,123,0,116,1,0,106,3,0,106,4,0,131, + 0,0,68,93,106,0,92,2,0,125,3,0,125,4,0,116, + 5,0,124,4,0,124,2,0,131,2,0,114,40,0,124,3, + 0,116,1,0,106,6,0,107,6,0,114,91,0,116,7,0, + 125,5,0,110,27,0,116,0,0,106,8,0,124,3,0,131, + 1,0,114,40,0,116,9,0,125,5,0,110,3,0,113,40, + 0,116,10,0,124,4,0,124,5,0,131,2,0,125,6,0, + 116,11,0,124,6,0,124,4,0,131,2,0,1,113,40,0, + 87,116,1,0,106,3,0,116,12,0,25,125,7,0,120,73, + 0,100,5,0,68,93,65,0,125,8,0,124,8,0,116,1, + 0,106,3,0,107,7,0,114,206,0,116,13,0,124,8,0, + 131,1,0,125,9,0,110,13,0,116,1,0,106,3,0,124, + 8,0,25,125,9,0,116,14,0,124,7,0,124,8,0,124, + 9,0,131,3,0,1,113,170,0,87,121,16,0,116,13,0, + 100,2,0,131,1,0,125,10,0,87,110,24,0,4,116,15, + 0,107,10,0,114,25,1,1,1,1,100,3,0,125,10,0, + 89,110,1,0,88,116,14,0,124,7,0,100,2,0,124,10, + 0,131,3,0,1,116,13,0,100,4,0,131,1,0,125,11, + 0,116,14,0,124,7,0,100,4,0,124,11,0,131,3,0, + 1,100,3,0,83,41,6,122,250,83,101,116,117,112,32,105, + 109,112,111,114,116,108,105,98,32,98,121,32,105,109,112,111, + 114,116,105,110,103,32,110,101,101,100,101,100,32,98,117,105, + 108,116,45,105,110,32,109,111,100,117,108,101,115,32,97,110, + 100,32,105,110,106,101,99,116,105,110,103,32,116,104,101,109, + 10,32,32,32,32,105,110,116,111,32,116,104,101,32,103,108, + 111,98,97,108,32,110,97,109,101,115,112,97,99,101,46,10, + 10,32,32,32,32,65,115,32,115,121,115,32,105,115,32,110, + 101,101,100,101,100,32,102,111,114,32,115,121,115,46,109,111, + 100,117,108,101,115,32,97,99,99,101,115,115,32,97,110,100, + 32,95,105,109,112,32,105,115,32,110,101,101,100,101,100,32, + 116,111,32,108,111,97,100,32,98,117,105,108,116,45,105,110, + 10,32,32,32,32,109,111,100,117,108,101,115,44,32,116,104, + 111,115,101,32,116,119,111,32,109,111,100,117,108,101,115,32, + 109,117,115,116,32,98,101,32,101,120,112,108,105,99,105,116, + 108,121,32,112,97,115,115,101,100,32,105,110,46,10,10,32, + 32,32,32,114,142,0,0,0,114,34,0,0,0,78,114,62, + 0,0,0,41,1,122,9,95,119,97,114,110,105,110,103,115, + 41,16,114,57,0,0,0,114,14,0,0,0,114,13,0,0, + 0,114,21,0,0,0,218,5,105,116,101,109,115,114,178,0, + 0,0,114,76,0,0,0,114,151,0,0,0,114,82,0,0, + 0,114,161,0,0,0,114,132,0,0,0,114,137,0,0,0, + 114,1,0,0,0,114,202,0,0,0,114,5,0,0,0,114, + 77,0,0,0,41,12,218,10,115,121,115,95,109,111,100,117, + 108,101,218,11,95,105,109,112,95,109,111,100,117,108,101,90, + 11,109,111,100,117,108,101,95,116,121,112,101,114,15,0,0, + 0,114,89,0,0,0,114,99,0,0,0,114,88,0,0,0, + 90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98, + 117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105, + 108,116,105,110,95,109,111,100,117,108,101,90,13,116,104,114, + 101,97,100,95,109,111,100,117,108,101,90,14,119,101,97,107, + 114,101,102,95,109,111,100,117,108,101,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,6,95,115,101,116,117, + 112,67,4,0,0,115,50,0,0,0,0,9,6,1,6,3, + 12,1,28,1,15,1,15,1,9,1,15,1,9,2,3,1, + 15,1,17,3,13,1,13,1,15,1,15,2,13,1,20,3, + 3,1,16,1,13,2,11,1,16,3,12,1,114,206,0,0, + 0,99,2,0,0,0,0,0,0,0,3,0,0,0,3,0, + 0,0,67,0,0,0,115,87,0,0,0,116,0,0,124,0, + 0,124,1,0,131,2,0,1,116,1,0,106,2,0,106,3, + 0,116,4,0,131,1,0,1,116,1,0,106,2,0,106,3, + 0,116,5,0,131,1,0,1,100,1,0,100,2,0,108,6, + 0,125,2,0,124,2,0,97,7,0,124,2,0,106,8,0, + 116,1,0,106,9,0,116,10,0,25,131,1,0,1,100,2, + 0,83,41,3,122,50,73,110,115,116,97,108,108,32,105,109, + 112,111,114,116,108,105,98,32,97,115,32,116,104,101,32,105, + 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, + 32,105,109,112,111,114,116,46,114,33,0,0,0,78,41,11, + 114,206,0,0,0,114,14,0,0,0,114,175,0,0,0,114, + 113,0,0,0,114,151,0,0,0,114,161,0,0,0,218,26, + 95,102,114,111,122,101,110,95,105,109,112,111,114,116,108,105, + 98,95,101,120,116,101,114,110,97,108,114,119,0,0,0,218, + 8,95,105,110,115,116,97,108,108,114,21,0,0,0,114,1, + 0,0,0,41,3,114,204,0,0,0,114,205,0,0,0,114, + 207,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,114,208,0,0,0,114,4,0,0,115,12,0,0, + 0,0,2,13,2,16,1,16,3,12,1,6,1,114,208,0, + 0,0,41,51,114,3,0,0,0,114,119,0,0,0,114,12, + 0,0,0,114,16,0,0,0,114,17,0,0,0,114,59,0, + 0,0,114,41,0,0,0,114,48,0,0,0,114,31,0,0, + 0,114,32,0,0,0,114,53,0,0,0,114,54,0,0,0, + 114,56,0,0,0,114,63,0,0,0,114,65,0,0,0,114, + 75,0,0,0,114,81,0,0,0,114,84,0,0,0,114,90, + 0,0,0,114,101,0,0,0,114,102,0,0,0,114,106,0, + 0,0,114,85,0,0,0,218,6,111,98,106,101,99,116,90, + 9,95,80,79,80,85,76,65,84,69,114,132,0,0,0,114, + 137,0,0,0,114,145,0,0,0,114,97,0,0,0,114,86, + 0,0,0,114,149,0,0,0,114,150,0,0,0,114,87,0, + 0,0,114,151,0,0,0,114,161,0,0,0,114,166,0,0, + 0,114,172,0,0,0,114,174,0,0,0,114,177,0,0,0, + 114,182,0,0,0,114,192,0,0,0,114,183,0,0,0,114, + 185,0,0,0,114,186,0,0,0,114,187,0,0,0,114,195, + 0,0,0,114,198,0,0,0,114,201,0,0,0,114,202,0, + 0,0,114,206,0,0,0,114,208,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, + 218,8,60,109,111,100,117,108,101,62,8,0,0,0,115,96, + 0,0,0,6,17,6,2,12,8,12,4,19,20,6,2,6, + 3,22,4,19,68,19,21,19,19,12,19,12,19,12,11,18, + 8,12,11,12,12,12,16,12,36,19,27,19,101,24,26,9, + 3,18,45,18,60,12,18,12,17,12,25,12,29,12,23,12, + 16,19,73,19,77,19,13,12,9,12,9,15,40,12,17,6, + 1,10,2,12,27,12,6,18,24,12,32,12,21,24,35,12, + 7,12,47, }; -- cgit v0.12 From 60255b67b986d4c448153bef16755599afbfdaa2 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Fri, 15 Jan 2016 15:01:33 -0800 Subject: revert change 87a9dff5106c: pure Enum members again evaluate to True; update Finer Points section of docs to cover boolean evaluation; add more tests for pure and mixed boolean evaluation --- Doc/library/enum.rst | 11 ++++++++++- Lib/enum.py | 3 --- Lib/test/test_enum.py | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index a76f5a3..377ac3e 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -257,7 +257,7 @@ members are not integers (but see `IntEnum`_ below):: >>> Color.red < Color.blue Traceback (most recent call last): File "", line 1, in - TypeError: '<' not supported between instances of 'Color' and 'Color' + TypeError: unorderable types: Color() < Color() Equality comparisons are defined though:: @@ -747,6 +747,15 @@ besides the :class:`Enum` member you looking for:: .. versionchanged:: 3.5 +Boolean evaluation: Enum classes that are mixed with non-Enum types (such as +:class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in +type's rules; otherwise, all members evaluate as ``True``. To make your own +Enum's boolean evaluation depend on the member's value add the following to +your class:: + + def __bool__(self): + return bool(self._value_) + The :attr:`__members__` attribute is only available on the class. If you give your :class:`Enum` subclass extra methods, like the `Planet`_ diff --git a/Lib/enum.py b/Lib/enum.py index 35a9c77..ac89d6b 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -482,9 +482,6 @@ class Enum(metaclass=EnumMeta): def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) - def __bool__(self): - return bool(self._value_) - def __dir__(self): added_behavior = [ m diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index e4e6c2b..7985948 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -272,11 +272,26 @@ class TestEnum(unittest.TestCase): _any_name_ = 9 def test_bool(self): + # plain Enum members are always True class Logic(Enum): true = True false = False self.assertTrue(Logic.true) - self.assertFalse(Logic.false) + self.assertTrue(Logic.false) + # unless overridden + class RealLogic(Enum): + true = True + false = False + def __bool__(self): + return bool(self._value_) + self.assertTrue(RealLogic.true) + self.assertFalse(RealLogic.false) + # mixed Enums depend on mixed-in type + class IntLogic(int, Enum): + true = 1 + false = 0 + self.assertTrue(IntLogic.true) + self.assertFalse(IntLogic.false) def test_contains(self): Season = self.Season -- cgit v0.12 From 7978e10441e84482d3d28e7c449d68b77c6310b6 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:26:54 +0000 Subject: Issue #23883: Missing fileinput.__all__ APIs; patch by Mauro SM Rodrigues --- Lib/fileinput.py | 3 ++- Lib/test/test_fileinput.py | 8 ++++++++ Misc/ACKS | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 021e39f..3543653 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -82,7 +82,8 @@ XXX Possible additions: import sys, os __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno", - "isfirstline", "isstdin", "FileInput"] + "fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed", + "hook_encoded"] _state = None diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 91c1166..ad81304 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -24,6 +24,7 @@ from fileinput import FileInput, hook_encoded from test.support import verbose, TESTFN, run_unittest, check_warnings from test.support import unlink as safe_unlink +from test import support from unittest import mock @@ -913,5 +914,12 @@ class Test_hook_encoded(unittest.TestCase): check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac']) +class MiscTest(unittest.TestCase): + + def test_all(self): + blacklist = {'DEFAULT_BUFSIZE'} + support.check__all__(self, fileinput, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index b243181..1df5d24 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1219,6 +1219,7 @@ Mark Roddy Kevin Rodgers Sean Rodman Giampaolo Rodola +Mauro S. M. Rodrigues Elson Rodriguez Adi Roiban Luis Rojas -- cgit v0.12 From 4eb376c441f99e9ea50beae5309020a1469393ac Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:49:30 +0000 Subject: Issue #23883: Add missing APIs to calendar.__all__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Joel Taddei and Jacek Kołodziej. --- Lib/calendar.py | 4 +++- Lib/test/test_calendar.py | 9 +++++++++ Misc/ACKS | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/calendar.py b/Lib/calendar.py index 196a075..85baf2e 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -12,7 +12,9 @@ import locale as _locale __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr"] + "timegm", "month_name", "month_abbr", "day_name", "day_abbr", + "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "LocaleHTMLCalendar", "weekheader"] # Exception raised for bad input (with string parameter for details) error = ValueError diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index d9d3128..d95ad04 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -815,5 +815,14 @@ class CommandLineTestCase(unittest.TestCase): b'href="custom.css" />', stdout) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'error', 'mdays', 'January', 'February', 'EPOCH', + 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', + 'SATURDAY', 'SUNDAY', 'different_locale', 'c', + 'prweek', 'week', 'format', 'formatstring', 'main'} + support.check__all__(self, calendar, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 1df5d24..ebc0d5d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1419,6 +1419,7 @@ Péter Szabó John Szakmeister Amir Szekely Maciej Szulik +Joel Taddei Arfrever Frehtes Taifersar Arahesis Hideaki Takahashi Takase Arihiro -- cgit v0.12 From 104dcdab593b50596666b9f27fa6bfedc376deae Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:59:13 +0000 Subject: Issue #23883: Add missing APIs to tarfile.__all__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Joel Taddei and Jacek Kołodziej. --- Lib/tarfile.py | 5 ++++- Lib/test/test_tarfile.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 80b6e35..7d43b1e 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -64,7 +64,10 @@ except NameError: pass # from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"] +__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", + "CompressionError", "StreamError", "ExtractError", "HeaderError", + "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", + "DEFAULT_FORMAT", "open"] #--------------------------------------------------------- # tar constants diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 1412cae..fee1a6c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1997,6 +1997,24 @@ class MiscTest(unittest.TestCase): with self.assertRaises(ValueError): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) + def test__all__(self): + blacklist = {'version', 'bltn_open', 'symlink_exception', + 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', + 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', + 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', + 'SYMTYPE', 'CHRTYPE', 'BLKTYPE', 'DIRTYPE', 'FIFOTYPE', + 'CONTTYPE', 'GNUTYPE_LONGNAME', 'GNUTYPE_LONGLINK', + 'GNUTYPE_SPARSE', 'XHDTYPE', 'XGLTYPE', 'SOLARIS_XHDTYPE', + 'SUPPORTED_TYPES', 'REGULAR_TYPES', 'GNU_TYPES', + 'PAX_FIELDS', 'PAX_NAME_FIELDS', 'PAX_NUMBER_FIELDS', + 'stn', 'nts', 'nti', 'itn', 'calc_chksums', 'copyfileobj', + 'filemode', + 'EmptyHeaderError', 'TruncatedHeaderError', + 'EOFHeaderError', 'InvalidHeaderError', + 'SubsequentHeaderError', 'ExFileObject', 'TarIter', + 'main'} + support.check__all__(self, tarfile, blacklist=blacklist) + class CommandLineTest(unittest.TestCase): -- cgit v0.12 From e8afd01db8c96ac686ab1834a04c0467e8d092e9 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 07:01:46 +0000 Subject: Issue #23883: Update news --- Doc/whatsnew/3.6.rst | 5 +++-- Misc/NEWS | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index a5b9be2..f54cf12 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -251,8 +251,9 @@ Changes in the Python API :exc:`PendingDeprecationWarning`. * The following modules have had missing APIs added to their :attr:`__all__` - attributes to match the documented APIs: :mod:`csv`, :mod:`enum`, - :mod:`ftplib`, :mod:`logging`, :mod:`optparse`, :mod:`threading` and + attributes to match the documented APIs: :mod:`calendar`, :mod:`csv`, + :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, + :mod:`optparse`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. diff --git a/Misc/NEWS b/Misc/NEWS index f87703f..0e46c8d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -275,9 +275,10 @@ Library anything as they could potentially signal a different process. - Issue #23883: Added missing APIs to __all__ to match the documented APIs - for the following modules: csv, enum, ftplib, logging, optparse, threading - and wave. Also added a test.support.check__all__() helper. Patches by - Jacek Kołodziej. + for the following modules: calendar, csv, enum, fileinput, ftplib, logging, + optparse, tarfile, threading and wave. Also added a + test.support.check__all__() helper. Patches by Jacek Kołodziej, Mauro + S. M. Rodrigues and Joel Taddei. - Issue #25590: In the Readline completer, only call getattr() once per attribute. Also complete names of attributes such as properties and slots -- cgit v0.12 From 2c1d3e320fa26d2f8da3196a4966d2c9ece77430 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Jan 2016 11:05:11 +0200 Subject: Issue #23883: Removed redundant names from blacklists. --- Lib/test/test_calendar.py | 2 +- Lib/test/test_tarfile.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index d95ad04..6dad058 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -817,7 +817,7 @@ class CommandLineTestCase(unittest.TestCase): class MiscTestCase(unittest.TestCase): def test__all__(self): - blacklist = {'error', 'mdays', 'January', 'February', 'EPOCH', + blacklist = {'mdays', 'January', 'February', 'EPOCH', 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY', 'SUNDAY', 'different_locale', 'c', 'prweek', 'week', 'format', 'formatstring', 'main'} diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index fee1a6c..9ffa2b5 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1998,7 +1998,7 @@ class MiscTest(unittest.TestCase): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) def test__all__(self): - blacklist = {'version', 'bltn_open', 'symlink_exception', + blacklist = {'version', 'symlink_exception', 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', @@ -2011,7 +2011,7 @@ class MiscTest(unittest.TestCase): 'filemode', 'EmptyHeaderError', 'TruncatedHeaderError', 'EOFHeaderError', 'InvalidHeaderError', - 'SubsequentHeaderError', 'ExFileObject', 'TarIter', + 'SubsequentHeaderError', 'ExFileObject', 'main'} support.check__all__(self, tarfile, blacklist=blacklist) -- cgit v0.12 From 5318d1048f701f5bfdc21aeca3bd6bc876d304b0 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 11:01:14 +0000 Subject: Issue #23883: grp and pwd are None on Windows --- Lib/test/test_tarfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 9ffa2b5..03875ab 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1998,7 +1998,7 @@ class MiscTest(unittest.TestCase): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) def test__all__(self): - blacklist = {'version', 'symlink_exception', + blacklist = {'version', 'grp', 'pwd', 'symlink_exception', 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', -- cgit v0.12 From 2ae4ea54a23468695b1bab4aca1befc3681e8bfb Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Sat, 16 Jan 2016 12:39:53 -0800 Subject: use public 'value' --- Doc/library/enum.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 377ac3e..1fa6e73 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -754,7 +754,7 @@ Enum's boolean evaluation depend on the member's value add the following to your class:: def __bool__(self): - return bool(self._value_) + return bool(self.value) The :attr:`__members__` attribute is only available on the class. -- cgit v0.12 From 5f6ccc787e3d554bf4d759eb532826151809a8dc Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Sun, 17 Jan 2016 12:28:43 +0100 Subject: Issue #26139: libmpdec: disable /W4 warning (non-standard dllimport behavior). --- Modules/_decimal/libmpdec/memory.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_decimal/libmpdec/memory.c b/Modules/_decimal/libmpdec/memory.c index 0f41fe5..61eb633 100644 --- a/Modules/_decimal/libmpdec/memory.c +++ b/Modules/_decimal/libmpdec/memory.c @@ -33,6 +33,11 @@ #include "memory.h" +#if defined(_MSC_VER) + #pragma warning(disable : 4232) +#endif + + /* Guaranteed minimum allocation for a coefficient. May be changed once at program start using mpd_setminalloc(). */ mpd_ssize_t MPD_MINALLOC = MPD_MINALLOC_MIN; -- cgit v0.12 From 613065b60d823efa6e2be0e0e53079a44305b6b2 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 17 Jan 2016 20:12:16 -0800 Subject: Issue26069 - Update whatsnew/3.6.rst on traceback module's api removals. --- Doc/whatsnew/3.6.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index f54cf12..0e15eda 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -227,6 +227,13 @@ API and Feature Removals :func:`inspect.getmodulename` should be used for obtaining the module name for a given path. +* ``traceback.Ignore`` class and ``traceback.usage``, ``traceback.modname``, + ``traceback.fullmodname``, ``traceback.find_lines_from_code``, + ``traceback.find_lines``, ``traceback.find_strings``, + ``traceback.find_executable_lines`` methods were removed from the + :mod:`traceback` module. They were undocumented methods deprecated since + Python 3.2 and equivalent functionality is available from private methods. + Porting to Python 3.6 ===================== -- cgit v0.12 From 5c60ea3fa1e40c6a3a2e0e9da71f55f8d7d6b2b1 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 18 Jan 2016 07:53:59 +0100 Subject: Fix two instances of wrong indentation. --- Python/ast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 33f2597..a5d8dba 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3214,14 +3214,14 @@ ast_for_import_stmt(struct compiling *c, const node *n) alias_ty import_alias = alias_for_import_name(c, n, 1); if (!import_alias) return NULL; - asdl_seq_SET(aliases, 0, import_alias); + asdl_seq_SET(aliases, 0, import_alias); } else { for (i = 0; i < NCH(n); i += 2) { alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1); if (!import_alias) return NULL; - asdl_seq_SET(aliases, i / 2, import_alias); + asdl_seq_SET(aliases, i / 2, import_alias); } } if (mod != NULL) -- cgit v0.12 From 4b5b06203e4dbe24579101e5d889080c43854453 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 18 Jan 2016 08:00:15 +0100 Subject: Fix indentation of continuation lines. --- Modules/_testcapimodule.c | 4 ++-- Python/pylifecycle.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index cc3f52d..fbb4aa4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1047,7 +1047,7 @@ test_k_code(PyObject *self) value = PyLong_AsUnsignedLongMask(num); if (value != ULONG_MAX) return raiseTestError("test_k_code", - "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); + "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); PyTuple_SET_ITEM(tuple, 0, num); @@ -1066,7 +1066,7 @@ test_k_code(PyObject *self) value = PyLong_AsUnsignedLongMask(num); if (value != (unsigned long)-0x42) return raiseTestError("test_k_code", - "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); + "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); PyTuple_SET_ITEM(tuple, 0, num); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c84c46a..e9db7f6 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -796,7 +796,7 @@ Py_NewInterpreter(void) if (initstdio() < 0) Py_FatalError( - "Py_Initialize: can't initialize sys standard streams"); + "Py_Initialize: can't initialize sys standard streams"); initmain(interp); if (!Py_NoSiteFlag) initsite(); -- cgit v0.12 From c437d0cb4e99bd58ff0150414b5d5f0b26605687 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Jan 2016 11:25:50 +0100 Subject: Fix test_compilepath() of test_compileall Issue #26101: Exclude Lib/test/ from sys.path in test_compilepath(). The directory contains invalid Python files like Lib/test/badsyntax_pep3120.py, whereas the test ensures that all files can be compiled. --- Lib/test/test_compileall.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 439c125..6d8a863 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -103,6 +103,18 @@ class CompileallTests(unittest.TestCase): force=False, quiet=2)) def test_compile_path(self): + # Exclude Lib/test/ which contains invalid Python files like + # Lib/test/badsyntax_pep3120.py + testdir = os.path.realpath(os.path.dirname(__file__)) + if testdir in sys.path: + self.addCleanup(setattr, sys, 'path', sys.path) + + sys.path = list(sys.path) + try: + sys.path.remove(testdir) + except ValueError: + pass + self.assertTrue(compileall.compile_path(quiet=2)) with test.test_importlib.util.import_state(path=[self.directory]): -- cgit v0.12 From 9def2843873edde3feec6eaf2ee60c4e48172164 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Jan 2016 12:15:08 +0100 Subject: subprocess._optim_args_from_interpreter_flags() Issue #26100: * Add subprocess._optim_args_from_interpreter_flags() * Add test.support.optim_args_from_interpreter_flags() * Use new functions in distutils, test_cmd_line_script, test_compileall and test_inspect The change enables test_details() test of test_inspect when -O or -OO command line option is used. --- Lib/distutils/util.py | 15 +++++++++------ Lib/subprocess.py | 14 ++++++++++++-- Lib/test/support/__init__.py | 5 +++++ Lib/test/test_cmd_line_script.py | 5 ++--- Lib/test/test_compileall.py | 7 +++---- Lib/test/test_inspect.py | 7 ++++--- 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index e423325..fdcf6fa 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -7,8 +7,8 @@ one of the other *util.py modules. import os import re import importlib.util -import sys import string +import sys from distutils.errors import DistutilsPlatformError from distutils.dep_util import newer from distutils.spawn import spawn @@ -350,6 +350,11 @@ def byte_compile (py_files, generated in indirect mode; unless you know what you're doing, leave it set to None. """ + + # Late import to fix a bootstrap issue: _posixsubprocess is built by + # setup.py, but setup.py uses distutils. + import subprocess + # nothing is done if sys.dont_write_bytecode is True if sys.dont_write_bytecode: raise DistutilsByteCompileError('byte-compiling is disabled.') @@ -412,11 +417,9 @@ byte_compile(files, optimize=%r, force=%r, script.close() - cmd = [sys.executable, script_name] - if optimize == 1: - cmd.insert(1, "-O") - elif optimize == 2: - cmd.insert(1, "-OO") + cmd = [sys.executable] + cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.append(script_name) spawn(cmd, dry_run=dry_run) execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index d1324b8..640519d 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -520,6 +520,16 @@ DEVNULL = -3 # but it's here so that it can be imported when Python is compiled without # threads. +def _optim_args_from_interpreter_flags(): + """Return a list of command-line arguments reproducing the current + optimization settings in sys.flags.""" + args = [] + value = sys.flags.optimize + if value > 0: + args.append('-' + 'O' * value) + return args + + def _args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current settings in sys.flags and sys.warnoptions.""" @@ -527,7 +537,6 @@ def _args_from_interpreter_flags(): 'debug': 'd', # 'inspect': 'i', # 'interactive': 'i', - 'optimize': 'O', 'dont_write_bytecode': 'B', 'no_user_site': 's', 'no_site': 'S', @@ -535,8 +544,9 @@ def _args_from_interpreter_flags(): 'verbose': 'v', 'bytes_warning': 'b', 'quiet': 'q', + # -O is handled in _optim_args_from_interpreter_flags() } - args = [] + args = _optim_args_from_interpreter_flags() for flag, opt in flag_opt_map.items(): v = getattr(sys.flags, flag) if v > 0: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2969b36..58dcc8a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2053,6 +2053,11 @@ def args_from_interpreter_flags(): settings in sys.flags and sys.warnoptions.""" return subprocess._args_from_interpreter_flags() +def optim_args_from_interpreter_flags(): + """Return a list of command-line arguments reproducing the current + optimization settings in sys.flags.""" + return subprocess._optim_args_from_interpreter_flags() + #============================================================ # Support for assertions about logging. #============================================================ diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index afac62a..9398040 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -138,9 +138,8 @@ class CmdLineTest(unittest.TestCase): expected_argv0, expected_path0, expected_package, expected_loader, *cmd_line_switches): - if not __debug__: - cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) - run_args = cmd_line_switches + (script_name,) + tuple(example_args) + run_args = [*support.optim_args_from_interpreter_flags(), + *cmd_line_switches, script_name, *example_args] rc, out, err = assert_python_ok(*run_args, __isolated=False) self._check_output(script_name, rc, out + err, expected_file, expected_argv0, expected_path0, diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 6d8a863..9b424a7 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -231,10 +231,9 @@ class CommandLineTests(unittest.TestCase): raise unittest.SkipTest('not all entries on sys.path are writable') def _get_run_args(self, args): - interp_args = ['-S'] - if sys.flags.optimize: - interp_args.append({1 : '-O', 2 : '-OO'}[sys.flags.optimize]) - return interp_args + ['-m', 'compileall'] + list(args) + return [*support.optim_args_from_interpreter_flags(), + '-S', '-m', 'compileall', + *args] def assertRunOK(self, *args, **env_vars): rc, out, err = script_helper.assert_python_ok( diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 283c922..422a3d6 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -30,6 +30,7 @@ from test.support import MISSING_C_DOCSTRINGS, cpython_only from test.support.script_helper import assert_python_ok, assert_python_failure from test import inspect_fodder as mod from test import inspect_fodder2 as mod2 +from test import support from test.test_import import _ready_to_import @@ -3536,14 +3537,14 @@ class TestMain(unittest.TestCase): def test_details(self): module = importlib.import_module('unittest') - rc, out, err = assert_python_ok('-m', 'inspect', + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', 'unittest', '--details') output = out.decode() # Just a quick sanity check on the output self.assertIn(module.__name__, output) self.assertIn(module.__file__, output) - if not sys.flags.optimize: - self.assertIn(module.__cached__, output) + self.assertIn(module.__cached__, output) self.assertEqual(err, b'') -- cgit v0.12 From 9cc4ed5c7a86b2aa58176222107dbb01d40a3680 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 18 Jan 2016 18:49:57 +0200 Subject: Issue #26129: Deprecated accepting non-integers in grp.getgrgid(). --- Doc/library/grp.rst | 3 +++ Lib/test/test_grp.py | 10 ++++++++++ Misc/NEWS | 2 ++ Modules/grpmodule.c | 21 ++++++++++++++++----- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index 8882140..c840cfe 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -42,6 +42,9 @@ It defines the following items: Return the group database entry for the given numeric group ID. :exc:`KeyError` is raised if the entry asked for cannot be found. + .. deprecated:: 3.6 + Since Python 3.6 the support of non-integer arguments like floats or + strings in :func:`getgrgid` is deprecated. .. function:: getgrnam(name) diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index 272b086..69095a3 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -92,5 +92,15 @@ class GroupDatabaseTestCase(unittest.TestCase): self.assertRaises(KeyError, grp.getgrgid, fakegid) + def test_noninteger_gid(self): + entries = grp.getgrall() + if not entries: + self.skipTest('no groups') + # Choose an existent gid. + gid = entries[0][2] + self.assertWarns(DeprecationWarning, grp.getgrgid, float(gid)) + self.assertWarns(DeprecationWarning, grp.getgrgid, str(gid)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 4564e29..d3edac7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -131,6 +131,8 @@ Core and Builtins Library ------- +- Issue #26129: Deprecated accepting non-integers in grp.getgrgid(). + - Issue #25850: Use cross-compilation by default for 64-bit Windows. - Issue #25822: Add docstrings to the fields of urllib.parse results. diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 403e434..5ad87f1 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -100,14 +100,25 @@ grp_getgrgid_impl(PyModuleDef *module, PyObject *id) gid_t gid; struct group *p; - py_int_id = PyNumber_Long(id); - if (!py_int_id) + if (!_Py_Gid_Converter(id, &gid)) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) { return NULL; - if (!_Py_Gid_Converter(py_int_id, &gid)) { + } + PyErr_Clear(); + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "group id must be int, not %.200", + id->ob_type->tp_name) < 0) { + return NULL; + } + py_int_id = PyNumber_Long(id); + if (!py_int_id) + return NULL; + if (!_Py_Gid_Converter(py_int_id, &gid)) { + Py_DECREF(py_int_id); + return NULL; + } Py_DECREF(py_int_id); - return NULL; } - Py_DECREF(py_int_id); if ((p = getgrgid(gid)) == NULL) { PyObject *gid_obj = _PyLong_FromGid(gid); -- cgit v0.12 From 31a858cbf1eca34f04dd425fa7f8d6f031e5de66 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 19 Jan 2016 14:09:33 +0200 Subject: Issue #16620: Got rid of using undocumented function glob.glob1(). --- Lib/msilib/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/msilib/__init__.py b/Lib/msilib/__init__.py index 6d0a28f..823644a 100644 --- a/Lib/msilib/__init__.py +++ b/Lib/msilib/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2005 Martin v. Löwis # Licensed to PSF under a Contributor Agreement. from _msi import * -import glob +import fnmatch import os import re import string @@ -379,7 +379,13 @@ class Directory: def glob(self, pattern, exclude = None): """Add a list of files to the current component as specified in the glob pattern. Individual files can be excluded in the exclude list.""" - files = glob.glob1(self.absolute, pattern) + try: + files = os.listdir(self.absolute) + except OSError: + return [] + if pattern[:1] != '.': + files = (f for f in files if f[0] != '.') + files = fnmatch.filter(files, pattern) for f in files: if exclude and f in exclude: continue self.add_file(f) -- cgit v0.12 From f3914eb16d28ad9eb20fe5133d9aa83658bcc27f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Jan 2016 12:16:21 +0100 Subject: co_lnotab supports negative line number delta Issue #26107: The format of the co_lnotab attribute of code objects changes to support negative line number delta. Changes: * assemble_lnotab(): if line number delta is less than -128 or greater than 127, emit multiple (offset_delta, lineno_delta) in co_lnotab * update functions decoding co_lnotab to use signed 8-bit integers - dis.findlinestarts() - PyCode_Addr2Line() - _PyCode_CheckLineNumber() - frame_setlineno() * update lnotab_notes.txt * increase importlib MAGIC_NUMBER to 3361 * document the change in What's New in Python 3.6 * cleanup also PyCode_Optimize() to use better variable names --- Doc/whatsnew/3.6.rst | 10 ++ Include/code.h | 2 +- Lib/dis.py | 7 +- Lib/importlib/_bootstrap_external.py | 5 +- Misc/NEWS | 3 + Objects/codeobject.c | 11 +- Objects/frameobject.c | 2 +- Objects/lnotab_notes.txt | 37 +++--- Python/compile.c | 25 +++-- Python/importlib_external.h | 211 ++++++++++++++++++----------------- Python/peephole.c | 51 +++++---- 11 files changed, 203 insertions(+), 161 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 0e15eda..8064537 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -244,6 +244,16 @@ that may require changes to your code. Changes in the Python API ------------------------- +* The format of the ``co_lnotab`` attribute of code objects changed to support + negative line number delta. By default, Python does not emit bytecode with + negative line number delta. Functions using ``frame.f_lineno``, + ``PyFrame_GetLineNumber()`` or ``PyCode_Addr2Line()`` are not affected. + Functions decoding directly ``co_lnotab`` should be updated to use a signed + 8-bit integer type for the line number delta, but it's only required to + support applications using negative line number delta. See + ``Objects/lnotab_notes.txt`` for the ``co_lnotab`` format and how to decode + it, and see the :pep:`511` for the rationale. + * The functions in the :mod:`compileall` module now return booleans instead of ``1`` or ``0`` to represent success or failure, respectively. Thanks to booleans being a subclass of integers, this should only be an issue if you diff --git a/Include/code.h b/Include/code.h index 56e6ec1..3ce2084 100644 --- a/Include/code.h +++ b/Include/code.h @@ -117,7 +117,7 @@ PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co, #endif PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, - PyObject *names, PyObject *lineno_obj); + PyObject *names, PyObject *lnotab); #ifdef __cplusplus } diff --git a/Lib/dis.py b/Lib/dis.py index 3540b47..2e80e17 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -397,8 +397,8 @@ def findlinestarts(code): Generate pairs (offset, lineno) as described in Python/compile.c. """ - byte_increments = list(code.co_lnotab[0::2]) - line_increments = list(code.co_lnotab[1::2]) + byte_increments = code.co_lnotab[0::2] + line_increments = code.co_lnotab[1::2] lastlineno = None lineno = code.co_firstlineno @@ -409,6 +409,9 @@ def findlinestarts(code): yield (addr, lineno) lastlineno = lineno addr += byte_incr + if line_incr >= 0x80: + # line_increments is an array of 8-bit signed integers + line_incr -= 0x100 lineno += line_incr if lineno != lastlineno: yield (addr, lineno) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f274501..71098f1 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -223,13 +223,14 @@ _code_type = type(_write_atomic.__code__) # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations) # Python 3.5b2 3340 (fix dictionary display evaluation order #11205) # Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483 +# Python 3.6a0 3361 (lineno delta of code.co_lnotab becomes signed) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). -MAGIC_NUMBER = (3360).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3361).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Misc/NEWS b/Misc/NEWS index 46464de..8c6864c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #26107: The format of the ``co_lnotab`` attribute of code objects + changes to support negative line number delta. + - Issue #26154: Add a new private _PyThreadState_UncheckedGet() function to get the current Python thread state, but don't issue a fatal error if it is NULL. This new function must be used instead of accessing directly the diff --git a/Objects/codeobject.c b/Objects/codeobject.c index b0e3446..0f03dfe 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -557,7 +557,8 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) addr += *p++; if (addr > addrq) break; - line += *p++; + line += (signed char)*p; + p++; } return line; } @@ -592,17 +593,19 @@ _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds) if (addr + *p > lasti) break; addr += *p++; - if (*p) + if ((signed char)*p) bounds->ap_lower = addr; - line += *p++; + line += (signed char)*p; + p++; --size; } if (size > 0) { while (--size >= 0) { addr += *p++; - if (*p++) + if ((signed char)*p) break; + p++; } bounds->ap_upper = addr; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 425a9ee..da2d2ed 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -137,7 +137,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) new_lasti = -1; for (offset = 0; offset < lnotab_len; offset += 2) { addr += lnotab[offset]; - line += lnotab[offset+1]; + line += (signed char)lnotab[offset+1]; if (line >= new_lineno) { new_lasti = addr; new_lineno = line; diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index d247edd..8af10db 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -12,42 +12,47 @@ pairs. The details are important and delicate, best illustrated by example: 0 1 6 2 50 7 - 350 307 - 361 308 + 350 207 + 361 208 Instead of storing these numbers literally, we compress the list by storing only -the increments from one row to the next. Conceptually, the stored list might +the difference from one row to the next. Conceptually, the stored list might look like: - 0, 1, 6, 1, 44, 5, 300, 300, 11, 1 + 0, 1, 6, 1, 44, 5, 300, 200, 11, 1 -The above doesn't really work, but it's a start. Note that an unsigned byte -can't hold negative values, or values larger than 255, and the above example -contains two such values. So we make two tweaks: +The above doesn't really work, but it's a start. An unsigned byte (byte code +offset)) can't hold negative values, or values larger than 255, a signed byte +(line number) can't hold values larger than 127 or less than -128, and the +above example contains two such values. So we make two tweaks: - (a) there's a deep assumption that byte code offsets and their corresponding - line #s both increase monotonically, and - (b) if at least one column jumps by more than 255 from one row to the next, - more than one pair is written to the table. In case #b, there's no way to know - from looking at the table later how many were written. That's the delicate - part. A user of co_lnotab desiring to find the source line number - corresponding to a bytecode address A should do something like this + (a) there's a deep assumption that byte code offsets increase monotonically, + and + (b) if byte code offset jumps by more than 255 from one row to the next, or if + source code line number jumps by more than 127 or less than -128 from one row + to the next, more than one pair is written to the table. In case #b, + there's no way to know from looking at the table later how many were written. + That's the delicate part. A user of co_lnotab desiring to find the source + line number corresponding to a bytecode address A should do something like + this: lineno = addr = 0 for addr_incr, line_incr in co_lnotab: addr += addr_incr if addr > A: return lineno + if line_incr >= 0x80: + line_incr -= 0x100 lineno += line_incr (In C, this is implemented by PyCode_Addr2Line().) In order for this to work, when the addr field increments by more than 255, the line # increment in each pair generated must be 0 until the remaining addr increment is < 256. So, in the example above, assemble_lnotab in compile.c should not (as was actually done -until 2.2) expand 300, 300 to +until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 255, 0, 45. + 255, 0, 45, 128, 0, 72. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c diff --git a/Python/compile.c b/Python/compile.c index 0f619c4..f362ac6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4452,7 +4452,6 @@ assemble_lnotab(struct assembler *a, struct instr *i) d_lineno = i->i_lineno - a->a_lineno; assert(d_bytecode >= 0); - assert(d_lineno >= 0); if(d_bytecode == 0 && d_lineno == 0) return 1; @@ -4482,9 +4481,21 @@ assemble_lnotab(struct assembler *a, struct instr *i) d_bytecode -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } - assert(d_bytecode <= 255); - if (d_lineno > 255) { - int j, nbytes, ncodes = d_lineno / 255; + assert(0 <= d_bytecode && d_bytecode <= 255); + + if (d_lineno < -128 || 127 < d_lineno) { + int j, nbytes, ncodes, k; + if (d_lineno < 0) { + k = -128; + /* use division on positive numbers */ + ncodes = (-d_lineno) / 128; + } + else { + k = 127; + ncodes = d_lineno / 127; + } + d_lineno -= ncodes * k; + assert(ncodes >= 1); nbytes = a->a_lnotab_off + 2 * ncodes; len = PyBytes_GET_SIZE(a->a_lnotab); if (nbytes >= len) { @@ -4502,15 +4513,15 @@ assemble_lnotab(struct assembler *a, struct instr *i) lnotab = (unsigned char *) PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off; *lnotab++ = d_bytecode; - *lnotab++ = 255; + *lnotab++ = k; d_bytecode = 0; for (j = 1; j < ncodes; j++) { *lnotab++ = 0; - *lnotab++ = 255; + *lnotab++ = k; } - d_lineno -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } + assert(-128 <= d_lineno && d_lineno <= 127); len = PyBytes_GET_SIZE(a->a_lnotab); if (a->a_lnotab_off + 2 >= len) { diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 5828ceb..97c4948 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -258,7 +258,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,5,0,0,0,218,13,95,119,114,105,116,101,95,97,116, 111,109,105,99,99,0,0,0,115,26,0,0,0,0,5,24, 1,9,1,33,1,3,3,21,1,20,1,20,1,13,1,3, - 1,17,1,13,1,5,1,114,55,0,0,0,105,32,13,0, + 1,17,1,13,1,5,1,114,55,0,0,0,105,33,13,0, 0,233,2,0,0,0,114,13,0,0,0,115,2,0,0,0, 13,10,90,11,95,95,112,121,99,97,99,104,101,95,95,122, 4,111,112,116,45,122,3,46,112,121,122,4,46,112,121,99, @@ -368,7 +368,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,90,15,97,108,109,111,115,116,95,102,105,108,101,110,97, 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, 0,218,17,99,97,99,104,101,95,102,114,111,109,95,115,111, - 117,114,99,101,244,0,0,0,115,46,0,0,0,0,18,12, + 117,114,99,101,245,0,0,0,115,46,0,0,0,0,18,12, 1,9,1,7,1,12,1,6,1,12,1,18,1,18,1,24, 1,12,1,12,1,12,1,36,1,12,1,18,1,9,2,12, 1,12,1,12,1,12,1,21,1,21,1,114,79,0,0,0, @@ -447,7 +447,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,95, - 99,97,99,104,101,32,1,0,0,115,44,0,0,0,0,9, + 99,97,99,104,101,33,1,0,0,115,44,0,0,0,0,9, 18,1,12,1,18,1,18,1,12,1,9,1,15,1,15,1, 12,1,9,1,15,1,12,1,22,1,15,1,9,1,12,1, 22,1,12,1,9,1,12,1,19,1,114,85,0,0,0,99, @@ -484,7 +484,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,36,0,0,0,90,9,101,120,116,101,110,115,105, 111,110,218,11,115,111,117,114,99,101,95,112,97,116,104,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,15, - 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,65, + 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,66, 1,0,0,115,20,0,0,0,0,7,18,1,4,1,24,1, 35,1,4,1,3,1,16,1,19,1,21,1,114,91,0,0, 0,99,1,0,0,0,0,0,0,0,1,0,0,0,11,0, @@ -499,7 +499,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,79,0,0,0,114,66,0,0,0,114,74,0,0, 0,41,1,218,8,102,105,108,101,110,97,109,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,103, - 101,116,95,99,97,99,104,101,100,84,1,0,0,115,16,0, + 101,116,95,99,97,99,104,101,100,85,1,0,0,115,16,0, 0,0,0,1,21,1,3,1,14,1,13,1,8,1,21,1, 4,2,114,95,0,0,0,99,1,0,0,0,0,0,0,0, 2,0,0,0,11,0,0,0,67,0,0,0,115,60,0,0, @@ -514,7 +514,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,39,0,0,0,114,41,0,0,0,114,40,0,0,0,41, 2,114,35,0,0,0,114,42,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,10,95,99,97,108, - 99,95,109,111,100,101,96,1,0,0,115,12,0,0,0,0, + 99,95,109,111,100,101,97,1,0,0,115,12,0,0,0,0, 2,3,1,19,1,13,1,11,3,10,1,114,97,0,0,0, 99,1,0,0,0,0,0,0,0,3,0,0,0,11,0,0, 0,3,0,0,0,115,84,0,0,0,100,1,0,135,0,0, @@ -554,7 +554,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,115,90,6,107,119,97,114,103,115,41,1,218,6,109,101, 116,104,111,100,114,4,0,0,0,114,5,0,0,0,218,19, 95,99,104,101,99,107,95,110,97,109,101,95,119,114,97,112, - 112,101,114,116,1,0,0,115,12,0,0,0,0,1,12,1, + 112,101,114,117,1,0,0,115,12,0,0,0,0,1,12,1, 12,1,15,1,6,1,25,1,122,40,95,99,104,101,99,107, 95,110,97,109,101,46,60,108,111,99,97,108,115,62,46,95, 99,104,101,99,107,95,110,97,109,101,95,119,114,97,112,112, @@ -573,7 +573,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,97,116,116,114,218,8,95,95,100,105,99,116,95,95,218, 6,117,112,100,97,116,101,41,3,90,3,110,101,119,90,3, 111,108,100,114,52,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,5,95,119,114,97,112,127,1, + 0,0,114,5,0,0,0,218,5,95,119,114,97,112,128,1, 0,0,115,8,0,0,0,0,1,25,1,15,1,29,1,122, 26,95,99,104,101,99,107,95,110,97,109,101,46,60,108,111, 99,97,108,115,62,46,95,119,114,97,112,41,3,218,10,95, @@ -581,7 +581,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,97,109,101,69,114,114,111,114,41,3,114,102,0,0,0, 114,103,0,0,0,114,113,0,0,0,114,4,0,0,0,41, 1,114,102,0,0,0,114,5,0,0,0,218,11,95,99,104, - 101,99,107,95,110,97,109,101,108,1,0,0,115,14,0,0, + 101,99,107,95,110,97,109,101,109,1,0,0,115,14,0,0, 0,0,8,21,7,3,1,13,1,13,2,17,5,13,1,114, 116,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, 0,4,0,0,0,67,0,0,0,115,84,0,0,0,124,0, @@ -611,7 +611,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,112,111,114,116,105,111,110,115,218,3,109,115,103,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,17, 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, - 109,136,1,0,0,115,10,0,0,0,0,10,21,1,24,1, + 109,137,1,0,0,115,10,0,0,0,0,10,21,1,24,1, 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,252, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, @@ -698,7 +698,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,11,115,111,117,114,99,101,95,115,105,122,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,25,95,118, 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, - 95,104,101,97,100,101,114,153,1,0,0,115,76,0,0,0, + 95,104,101,97,100,101,114,154,1,0,0,115,76,0,0,0, 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, 16,1,16,1,12,1,18,1,16,1,18,1,18,1,15,1, 16,1,15,1,18,1,15,1,16,1,12,1,12,1,3,1, @@ -729,7 +729,7 @@ const unsigned char _Py_M__importlib_external[] = { 5,114,53,0,0,0,114,98,0,0,0,114,89,0,0,0, 114,90,0,0,0,218,4,99,111,100,101,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,109, - 112,105,108,101,95,98,121,116,101,99,111,100,101,208,1,0, + 112,105,108,101,95,98,121,116,101,99,111,100,101,209,1,0, 0,115,16,0,0,0,0,2,15,1,15,1,16,1,12,1, 16,1,4,2,18,1,114,141,0,0,0,114,59,0,0,0, 99,3,0,0,0,0,0,0,0,4,0,0,0,3,0,0, @@ -749,7 +749,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,117,109,112,115,41,4,114,140,0,0,0,114,126,0,0, 0,114,134,0,0,0,114,53,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,100, - 101,95,116,111,95,98,121,116,101,99,111,100,101,220,1,0, + 101,95,116,111,95,98,121,116,101,99,111,100,101,221,1,0, 0,115,10,0,0,0,0,3,12,1,19,1,19,1,22,1, 114,144,0,0,0,99,1,0,0,0,0,0,0,0,5,0, 0,0,4,0,0,0,67,0,0,0,115,89,0,0,0,100, @@ -778,7 +778,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,101,110,99,111,100,105,110,103,90,15,110,101,119,108, 105,110,101,95,100,101,99,111,100,101,114,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,13,100,101,99,111, - 100,101,95,115,111,117,114,99,101,230,1,0,0,115,10,0, + 100,101,95,115,111,117,114,99,101,231,1,0,0,115,10,0, 0,0,0,5,12,1,18,1,15,1,18,1,114,149,0,0, 0,114,120,0,0,0,218,26,115,117,98,109,111,100,117,108, 101,95,115,101,97,114,99,104,95,108,111,99,97,116,105,111, @@ -843,7 +843,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,105,120,101,115,114,153,0,0,0,90,7,100,105,114,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,23,115,112,101,99,95,102,114,111,109,95,102,105, - 108,101,95,108,111,99,97,116,105,111,110,247,1,0,0,115, + 108,101,95,108,111,99,97,116,105,111,110,248,1,0,0,115, 60,0,0,0,0,12,12,4,6,1,15,2,3,1,19,1, 13,1,5,8,24,1,9,3,12,1,22,1,21,1,15,1, 9,1,5,2,4,3,12,2,15,1,3,1,19,1,13,1, @@ -883,7 +883,7 @@ const unsigned char _Py_M__importlib_external[] = { 72,75,69,89,95,76,79,67,65,76,95,77,65,67,72,73, 78,69,41,2,218,3,99,108,115,218,3,107,101,121,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,218,14,95, - 111,112,101,110,95,114,101,103,105,115,116,114,121,69,2,0, + 111,112,101,110,95,114,101,103,105,115,116,114,121,70,2,0, 0,115,8,0,0,0,0,2,3,1,23,1,13,1,122,36, 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, @@ -910,7 +910,7 @@ const unsigned char _Py_M__importlib_external[] = { 121,95,107,101,121,114,165,0,0,0,90,4,104,107,101,121, 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, - 99,104,95,114,101,103,105,115,116,114,121,76,2,0,0,115, + 99,104,95,114,101,103,105,115,116,114,121,77,2,0,0,115, 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, @@ -935,7 +935,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,101,116,114,171,0,0,0,114,120,0,0,0,114,160,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,9,102,105,110,100,95,115,112,101, - 99,91,2,0,0,115,26,0,0,0,0,2,15,1,12,1, + 99,92,2,0,0,115,26,0,0,0,0,2,15,1,12,1, 4,1,3,1,14,1,13,1,9,1,22,1,21,1,9,1, 15,1,9,1,122,31,87,105,110,100,111,119,115,82,101,103, 105,115,116,114,121,70,105,110,100,101,114,46,102,105,110,100, @@ -954,7 +954,7 @@ const unsigned char _Py_M__importlib_external[] = { 175,0,0,0,114,120,0,0,0,41,4,114,164,0,0,0, 114,119,0,0,0,114,35,0,0,0,114,158,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 102,105,110,100,95,109,111,100,117,108,101,107,2,0,0,115, + 102,105,110,100,95,109,111,100,117,108,101,108,2,0,0,115, 8,0,0,0,0,7,18,1,12,1,7,2,122,33,87,105, 110,100,111,119,115,82,101,103,105,115,116,114,121,70,105,110, 100,101,114,46,102,105,110,100,95,109,111,100,117,108,101,41, @@ -963,7 +963,7 @@ const unsigned char _Py_M__importlib_external[] = { 167,0,0,0,218,11,99,108,97,115,115,109,101,116,104,111, 100,114,166,0,0,0,114,172,0,0,0,114,175,0,0,0, 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,162,0,0,0,57,2, + 4,0,0,0,114,5,0,0,0,114,162,0,0,0,58,2, 0,0,115,20,0,0,0,12,2,6,3,6,3,6,2,6, 2,18,7,18,15,3,1,21,15,3,1,114,162,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, @@ -1001,7 +1001,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,0,0,0,114,119,0,0,0,114,94,0,0,0,90,13, 102,105,108,101,110,97,109,101,95,98,97,115,101,90,9,116, 97,105,108,95,110,97,109,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,153,0,0,0,126,2,0,0, + 0,0,114,5,0,0,0,114,153,0,0,0,127,2,0,0, 115,8,0,0,0,0,3,25,1,22,1,19,1,122,24,95, 76,111,97,100,101,114,66,97,115,105,99,115,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, @@ -1012,7 +1012,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,110,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, 5,0,0,0,218,13,99,114,101,97,116,101,95,109,111,100, - 117,108,101,134,2,0,0,115,0,0,0,0,122,27,95,76, + 117,108,101,135,2,0,0,115,0,0,0,0,122,27,95,76, 111,97,100,101,114,66,97,115,105,99,115,46,99,114,101,97, 116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,0, 0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,80, @@ -1033,7 +1033,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,99,114,111,0,0,0,41,3,114,100,0,0,0,218,6, 109,111,100,117,108,101,114,140,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,11,101,120,101,99, - 95,109,111,100,117,108,101,137,2,0,0,115,10,0,0,0, + 95,109,111,100,117,108,101,138,2,0,0,115,10,0,0,0, 0,2,18,1,12,1,9,1,15,1,122,25,95,76,111,97, 100,101,114,66,97,115,105,99,115,46,101,120,101,99,95,109, 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1044,14 +1044,14 @@ const unsigned char _Py_M__importlib_external[] = { 114,0,0,0,218,17,95,108,111,97,100,95,109,111,100,117, 108,101,95,115,104,105,109,41,2,114,100,0,0,0,114,119, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,11,108,111,97,100,95,109,111,100,117,108,101,145, + 0,0,218,11,108,111,97,100,95,109,111,100,117,108,101,146, 2,0,0,115,2,0,0,0,0,2,122,25,95,76,111,97, 100,101,114,66,97,115,105,99,115,46,108,111,97,100,95,109, 111,100,117,108,101,78,41,8,114,105,0,0,0,114,104,0, 0,0,114,106,0,0,0,114,107,0,0,0,114,153,0,0, 0,114,180,0,0,0,114,185,0,0,0,114,187,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,178,0,0,0,121,2,0,0,115,10,0, + 5,0,0,0,114,178,0,0,0,122,2,0,0,115,10,0, 0,0,12,3,6,2,12,8,12,3,12,8,114,178,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, 0,0,64,0,0,0,115,106,0,0,0,101,0,0,90,1, @@ -1079,7 +1079,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,1,218,7,73,79,69,114,114,111,114,41,2,114,100,0, 0,0,114,35,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,10,112,97,116,104,95,109,116,105, - 109,101,152,2,0,0,115,2,0,0,0,0,6,122,23,83, + 109,101,153,2,0,0,115,2,0,0,0,0,6,122,23,83, 111,117,114,99,101,76,111,97,100,101,114,46,112,97,116,104, 95,109,116,105,109,101,99,2,0,0,0,0,0,0,0,2, 0,0,0,3,0,0,0,67,0,0,0,115,19,0,0,0, @@ -1114,7 +1114,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,114,126,0,0,0,41,1,114,190,0,0,0, 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, - 104,95,115,116,97,116,115,160,2,0,0,115,2,0,0,0, + 104,95,115,116,97,116,115,161,2,0,0,115,2,0,0,0, 0,11,122,23,83,111,117,114,99,101,76,111,97,100,101,114, 46,112,97,116,104,95,115,116,97,116,115,99,4,0,0,0, 0,0,0,0,4,0,0,0,3,0,0,0,67,0,0,0, @@ -1138,7 +1138,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,90,0,0,0,90,10,99,97,99,104,101,95,112,97,116, 104,114,53,0,0,0,114,4,0,0,0,114,4,0,0,0, 114,5,0,0,0,218,15,95,99,97,99,104,101,95,98,121, - 116,101,99,111,100,101,173,2,0,0,115,2,0,0,0,0, + 116,101,99,111,100,101,174,2,0,0,115,2,0,0,0,0, 8,122,28,83,111,117,114,99,101,76,111,97,100,101,114,46, 95,99,97,99,104,101,95,98,121,116,101,99,111,100,101,99, 3,0,0,0,0,0,0,0,3,0,0,0,1,0,0,0, @@ -1155,7 +1155,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,32,32,78,114,4,0,0,0,41,3,114, 100,0,0,0,114,35,0,0,0,114,53,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,192,0, - 0,0,183,2,0,0,115,0,0,0,0,122,21,83,111,117, + 0,0,184,2,0,0,115,0,0,0,0,122,21,83,111,117, 114,99,101,76,111,97,100,101,114,46,115,101,116,95,100,97, 116,97,99,2,0,0,0,0,0,0,0,5,0,0,0,16, 0,0,0,67,0,0,0,115,105,0,0,0,124,0,0,106, @@ -1177,7 +1177,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,119,0,0,0,114,35,0,0,0,114,147,0, 0,0,218,3,101,120,99,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,10,103,101,116,95,115,111,117,114, - 99,101,190,2,0,0,115,14,0,0,0,0,2,15,1,3, + 99,101,191,2,0,0,115,14,0,0,0,0,2,15,1,3, 1,19,1,18,1,9,1,31,1,122,23,83,111,117,114,99, 101,76,111,97,100,101,114,46,103,101,116,95,115,111,117,114, 99,101,218,9,95,111,112,116,105,109,105,122,101,114,29,0, @@ -1199,7 +1199,7 @@ const unsigned char _Py_M__importlib_external[] = { 108,101,41,4,114,100,0,0,0,114,53,0,0,0,114,35, 0,0,0,114,197,0,0,0,114,4,0,0,0,114,4,0, 0,0,114,5,0,0,0,218,14,115,111,117,114,99,101,95, - 116,111,95,99,111,100,101,200,2,0,0,115,4,0,0,0, + 116,111,95,99,111,100,101,201,2,0,0,115,4,0,0,0, 0,5,21,1,122,27,83,111,117,114,99,101,76,111,97,100, 101,114,46,115,111,117,114,99,101,95,116,111,95,99,111,100, 101,99,2,0,0,0,0,0,0,0,10,0,0,0,43,0, @@ -1260,7 +1260,7 @@ const unsigned char _Py_M__importlib_external[] = { 89,0,0,0,218,2,115,116,114,53,0,0,0,218,10,98, 121,116,101,115,95,100,97,116,97,114,147,0,0,0,90,11, 99,111,100,101,95,111,98,106,101,99,116,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,181,0,0,0,208, + 114,4,0,0,0,114,5,0,0,0,114,181,0,0,0,209, 2,0,0,115,78,0,0,0,0,7,15,1,6,1,3,1, 16,1,13,1,11,2,3,1,19,1,13,1,5,2,16,1, 3,1,19,1,13,1,5,2,3,1,9,1,12,1,13,1, @@ -1273,7 +1273,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,192,0,0,0,114,196,0,0,0,114,200,0, 0,0,114,181,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,188,0,0,0, - 150,2,0,0,115,14,0,0,0,12,2,12,8,12,13,12, + 151,2,0,0,115,14,0,0,0,12,2,12,8,12,13,12, 10,12,7,12,10,18,8,114,188,0,0,0,99,0,0,0, 0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0, 0,115,112,0,0,0,101,0,0,90,1,0,100,0,0,90, @@ -1301,7 +1301,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,102,105,110,100,101,114,46,78,41,2,114, 98,0,0,0,114,35,0,0,0,41,3,114,100,0,0,0, 114,119,0,0,0,114,35,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,179,0,0,0,9,3, + 4,0,0,0,114,5,0,0,0,114,179,0,0,0,10,3, 0,0,115,4,0,0,0,0,3,9,1,122,19,70,105,108, 101,76,111,97,100,101,114,46,95,95,105,110,105,116,95,95, 99,2,0,0,0,0,0,0,0,2,0,0,0,2,0,0, @@ -1311,7 +1311,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,218,9,95,95,99,108,97,115,115,95,95,114,111,0,0, 0,41,2,114,100,0,0,0,218,5,111,116,104,101,114,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, - 95,95,101,113,95,95,15,3,0,0,115,4,0,0,0,0, + 95,95,101,113,95,95,16,3,0,0,115,4,0,0,0,0, 1,18,1,122,17,70,105,108,101,76,111,97,100,101,114,46, 95,95,101,113,95,95,99,1,0,0,0,0,0,0,0,1, 0,0,0,3,0,0,0,67,0,0,0,115,26,0,0,0, @@ -1319,7 +1319,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,106,2,0,131,1,0,65,83,41,1,78,41,3,218, 4,104,97,115,104,114,98,0,0,0,114,35,0,0,0,41, 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,95,104,97,115,104,95,95,19, + 114,5,0,0,0,218,8,95,95,104,97,115,104,95,95,20, 3,0,0,115,2,0,0,0,0,1,122,19,70,105,108,101, 76,111,97,100,101,114,46,95,95,104,97,115,104,95,95,99, 2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, @@ -1334,7 +1334,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,32,32,41,3,218,5,115,117,112,101,114, 114,204,0,0,0,114,187,0,0,0,41,2,114,100,0,0, 0,114,119,0,0,0,41,1,114,205,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,187,0,0,0,22,3,0,0, + 0,0,114,5,0,0,0,114,187,0,0,0,23,3,0,0, 115,2,0,0,0,0,10,122,22,70,105,108,101,76,111,97, 100,101,114,46,108,111,97,100,95,109,111,100,117,108,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, @@ -1345,7 +1345,7 @@ const unsigned char _Py_M__importlib_external[] = { 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,151,0,0,0,34,3,0,0,115,2,0,0,0,0,3, + 114,151,0,0,0,35,3,0,0,115,2,0,0,0,0,3, 122,23,70,105,108,101,76,111,97,100,101,114,46,103,101,116, 95,102,105,108,101,110,97,109,101,99,2,0,0,0,0,0, 0,0,3,0,0,0,9,0,0,0,67,0,0,0,115,42, @@ -1358,14 +1358,14 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,50,0,0,0,90,4,114,101,97,100,41,3, 114,100,0,0,0,114,35,0,0,0,114,54,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,194, - 0,0,0,39,3,0,0,115,4,0,0,0,0,2,21,1, + 0,0,0,40,3,0,0,115,4,0,0,0,0,2,21,1, 122,19,70,105,108,101,76,111,97,100,101,114,46,103,101,116, 95,100,97,116,97,41,11,114,105,0,0,0,114,104,0,0, 0,114,106,0,0,0,114,107,0,0,0,114,179,0,0,0, 114,207,0,0,0,114,209,0,0,0,114,116,0,0,0,114, 187,0,0,0,114,151,0,0,0,114,194,0,0,0,114,4, 0,0,0,114,4,0,0,0,41,1,114,205,0,0,0,114, - 5,0,0,0,114,204,0,0,0,4,3,0,0,115,14,0, + 5,0,0,0,114,204,0,0,0,5,3,0,0,115,14,0, 0,0,12,3,6,2,12,6,12,4,12,3,24,12,18,5, 114,204,0,0,0,99,0,0,0,0,0,0,0,0,0,0, 0,0,4,0,0,0,64,0,0,0,115,64,0,0,0,101, @@ -1388,7 +1388,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,39,0,0,0,218,8,115,116,95,109,116,105,109,101,90, 7,115,116,95,115,105,122,101,41,3,114,100,0,0,0,114, 35,0,0,0,114,202,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,191,0,0,0,49,3,0, + 0,0,0,114,5,0,0,0,114,191,0,0,0,50,3,0, 0,115,4,0,0,0,0,2,12,1,122,27,83,111,117,114, 99,101,70,105,108,101,76,111,97,100,101,114,46,112,97,116, 104,95,115,116,97,116,115,99,4,0,0,0,0,0,0,0, @@ -1399,7 +1399,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,0,0,0,114,192,0,0,0,41,5,114,100,0,0,0, 114,90,0,0,0,114,89,0,0,0,114,53,0,0,0,114, 42,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,193,0,0,0,54,3,0,0,115,4,0,0, + 0,0,0,114,193,0,0,0,55,3,0,0,115,4,0,0, 0,0,2,12,1,122,32,83,111,117,114,99,101,70,105,108, 101,76,111,97,100,101,114,46,95,99,97,99,104,101,95,98, 121,116,101,99,111,100,101,114,214,0,0,0,105,182,1,0, @@ -1438,7 +1438,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,214,0,0,0,218,6,112,97,114,101,110,116,114,94, 0,0,0,114,27,0,0,0,114,23,0,0,0,114,195,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,192,0,0,0,59,3,0,0,115,42,0,0,0,0, + 0,114,192,0,0,0,60,3,0,0,115,42,0,0,0,0, 2,18,1,6,2,22,1,18,1,17,2,19,1,15,1,3, 1,17,1,13,2,7,1,18,3,9,1,10,1,27,1,3, 1,16,1,20,1,18,2,12,1,122,25,83,111,117,114,99, @@ -1447,7 +1447,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,106,0,0,0,114,107,0,0,0,114,191,0,0,0, 114,193,0,0,0,114,192,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,212, - 0,0,0,45,3,0,0,115,8,0,0,0,12,2,6,2, + 0,0,0,46,3,0,0,115,8,0,0,0,12,2,6,2, 12,5,12,5,114,212,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,2,0,0,0,64,0,0,0,115,46, 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, @@ -1469,7 +1469,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,141,0,0,0,41,5,114,100,0,0,0,114,119,0, 0,0,114,35,0,0,0,114,53,0,0,0,114,203,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,181,0,0,0,94,3,0,0,115,8,0,0,0,0,1, + 114,181,0,0,0,95,3,0,0,115,8,0,0,0,0,1, 15,1,15,1,24,1,122,29,83,111,117,114,99,101,108,101, 115,115,70,105,108,101,76,111,97,100,101,114,46,103,101,116, 95,99,111,100,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1479,13 +1479,13 @@ const unsigned char _Py_M__importlib_external[] = { 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 196,0,0,0,100,3,0,0,115,2,0,0,0,0,2,122, + 196,0,0,0,101,3,0,0,115,2,0,0,0,0,2,122, 31,83,111,117,114,99,101,108,101,115,115,70,105,108,101,76, 111,97,100,101,114,46,103,101,116,95,115,111,117,114,99,101, 78,41,6,114,105,0,0,0,114,104,0,0,0,114,106,0, 0,0,114,107,0,0,0,114,181,0,0,0,114,196,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,217,0,0,0,90,3,0,0,115,6, + 114,5,0,0,0,114,217,0,0,0,91,3,0,0,115,6, 0,0,0,12,2,6,2,12,6,114,217,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, 0,0,0,115,136,0,0,0,101,0,0,90,1,0,100,0, @@ -1510,7 +1510,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,0,100,0,0,83,41,1,78,41,2,114,98,0,0,0, 114,35,0,0,0,41,3,114,100,0,0,0,114,98,0,0, 0,114,35,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,179,0,0,0,117,3,0,0,115,4, + 114,5,0,0,0,114,179,0,0,0,118,3,0,0,115,4, 0,0,0,0,1,9,1,122,28,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,105, 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, @@ -1520,7 +1520,7 @@ const unsigned char _Py_M__importlib_external[] = { 83,41,1,78,41,2,114,205,0,0,0,114,111,0,0,0, 41,2,114,100,0,0,0,114,206,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,207,0,0,0, - 121,3,0,0,115,4,0,0,0,0,1,18,1,122,26,69, + 122,3,0,0,115,4,0,0,0,0,1,18,1,122,26,69, 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, 101,114,46,95,95,101,113,95,95,99,1,0,0,0,0,0, 0,0,1,0,0,0,3,0,0,0,67,0,0,0,115,26, @@ -1528,7 +1528,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,124,0,0,106,2,0,131,1,0,65,83,41,1,78, 41,3,114,208,0,0,0,114,98,0,0,0,114,35,0,0, 0,41,1,114,100,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,209,0,0,0,125,3,0,0, + 0,0,114,5,0,0,0,114,209,0,0,0,126,3,0,0, 115,2,0,0,0,0,1,122,28,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,104, 97,115,104,95,95,99,2,0,0,0,0,0,0,0,3,0, @@ -1546,7 +1546,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, 0,0,0,41,3,114,100,0,0,0,114,158,0,0,0,114, 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,180,0,0,0,128,3,0,0,115,10,0,0, + 0,0,0,114,180,0,0,0,129,3,0,0,115,10,0,0, 0,0,2,6,1,15,1,9,1,16,1,122,33,69,120,116, 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, 46,99,114,101,97,116,101,95,109,111,100,117,108,101,99,2, @@ -1564,7 +1564,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, 0,0,0,41,2,114,100,0,0,0,114,184,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,185, - 0,0,0,136,3,0,0,115,6,0,0,0,0,2,19,1, + 0,0,0,137,3,0,0,115,6,0,0,0,0,2,19,1, 9,1,122,31,69,120,116,101,110,115,105,111,110,70,105,108, 101,76,111,97,100,101,114,46,101,120,101,99,95,109,111,100, 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, @@ -1582,7 +1582,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,2,114,179,0,0,0,78,114,4,0,0,0,41,2,114, 22,0,0,0,218,6,115,117,102,102,105,120,41,1,218,9, 102,105,108,101,95,110,97,109,101,114,4,0,0,0,114,5, - 0,0,0,250,9,60,103,101,110,101,120,112,114,62,145,3, + 0,0,0,250,9,60,103,101,110,101,120,112,114,62,146,3, 0,0,115,2,0,0,0,6,1,122,49,69,120,116,101,110, 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,105, 115,95,112,97,99,107,97,103,101,46,60,108,111,99,97,108, @@ -1591,7 +1591,7 @@ const unsigned char _Py_M__importlib_external[] = { 88,84,69,78,83,73,79,78,95,83,85,70,70,73,88,69, 83,41,2,114,100,0,0,0,114,119,0,0,0,114,4,0, 0,0,41,1,114,220,0,0,0,114,5,0,0,0,114,153, - 0,0,0,142,3,0,0,115,6,0,0,0,0,2,19,1, + 0,0,0,143,3,0,0,115,6,0,0,0,0,2,19,1, 18,1,122,30,69,120,116,101,110,115,105,111,110,70,105,108, 101,76,111,97,100,101,114,46,105,115,95,112,97,99,107,97, 103,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, @@ -1602,7 +1602,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,97,116,101,32,97,32,99,111,100,101,32,111,98,106,101, 99,116,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,181,0,0,0,148,3,0,0,115,2,0, + 5,0,0,0,114,181,0,0,0,149,3,0,0,115,2,0, 0,0,0,2,122,28,69,120,116,101,110,115,105,111,110,70, 105,108,101,76,111,97,100,101,114,46,103,101,116,95,99,111, 100,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, @@ -1612,7 +1612,7 @@ const unsigned char _Py_M__importlib_external[] = { 117,108,101,115,32,104,97,118,101,32,110,111,32,115,111,117, 114,99,101,32,99,111,100,101,46,78,114,4,0,0,0,41, 2,114,100,0,0,0,114,119,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,196,0,0,0,152, + 114,4,0,0,0,114,5,0,0,0,114,196,0,0,0,153, 3,0,0,115,2,0,0,0,0,2,122,30,69,120,116,101, 110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,46, 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, @@ -1624,7 +1624,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,32,102,105,110,100,101,114,46,41,1,114,35,0,0,0, 41,2,114,100,0,0,0,114,119,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,151,0,0,0, - 156,3,0,0,115,2,0,0,0,0,3,122,32,69,120,116, + 157,3,0,0,115,2,0,0,0,0,3,122,32,69,120,116, 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, 46,103,101,116,95,102,105,108,101,110,97,109,101,78,41,14, 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, @@ -1632,7 +1632,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,180,0,0,0,114,185,0,0,0,114,153,0, 0,0,114,181,0,0,0,114,196,0,0,0,114,116,0,0, 0,114,151,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,218,0,0,0,109, + 114,4,0,0,0,114,5,0,0,0,114,218,0,0,0,110, 3,0,0,115,20,0,0,0,12,6,6,2,12,4,12,4, 12,3,12,8,12,6,12,6,12,4,12,4,114,218,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0, @@ -1677,7 +1677,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,101,114,41,4,114,100,0,0,0,114,98,0,0,0,114, 35,0,0,0,218,11,112,97,116,104,95,102,105,110,100,101, 114,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,169,3,0,0,115,8,0,0,0,0,1, + 114,179,0,0,0,170,3,0,0,115,8,0,0,0,0,1, 9,1,9,1,21,1,122,23,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,99, 1,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, @@ -1696,7 +1696,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,218,3,100,111,116,90,2,109,101,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,23,95,102,105, 110,100,95,112,97,114,101,110,116,95,112,97,116,104,95,110, - 97,109,101,115,175,3,0,0,115,8,0,0,0,0,2,27, + 97,109,101,115,176,3,0,0,115,8,0,0,0,0,2,27, 1,12,2,4,3,122,38,95,78,97,109,101,115,112,97,99, 101,80,97,116,104,46,95,102,105,110,100,95,112,97,114,101, 110,116,95,112,97,116,104,95,110,97,109,101,115,99,1,0, @@ -1709,7 +1709,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,90,18,112,97,114,101,110,116,95,109,111,100,117,108,101, 95,110,97,109,101,90,14,112,97,116,104,95,97,116,116,114, 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,227,0,0,0,185,3,0,0,115,4,0, + 5,0,0,0,114,227,0,0,0,186,3,0,0,115,4,0, 0,0,0,1,18,1,122,31,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,103,101,116,95,112,97,114,101, 110,116,95,112,97,116,104,99,1,0,0,0,0,0,0,0, @@ -1727,7 +1727,7 @@ const unsigned char _Py_M__importlib_external[] = { 226,0,0,0,41,3,114,100,0,0,0,90,11,112,97,114, 101,110,116,95,112,97,116,104,114,158,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,114, - 101,99,97,108,99,117,108,97,116,101,189,3,0,0,115,16, + 101,99,97,108,99,117,108,97,116,101,190,3,0,0,115,16, 0,0,0,0,2,18,1,15,1,21,3,27,1,9,1,12, 1,9,1,122,27,95,78,97,109,101,115,112,97,99,101,80, 97,116,104,46,95,114,101,99,97,108,99,117,108,97,116,101, @@ -1736,7 +1736,7 @@ const unsigned char _Py_M__importlib_external[] = { 106,1,0,131,0,0,131,1,0,83,41,1,78,41,2,218, 4,105,116,101,114,114,234,0,0,0,41,1,114,100,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,8,95,95,105,116,101,114,95,95,202,3,0,0,115,2, + 218,8,95,95,105,116,101,114,95,95,203,3,0,0,115,2, 0,0,0,0,1,122,23,95,78,97,109,101,115,112,97,99, 101,80,97,116,104,46,95,95,105,116,101,114,95,95,99,1, 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, @@ -1744,7 +1744,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,131,0,0,131,1,0,83,41,1,78,41,2,114,31,0, 0,0,114,234,0,0,0,41,1,114,100,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,218,7,95, - 95,108,101,110,95,95,205,3,0,0,115,2,0,0,0,0, + 95,108,101,110,95,95,206,3,0,0,115,2,0,0,0,0, 1,122,22,95,78,97,109,101,115,112,97,99,101,80,97,116, 104,46,95,95,108,101,110,95,95,99,1,0,0,0,0,0, 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, @@ -1753,7 +1753,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,101,80,97,116,104,40,123,33,114,125,41,41,2,114,47, 0,0,0,114,226,0,0,0,41,1,114,100,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,95,114,101,112,114,95,95,208,3,0,0,115,2,0,0, + 95,95,114,101,112,114,95,95,209,3,0,0,115,2,0,0, 0,0,1,122,23,95,78,97,109,101,115,112,97,99,101,80, 97,116,104,46,95,95,114,101,112,114,95,95,99,2,0,0, 0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,0, @@ -1761,7 +1761,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,107,6,0,83,41,1,78,41,1,114,234,0,0,0, 41,2,114,100,0,0,0,218,4,105,116,101,109,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,95, - 99,111,110,116,97,105,110,115,95,95,211,3,0,0,115,2, + 99,111,110,116,97,105,110,115,95,95,212,3,0,0,115,2, 0,0,0,0,1,122,27,95,78,97,109,101,115,112,97,99, 101,80,97,116,104,46,95,95,99,111,110,116,97,105,110,115, 95,95,99,2,0,0,0,0,0,0,0,2,0,0,0,2, @@ -1769,7 +1769,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,106,1,0,124,1,0,131,1,0,1,100,0,0,83, 41,1,78,41,2,114,226,0,0,0,114,157,0,0,0,41, 2,114,100,0,0,0,114,239,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,214, + 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,215, 3,0,0,115,2,0,0,0,0,1,122,21,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,97,112,112,101,110, 100,78,41,13,114,105,0,0,0,114,104,0,0,0,114,106, @@ -1777,7 +1777,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,227,0,0,0,114,234,0,0,0,114,236,0,0, 0,114,237,0,0,0,114,238,0,0,0,114,240,0,0,0, 114,157,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,224,0,0,0,162,3, + 4,0,0,0,114,5,0,0,0,114,224,0,0,0,163,3, 0,0,115,20,0,0,0,12,5,6,2,12,6,12,10,12, 4,12,13,12,3,12,3,12,3,12,3,114,224,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, @@ -1797,7 +1797,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,226,0,0,0,41,4,114,100,0,0,0,114,98,0, 0,0,114,35,0,0,0,114,230,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,179,0,0,0, - 220,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, + 221,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, 109,101,115,112,97,99,101,76,111,97,100,101,114,46,95,95, 105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,2, 0,0,0,2,0,0,0,67,0,0,0,115,16,0,0,0, @@ -1814,21 +1814,21 @@ const unsigned char _Py_M__importlib_external[] = { 41,62,41,2,114,47,0,0,0,114,105,0,0,0,41,2, 114,164,0,0,0,114,184,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,11,109,111,100,117,108, - 101,95,114,101,112,114,223,3,0,0,115,2,0,0,0,0, + 101,95,114,101,112,114,224,3,0,0,115,2,0,0,0,0, 7,122,28,95,78,97,109,101,115,112,97,99,101,76,111,97, 100,101,114,46,109,111,100,117,108,101,95,114,101,112,114,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,78, 84,114,4,0,0,0,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,153,0,0,0,232,3,0,0,115,2,0,0,0,0, + 0,114,153,0,0,0,233,3,0,0,115,2,0,0,0,0, 1,122,27,95,78,97,109,101,115,112,97,99,101,76,111,97, 100,101,114,46,105,115,95,112,97,99,107,97,103,101,99,2, 0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67, 0,0,0,115,4,0,0,0,100,1,0,83,41,2,78,114, 30,0,0,0,114,4,0,0,0,41,2,114,100,0,0,0, 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,235,3,0,0,115,2,0, + 5,0,0,0,114,196,0,0,0,236,3,0,0,115,2,0, 0,0,0,1,122,27,95,78,97,109,101,115,112,97,99,101, 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, 101,99,2,0,0,0,0,0,0,0,2,0,0,0,6,0, @@ -1838,7 +1838,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,103,62,114,183,0,0,0,114,198,0,0,0,84,41,1, 114,199,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,181,0,0,0,238,3,0,0,115,2,0,0,0,0,1, + 114,181,0,0,0,239,3,0,0,115,2,0,0,0,0,1, 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, @@ -1847,14 +1847,14 @@ const unsigned char _Py_M__importlib_external[] = { 99,115,32,102,111,114,32,109,111,100,117,108,101,32,99,114, 101,97,116,105,111,110,46,78,114,4,0,0,0,41,2,114, 100,0,0,0,114,158,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,241,3,0, + 0,0,0,114,5,0,0,0,114,180,0,0,0,242,3,0, 0,115,0,0,0,0,122,30,95,78,97,109,101,115,112,97, 99,101,76,111,97,100,101,114,46,99,114,101,97,116,101,95, 109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,2, 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,100, 0,0,0,114,184,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,185,0,0,0,244,3,0,0, + 0,0,114,5,0,0,0,114,185,0,0,0,245,3,0,0, 115,2,0,0,0,0,1,122,28,95,78,97,109,101,115,112, 97,99,101,76,111,97,100,101,114,46,101,120,101,99,95,109, 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1873,7 +1873,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,4,114,114,0,0,0,114,129,0,0,0,114,226,0,0, 0,114,186,0,0,0,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,187,0,0,0,247,3,0,0,115,6,0,0,0,0, + 0,114,187,0,0,0,248,3,0,0,115,6,0,0,0,0, 7,9,1,10,1,122,28,95,78,97,109,101,115,112,97,99, 101,76,111,97,100,101,114,46,108,111,97,100,95,109,111,100, 117,108,101,78,41,12,114,105,0,0,0,114,104,0,0,0, @@ -1881,7 +1881,7 @@ const unsigned char _Py_M__importlib_external[] = { 242,0,0,0,114,153,0,0,0,114,196,0,0,0,114,181, 0,0,0,114,180,0,0,0,114,185,0,0,0,114,187,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,241,0,0,0,219,3,0,0,115, + 0,114,5,0,0,0,114,241,0,0,0,220,3,0,0,115, 16,0,0,0,12,1,12,3,18,9,12,3,12,3,12,3, 12,3,12,3,114,241,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,5,0,0,0,64,0,0,0,115,160, @@ -1919,7 +1919,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,104,101,218,6,118,97,108,117,101,115,114,108,0,0,0, 114,244,0,0,0,41,2,114,164,0,0,0,218,6,102,105, 110,100,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,244,0,0,0,9,4,0,0,115,6,0,0, + 0,0,0,114,244,0,0,0,10,4,0,0,115,6,0,0, 0,0,4,22,1,15,1,122,28,80,97,116,104,70,105,110, 100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99, 97,99,104,101,115,99,2,0,0,0,0,0,0,0,3,0, @@ -1944,7 +1944,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,61,0,0,0,114,118,0,0,0,114,99,0,0,0, 41,3,114,164,0,0,0,114,35,0,0,0,90,4,104,111, 111,107,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,11,95,112,97,116,104,95,104,111,111,107,115,17,4, + 0,218,11,95,112,97,116,104,95,104,111,111,107,115,18,4, 0,0,115,16,0,0,0,0,7,25,1,16,1,16,1,3, 1,14,1,13,1,12,2,122,22,80,97,116,104,70,105,110, 100,101,114,46,95,112,97,116,104,95,104,111,111,107,115,99, @@ -1977,7 +1977,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,3,114,164,0,0,0,114,35,0,0,0,114, 247,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, 0,0,0,218,20,95,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,34,4,0,0,115,22,0, + 116,101,114,95,99,97,99,104,101,35,4,0,0,115,22,0, 0,0,0,8,12,1,3,1,16,1,13,3,9,1,3,1, 17,1,13,1,15,1,18,1,122,31,80,97,116,104,70,105, 110,100,101,114,46,95,112,97,116,104,95,105,109,112,111,114, @@ -1997,7 +1997,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,247,0,0,0,114,120,0,0,0,114,121,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,16,95,108,101,103,97,99,121,95, - 103,101,116,95,115,112,101,99,56,4,0,0,115,18,0,0, + 103,101,116,95,115,112,101,99,57,4,0,0,115,18,0,0, 0,0,4,15,1,24,2,15,1,6,1,12,1,16,1,18, 1,9,1,122,27,80,97,116,104,70,105,110,100,101,114,46, 95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,99, @@ -2033,7 +2033,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,101,95,112,97,116,104,90,5,101,110,116,114,121,114,247, 0,0,0,114,158,0,0,0,114,121,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,9,95,103, - 101,116,95,115,112,101,99,71,4,0,0,115,40,0,0,0, + 101,116,95,115,112,101,99,72,4,0,0,115,40,0,0,0, 0,5,6,1,13,1,21,1,3,1,15,1,12,1,15,1, 21,2,18,1,12,1,3,1,15,1,4,1,9,1,12,1, 12,5,17,2,18,1,9,1,122,20,80,97,116,104,70,105, @@ -2060,7 +2060,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,152,0,0,0,114,224,0,0,0,41,6,114,164,0, 0,0,114,119,0,0,0,114,35,0,0,0,114,174,0,0, 0,114,158,0,0,0,114,254,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,103, + 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,104, 4,0,0,115,26,0,0,0,0,4,12,1,9,1,21,1, 12,1,4,1,15,1,9,1,6,3,9,1,24,1,4,2, 7,2,122,20,80,97,116,104,70,105,110,100,101,114,46,102, @@ -2083,7 +2083,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,120,0,0,0,41,4,114,164,0,0,0,114,119,0, 0,0,114,35,0,0,0,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,176,0,0,0, - 125,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, + 126,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, 1,122,22,80,97,116,104,70,105,110,100,101,114,46,102,105, 110,100,95,109,111,100,117,108,101,41,12,114,105,0,0,0, 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, @@ -2091,7 +2091,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,252,0,0,0,114,255,0,0,0,114,175,0, 0,0,114,176,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,243,0,0,0, - 5,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, + 6,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, 17,18,22,18,15,3,1,18,31,3,1,21,21,3,1,114, 243,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, 0,3,0,0,0,64,0,0,0,115,133,0,0,0,101,0, @@ -2140,7 +2140,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,0,100,0,0,83,41,1,78,114,4,0,0,0,41,2, 114,22,0,0,0,114,219,0,0,0,41,1,114,120,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 154,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, + 155,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, 46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120, 112,114,62,114,58,0,0,0,114,29,0,0,0,78,114,87, @@ -2152,7 +2152,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,0,0,0,114,35,0,0,0,218,14,108,111,97,100,101, 114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101, 114,115,114,160,0,0,0,114,4,0,0,0,41,1,114,120, - 0,0,0,114,5,0,0,0,114,179,0,0,0,148,4,0, + 0,0,0,114,5,0,0,0,114,179,0,0,0,149,4,0, 0,115,16,0,0,0,0,4,6,1,19,1,36,1,9,2, 15,1,9,1,12,1,122,19,70,105,108,101,70,105,110,100, 101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0, @@ -2163,7 +2163,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,105,109,101,46,114,29,0,0,0,78,114,87,0,0,0, 41,1,114,2,1,0,0,41,1,114,100,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,244,0, - 0,0,162,4,0,0,115,2,0,0,0,0,2,122,28,70, + 0,0,163,4,0,0,115,2,0,0,0,0,2,122,28,70, 105,108,101,70,105,110,100,101,114,46,105,110,118,97,108,105, 100,97,116,101,95,99,97,99,104,101,115,99,2,0,0,0, 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, @@ -2187,7 +2187,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,120,0,0,0,114,150,0,0,0,41,3,114,100,0, 0,0,114,119,0,0,0,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,117,0,0,0, - 168,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, + 169,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, 1,122,22,70,105,108,101,70,105,110,100,101,114,46,102,105, 110,100,95,108,111,97,100,101,114,99,6,0,0,0,0,0, 0,0,7,0,0,0,7,0,0,0,67,0,0,0,115,40, @@ -2198,7 +2198,7 @@ const unsigned char _Py_M__importlib_external[] = { 7,114,100,0,0,0,114,159,0,0,0,114,119,0,0,0, 114,35,0,0,0,90,4,115,109,115,108,114,174,0,0,0, 114,120,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,255,0,0,0,180,4,0,0,115,6,0, + 5,0,0,0,114,255,0,0,0,181,4,0,0,115,6,0, 0,0,0,1,15,1,18,1,122,20,70,105,108,101,70,105, 110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99, 3,0,0,0,0,0,0,0,14,0,0,0,15,0,0,0, @@ -2262,7 +2262,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,117, 108,108,95,112,97,116,104,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,175,0,0,0, - 185,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, + 186,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, 1,34,1,13,1,11,1,15,1,10,1,9,2,9,1,9, 1,15,2,9,1,6,2,12,1,18,1,22,1,10,1,15, 1,12,1,32,4,12,2,22,1,22,1,22,1,16,1,12, @@ -2298,7 +2298,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,146,2,0,113,6,0,83,114,4,0,0,0,41,1, 114,88,0,0,0,41,2,114,22,0,0,0,90,2,102,110, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,250, - 9,60,115,101,116,99,111,109,112,62,4,5,0,0,115,2, + 9,60,115,101,116,99,111,109,112,62,5,5,0,0,115,2, 0,0,0,9,0,122,41,70,105,108,101,70,105,110,100,101, 114,46,95,102,105,108,108,95,99,97,99,104,101,46,60,108, 111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,62, @@ -2315,7 +2315,7 @@ const unsigned char _Py_M__importlib_external[] = { 95,99,111,110,116,101,110,116,115,114,239,0,0,0,114,98, 0,0,0,114,231,0,0,0,114,219,0,0,0,90,8,110, 101,119,95,110,97,109,101,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,7,1,0,0,231,4,0,0,115, + 0,114,5,0,0,0,114,7,1,0,0,232,4,0,0,115, 34,0,0,0,0,2,9,1,3,1,31,1,22,3,11,3, 18,1,18,7,9,1,13,1,24,1,6,1,27,2,6,1, 17,1,9,1,18,1,122,22,70,105,108,101,70,105,110,100, @@ -2354,7 +2354,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,1,114,35,0,0,0,41,2,114,164,0,0, 0,114,6,1,0,0,114,4,0,0,0,114,5,0,0,0, 218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95, - 70,105,108,101,70,105,110,100,101,114,16,5,0,0,115,6, + 70,105,108,101,70,105,110,100,101,114,17,5,0,0,115,6, 0,0,0,0,2,12,1,18,1,122,54,70,105,108,101,70, 105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46, 60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111, @@ -2362,7 +2362,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,114,4,0,0,0,41,3,114,164,0,0,0,114,6,1, 0,0,114,12,1,0,0,114,4,0,0,0,41,2,114,164, 0,0,0,114,6,1,0,0,114,5,0,0,0,218,9,112, - 97,116,104,95,104,111,111,107,6,5,0,0,115,4,0,0, + 97,116,104,95,104,111,111,107,7,5,0,0,115,4,0,0, 0,0,10,21,6,122,20,70,105,108,101,70,105,110,100,101, 114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,0, 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, @@ -2371,7 +2371,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,100,101,114,40,123,33,114,125,41,41,2,114,47,0,0, 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,238,0,0, - 0,24,5,0,0,115,2,0,0,0,0,1,122,19,70,105, + 0,25,5,0,0,115,2,0,0,0,0,1,122,19,70,105, 108,101,70,105,110,100,101,114,46,95,95,114,101,112,114,95, 95,41,15,114,105,0,0,0,114,104,0,0,0,114,106,0, 0,0,114,107,0,0,0,114,179,0,0,0,114,244,0,0, @@ -2379,7 +2379,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,255,0,0,0,114,175,0,0,0,114,7,1,0,0,114, 177,0,0,0,114,13,1,0,0,114,238,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,0,1,0,0,139,4,0,0,115,20,0,0,0, + 0,0,114,0,1,0,0,140,4,0,0,115,20,0,0,0, 12,7,6,2,12,14,12,4,6,2,12,12,12,5,15,46, 12,31,18,18,114,0,1,0,0,99,4,0,0,0,0,0, 0,0,6,0,0,0,11,0,0,0,67,0,0,0,115,195, @@ -2405,7 +2405,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,110,97,109,101,90,9,99,112,97,116,104,110,97,109,101, 114,120,0,0,0,114,158,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,14,95,102,105,120,95, - 117,112,95,109,111,100,117,108,101,30,5,0,0,115,34,0, + 117,112,95,109,111,100,117,108,101,31,5,0,0,115,34,0, 0,0,0,2,15,1,15,1,6,1,6,1,12,1,12,1, 18,2,15,1,6,1,21,1,3,1,10,1,10,1,10,1, 14,1,13,2,114,18,1,0,0,99,0,0,0,0,0,0, @@ -2426,7 +2426,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,90, 6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,100, 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,155,0,0,0,53,5,0,0,115,8,0,0,0,0,5, + 114,155,0,0,0,54,5,0,0,115,8,0,0,0,0,5, 18,1,12,1,12,1,114,155,0,0,0,99,1,0,0,0, 0,0,0,0,12,0,0,0,12,0,0,0,67,0,0,0, 115,70,2,0,0,124,0,0,97,0,0,116,0,0,106,1, @@ -2488,7 +2488,7 @@ const unsigned char _Py_M__importlib_external[] = { 83,41,2,114,29,0,0,0,78,41,1,114,31,0,0,0, 41,2,114,22,0,0,0,114,77,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 89,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, + 90,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, 116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,101, 110,101,120,112,114,62,114,59,0,0,0,122,30,105,109,112, 111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,32, @@ -2518,7 +2518,7 @@ const unsigned char _Py_M__importlib_external[] = { 90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,101, 90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, - 95,115,101,116,117,112,64,5,0,0,115,82,0,0,0,0, + 95,115,101,116,117,112,65,5,0,0,115,82,0,0,0,0, 8,6,1,9,1,9,3,13,1,13,1,15,1,18,2,13, 1,20,3,33,1,19,2,31,1,10,1,15,1,13,1,4, 2,3,1,15,1,5,1,13,1,12,2,12,1,16,1,16, @@ -2544,7 +2544,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,212,0,0,0,41,2,114,26,1,0,0,90,17,115,117, 112,112,111,114,116,101,100,95,108,111,97,100,101,114,115,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,105,110,115,116,97,108,108,132,5,0,0,115,16,0,0, + 95,105,110,115,116,97,108,108,133,5,0,0,115,16,0,0, 0,0,2,10,1,9,1,28,1,15,1,16,1,16,4,9, 1,114,29,1,0,0,41,3,122,3,119,105,110,114,1,0, 0,0,114,2,0,0,0,41,56,114,107,0,0,0,114,10, @@ -2571,11 +2571,12 @@ const unsigned char _Py_M__importlib_external[] = { 114,18,1,0,0,114,155,0,0,0,114,27,1,0,0,114, 29,1,0,0,114,4,0,0,0,114,4,0,0,0,114,4, 0,0,0,114,5,0,0,0,218,8,60,109,111,100,117,108, - 101,62,8,0,0,0,115,98,0,0,0,6,17,6,3,12, + 101,62,8,0,0,0,115,102,0,0,0,6,17,6,3,12, 12,12,5,12,5,12,6,12,12,12,10,12,9,12,5,12, - 7,15,22,15,111,22,1,18,2,6,1,6,2,9,2,9, + 7,15,22,15,112,22,1,18,2,6,1,6,2,9,2,9, 2,10,2,21,44,12,33,12,19,12,12,12,12,12,28,12, 17,21,55,21,12,18,10,12,14,9,3,12,1,15,65,19, 64,19,29,22,110,19,41,25,45,25,16,6,3,25,53,19, - 57,19,42,19,134,19,147,15,23,12,11,12,68, + 57,19,42,19,127,0,7,19,127,0,20,15,23,12,11,12, + 68, }; diff --git a/Python/peephole.c b/Python/peephole.c index 4bf786e..c33bf1a 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -346,19 +346,19 @@ markblocks(unsigned char *code, Py_ssize_t len) single basic block. All transformations keep the code size the same or smaller. For those that reduce size, the gaps are initially filled with NOPs. Later those NOPs are removed and the jump addresses retargeted in - a single pass. Line numbering is adjusted accordingly. */ + a single pass. Code offset is adjusted accordingly. */ PyObject * PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, - PyObject *lineno_obj) + PyObject *lnotab_obj) { Py_ssize_t i, j, codelen; int nops, h, adj; int tgt, tgttgt, opcode; unsigned char *codestr = NULL; - unsigned char *lineno; + unsigned char *lnotab; int *addrmap = NULL; - int new_line, cum_orig_line, last_line; + int cum_orig_offset, last_offset; Py_ssize_t tabsiz; PyObject **const_stack = NULL; Py_ssize_t *load_const_stack = NULL; @@ -371,12 +371,17 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, if (PyErr_Occurred()) goto exitError; - /* Bypass optimization when the lineno table is too complex */ - assert(PyBytes_Check(lineno_obj)); - lineno = (unsigned char*)PyBytes_AS_STRING(lineno_obj); - tabsiz = PyBytes_GET_SIZE(lineno_obj); - if (memchr(lineno, 255, tabsiz) != NULL) + /* Bypass optimization when the lnotab table is too complex */ + assert(PyBytes_Check(lnotab_obj)); + lnotab = (unsigned char*)PyBytes_AS_STRING(lnotab_obj); + tabsiz = PyBytes_GET_SIZE(lnotab_obj); + assert(tabsiz == 0 || Py_REFCNT(lnotab_obj) == 1); + if (memchr(lnotab, 255, tabsiz) != NULL) { + /* 255 value are used for multibyte bytecode instructions */ goto exitUnchanged; + } + /* Note: -128 and 127 special values for line number delta are ok, + the peephole optimizer doesn't modify line numbers. */ /* Avoid situations where jump retargeting could overflow */ assert(PyBytes_Check(code)); @@ -663,21 +668,24 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, } } - /* Fixup linenotab */ + /* Fixup lnotab */ for (i=0, nops=0 ; i new code offset */ addrmap[i] = (int)(i - nops); if (codestr[i] == NOP) nops++; } - cum_orig_line = 0; - last_line = 0; + cum_orig_offset = 0; + last_offset = 0; for (i=0 ; i < tabsiz ; i+=2) { - cum_orig_line += lineno[i]; - new_line = addrmap[cum_orig_line]; - assert (new_line - last_line < 255); - lineno[i] =((unsigned char)(new_line - last_line)); - last_line = new_line; + int offset_delta, new_offset; + cum_orig_offset += lnotab[i]; + new_offset = addrmap[cum_orig_offset]; + offset_delta = new_offset - last_offset; + assert(0 <= offset_delta && offset_delta <= 255); + lnotab[i] = (unsigned char)offset_delta; + last_offset = new_offset; } /* Remove NOPs and fixup jump targets */ @@ -727,12 +735,9 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, exitUnchanged: CONST_STACK_DELETE(); - if (blocks != NULL) - PyMem_Free(blocks); - if (addrmap != NULL) - PyMem_Free(addrmap); - if (codestr != NULL) - PyMem_Free(codestr); + PyMem_Free(blocks); + PyMem_Free(addrmap); + PyMem_Free(codestr); Py_XINCREF(code); return code; } -- cgit v0.12 From 9f7893955298a5691ee9d9c65d98bfc7d1eb34f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Jan 2016 18:12:29 +0100 Subject: Issue #26107: Fix typo in Objects/lnotab_notes.txt Double parenthesis --- Objects/lnotab_notes.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 8af10db..5153757 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -22,7 +22,7 @@ look like: 0, 1, 6, 1, 44, 5, 300, 200, 11, 1 The above doesn't really work, but it's a start. An unsigned byte (byte code -offset)) can't hold negative values, or values larger than 255, a signed byte +offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the above example contains two such values. So we make two tweaks: @@ -95,16 +95,16 @@ which compiles to this: 6 POP_JUMP_IF_FALSE 17 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 12 PRINT_ITEM - 4 13 BREAK_LOOP + 4 13 BREAK_LOOP 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + >> 17 POP_BLOCK 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM + 21 PRINT_ITEM >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 25 RETURN_VALUE If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 and the co_lnotab will claim that execution has moved to line 4, which is wrong. -- cgit v0.12 From e3560a7dc9eeac324ff407588cb3f0b36ffe5c6e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 12:22:07 +0100 Subject: site: error on sitecustomize import error Issue #26099: The site module now writes an error into stderr if sitecustomize module can be imported but executing the module raise an ImportError. Same change for usercustomize. --- Lib/site.py | 20 ++++++++++++++------ Misc/NEWS | 4 ++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 3a8d1c3..13ec8fa 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -504,9 +504,13 @@ def venv(known_paths): def execsitecustomize(): """Run custom site specific code, if available.""" try: - import sitecustomize - except ImportError: - pass + try: + import sitecustomize + except ImportError as exc: + if exc.name == 'sitecustomize': + pass + else: + raise except Exception as err: if os.environ.get("PYTHONVERBOSE"): sys.excepthook(*sys.exc_info()) @@ -520,9 +524,13 @@ def execsitecustomize(): def execusercustomize(): """Run custom user specific code, if available.""" try: - import usercustomize - except ImportError: - pass + try: + import usercustomize + except ImportError as exc: + if exc.name == 'usercustomize': + pass + else: + raise except Exception as err: if os.environ.get("PYTHONVERBOSE"): sys.excepthook(*sys.exc_info()) diff --git a/Misc/NEWS b/Misc/NEWS index 01e933c..4e3cc96 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -146,6 +146,10 @@ Core and Builtins Library ------- +- Issue #26099: The site module now writes an error into stderr if + sitecustomize module can be imported but executing the module raise an + ImportError. Same change for usercustomize. + - Issue #26147: xmlrpc now works with strings not encodable with used non-UTF-8 encoding. -- cgit v0.12 From efb2413ce82acaa5dec43a8cb14aa7cdf2352fb1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 12:33:12 +0100 Subject: code_richcompare() now uses the constants types Issue #25843: When compiling code, don't merge constants if they are equal but have a different types. For example, "f1, f2 = lambda: 1, lambda: 1.0" is now correctly compiled to two different functions: f1() returns 1 (int) and f2() returns 1.0 (int), even if 1 and 1.0 are equal. Add a new _PyCode_ConstantKey() private function. --- Include/code.h | 11 +++- Lib/test/test_compile.py | 82 ++++++++++++++++++++++++++++ Misc/NEWS | 6 ++ Objects/codeobject.c | 139 ++++++++++++++++++++++++++++++++++++++++++++++- Python/compile.c | 58 ++++---------------- 5 files changed, 246 insertions(+), 50 deletions(-) diff --git a/Include/code.h b/Include/code.h index 3ce2084..a300ead 100644 --- a/Include/code.h +++ b/Include/code.h @@ -108,12 +108,21 @@ typedef struct _addr_pair { int ap_upper; } PyAddrPair; +#ifndef Py_LIMITED_API /* Update *bounds to describe the first and one-past-the-last instructions in the same line as lasti. Return the number of that line. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds); + +/* Create a comparable key used to compare constants taking in account the + * object type. It is used to make sure types are not coerced (e.g., float and + * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms + * + * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) + * depending on the type and the value. The type is the first item to not + * compare bytes and str which can raise a BytesWarning exception. */ +PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); #endif PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 8935c65..e0fdee3 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -572,6 +572,88 @@ if 1: exec(memoryview(b"ax = 123")[1:-1], namespace) self.assertEqual(namespace['x'], 12) + def check_constant(self, func, expected): + for const in func.__code__.co_consts: + if repr(const) == repr(expected): + break + else: + self.fail("unable to find constant %r in %r" + % (expected, func.__code__.co_consts)) + + # Merging equal constants is not a strict requirement for the Python + # semantics, it's a more an implementation detail. + @support.cpython_only + def test_merge_constants(self): + # Issue #25843: compile() must merge constants which are equal + # and have the same type. + + def check_same_constant(const): + ns = {} + code = "f1, f2 = lambda: %r, lambda: %r" % (const, const) + exec(code, ns) + f1 = ns['f1'] + f2 = ns['f2'] + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, const) + self.assertEqual(repr(f1()), repr(const)) + + check_same_constant(None) + check_same_constant(0) + check_same_constant(0.0) + check_same_constant(b'abc') + check_same_constant('abc') + + # Note: "lambda: ..." emits "LOAD_CONST Ellipsis", + # whereas "lambda: Ellipsis" emits "LOAD_GLOBAL Ellipsis" + f1, f2 = lambda: ..., lambda: ... + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, Ellipsis) + self.assertEqual(repr(f1()), repr(Ellipsis)) + + # {0} is converted to a constant frozenset({0}) by the peephole + # optimizer + f1, f2 = lambda x: x in {0}, lambda x: x in {0} + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, frozenset({0})) + self.assertTrue(f1(0)) + + def test_dont_merge_constants(self): + # Issue #25843: compile() must not merge constants which are equal + # but have a different type. + + def check_different_constants(const1, const2): + ns = {} + exec("f1, f2 = lambda: %r, lambda: %r" % (const1, const2), ns) + f1 = ns['f1'] + f2 = ns['f2'] + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, const1) + self.check_constant(f2, const2) + self.assertEqual(repr(f1()), repr(const1)) + self.assertEqual(repr(f2()), repr(const2)) + + check_different_constants(0, 0.0) + check_different_constants(+0.0, -0.0) + check_different_constants((0,), (0.0,)) + + # check_different_constants() cannot be used because repr(-0j) is + # '(-0-0j)', but when '(-0-0j)' is evaluated to 0j: we loose the sign. + f1, f2 = lambda: +0.0j, lambda: -0.0j + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, +0.0j) + self.check_constant(f2, -0.0j) + self.assertEqual(repr(f1()), repr(+0.0j)) + self.assertEqual(repr(f2()), repr(-0.0j)) + + # {0} is converted to a constant frozenset({0}) by the peephole + # optimizer + f1, f2 = lambda x: x in {0}, lambda x: x in {0.0} + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, frozenset({0})) + self.check_constant(f2, frozenset({0.0})) + self.assertTrue(f1(0)) + self.assertTrue(f2(0.0)) + class TestStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object diff --git a/Misc/NEWS b/Misc/NEWS index 4e3cc96..409feae 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,12 @@ Release date: tba Core and Builtins ----------------- +- Issue #25843: When compiling code, don't merge constants if they are equal + but have a different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` + is now correctly compiled to two different functions: ``f1()`` returns ``1`` + (``int``) and ``f2()`` returns ``1.0`` (``int``), even if ``1`` and ``1.0`` + are equal. + - Issue #26107: The format of the ``co_lnotab`` attribute of code objects changes to support negative line number delta. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0f03dfe..e3d28e3 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -409,11 +409,135 @@ code_repr(PyCodeObject *co) } } +PyObject* +_PyCode_ConstantKey(PyObject *op) +{ + PyObject *key; + + /* Py_None and Py_Ellipsis are singleton */ + if (op == Py_None || op == Py_Ellipsis + || PyLong_CheckExact(op) + || PyBool_Check(op) + || PyBytes_CheckExact(op) + || PyUnicode_CheckExact(op) + /* code_richcompare() uses _PyCode_ConstantKey() internally */ + || PyCode_Check(op)) { + key = PyTuple_Pack(2, Py_TYPE(op), op); + } + else if (PyFloat_CheckExact(op)) { + double d = PyFloat_AS_DOUBLE(op); + /* all we need is to make the tuple different in either the 0.0 + * or -0.0 case from all others, just to avoid the "coercion". + */ + if (d == 0.0 && copysign(1.0, d) < 0.0) + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_None); + else + key = PyTuple_Pack(2, Py_TYPE(op), op); + } + else if (PyComplex_CheckExact(op)) { + Py_complex z; + int real_negzero, imag_negzero; + /* For the complex case we must make complex(x, 0.) + different from complex(x, -0.) and complex(0., y) + different from complex(-0., y), for any x and y. + All four complex zeros must be distinguished.*/ + z = PyComplex_AsCComplex(op); + real_negzero = z.real == 0.0 && copysign(1.0, z.real) < 0.0; + imag_negzero = z.imag == 0.0 && copysign(1.0, z.imag) < 0.0; + /* use True, False and None singleton as tags for the real and imag + * sign, to make tuples different */ + if (real_negzero && imag_negzero) { + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_True); + } + else if (imag_negzero) { + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_False); + } + else if (real_negzero) { + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_None); + } + else { + key = PyTuple_Pack(2, Py_TYPE(op), op); + } + } + else if (PyTuple_CheckExact(op)) { + Py_ssize_t i, len; + PyObject *tuple; + + len = PyTuple_GET_SIZE(op); + tuple = PyTuple_New(len); + if (tuple == NULL) + return NULL; + + for (i=0; i < len; i++) { + PyObject *item, *item_key; + + item = PyTuple_GET_ITEM(op, i); + item_key = _PyCode_ConstantKey(item); + if (item_key == NULL) { + Py_DECREF(tuple); + return NULL; + } + + PyTuple_SET_ITEM(tuple, i, item_key); + } + + key = PyTuple_Pack(3, Py_TYPE(op), op, tuple); + Py_DECREF(tuple); + } + else if (PyFrozenSet_CheckExact(op)) { + Py_ssize_t pos = 0; + PyObject *item; + Py_hash_t hash; + Py_ssize_t i, len; + PyObject *tuple, *set; + + len = PySet_GET_SIZE(op); + tuple = PyTuple_New(len); + if (tuple == NULL) + return NULL; + + i = 0; + while (_PySet_NextEntry(op, &pos, &item, &hash)) { + PyObject *item_key; + + item_key = _PyCode_ConstantKey(item); + if (item_key == NULL) { + Py_DECREF(tuple); + return NULL; + } + + assert(i < len); + PyTuple_SET_ITEM(tuple, i, item_key); + i++; + } + set = PyFrozenSet_New(tuple); + Py_DECREF(tuple); + if (set == NULL) + return NULL; + + key = PyTuple_Pack(3, Py_TYPE(op), op, set); + Py_DECREF(set); + return key; + } + else { + /* for other types, use the object identifier as an unique identifier + * to ensure that they are seen as unequal. */ + PyObject *obj_id = PyLong_FromVoidPtr(op); + if (obj_id == NULL) + return NULL; + + key = PyTuple_Pack(3, Py_TYPE(op), op, obj_id); + Py_DECREF(obj_id); + } + return key; +} + static PyObject * code_richcompare(PyObject *self, PyObject *other, int op) { PyCodeObject *co, *cp; int eq; + PyObject *consts1, *consts2; PyObject *res; if ((op != Py_EQ && op != Py_NE) || @@ -439,8 +563,21 @@ code_richcompare(PyObject *self, PyObject *other, int op) if (!eq) goto unequal; eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ); if (eq <= 0) goto unequal; - eq = PyObject_RichCompareBool(co->co_consts, cp->co_consts, Py_EQ); + + /* compare constants */ + consts1 = _PyCode_ConstantKey(co->co_consts); + if (!consts1) + return NULL; + consts2 = _PyCode_ConstantKey(cp->co_consts); + if (!consts2) { + Py_DECREF(consts1); + return NULL; + } + eq = PyObject_RichCompareBool(consts1, consts2, Py_EQ); + Py_DECREF(consts1); + Py_DECREF(consts2); if (eq <= 0) goto unequal; + eq = PyObject_RichCompareBool(co->co_names, cp->co_names, Py_EQ); if (eq <= 0) goto unequal; eq = PyObject_RichCompareBool(co->co_varnames, cp->co_varnames, Py_EQ); diff --git a/Python/compile.c b/Python/compile.c index f362ac6..a710e82 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -393,7 +393,7 @@ list2dict(PyObject *list) return NULL; } k = PyList_GET_ITEM(list, i); - k = PyTuple_Pack(2, k, k->ob_type); + k = _PyCode_ConstantKey(k); if (k == NULL || PyDict_SetItem(dict, k, v) < 0) { Py_XDECREF(k); Py_DECREF(v); @@ -456,7 +456,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) return NULL; } i++; - tuple = PyTuple_Pack(2, k, k->ob_type); + tuple = _PyCode_ConstantKey(k); if (!tuple || PyDict_SetItem(dest, tuple, item) < 0) { Py_DECREF(sorted_keys); Py_DECREF(item); @@ -559,7 +559,7 @@ compiler_enter_scope(struct compiler *c, identifier name, compiler_unit_free(u); return 0; } - tuple = PyTuple_Pack(2, name, Py_TYPE(name)); + tuple = _PyCode_ConstantKey(name); if (!tuple) { compiler_unit_free(u); return 0; @@ -1105,47 +1105,8 @@ compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o) { PyObject *t, *v; Py_ssize_t arg; - double d; - - /* necessary to make sure types aren't coerced (e.g., float and complex) */ - /* _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms */ - if (PyFloat_Check(o)) { - d = PyFloat_AS_DOUBLE(o); - /* all we need is to make the tuple different in either the 0.0 - * or -0.0 case from all others, just to avoid the "coercion". - */ - if (d == 0.0 && copysign(1.0, d) < 0.0) - t = PyTuple_Pack(3, o, o->ob_type, Py_None); - else - t = PyTuple_Pack(2, o, o->ob_type); - } - else if (PyComplex_Check(o)) { - Py_complex z; - int real_negzero, imag_negzero; - /* For the complex case we must make complex(x, 0.) - different from complex(x, -0.) and complex(0., y) - different from complex(-0., y), for any x and y. - All four complex zeros must be distinguished.*/ - z = PyComplex_AsCComplex(o); - real_negzero = z.real == 0.0 && copysign(1.0, z.real) < 0.0; - imag_negzero = z.imag == 0.0 && copysign(1.0, z.imag) < 0.0; - if (real_negzero && imag_negzero) { - t = PyTuple_Pack(5, o, o->ob_type, - Py_None, Py_None, Py_None); - } - else if (imag_negzero) { - t = PyTuple_Pack(4, o, o->ob_type, Py_None, Py_None); - } - else if (real_negzero) { - t = PyTuple_Pack(3, o, o->ob_type, Py_None); - } - else { - t = PyTuple_Pack(2, o, o->ob_type); - } - } - else { - t = PyTuple_Pack(2, o, o->ob_type); - } + + t = _PyCode_ConstantKey(o); if (t == NULL) return -1; @@ -1459,7 +1420,7 @@ static int compiler_lookup_arg(PyObject *dict, PyObject *name) { PyObject *k, *v; - k = PyTuple_Pack(2, name, name->ob_type); + k = _PyCode_ConstantKey(name); if (k == NULL) return -1; v = PyDict_GetItem(dict, k); @@ -4657,9 +4618,10 @@ dict_keys_inorder(PyObject *dict, Py_ssize_t offset) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { i = PyLong_AS_LONG(v); - /* The keys of the dictionary are tuples. (see compiler_add_o) - The object we want is always first, though. */ - k = PyTuple_GET_ITEM(k, 0); + /* The keys of the dictionary are tuples. (see compiler_add_o + * and _PyCode_ConstantKey). The object we want is always second, + * though. */ + k = PyTuple_GET_ITEM(k, 1); Py_INCREF(k); assert((i - offset) < size); assert((i - offset) >= 0); -- cgit v0.12 From b02ef715a303ab09a39ef994d8f5acc4eb572376 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 14:09:55 +0100 Subject: Use Py_uintptr_t for atomic pointers Issue #26161: Use Py_uintptr_t instead of void* for atomic pointers in pyatomic.h. Use atomic_uintptr_t when is used. Using void* causes compilation warnings depending on which implementation of atomic types is used. --- Include/pyatomic.h | 6 +++--- Python/ceval_gil.h | 8 ++++---- Python/pystate.c | 47 ++++++++++++++++++++++++----------------------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/Include/pyatomic.h b/Include/pyatomic.h index 892a217..89028ef 100644 --- a/Include/pyatomic.h +++ b/Include/pyatomic.h @@ -30,7 +30,7 @@ typedef enum _Py_memory_order { } _Py_memory_order; typedef struct _Py_atomic_address { - _Atomic void *_value; + atomic_uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { @@ -61,7 +61,7 @@ typedef enum _Py_memory_order { } _Py_memory_order; typedef struct _Py_atomic_address { - void *_value; + Py_uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { @@ -98,7 +98,7 @@ typedef enum _Py_memory_order { } _Py_memory_order; typedef struct _Py_atomic_address { - void *_value; + Py_uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h index aafcbc2..8d38ee9 100644 --- a/Python/ceval_gil.h +++ b/Python/ceval_gil.h @@ -111,7 +111,7 @@ static _Py_atomic_int gil_locked = {-1}; static unsigned long gil_switch_number = 0; /* Last PyThreadState holding / having held the GIL. This helps us know whether anyone else was scheduled after we dropped the GIL. */ -static _Py_atomic_address gil_last_holder = {NULL}; +static _Py_atomic_address gil_last_holder = {0}; /* This condition variable allows one or several threads to wait until the GIL is released. In addition, the mutex also protects the above @@ -142,7 +142,7 @@ static void create_gil(void) #ifdef FORCE_SWITCHING COND_INIT(switch_cond); #endif - _Py_atomic_store_relaxed(&gil_last_holder, NULL); + _Py_atomic_store_relaxed(&gil_last_holder, 0); _Py_ANNOTATE_RWLOCK_CREATE(&gil_locked); _Py_atomic_store_explicit(&gil_locked, 0, _Py_memory_order_release); } @@ -178,7 +178,7 @@ static void drop_gil(PyThreadState *tstate) /* Sub-interpreter support: threads might have been switched under our feet using PyThreadState_Swap(). Fix the GIL last holder variable so that our heuristics work. */ - _Py_atomic_store_relaxed(&gil_last_holder, tstate); + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); } MUTEX_LOCK(gil_mutex); @@ -240,7 +240,7 @@ _ready: _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil_locked, /*is_write=*/1); if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder)) { - _Py_atomic_store_relaxed(&gil_last_holder, tstate); + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); ++gil_switch_number; } diff --git a/Python/pystate.c b/Python/pystate.c index 6d0696d..853e5c7 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3,11 +3,13 @@ #include "Python.h" -#ifndef Py_BUILD_CORE -/* ensure that PyThreadState_GET() is a macro, not an alias to - * PyThreadState_Get() */ -# error "pystate.c must be compiled with Py_BUILD_CORE defined" -#endif +#define GET_TSTATE() \ + ((PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current)) +#define SET_TSTATE(value) \ + _Py_atomic_store_relaxed(&_PyThreadState_Current, (Py_uintptr_t)(value)) +#define GET_INTERP_STATE() \ + (GET_TSTATE()->interp) + /* -------------------------------------------------------------------------- CAUTION @@ -54,7 +56,7 @@ static PyInterpreterState *interp_head = NULL; /* Assuming the current thread holds the GIL, this is the PyThreadState for the current thread. */ -_Py_atomic_address _PyThreadState_Current = {NULL}; +_Py_atomic_address _PyThreadState_Current = {0}; PyThreadFrameGetter _PyThreadState_GetFrame = NULL; #ifdef WITH_THREAD @@ -260,7 +262,7 @@ PyObject* PyState_FindModule(struct PyModuleDef* module) { Py_ssize_t index = module->m_base.m_index; - PyInterpreterState *state = PyThreadState_GET()->interp; + PyInterpreterState *state = GET_INTERP_STATE(); PyObject *res; if (module->m_slots) { return NULL; @@ -284,7 +286,7 @@ _PyState_AddModule(PyObject* module, struct PyModuleDef* def) "PyState_AddModule called on module with slots"); return -1; } - state = PyThreadState_GET()->interp; + state = GET_INTERP_STATE(); if (!def) return -1; if (!state->modules_by_index) { @@ -304,7 +306,7 @@ int PyState_AddModule(PyObject* module, struct PyModuleDef* def) { Py_ssize_t index; - PyInterpreterState *state = PyThreadState_GET()->interp; + PyInterpreterState *state = GET_INTERP_STATE(); if (!def) { Py_FatalError("PyState_AddModule: Module Definition is NULL"); return -1; @@ -331,7 +333,7 @@ PyState_RemoveModule(struct PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - state = PyThreadState_GET()->interp; + state = GET_INTERP_STATE(); if (index == 0) { Py_FatalError("PyState_RemoveModule: Module index invalid."); return -1; @@ -351,7 +353,7 @@ PyState_RemoveModule(struct PyModuleDef* def) void _PyState_ClearModules(void) { - PyInterpreterState *state = PyThreadState_GET()->interp; + PyInterpreterState *state = GET_INTERP_STATE(); if (state->modules_by_index) { Py_ssize_t i; for (i = 0; i < PyList_GET_SIZE(state->modules_by_index); i++) { @@ -429,7 +431,7 @@ tstate_delete_common(PyThreadState *tstate) void PyThreadState_Delete(PyThreadState *tstate) { - if (tstate == PyThreadState_GET()) + if (tstate == GET_TSTATE()) Py_FatalError("PyThreadState_Delete: tstate is still current"); #ifdef WITH_THREAD if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) @@ -443,11 +445,11 @@ PyThreadState_Delete(PyThreadState *tstate) void PyThreadState_DeleteCurrent() { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); if (tstate == NULL) Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); - _Py_atomic_store_relaxed(&_PyThreadState_Current, NULL); + SET_TSTATE(NULL); if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) PyThread_delete_key_value(autoTLSkey); tstate_delete_common(tstate); @@ -496,14 +498,14 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) PyThreadState * _PyThreadState_UncheckedGet(void) { - return PyThreadState_GET(); + return GET_TSTATE(); } PyThreadState * PyThreadState_Get(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); if (tstate == NULL) Py_FatalError("PyThreadState_Get: no current thread"); @@ -514,9 +516,9 @@ PyThreadState_Get(void) PyThreadState * PyThreadState_Swap(PyThreadState *newts) { - PyThreadState *oldts = PyThreadState_GET(); + PyThreadState *oldts = GET_TSTATE(); - _Py_atomic_store_relaxed(&_PyThreadState_Current, newts); + SET_TSTATE(newts); /* It should not be possible for more than one thread state to be used for a thread. Check this the best we can in debug builds. @@ -545,7 +547,7 @@ PyThreadState_Swap(PyThreadState *newts) PyObject * PyThreadState_GetDict(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); if (tstate == NULL) return NULL; @@ -569,8 +571,7 @@ PyThreadState_GetDict(void) int PyThreadState_SetAsyncExc(long id, PyObject *exc) { - PyThreadState *tstate = PyThreadState_GET(); - PyInterpreterState *interp = tstate->interp; + PyInterpreterState *interp = GET_INTERP_STATE(); PyThreadState *p; /* Although the GIL is held, a few C API functions can be called @@ -691,7 +692,7 @@ PyThreadState_IsCurrent(PyThreadState *tstate) { /* Must be the tstate for this thread */ assert(PyGILState_GetThisThreadState()==tstate); - return tstate == PyThreadState_GET(); + return tstate == GET_TSTATE(); } /* Internal initialization/finalization functions called by @@ -783,7 +784,7 @@ PyGILState_GetThisThreadState(void) int PyGILState_Check(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); return tstate && (tstate == PyGILState_GetThisThreadState()); } -- cgit v0.12 From 22756f18bca783505d864d675d984444ae5633a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 14:16:47 +0100 Subject: Issue #25876: test_gdb: use subprocess._args_from_interpreter_flags() to test Python with more options. --- Lib/test/test_gdb.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 78fc55c..cc5aab1 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -176,6 +176,7 @@ class DebuggerTests(unittest.TestCase): args = ['--eval-command=%s' % cmd for cmd in commands] args += ["--args", sys.executable] + args.extend(subprocess._args_from_interpreter_flags()) if not import_site: # -S suppresses the default 'import site' @@ -301,7 +302,9 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of dictionaries' self.assertGdbRepr({}) self.assertGdbRepr({'foo': 'bar'}, "{'foo': 'bar'}") - self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}") + # PYTHONHASHSEED is need to get the exact item order + if not sys.flags.ignore_environment: + self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}") def test_lists(self): 'Verify the pretty-printing of lists' @@ -379,9 +382,12 @@ id(s)''') 'Verify the pretty-printing of frozensets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") - self.assertGdbRepr(frozenset(), 'frozenset()') - self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") - self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") + self.assertGdbRepr(frozenset(), "frozenset()") + self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") + self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") def test_exceptions(self): # Test a RuntimeError @@ -510,6 +516,10 @@ id(foo)''') def test_builtins_help(self): 'Ensure that the new-style class _Helper in site.py can be handled' + + if sys.flags.no_site: + self.skipTest("need site module, but -S option was used") + # (this was the issue causing tracebacks in # http://bugs.python.org/issue8032#msg100537 ) gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True) -- cgit v0.12 From 5a701f03047dbf14132e447e6bb6bdfe8ea9c33e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 15:04:27 +0100 Subject: Issue #25876: Fix also test_set() of test_gdb when -E command line is used --- Lib/test/test_gdb.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index cc5aab1..3fe15e4 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -367,9 +367,12 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of sets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of sets needs gdb 7.3 or later") - self.assertGdbRepr(set(), 'set()') - self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") - self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") + self.assertGdbRepr(set(), "set()") + self.assertGdbRepr(set(['a']), "{'a'}") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") + self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") # Ensure that we handle sets containing the "dummy" key value, # which happens on deletion: -- cgit v0.12 From 849113af6ba96c36ffaaa9896c9a337eb3253b89 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:25:50 -0800 Subject: Issue #25791: Warn when __package__ != __spec__.parent. In a previous change, __spec__.parent was prioritized over __package__. That is a backwards-compatibility break, but we do eventually want __spec__ to be the ground truth for module details. So this change reverts the change in semantics and instead raises an ImportWarning when __package__ != __spec__.parent to give people time to adjust to using spec objects. --- Doc/reference/import.rst | 18 +++- Doc/whatsnew/3.6.rst | 6 +- Lib/importlib/_bootstrap.py | 12 ++- .../test_importlib/import_/test___package__.py | 33 ++++-- .../import_/test_relative_imports.py | 16 ++- Misc/NEWS | 4 +- Python/import.c | 97 ++++++++++------- Python/importlib.h | 117 +++++++++++---------- 8 files changed, 183 insertions(+), 120 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index a162851..56049ce 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -554,20 +554,30 @@ the module. details. This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules -- as defined in :pep:`366` -- - when ``__spec__`` is not defined. + relative imports for main modules, as defined in :pep:`366`. It is + expected to have the same value as ``__spec__.parent``. + + .. versionchanged:: 3.6 + The value of ``__package__`` is expected to be the same as + ``__spec__.parent``. .. attribute:: __spec__ The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. This is used primarily for - introspection and during reloading. Setting ``__spec__`` + used when importing the module. Setting ``__spec__`` appropriately applies equally to :ref:`modules initialized during interpreter startup `. The one exception is ``__main__``, where ``__spec__`` is :ref:`set to None in some cases `. + When ``__package__`` is not defined, ``__spec__.parent`` is used as + a fallback. + .. versionadded:: 3.4 + .. versionchanged:: 3.6 + ``__spec__.parent`` is used as a fallback when ``__package__`` is + not defined. + .. attribute:: __path__ If the module is a package (either regular or namespace), the module diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8064537..5872cc0 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -274,9 +274,9 @@ Changes in the Python API :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. -* When performing a relative import, ``__spec__.parent`` is used - is ``__spec__`` is defined instead of ``__package__``. - (Contributed by Rose Ames in :issue:`25791`.) +* When performing a relative import, if ``__package__`` does not compare equal + to ``__spec__.parent`` then :exc:`ImportWarning` is raised. + (Contributed by Brett Cannon in :issue:`25791`.) Changes in the C API diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9adcf7b..f0a4e1c 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1032,11 +1032,17 @@ def _calc___package__(globals): to represent that its proper value is unknown. """ + package = globals.get('__package__') spec = globals.get('__spec__') - if spec is not None: + if package is not None: + if spec is not None and package != spec.parent: + _warnings.warn("__package__ != __spec__.parent " + f"({package!r} != {spec.parent!r})", + ImportWarning, stacklevel=3) + return package + elif spec is not None: return spec.parent - package = globals.get('__package__') - if package is None: + else: _warnings.warn("can't resolve package from __spec__ or __package__, " "falling back on __name__ and __path__", ImportWarning, stacklevel=3) diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 08e41c9..ddeeaca 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -5,6 +5,7 @@ of using the typical __path__/__name__ test). """ import unittest +import warnings from .. import util @@ -38,8 +39,8 @@ class Using__package__: with util.import_state(meta_path=[importer]): self.__import__('pkg.fake') module = self.__import__('', - globals=globals_, - fromlist=['attr'], level=2) + globals=globals_, + fromlist=['attr'], level=2) return module def test_using___package__(self): @@ -49,7 +50,10 @@ class Using__package__: def test_using___name__(self): # [__name__] - module = self.import_module({'__name__': 'pkg.fake', '__path__': []}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({'__name__': 'pkg.fake', + '__path__': []}) self.assertEqual(module.__name__, 'pkg') def test_warn_when_using___name__(self): @@ -58,14 +62,22 @@ class Using__package__: def test_None_as___package__(self): # [None] - module = self.import_module({ - '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) self.assertEqual(module.__name__, 'pkg') - def test_prefers___spec__(self): - globals = {'__spec__': FakeSpec()} - with self.assertRaises(SystemError): - self.__import__('', globals, {}, ['relimport'], 1) + def test_spec_fallback(self): + # If __package__ isn't defined, fall back on __spec__.parent. + module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_package_and_spec_disagree(self): + # Raise an ImportWarning if __package__ != __spec__.parent. + with self.assertWarns(ImportWarning): + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) def test_bad__package__(self): globals = {'__package__': ''} @@ -79,7 +91,8 @@ class Using__package__: class FakeSpec: - parent = '' + def __init__(self, parent): + self.parent = parent class Using__package__PEP302(Using__package__): diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 28bb6f7..0409f22 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -2,6 +2,8 @@ from .. import util import sys import unittest +import warnings + class RelativeImports: @@ -65,9 +67,11 @@ class RelativeImports: uncache_names.append(name[:-len('.__init__')]) with util.mock_spec(*create) as importer: with util.import_state(meta_path=[importer]): - for global_ in globals_: - with util.uncache(*uncache_names): - callback(global_) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for global_ in globals_: + with util.uncache(*uncache_names): + callback(global_) def test_module_from_module(self): @@ -204,8 +208,10 @@ class RelativeImports: def test_relative_import_no_globals(self): # No globals for a relative import is an error. - with self.assertRaises(KeyError): - self.__import__('sys', level=1) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with self.assertRaises(KeyError): + self.__import__('sys', level=1) (Frozen_RelativeImports, diff --git a/Misc/NEWS b/Misc/NEWS index 409feae..165b428 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,8 +26,8 @@ Core and Builtins Python 3.5.1 to hide the exact implementation of atomic C types, to avoid compiler issues. -- Issue #25791: Trying to resolve a relative import without __spec__ or - __package__ defined now raises an ImportWarning +- Issue #25791: If __package__ != __spec__.parent or if neither __package__ or + __spec__ are defined then ImportWarning is raised. - Issue #25731: Fix set and deleting __new__ on a class. diff --git a/Python/import.c b/Python/import.c index c1071c0..22f9d21 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1415,60 +1415,79 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, goto error; } else if (level > 0) { + package = _PyDict_GetItemId(globals, &PyId___package__); spec = _PyDict_GetItemId(globals, &PyId___spec__); - if (spec != NULL) { - package = PyObject_GetAttrString(spec, "parent"); - } + if (package != NULL && package != Py_None) { + Py_INCREF(package); if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "package must be a string"); + goto error; + } + else if (spec != NULL) { + int equal; + PyObject *parent = PyObject_GetAttrString(spec, "parent"); + if (parent == NULL) { + goto error; + } + + equal = PyObject_RichCompareBool(package, parent, Py_EQ); + Py_DECREF(parent); + if (equal < 0) { + goto error; + } + else if (equal == 0) { + if (PyErr_WarnEx(PyExc_ImportWarning, + "__package__ != __spec__.parent", 1) < 0) { + goto error; + } + } + } + } + else if (spec != NULL) { + package = PyObject_GetAttrString(spec, "parent"); + if (package == NULL) { + goto error; + } + else if (!PyUnicode_Check(package)) { PyErr_SetString(PyExc_TypeError, "__spec__.parent must be a string"); goto error; } } else { - package = _PyDict_GetItemId(globals, &PyId___package__); - if (package != NULL && package != Py_None) { - Py_INCREF(package); - if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "package must be a string"); - goto error; - } + if (PyErr_WarnEx(PyExc_ImportWarning, + "can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", 1) < 0) { + goto error; } - else { - if (PyErr_WarnEx(PyExc_ImportWarning, - "can't resolve package from __spec__ or __package__, " - "falling back on __name__ and __path__", 1) < 0) { - goto error; - } - package = _PyDict_GetItemId(globals, &PyId___name__); - if (package == NULL) { - PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); - goto error; - } + package = _PyDict_GetItemId(globals, &PyId___name__); + if (package == NULL) { + PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); + goto error; + } - Py_INCREF(package); - if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + Py_INCREF(package); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + goto error; + } + + if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *partition = NULL; + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL) { goto error; } - - if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { - PyObject *partition = NULL; - PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); - if (borrowed_dot == NULL) { - goto error; - } - partition = PyUnicode_RPartition(package, borrowed_dot); - Py_DECREF(package); - if (partition == NULL) { - goto error; - } - package = PyTuple_GET_ITEM(partition, 0); - Py_INCREF(package); - Py_DECREF(partition); + partition = PyUnicode_RPartition(package, borrowed_dot); + Py_DECREF(package); + if (partition == NULL) { + goto error; } + package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); + Py_DECREF(partition); } } diff --git a/Python/importlib.h b/Python/importlib.h index 709c5fd..e4ef835 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1787,43 +1787,52 @@ const unsigned char _Py_M__importlib[] = { 115,116,228,3,0,0,115,34,0,0,0,0,10,15,1,12, 1,12,1,13,1,15,1,16,1,13,1,15,1,21,1,3, 1,17,1,18,4,21,1,15,1,3,1,26,1,114,195,0, - 0,0,99,1,0,0,0,0,0,0,0,3,0,0,0,5, - 0,0,0,67,0,0,0,115,128,0,0,0,124,0,0,106, - 0,0,100,1,0,131,1,0,125,1,0,124,1,0,100,2, - 0,107,9,0,114,34,0,124,1,0,106,1,0,83,124,0, - 0,106,0,0,100,3,0,131,1,0,125,2,0,124,2,0, - 100,2,0,107,8,0,114,124,0,116,2,0,106,3,0,100, - 4,0,116,4,0,100,5,0,100,6,0,131,2,1,1,124, - 0,0,100,7,0,25,125,2,0,100,8,0,124,0,0,107, - 7,0,114,124,0,124,2,0,106,5,0,100,9,0,131,1, - 0,100,10,0,25,125,2,0,124,2,0,83,41,11,122,167, - 67,97,108,99,117,108,97,116,101,32,119,104,97,116,32,95, - 95,112,97,99,107,97,103,101,95,95,32,115,104,111,117,108, - 100,32,98,101,46,10,10,32,32,32,32,95,95,112,97,99, - 107,97,103,101,95,95,32,105,115,32,110,111,116,32,103,117, - 97,114,97,110,116,101,101,100,32,116,111,32,98,101,32,100, - 101,102,105,110,101,100,32,111,114,32,99,111,117,108,100,32, - 98,101,32,115,101,116,32,116,111,32,78,111,110,101,10,32, - 32,32,32,116,111,32,114,101,112,114,101,115,101,110,116,32, - 116,104,97,116,32,105,116,115,32,112,114,111,112,101,114,32, - 118,97,108,117,101,32,105,115,32,117,110,107,110,111,119,110, - 46,10,10,32,32,32,32,114,95,0,0,0,78,114,134,0, - 0,0,122,89,99,97,110,39,116,32,114,101,115,111,108,118, - 101,32,112,97,99,107,97,103,101,32,102,114,111,109,32,95, - 95,115,112,101,99,95,95,32,111,114,32,95,95,112,97,99, - 107,97,103,101,95,95,44,32,102,97,108,108,105,110,103,32, - 98,97,99,107,32,111,110,32,95,95,110,97,109,101,95,95, - 32,97,110,100,32,95,95,112,97,116,104,95,95,114,140,0, - 0,0,233,3,0,0,0,114,1,0,0,0,114,131,0,0, - 0,114,121,0,0,0,114,33,0,0,0,41,6,114,42,0, - 0,0,114,123,0,0,0,114,142,0,0,0,114,143,0,0, - 0,114,176,0,0,0,114,122,0,0,0,41,3,218,7,103, - 108,111,98,97,108,115,114,88,0,0,0,114,170,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, - 17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,101, - 95,95,4,4,0,0,115,22,0,0,0,0,7,15,1,12, - 1,7,1,15,1,12,1,9,2,13,1,10,1,12,1,19, - 1,114,198,0,0,0,99,5,0,0,0,0,0,0,0,9, + 0,0,99,1,0,0,0,0,0,0,0,3,0,0,0,7, + 0,0,0,67,0,0,0,115,214,0,0,0,124,0,0,106, + 0,0,100,1,0,131,1,0,125,1,0,124,0,0,106,0, + 0,100,2,0,131,1,0,125,2,0,124,1,0,100,3,0, + 107,9,0,114,128,0,124,2,0,100,3,0,107,9,0,114, + 124,0,124,1,0,124,2,0,106,1,0,107,3,0,114,124, + 0,116,2,0,106,3,0,100,4,0,106,4,0,100,5,0, + 124,1,0,155,2,0,100,6,0,124,2,0,106,1,0,155, + 2,0,100,7,0,103,5,0,131,1,0,116,5,0,100,8, + 0,100,9,0,131,2,1,1,124,1,0,83,124,2,0,100, + 3,0,107,9,0,114,147,0,124,2,0,106,1,0,83,116, + 2,0,106,3,0,100,10,0,116,5,0,100,8,0,100,9, + 0,131,2,1,1,124,0,0,100,11,0,25,125,1,0,100, + 12,0,124,0,0,107,7,0,114,210,0,124,1,0,106,6, + 0,100,13,0,131,1,0,100,14,0,25,125,1,0,124,1, + 0,83,41,15,122,167,67,97,108,99,117,108,97,116,101,32, + 119,104,97,116,32,95,95,112,97,99,107,97,103,101,95,95, + 32,115,104,111,117,108,100,32,98,101,46,10,10,32,32,32, + 32,95,95,112,97,99,107,97,103,101,95,95,32,105,115,32, + 110,111,116,32,103,117,97,114,97,110,116,101,101,100,32,116, + 111,32,98,101,32,100,101,102,105,110,101,100,32,111,114,32, + 99,111,117,108,100,32,98,101,32,115,101,116,32,116,111,32, + 78,111,110,101,10,32,32,32,32,116,111,32,114,101,112,114, + 101,115,101,110,116,32,116,104,97,116,32,105,116,115,32,112, + 114,111,112,101,114,32,118,97,108,117,101,32,105,115,32,117, + 110,107,110,111,119,110,46,10,10,32,32,32,32,114,134,0, + 0,0,114,95,0,0,0,78,218,0,122,32,95,95,112,97, + 99,107,97,103,101,95,95,32,33,61,32,95,95,115,112,101, + 99,95,95,46,112,97,114,101,110,116,32,40,122,4,32,33, + 61,32,250,1,41,114,140,0,0,0,233,3,0,0,0,122, + 89,99,97,110,39,116,32,114,101,115,111,108,118,101,32,112, + 97,99,107,97,103,101,32,102,114,111,109,32,95,95,115,112, + 101,99,95,95,32,111,114,32,95,95,112,97,99,107,97,103, + 101,95,95,44,32,102,97,108,108,105,110,103,32,98,97,99, + 107,32,111,110,32,95,95,110,97,109,101,95,95,32,97,110, + 100,32,95,95,112,97,116,104,95,95,114,1,0,0,0,114, + 131,0,0,0,114,121,0,0,0,114,33,0,0,0,41,7, + 114,42,0,0,0,114,123,0,0,0,114,142,0,0,0,114, + 143,0,0,0,114,115,0,0,0,114,176,0,0,0,114,122, + 0,0,0,41,3,218,7,103,108,111,98,97,108,115,114,170, + 0,0,0,114,88,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,17,95,99,97,108,99,95,95, + 95,112,97,99,107,97,103,101,95,95,4,4,0,0,115,30, + 0,0,0,0,7,15,1,15,1,12,1,27,1,42,2,13, + 1,4,1,12,1,7,2,9,2,13,1,10,1,12,1,19, + 1,114,200,0,0,0,99,5,0,0,0,0,0,0,0,9, 0,0,0,5,0,0,0,67,0,0,0,115,227,0,0,0, 124,4,0,100,1,0,107,2,0,114,27,0,116,0,0,124, 0,0,131,1,0,125,5,0,110,54,0,124,1,0,100,2, @@ -1870,17 +1879,17 @@ const unsigned char _Py_M__importlib[] = { 111,117,108,100,32,104,97,118,101,32,97,32,39,108,101,118, 101,108,39,32,111,102,32,50,41,46,10,10,32,32,32,32, 114,33,0,0,0,78,114,121,0,0,0,41,8,114,187,0, - 0,0,114,198,0,0,0,218,9,112,97,114,116,105,116,105, + 0,0,114,200,0,0,0,218,9,112,97,114,116,105,116,105, 111,110,114,168,0,0,0,114,14,0,0,0,114,21,0,0, 0,114,1,0,0,0,114,195,0,0,0,41,9,114,15,0, - 0,0,114,197,0,0,0,218,6,108,111,99,97,108,115,114, + 0,0,114,199,0,0,0,218,6,108,111,99,97,108,115,114, 193,0,0,0,114,171,0,0,0,114,89,0,0,0,90,8, 103,108,111,98,97,108,115,95,114,170,0,0,0,90,7,99, 117,116,95,111,102,102,114,10,0,0,0,114,10,0,0,0, 114,11,0,0,0,218,10,95,95,105,109,112,111,114,116,95, - 95,25,4,0,0,115,26,0,0,0,0,11,12,1,15,2, + 95,31,4,0,0,115,26,0,0,0,0,11,12,1,15,2, 24,1,12,1,18,1,6,3,12,1,23,1,6,1,4,4, - 35,3,40,2,114,201,0,0,0,99,1,0,0,0,0,0, + 35,3,40,2,114,203,0,0,0,99,1,0,0,0,0,0, 0,0,2,0,0,0,3,0,0,0,67,0,0,0,115,53, 0,0,0,116,0,0,106,1,0,124,0,0,131,1,0,125, 1,0,124,1,0,100,0,0,107,8,0,114,43,0,116,2, @@ -1891,8 +1900,8 @@ const unsigned char _Py_M__importlib[] = { 0,0,114,77,0,0,0,114,150,0,0,0,41,2,114,15, 0,0,0,114,88,0,0,0,114,10,0,0,0,114,10,0, 0,0,114,11,0,0,0,218,18,95,98,117,105,108,116,105, - 110,95,102,114,111,109,95,110,97,109,101,60,4,0,0,115, - 8,0,0,0,0,1,15,1,12,1,16,1,114,202,0,0, + 110,95,102,114,111,109,95,110,97,109,101,66,4,0,0,115, + 8,0,0,0,0,1,15,1,12,1,16,1,114,204,0,0, 0,99,2,0,0,0,0,0,0,0,12,0,0,0,12,0, 0,0,67,0,0,0,115,74,1,0,0,124,1,0,97,0, 0,124,0,0,97,1,0,116,2,0,116,1,0,131,1,0, @@ -1937,7 +1946,7 @@ const unsigned char _Py_M__importlib[] = { 0,114,21,0,0,0,218,5,105,116,101,109,115,114,178,0, 0,0,114,76,0,0,0,114,151,0,0,0,114,82,0,0, 0,114,161,0,0,0,114,132,0,0,0,114,137,0,0,0, - 114,1,0,0,0,114,202,0,0,0,114,5,0,0,0,114, + 114,1,0,0,0,114,204,0,0,0,114,5,0,0,0,114, 77,0,0,0,41,12,218,10,115,121,115,95,109,111,100,117, 108,101,218,11,95,105,109,112,95,109,111,100,117,108,101,90, 11,109,111,100,117,108,101,95,116,121,112,101,114,15,0,0, @@ -1948,10 +1957,10 @@ const unsigned char _Py_M__importlib[] = { 101,97,100,95,109,111,100,117,108,101,90,14,119,101,97,107, 114,101,102,95,109,111,100,117,108,101,114,10,0,0,0,114, 10,0,0,0,114,11,0,0,0,218,6,95,115,101,116,117, - 112,67,4,0,0,115,50,0,0,0,0,9,6,1,6,3, + 112,73,4,0,0,115,50,0,0,0,0,9,6,1,6,3, 12,1,28,1,15,1,15,1,9,1,15,1,9,2,3,1, 15,1,17,3,13,1,13,1,15,1,15,2,13,1,20,3, - 3,1,16,1,13,2,11,1,16,3,12,1,114,206,0,0, + 3,1,16,1,13,2,11,1,16,3,12,1,114,208,0,0, 0,99,2,0,0,0,0,0,0,0,3,0,0,0,3,0, 0,0,67,0,0,0,115,87,0,0,0,116,0,0,124,0, 0,124,1,0,131,2,0,1,116,1,0,106,2,0,106,3, @@ -1963,15 +1972,15 @@ const unsigned char _Py_M__importlib[] = { 112,111,114,116,108,105,98,32,97,115,32,116,104,101,32,105, 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, 32,105,109,112,111,114,116,46,114,33,0,0,0,78,41,11, - 114,206,0,0,0,114,14,0,0,0,114,175,0,0,0,114, + 114,208,0,0,0,114,14,0,0,0,114,175,0,0,0,114, 113,0,0,0,114,151,0,0,0,114,161,0,0,0,218,26, 95,102,114,111,122,101,110,95,105,109,112,111,114,116,108,105, 98,95,101,120,116,101,114,110,97,108,114,119,0,0,0,218, 8,95,105,110,115,116,97,108,108,114,21,0,0,0,114,1, - 0,0,0,41,3,114,204,0,0,0,114,205,0,0,0,114, - 207,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,114,208,0,0,0,114,4,0,0,115,12,0,0, - 0,0,2,13,2,16,1,16,3,12,1,6,1,114,208,0, + 0,0,0,41,3,114,206,0,0,0,114,207,0,0,0,114, + 209,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,114,210,0,0,0,120,4,0,0,115,12,0,0, + 0,0,2,13,2,16,1,16,3,12,1,6,1,114,210,0, 0,0,41,51,114,3,0,0,0,114,119,0,0,0,114,12, 0,0,0,114,16,0,0,0,114,17,0,0,0,114,59,0, 0,0,114,41,0,0,0,114,48,0,0,0,114,31,0,0, @@ -1987,8 +1996,8 @@ const unsigned char _Py_M__importlib[] = { 0,114,172,0,0,0,114,174,0,0,0,114,177,0,0,0, 114,182,0,0,0,114,192,0,0,0,114,183,0,0,0,114, 185,0,0,0,114,186,0,0,0,114,187,0,0,0,114,195, - 0,0,0,114,198,0,0,0,114,201,0,0,0,114,202,0, - 0,0,114,206,0,0,0,114,208,0,0,0,114,10,0,0, + 0,0,0,114,200,0,0,0,114,203,0,0,0,114,204,0, + 0,0,114,208,0,0,0,114,210,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, 218,8,60,109,111,100,117,108,101,62,8,0,0,0,115,96, 0,0,0,6,17,6,2,12,8,12,4,19,20,6,2,6, @@ -1996,6 +2005,6 @@ const unsigned char _Py_M__importlib[] = { 8,12,11,12,12,12,16,12,36,19,27,19,101,24,26,9, 3,18,45,18,60,12,18,12,17,12,25,12,29,12,23,12, 16,19,73,19,77,19,13,12,9,12,9,15,40,12,17,6, - 1,10,2,12,27,12,6,18,24,12,32,12,21,24,35,12, + 1,10,2,12,27,12,6,18,24,12,32,12,27,24,35,12, 7,12,47, }; -- cgit v0.12 From b0db3718d44d4ef8aa257a166299bc35f0e08b87 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:26:56 -0800 Subject: whitespace cleanup --- Lib/test/test_importlib/import_/test___package__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index ddeeaca..7f64548 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -76,8 +76,8 @@ class Using__package__: def test_warn_when_package_and_spec_disagree(self): # Raise an ImportWarning if __package__ != __spec__.parent. with self.assertWarns(ImportWarning): - self.import_module({'__package__': 'pkg.fake', - '__spec__': FakeSpec('pkg.fakefake')}) + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) def test_bad__package__(self): globals = {'__package__': ''} -- cgit v0.12 From 4b18dd339a9919e6f5fa3485c8b871ed19d9e391 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:55:56 -0800 Subject: Issue #25234: Skip test_eintr.test_open() under OS X to avoid hanging --- Lib/test/eintrdata/eintr_tester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1ed3da..1cb86e5 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -341,6 +341,7 @@ class SocketEINTRTest(EINTRBaseTest): self._test_open("fp = open(path, 'r')\nfp.close()", self.python_open) + @unittest.skipIf(sys.platform == 'darwin', "hangs under OS X; see issue #25234") def os_open(self, path): fd = os.open(path, os.O_WRONLY) os.close(fd) -- cgit v0.12 From 9fa812668faf0d2d7839845e1a0da19b87bdbc29 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 16:39:02 -0800 Subject: Issue #18018: Raise an ImportError if a relative import is attempted with no known parent package. Previously SystemError was raised if the parent package didn't exist (e.g., __package__ was set to ''). Thanks to Florent Xicluna and Yongzhi Pan for reporting the issue. --- Doc/whatsnew/3.6.rst | 4 ++++ Lib/test/test_importlib/import_/test_relative_imports.py | 5 +++++ Misc/NEWS | 3 +++ Python/import.c | 11 ++++++++--- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 5872cc0..158dad3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -278,6 +278,10 @@ Changes in the Python API to ``__spec__.parent`` then :exc:`ImportWarning` is raised. (Contributed by Brett Cannon in :issue:`25791`.) +* When a relative import is performed and no parent package is known, then + :exc:`ImportError` will be raised. Previously, :exc:`SystemError` could be + raised. (Contribute by Brett Cannon in :issue:`18018`.) + Changes in the C API -------------------- diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 0409f22..1cad6b3 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -213,6 +213,11 @@ class RelativeImports: with self.assertRaises(KeyError): self.__import__('sys', level=1) + def test_relative_import_no_package(self): + with self.assertRaises(ImportError): + self.__import__('a', {'__package__': '', '__spec__': None}, + level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Misc/NEWS b/Misc/NEWS index 165b428..676f2ca 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #18018: Import raises ImportError instead of SystemError if a relative + import is attempted without a known parent package. + - Issue #25843: When compiling code, don't merge constants if they are equal but have a different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` is now correctly compiled to two different functions: ``f1()`` returns ``1`` diff --git a/Python/import.c b/Python/import.c index 22f9d21..8ad5b4c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1424,7 +1424,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyErr_SetString(PyExc_TypeError, "package must be a string"); goto error; } - else if (spec != NULL) { + else if (spec != NULL && spec != Py_None) { int equal; PyObject *parent = PyObject_GetAttrString(spec, "parent"); if (parent == NULL) { @@ -1444,7 +1444,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } } - else if (spec != NULL) { + else if (spec != NULL && spec != Py_None) { package = PyObject_GetAttrString(spec, "parent"); if (package == NULL) { goto error; @@ -1491,7 +1491,12 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } - if (PyDict_GetItem(interp->modules, package) == NULL) { + if (PyUnicode_CompareWithASCIIString(package, "") == 0) { + PyErr_SetString(PyExc_ImportError, + "attempted relative import with no known parent package"); + goto error; + } + else if (PyDict_GetItem(interp->modules, package) == NULL) { PyErr_Format(PyExc_SystemError, "Parent module %R not loaded, cannot perform relative " "import", package); -- cgit v0.12 From 843a1fb1e78bf2d374daccaaa5c616f63afdf053 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 13:29:02 +0100 Subject: test_gc: remove unused imports --- Lib/test/test_gc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 1f0867d..872ca62 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -682,7 +682,6 @@ class GCTests(unittest.TestCase): # Create a reference cycle through the __main__ module and check # it gets collected at interpreter shutdown. code = """if 1: - import weakref class C: def __del__(self): print('__del__ called') @@ -696,7 +695,6 @@ class GCTests(unittest.TestCase): # Same as above, but with a non-__main__ module. with temp_dir() as script_dir: module = """if 1: - import weakref class C: def __del__(self): print('__del__ called') -- cgit v0.12 From 5ebe2c89fe47501db64e1a82ac6a3bcbd21f6e70 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 13:52:05 +0100 Subject: Cleanup test_dict * Write one import per line * Sort imports by name * Add an empty line: 2 empty lines between code blocks at the module level (PEP 8) --- Lib/test/test_dict.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 3b42414..1049be5 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1,10 +1,12 @@ -import unittest -from test import support - -import collections, random, string +import collections import collections.abc -import gc, weakref +import gc import pickle +import random +import string +import unittest +import weakref +from test import support class DictTest(unittest.TestCase): @@ -963,5 +965,6 @@ class Dict(dict): class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = Dict + if __name__ == "__main__": unittest.main() -- cgit v0.12 From 1aa78938b0d019b7c9cbb153c2f35709ee01a19a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 14:15:48 +0100 Subject: Issue #26146: marshal.loads() now uses the empty frozenset singleton --- Lib/test/test_marshal.py | 7 +++++ Python/marshal.c | 69 ++++++++++++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index c7def9a..b378ffe 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -135,6 +135,13 @@ class ContainerTestCase(unittest.TestCase, HelperMixin): for constructor in (set, frozenset): self.helper(constructor(self.d.keys())) + @support.cpython_only + def test_empty_frozenset_singleton(self): + # marshal.loads() must reuse the empty frozenset singleton + obj = frozenset() + obj2 = marshal.loads(marshal.dumps(obj)) + self.assertIs(obj2, obj) + class BufferTestCase(unittest.TestCase, HelperMixin): diff --git a/Python/marshal.c b/Python/marshal.c index 589eb80..7a4b9d2 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1266,41 +1266,52 @@ r_object(RFILE *p) PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); break; } - v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); - if (type == TYPE_SET) { - R_REF(v); - } else { - /* must use delayed registration of frozensets because they must - * be init with a refcount of 1 - */ - idx = r_ref_reserve(flag, p); - if (idx < 0) - Py_CLEAR(v); /* signal error */ - } - if (v == NULL) - break; - for (i = 0; i < n; i++) { - v2 = r_object(p); - if ( v2 == NULL ) { - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data for set"); - Py_DECREF(v); - v = NULL; + if (n == 0 && type == TYPE_FROZENSET) { + /* call frozenset() to get the empty frozenset singleton */ + v = PyObject_CallFunction((PyObject*)&PyFrozenSet_Type, NULL); + if (v == NULL) break; + R_REF(v); + retval = v; + } + else { + v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); + if (type == TYPE_SET) { + R_REF(v); + } else { + /* must use delayed registration of frozensets because they must + * be init with a refcount of 1 + */ + idx = r_ref_reserve(flag, p); + if (idx < 0) + Py_CLEAR(v); /* signal error */ } - if (PySet_Add(v, v2) == -1) { - Py_DECREF(v); - Py_DECREF(v2); - v = NULL; + if (v == NULL) break; + + for (i = 0; i < n; i++) { + v2 = r_object(p); + if ( v2 == NULL ) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, + "NULL object in marshal data for set"); + Py_DECREF(v); + v = NULL; + break; + } + if (PySet_Add(v, v2) == -1) { + Py_DECREF(v); + Py_DECREF(v2); + v = NULL; + break; + } + Py_DECREF(v2); } - Py_DECREF(v2); + if (type != TYPE_SET) + v = r_ref_insert(v, idx, flag, p); + retval = v; } - if (type != TYPE_SET) - v = r_ref_insert(v, idx, flag, p); - retval = v; break; case TYPE_CODE: -- cgit v0.12 From d84ec225bdf88a8ad54c57b78beb6c32ae9fffde Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 09:12:06 -0800 Subject: Miscellaneous refactorings * Add comment to the maxlen structure entry about the meaning of maxlen == -1 * Factor-out code common to deque_append(left) and deque_extend(left) * Factor inner-loop in deque_clear() to use only 1 test per loop instead of 2 * Tighten inner-loops for deque_item() and deque_ass_item() so that the compiler can combine the decrement and test into a single step. --- Modules/_collectionsmodule.c | 123 ++++++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 65 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index fbf07a8..3ab987d 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -81,7 +81,7 @@ typedef struct { Py_ssize_t leftindex; /* 0 <= leftindex < BLOCKLEN */ Py_ssize_t rightindex; /* 0 <= rightindex < BLOCKLEN */ size_t state; /* incremented whenever the indices move */ - Py_ssize_t maxlen; + Py_ssize_t maxlen; /* maxlen is -1 for unbounded deques */ PyObject *weakreflist; } dequeobject; @@ -264,13 +264,13 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); #define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) -static PyObject * -deque_append(dequeobject *deque, PyObject *item) +int +deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(); if (b == NULL) - return NULL; + return -1; b->leftlink = deque->rightblock; CHECK_END(deque->rightblock->rightlink); deque->rightblock->rightlink = b; @@ -279,27 +279,35 @@ deque_append(dequeobject *deque, PyObject *item) deque->rightindex = -1; } Py_SIZE(deque)++; - Py_INCREF(item); deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - if (NEEDS_TRIM(deque, deque->maxlen)) { + if (NEEDS_TRIM(deque, maxlen)) { PyObject *olditem = deque_popleft(deque, NULL); Py_DECREF(olditem); } else { deque->state++; } + return 0; +} + +static PyObject * +deque_append(dequeobject *deque, PyObject *item) +{ + Py_INCREF(item); + if (deque_append_internal(deque, item, deque->maxlen) < 0) + return NULL; Py_RETURN_NONE; } PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); -static PyObject * -deque_appendleft(dequeobject *deque, PyObject *item) +int +deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->leftindex == 0) { block *b = newblock(); if (b == NULL) - return NULL; + return -1; b->rightlink = deque->leftblock; CHECK_END(deque->leftblock->leftlink); deque->leftblock->leftlink = b; @@ -308,7 +316,6 @@ deque_appendleft(dequeobject *deque, PyObject *item) deque->leftindex = BLOCKLEN; } Py_SIZE(deque)++; - Py_INCREF(item); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { @@ -317,6 +324,15 @@ deque_appendleft(dequeobject *deque, PyObject *item) } else { deque->state++; } + return 0; +} + +static PyObject * +deque_appendleft(dequeobject *deque, PyObject *item) +{ + Py_INCREF(item); + if (deque_appendleft_internal(deque, item, deque->maxlen) < 0) + return NULL; Py_RETURN_NONE; } @@ -387,28 +403,10 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(); - if (b == NULL) { - Py_DECREF(item); - Py_DECREF(it); - return NULL; - } - b->leftlink = deque->rightblock; - CHECK_END(deque->rightblock->rightlink); - deque->rightblock->rightlink = b; - deque->rightblock = b; - MARK_END(b->rightlink); - deque->rightindex = -1; - } - Py_SIZE(deque)++; - deque->rightindex++; - deque->rightblock->data[deque->rightindex] = item; - if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_popleft(deque, NULL); - Py_DECREF(olditem); - } else { - deque->state++; + if (deque_append_internal(deque, item, maxlen) < 0) { + Py_DECREF(item); + Py_DECREF(it); + return NULL; } } return finalize_iterator(it); @@ -452,28 +450,10 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque->leftindex == 0) { - block *b = newblock(); - if (b == NULL) { - Py_DECREF(item); - Py_DECREF(it); - return NULL; - } - b->rightlink = deque->leftblock; - CHECK_END(deque->leftblock->leftlink); - deque->leftblock->leftlink = b; - deque->leftblock = b; - MARK_END(b->leftlink); - deque->leftindex = BLOCKLEN; - } - Py_SIZE(deque)++; - deque->leftindex--; - deque->leftblock->data[deque->leftindex] = item; - if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_pop(deque, NULL); - Py_DECREF(olditem); - } else { - deque->state++; + if (deque_appendleft_internal(deque, item, maxlen) < 0) { + Py_DECREF(item); + Py_DECREF(it); + return NULL; } } return finalize_iterator(it); @@ -565,8 +545,9 @@ deque_clear(dequeobject *deque) block *prevblock; block *leftblock; Py_ssize_t leftindex; - Py_ssize_t n; + Py_ssize_t n, m; PyObject *item; + PyObject **itemptr, **limit; if (Py_SIZE(deque) == 0) return; @@ -608,17 +589,25 @@ deque_clear(dequeobject *deque) /* Now the old size, leftblock, and leftindex are disconnected from the empty deque and we can use them to decref the pointers. */ - while (n--) { - item = leftblock->data[leftindex]; - Py_DECREF(item); - leftindex++; - if (leftindex == BLOCKLEN && n) { + itemptr = &leftblock->data[leftindex]; + m = (BLOCKLEN - leftindex > n) ? n : BLOCKLEN - leftindex; + limit = &leftblock->data[leftindex + m]; + n -= m; + while (1) { + if (itemptr == limit) { + if (n == 0) + break; CHECK_NOT_END(leftblock->rightlink); prevblock = leftblock; leftblock = leftblock->rightlink; - leftindex = 0; + itemptr = leftblock->data; + m = (n > BLOCKLEN) ? BLOCKLEN : n; + limit = &leftblock->data[m]; + n -= m; freeblock(prevblock); } + item = *(itemptr++); + Py_DECREF(item); } CHECK_END(leftblock->rightlink); freeblock(leftblock); @@ -1185,14 +1174,16 @@ deque_item(dequeobject *deque, Py_ssize_t i) i = (Py_ssize_t)((size_t) i % BLOCKLEN); if (index < (Py_SIZE(deque) >> 1)) { b = deque->leftblock; - while (n--) + n++; + while (--n) b = b->rightlink; } else { n = (Py_ssize_t)( ((size_t)(deque->leftindex + Py_SIZE(deque) - 1)) / BLOCKLEN - n); b = deque->rightblock; - while (n--) + n++; + while (--n) b = b->leftlink; } } @@ -1235,14 +1226,16 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) i = (Py_ssize_t)((size_t) i % BLOCKLEN); if (index <= halflen) { b = deque->leftblock; - while (n--) + n++; + while (--n) b = b->rightlink; } else { n = (Py_ssize_t)( ((size_t)(deque->leftindex + Py_SIZE(deque) - 1)) / BLOCKLEN - n); b = deque->rightblock; - while (n--) + n++; + while (--n) b = b->leftlink; } Py_INCREF(v); -- cgit v0.12 From 165eee214bc388eb588db33385ca49ddbb305565 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 11:32:07 -0800 Subject: Convert two other post-decrement while-loops to pre-decrements for consistency and for better code generation. --- Modules/_collectionsmodule.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 3ab987d..cc9e4e8 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -937,7 +937,8 @@ deque_count(dequeobject *deque, PyObject *v) PyObject *item; int cmp; - while (n--) { + n++; + while (--n) { CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); @@ -974,7 +975,8 @@ deque_contains(dequeobject *deque, PyObject *v) PyObject *item; int cmp; - while (n--) { + n++; + while (--n) { CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); -- cgit v0.12 From 306d6b1ea6bf2582b9284be2fd27275abbade3e1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 12:40:42 -0800 Subject: Convert another post-decrement while-loop to pre-decrement for consistency and better generated code (on both GCC and CLang). --- Modules/_collectionsmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index cc9e4e8..b77ea65 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -895,7 +895,8 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_ssize_t n = Py_SIZE(deque) >> 1; PyObject *tmp; - while (n-- > 0) { + n++; + while (--n) { /* Validate that pointers haven't met in the middle */ assert(leftblock != rightblock || leftindex < rightindex); CHECK_NOT_END(leftblock); -- cgit v0.12 From 1ce6885f8a5bc2cca1fb03fa0b0bd953407f3c25 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 22:15:20 -0800 Subject: Fix compiler warning about obviously unreachable code. --- Modules/socketmodule.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7ab534e..a6143c6 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -5411,10 +5411,6 @@ socket_inet_ntop(PyObject *self, PyObject *args) } else { return PyUnicode_FromString(retval); } - - /* NOTREACHED */ - PyErr_SetString(PyExc_RuntimeError, "invalid handling of inet_ntop"); - return NULL; } #elif defined(MS_WINDOWS) -- cgit v0.12 From 0dceb918668399bdcbe27d1c59d01c4c9228c1a6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Jan 2016 10:33:30 -0800 Subject: Tests versus zero are more compact than tests versus -1. --- Objects/listobject.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index fcc21cb..bcf587c 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -248,7 +248,7 @@ ins1(PyListObject *self, Py_ssize_t where, PyObject *v) return -1; } - if (list_resize(self, n+1) == -1) + if (list_resize(self, n+1) < 0) return -1; if (where < 0) { @@ -288,7 +288,7 @@ app1(PyListObject *self, PyObject *v) return -1; } - if (list_resize(self, n+1) == -1) + if (list_resize(self, n+1) < 0) return -1; Py_INCREF(v); @@ -708,7 +708,7 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) return PyErr_NoMemory(); } - if (list_resize(self, size*n) == -1) + if (list_resize(self, size*n) < 0) return NULL; p = size; @@ -798,7 +798,7 @@ listextend(PyListObject *self, PyObject *b) Py_RETURN_NONE; } m = Py_SIZE(self); - if (list_resize(self, m + n) == -1) { + if (list_resize(self, m + n) < 0) { Py_DECREF(b); return NULL; } @@ -826,7 +826,7 @@ listextend(PyListObject *self, PyObject *b) /* Guess a result list size. */ n = PyObject_LengthHint(b, 8); - if (n == -1) { + if (n < 0) { Py_DECREF(it); return NULL; } @@ -834,7 +834,7 @@ listextend(PyListObject *self, PyObject *b) mn = m + n; if (mn >= m) { /* Make room. */ - if (list_resize(self, mn) == -1) + if (list_resize(self, mn) < 0) goto error; /* Make the list sane again. */ Py_SIZE(self) = m; -- cgit v0.12 From f2c1aa1661edb3e14ff8b7b9995f93e303c8acbb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Jan 2016 00:40:57 +0100 Subject: Add ast.Constant Issue #26146: Add a new kind of AST node: ast.Constant. It can be used by external AST optimizers, but the compiler does not emit directly such node. An optimizer can replace the following AST nodes with ast.Constant: * ast.NameConstant: None, False, True * ast.Num: int, float, complex * ast.Str: str * ast.Bytes: bytes * ast.Tuple if items are constants too: tuple * frozenset Update code to accept ast.Constant instead of ast.Num and/or ast.Str: * compiler * docstrings * ast.literal_eval() * Tools/parser/unparse.py --- Include/Python-ast.h | 13 ++++-- Include/asdl.h | 1 + Lib/ast.py | 52 ++++++++++++--------- Lib/test/test_ast.py | 120 +++++++++++++++++++++++++++++++++++++++++++++++- Misc/NEWS | 4 ++ Parser/Python.asdl | 7 ++- Parser/asdl.py | 3 +- Parser/asdl_c.py | 21 +++++++++ Python/Python-ast.c | 79 +++++++++++++++++++++++++++++++ Python/ast.c | 50 ++++++++++++++++++++ Python/compile.c | 60 ++++++++++++++++++------ Python/future.c | 5 +- Python/symtable.c | 1 + Tools/parser/unparse.py | 29 +++++++++++- 14 files changed, 401 insertions(+), 44 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 175e380..1ab376a 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -202,9 +202,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, Await_kind=12, Yield_kind=13, YieldFrom_kind=14, Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18, FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21, - NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24, - Subscript_kind=25, Starred_kind=26, Name_kind=27, - List_kind=28, Tuple_kind=29}; + NameConstant_kind=22, Ellipsis_kind=23, Constant_kind=24, + Attribute_kind=25, Subscript_kind=26, Starred_kind=27, + Name_kind=28, List_kind=29, Tuple_kind=30}; struct _expr { enum _expr_kind kind; union { @@ -316,6 +316,10 @@ struct _expr { } NameConstant; struct { + constant value; + } Constant; + + struct { expr_ty value; identifier attr; expr_context_ty ctx; @@ -567,6 +571,9 @@ expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena *arena); #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2) expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena); +#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3) +expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena + *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena); diff --git a/Include/asdl.h b/Include/asdl.h index 495153c..35e9fa1 100644 --- a/Include/asdl.h +++ b/Include/asdl.h @@ -6,6 +6,7 @@ typedef PyObject * string; typedef PyObject * bytes; typedef PyObject * object; typedef PyObject * singleton; +typedef PyObject * constant; /* It would be nice if the code generated by asdl_c.py was completely independent of Python, but it is a goal the requires too much work diff --git a/Lib/ast.py b/Lib/ast.py index 0170472..156a1f2 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -35,6 +35,8 @@ def parse(source, filename='', mode='exec'): return compile(source, filename, mode, PyCF_ONLY_AST) +_NUM_TYPES = (int, float, complex) + def literal_eval(node_or_string): """ Safely evaluate an expression node or a string containing a Python @@ -47,7 +49,9 @@ def literal_eval(node_or_string): if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert(node): - if isinstance(node, (Str, Bytes)): + if isinstance(node, Constant): + return node.value + elif isinstance(node, (Str, Bytes)): return node.s elif isinstance(node, Num): return node.n @@ -62,24 +66,21 @@ def literal_eval(node_or_string): in zip(node.keys, node.values)) elif isinstance(node, NameConstant): return node.value - elif isinstance(node, UnaryOp) and \ - isinstance(node.op, (UAdd, USub)) and \ - isinstance(node.operand, (Num, UnaryOp, BinOp)): + elif isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - elif isinstance(node, BinOp) and \ - isinstance(node.op, (Add, Sub)) and \ - isinstance(node.right, (Num, UnaryOp, BinOp)) and \ - isinstance(node.left, (Num, UnaryOp, BinOp)): + if isinstance(operand, _NUM_TYPES): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): left = _convert(node.left) right = _convert(node.right) - if isinstance(node.op, Add): - return left + right - else: - return left - right + if isinstance(left, _NUM_TYPES) and isinstance(right, _NUM_TYPES): + if isinstance(node.op, Add): + return left + right + else: + return left - right raise ValueError('malformed node or string: ' + repr(node)) return _convert(node_or_string) @@ -196,12 +197,19 @@ def get_docstring(node, clean=True): """ if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - if clean: - import inspect - return inspect.cleandoc(node.body[0].value.s) - return node.body[0].value.s + if not(node.body and isinstance(node.body[0], Expr)): + return + node = node.body[0].value + if isinstance(node, Str): + text = node.s + elif isinstance(node, Constant) and isinstance(node.value, str): + text = node.value + else: + return + if clean: + import inspect + text = inspect.cleandoc(text) + return text def walk(node): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index d3e6d35..6d6c9bd 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,7 +1,8 @@ +import ast +import dis import os import sys import unittest -import ast import weakref from test import support @@ -933,6 +934,123 @@ class ASTValidatorTests(unittest.TestCase): compile(mod, fn, "exec") +class ConstantTests(unittest.TestCase): + """Tests on the ast.Constant node type.""" + + def compile_constant(self, value): + tree = ast.parse("x = 123") + + node = tree.body[0].value + new_node = ast.Constant(value=value) + ast.copy_location(new_node, node) + tree.body[0].value = new_node + + code = compile(tree, "", "exec") + + ns = {} + exec(code, ns) + return ns['x'] + + def test_singletons(self): + for const in (None, False, True, Ellipsis, b'', frozenset()): + with self.subTest(const=const): + value = self.compile_constant(const) + self.assertIs(value, const) + + def test_values(self): + nested_tuple = (1,) + nested_frozenset = frozenset({1}) + for level in range(3): + nested_tuple = (nested_tuple, 2) + nested_frozenset = frozenset({nested_frozenset, 2}) + values = (123, 123.0, 123j, + "unicode", b'bytes', + tuple("tuple"), frozenset("frozenset"), + nested_tuple, nested_frozenset) + for value in values: + with self.subTest(value=value): + result = self.compile_constant(value) + self.assertEqual(result, value) + + def test_assign_to_constant(self): + tree = ast.parse("x = 1") + + target = tree.body[0].targets[0] + new_target = ast.Constant(value=1) + ast.copy_location(new_target, target) + tree.body[0].targets[0] = new_target + + with self.assertRaises(ValueError) as cm: + compile(tree, "string", "exec") + self.assertEqual(str(cm.exception), + "expression which can't be assigned " + "to in Store context") + + def test_get_docstring(self): + tree = ast.parse("'docstring'\nx = 1") + self.assertEqual(ast.get_docstring(tree), 'docstring') + + tree.body[0].value = ast.Constant(value='constant docstring') + self.assertEqual(ast.get_docstring(tree), 'constant docstring') + + def get_load_const(self, tree): + # Compile to bytecode, disassemble and get parameter of LOAD_CONST + # instructions + co = compile(tree, '', 'exec') + consts = [] + for instr in dis.get_instructions(co): + if instr.opname == 'LOAD_CONST': + consts.append(instr.argval) + return consts + + @support.cpython_only + def test_load_const(self): + consts = [None, + True, False, + 124, + 2.0, + 3j, + "unicode", + b'bytes', + (1, 2, 3)] + + code = '\n'.join(map(repr, consts)) + code += '\n...' + + code_consts = [const for const in consts + if (not isinstance(const, (str, int, float, complex)) + or isinstance(const, bool))] + code_consts.append(Ellipsis) + # the compiler adds a final "LOAD_CONST None" + code_consts.append(None) + + tree = ast.parse(code) + self.assertEqual(self.get_load_const(tree), code_consts) + + # Replace expression nodes with constants + for expr_node, const in zip(tree.body, consts): + assert isinstance(expr_node, ast.Expr) + new_node = ast.Constant(value=const) + ast.copy_location(new_node, expr_node.value) + expr_node.value = new_node + + self.assertEqual(self.get_load_const(tree), code_consts) + + def test_literal_eval(self): + tree = ast.parse("1 + 2") + binop = tree.body[0].value + + new_left = ast.Constant(value=10) + ast.copy_location(new_left, binop.left) + binop.left = new_left + + new_right = ast.Constant(value=20) + ast.copy_location(new_right, binop.right) + binop.right = new_right + + self.assertEqual(ast.literal_eval(binop), 30) + + def main(): if __name__ != '__main__': return diff --git a/Misc/NEWS b/Misc/NEWS index 676f2ca..6e1a5a0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used + by external AST optimizers, but the compiler does not emit directly such + node. + - Issue #18018: Import raises ImportError instead of SystemError if a relative import is attempted without a known parent package. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 22775c6..aaddf61 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -1,4 +1,8 @@ --- ASDL's six builtin types are identifier, int, string, bytes, object, singleton +-- ASDL's 7 builtin types are: +-- identifier, int, string, bytes, object, singleton, constant +-- +-- singleton: None, True or False +-- constant can be None, whereas None means "no value" for object. module Python { @@ -76,6 +80,7 @@ module Python | Bytes(bytes s) | NameConstant(singleton value) | Ellipsis + | Constant(constant value) -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) diff --git a/Parser/asdl.py b/Parser/asdl.py index 121cdab..62f5c19 100644 --- a/Parser/asdl.py +++ b/Parser/asdl.py @@ -33,7 +33,8 @@ __all__ = [ # See the EBNF at the top of the file to understand the logical connection # between the various node types. -builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'} +builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton', + 'constant'} class AST: def __repr__(self): diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index eedd89b..a81644d 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -834,6 +834,7 @@ static PyObject* ast2obj_object(void *o) return (PyObject*)o; } #define ast2obj_singleton ast2obj_object +#define ast2obj_constant ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object #define ast2obj_bytes ast2obj_object @@ -871,6 +872,26 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) return 0; } +static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (obj == Py_None || obj == Py_True || obj == Py_False) { + /* don't increment the reference counter, Constant uses a borrowed + * reference, not a strong reference */ + *out = obj; + return 0; + } + + if (obj) { + if (PyArena_AddPyObject(arena, obj) < 0) { + *out = NULL; + return -1; + } + Py_INCREF(obj); + } + *out = obj; + return 0; +} + static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 07d9b3e..4dde11f 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -306,6 +306,10 @@ static char *NameConstant_fields[]={ "value", }; static PyTypeObject *Ellipsis_type; +static PyTypeObject *Constant_type; +static char *Constant_fields[]={ + "value", +}; static PyTypeObject *Attribute_type; _Py_IDENTIFIER(attr); _Py_IDENTIFIER(ctx); @@ -709,6 +713,7 @@ static PyObject* ast2obj_object(void *o) return (PyObject*)o; } #define ast2obj_singleton ast2obj_object +#define ast2obj_constant ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object #define ast2obj_bytes ast2obj_object @@ -746,6 +751,26 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) return 0; } +static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (obj == Py_None || obj == Py_True || obj == Py_False) { + /* don't increment the reference counter, Constant uses a borrowed + * reference, not a strong reference */ + *out = obj; + return 0; + } + + if (obj) { + if (PyArena_AddPyObject(arena, obj) < 0) { + *out = NULL; + return -1; + } + Py_INCREF(obj); + } + *out = obj; + return 0; +} + static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { @@ -941,6 +966,8 @@ static int init_types(void) if (!NameConstant_type) return 0; Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0); if (!Ellipsis_type) return 0; + Constant_type = make_type("Constant", expr_type, Constant_fields, 1); + if (!Constant_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3); @@ -2167,6 +2194,25 @@ Ellipsis(int lineno, int col_offset, PyArena *arena) } expr_ty +Constant(constant value, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Constant"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = Constant_kind; + p->v.Constant.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena) { @@ -3267,6 +3313,15 @@ ast2obj_expr(void* _o) result = PyType_GenericNew(Ellipsis_type, NULL, NULL); if (!result) goto failed; break; + case Constant_kind: + result = PyType_GenericNew(Constant_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_constant(o->v.Constant.value); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) + goto failed; + Py_DECREF(value); + break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); if (!result) goto failed; @@ -6240,6 +6295,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)Constant_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + constant value; + + if (_PyObject_HasAttrId(obj, &PyId_value)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp == NULL) goto failed; + res = obj2ast_constant(tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant"); + return 1; + } + *out = Constant(value, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type); if (isinstance == -1) { return 1; @@ -7517,6 +7594,8 @@ PyInit__ast(void) 0) return NULL; if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0) return NULL; + if (PyDict_SetItemString(d, "Constant", (PyObject*)Constant_type) < 0) + return NULL; if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0) return NULL; if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < 0) diff --git a/Python/ast.c b/Python/ast.c index a5d8dba..5422e9c 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -132,6 +132,50 @@ validate_arguments(arguments_ty args) } static int +validate_constant(PyObject *value) +{ + if (value == Py_None || value == Py_Ellipsis) + return 1; + + if (PyLong_CheckExact(value) + || PyFloat_CheckExact(value) + || PyComplex_CheckExact(value) + || PyBool_Check(value) + || PyUnicode_CheckExact(value) + || PyBytes_CheckExact(value)) + return 1; + + if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) { + PyObject *it; + + it = PyObject_GetIter(value); + if (it == NULL) + return 0; + + while (1) { + PyObject *item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(it); + return 0; + } + break; + } + + if (!validate_constant(item)) { + Py_DECREF(it); + return 0; + } + } + + Py_DECREF(it); + return 1; + } + + return 0; +} + +static int validate_expr(expr_ty exp, expr_context_ty ctx) { int check_ctx = 1; @@ -240,6 +284,12 @@ validate_expr(expr_ty exp, expr_context_ty ctx) return validate_expr(exp->v.Call.func, Load) && validate_exprs(exp->v.Call.args, Load, 0) && validate_keywords(exp->v.Call.keywords); + case Constant_kind: + if (!validate_constant(exp->v.Constant.value)) { + PyErr_SetString(PyExc_TypeError, "invalid type in Constant"); + return 0; + } + return 1; case Num_kind: { PyObject *n = exp->v.Num.n; if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) && diff --git a/Python/compile.c b/Python/compile.c index a710e82..ccb05cf 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1314,7 +1314,11 @@ compiler_isdocstring(stmt_ty s) { if (s->kind != Expr_kind) return 0; - return s->v.Expr.value->kind == Str_kind; + if (s->v.Expr.value->kind == Str_kind) + return 1; + if (s->v.Expr.value->kind == Constant_kind) + return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value); + return 0; } /* Compile a sequence of statements, checking for a docstring. */ @@ -1688,8 +1692,12 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) st = (stmt_ty)asdl_seq_GET(body, 0); docstring = compiler_isdocstring(st); - if (docstring && c->c_optimize < 2) - first_const = st->v.Expr.value->v.Str.s; + if (docstring && c->c_optimize < 2) { + if (st->v.Expr.value->kind == Constant_kind) + first_const = st->v.Expr.value->v.Constant.value; + else + first_const = st->v.Expr.value->v.Str.s; + } if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); return 0; @@ -2600,6 +2608,36 @@ compiler_assert(struct compiler *c, stmt_ty s) } static int +compiler_visit_stmt_expr(struct compiler *c, expr_ty value) +{ + if (c->c_interactive && c->c_nestlevel <= 1) { + VISIT(c, expr, value); + ADDOP(c, PRINT_EXPR); + return 1; + } + + if (value->kind == Str_kind || value->kind == Num_kind) { + /* ignore strings and numbers */ + return 1; + } + + if (value->kind == Constant_kind) { + PyObject *cst = value->v.Constant.value; + if (PyUnicode_CheckExact(cst) + || PyLong_CheckExact(cst) + || PyFloat_CheckExact(cst) + || PyComplex_CheckExact(cst)) { + /* ignore strings and numbers */ + return 1; + } + } + + VISIT(c, expr, value); + ADDOP(c, POP_TOP); + return 1; +} + +static int compiler_visit_stmt(struct compiler *c, stmt_ty s) { Py_ssize_t i, n; @@ -2669,16 +2707,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Nonlocal_kind: break; case Expr_kind: - if (c->c_interactive && c->c_nestlevel <= 1) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, PRINT_EXPR); - } - else if (s->v.Expr.value->kind != Str_kind && - s->v.Expr.value->kind != Num_kind) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, POP_TOP); - } - break; + return compiler_visit_stmt_expr(c, s->v.Expr.value); case Pass_kind: break; case Break_kind: @@ -3625,6 +3654,8 @@ expr_constant(struct compiler *c, expr_ty e) switch (e->kind) { case Ellipsis_kind: return 1; + case Constant_kind: + return PyObject_IsTrue(e->v.Constant.value); case Num_kind: return PyObject_IsTrue(e->v.Num.n); case Str_kind: @@ -3912,6 +3943,9 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); + case Constant_kind: + ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts); + break; case Num_kind: ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); break; diff --git a/Python/future.c b/Python/future.c index 163f87f..75f2107 100644 --- a/Python/future.c +++ b/Python/future.c @@ -79,7 +79,10 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) i = 0; first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i); - if (first->kind == Expr_kind && first->v.Expr.value->kind == Str_kind) + if (first->kind == Expr_kind + && (first->v.Expr.value->kind == Str_kind + || (first->v.Expr.value->kind == Constant_kind + && PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value)))) i++; diff --git a/Python/symtable.c b/Python/symtable.c index 9f5149f..558bb06 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1455,6 +1455,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case JoinedStr_kind: VISIT_SEQ(st, expr, e->v.JoinedStr.values); break; + case Constant_kind: case Num_kind: case Str_kind: case Bytes_kind: diff --git a/Tools/parser/unparse.py b/Tools/parser/unparse.py index 9972797..35ebc66 100644 --- a/Tools/parser/unparse.py +++ b/Tools/parser/unparse.py @@ -343,6 +343,11 @@ class Unparser: value = t.s.replace("{", "{{").replace("}", "}}") write(value) + def _fstring_Constant(self, t, write): + assert isinstance(t.value, str) + value = t.value.replace("{", "{{").replace("}", "}}") + write(value) + def _fstring_FormattedValue(self, t, write): write("{") expr = io.StringIO() @@ -364,6 +369,25 @@ class Unparser: def _Name(self, t): self.write(t.id) + def _write_constant(self, value): + if isinstance(value, (float, complex)): + self.write(repr(value).replace("inf", INFSTR)) + else: + self.write(repr(value)) + + def _Constant(self, t): + value = t.value + if isinstance(value, tuple): + self.write("(") + if len(value) == 1: + self._write_constant(value[0]) + self.write(",") + else: + interleave(lambda: self.write(", "), self._write_constant, value) + self.write(")") + else: + self._write_constant(t.value) + def _NameConstant(self, t): self.write(repr(t.value)) @@ -443,7 +467,7 @@ class Unparser: def _Tuple(self, t): self.write("(") if len(t.elts) == 1: - (elt,) = t.elts + elt = t.elts[0] self.dispatch(elt) self.write(",") else: @@ -490,7 +514,8 @@ class Unparser: # Special case: 3.__abs__() is a syntax error, so if t.value # is an integer literal then we need to either parenthesize # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): + if ((isinstance(t.value, ast.Num) and isinstance(t.value.n, int)) + or (isinstance(t.value, ast.Constant) and isinstance(t.value.value, int))): self.write(" ") self.write(".") self.write(t.attr) -- cgit v0.12 From 906d82db6d63c81b9d63a0c99039358bce2da6cb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Jan 2016 23:00:21 -0800 Subject: Fix typo --- Lib/test/test_deque.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index c61e80b..f75b3ff 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -289,7 +289,7 @@ class TestBasic(unittest.TestCase): else: self.assertEqual(d.index(element, start, stop), target) - def test_insert_bug_24913(self): + def test_index_bug_24913(self): d = deque('A' * 3) with self.assertRaises(ValueError): i = d.index("Hello world", 0, 4) -- cgit v0.12 From 726f6902ce9c4db8dc73074ed5323a3bdeb04194 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Jan 2016 00:11:47 +0100 Subject: Fix a refleak in validate_constant() Issue #26146. --- Python/ast.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/ast.c b/Python/ast.c index 5422e9c..ecfc14c 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -164,8 +164,10 @@ validate_constant(PyObject *value) if (!validate_constant(item)) { Py_DECREF(it); + Py_DECREF(item); return 0; } + Py_DECREF(item); } Py_DECREF(it); -- cgit v0.12 From 25219f596a069e8d4ed7114cd9b1bddc2a1de3b7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Jan 2016 00:37:59 +0100 Subject: Issue #26146: remove useless code obj2ast_constant() code is baesd on obj2ast_object() which has a special case for Py_None. But in practice, we don't need to have a special case for constants. Issue noticed by Joseph Jevnik on a review. --- Parser/asdl_c.py | 7 ------- Python/Python-ast.c | 7 ------- 2 files changed, 14 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index a81644d..17c8517 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -874,13 +874,6 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) { - if (obj == Py_None || obj == Py_True || obj == Py_False) { - /* don't increment the reference counter, Constant uses a borrowed - * reference, not a strong reference */ - *out = obj; - return 0; - } - if (obj) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 4dde11f..1193c7c 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -753,13 +753,6 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) { - if (obj == Py_None || obj == Py_True || obj == Py_False) { - /* don't increment the reference counter, Constant uses a borrowed - * reference, not a strong reference */ - *out = obj; - return 0; - } - if (obj) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; -- cgit v0.12 From be59d1489bdf150e05e67a89cc772628af7e8fd6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Jan 2016 00:39:12 +0100 Subject: Issue #26146: enhance ast.Constant error message Mention the name of the invalid type in error message of AST validation for constants. Suggestion made by Joseph Jevnik on a review. --- Lib/test/test_ast.py | 6 ++++++ Python/ast.c | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 6d6c9bd..57060d8 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -951,6 +951,12 @@ class ConstantTests(unittest.TestCase): exec(code, ns) return ns['x'] + def test_validation(self): + with self.assertRaises(TypeError) as cm: + self.compile_constant([1, 2, 3]) + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") + def test_singletons(self): for const in (None, False, True, Ellipsis, b'', frozenset()): with self.subTest(const=const): diff --git a/Python/ast.c b/Python/ast.c index ecfc14c..d19546a 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -288,7 +288,9 @@ validate_expr(expr_ty exp, expr_context_ty ctx) validate_keywords(exp->v.Call.keywords); case Constant_kind: if (!validate_constant(exp->v.Constant.value)) { - PyErr_SetString(PyExc_TypeError, "invalid type in Constant"); + PyErr_Format(PyExc_TypeError, + "got an invalid type in Constant: %s", + Py_TYPE(exp->v.Constant.value)->tp_name); return 0; } return 1; -- cgit v0.12 From 5586ba7643e608827fc7109a40e851f3cae0f2e4 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 30 Jan 2016 12:34:12 +0200 Subject: Simply docstrings of venv module This will hopefully make maintenance of venv documentation easier. For example, see commits a4f0d76af176 and 5764cc02244d. This patch has been reviewed by Vinaj Sajip, the maintainer of venv module. --- Lib/venv/__init__.py | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 74245ab..fa3d2a3 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -3,28 +3,6 @@ Virtual environment (venv) package for Python. Based on PEP 405. Copyright (C) 2011-2014 Vinay Sajip. Licensed to the PSF under a contributor agreement. - -usage: python -m venv [-h] [--system-site-packages] [--symlinks] [--clear] - [--upgrade] - ENV_DIR [ENV_DIR ...] - -Creates virtual Python environments in one or more target directories. - -positional arguments: - ENV_DIR A directory to create the environment in. - -optional arguments: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Attempt to symlink rather than copy. - --clear Delete the contents of the environment directory if it - already exists, before environment creation. - --upgrade Upgrade the environment directory to use this version - of Python, assuming Python has been upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) """ import logging import os @@ -349,23 +327,7 @@ class EnvBuilder: def create(env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False): - """ - Create a virtual environment in a directory. - - By default, makes the system (global) site-packages dir *un*available to - the created environment, and uses copying rather than symlinking for files - obtained from the source Python installation. - - :param env_dir: The target directory to create an environment in. - :param system_site_packages: If True, the system (global) site-packages - dir is available to the environment. - :param clear: If True, delete the contents of the environment directory if - it already exists, before environment creation. - :param symlinks: If True, attempt to symlink rather than copy files into - virtual environment. - :param with_pip: If True, ensure pip is installed in the virtual - environment - """ + """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip) builder.create(env_dir) -- cgit v0.12 From ce5179fcbab40df1bf0a74a9ebf96cc7c1e061b3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 31 Jan 2016 08:56:21 -0800 Subject: Issue #23601: Use small object allocator for dict key objects --- Misc/NEWS | 3 +++ Objects/dictobject.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index aac1bad..a182887 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,9 @@ Core and Builtins by external AST optimizers, but the compiler does not emit directly such node. +- Issue #23601: Sped-up allocation of dict key objects by using Python's + small object allocator. (Contributed by Julian Taylor.) + - Issue #18018: Import raises ImportError instead of SystemError if a relative import is attempted without a known parent package. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f6a98b4..31a6322 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -324,7 +324,7 @@ static PyDictKeysObject *new_keys_object(Py_ssize_t size) assert(size >= PyDict_MINSIZE_SPLIT); assert(IS_POWER_OF_2(size)); - dk = PyMem_MALLOC(sizeof(PyDictKeysObject) + + dk = PyObject_MALLOC(sizeof(PyDictKeysObject) + sizeof(PyDictKeyEntry) * (size-1)); if (dk == NULL) { PyErr_NoMemory(); @@ -353,7 +353,7 @@ free_keys_object(PyDictKeysObject *keys) Py_XDECREF(entries[i].me_key); Py_XDECREF(entries[i].me_value); } - PyMem_FREE(keys); + PyObject_FREE(keys); } #define new_values(size) PyMem_NEW(PyObject *, size) @@ -964,7 +964,7 @@ dictresize(PyDictObject *mp, Py_ssize_t minused) } } assert(oldkeys->dk_refcnt == 1); - DK_DEBUG_DECREF PyMem_FREE(oldkeys); + DK_DEBUG_DECREF PyObject_FREE(oldkeys); } return 0; } -- cgit v0.12 From a63897164eb1fb01bdc51fb8de0d6bc7eb791de9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 1 Feb 2016 21:21:19 -0800 Subject: merge --- Doc/library/collections.rst | 4 ++-- Lib/test/test_deque.py | 23 ++++++++++++----------- Modules/_collectionsmodule.c | 7 ++----- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 6a0c127..080f968 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -477,8 +477,8 @@ or subtracting from an empty counter. Insert *x* into the deque at position *i*. - If the insertion causes a bounded deque to grow beyond *maxlen*, the - rightmost element is then removed to restore the size limit. + If the insertion would cause a bounded deque to grow beyond *maxlen*, + an :exc:`IndexError` is raised. .. versionadded:: 3.5 diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 34b1be2..e2c6b6c 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -304,19 +304,20 @@ class TestBasic(unittest.TestCase): s.insert(i, 'Z') self.assertEqual(list(d), s) - def test_index_bug_26194(self): + def test_insert_bug_26194(self): data = 'ABC' - for i in range(len(data) + 1): - d = deque(data, len(data)) - d.insert(i, None) - s = list(data) - s.insert(i, None) - s.pop() - self.assertEqual(list(d), s) - if i < len(data): - self.assertIsNone(d[i]) + d = deque(data, maxlen=len(data)) + with self.assertRaises(IndexError): + d.insert(2, None) + + elements = 'ABCDEFGHI' + for i in range(-len(elements), len(elements)): + d = deque(elements, maxlen=len(elements)+1) + d.insert(i, 'Z') + if i >= 0: + self.assertEqual(d[i], 'Z') else: - self.assertTrue(None not in d) + self.assertEqual(d[i-1], 'Z') def test_imul(self): for n in (-10, -1, 0, 1, 2, 10, 1000): diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 69acc64..479b052 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1085,16 +1085,13 @@ deque_insert(dequeobject *deque, PyObject *args) Py_ssize_t index; Py_ssize_t n = Py_SIZE(deque); PyObject *value; - PyObject *oldvalue; PyObject *rv; if (!PyArg_ParseTuple(args, "nO:insert", &index, &value)) return NULL; if (deque->maxlen == Py_SIZE(deque)) { - if (index >= deque->maxlen || Py_SIZE(deque) == 0) - Py_RETURN_NONE; - oldvalue = deque_pop(deque, NULL); - Py_DECREF(oldvalue); + PyErr_SetString(PyExc_IndexError, "deque already at its maximum size"); + return NULL; } if (index >= n) return deque_append(deque, value); -- cgit v0.12 From c9deece272abe87865d91f3077fe4591e57303ad Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 3 Feb 2016 05:19:44 +0000 Subject: Issue #24421: Compile _math.c separately to avoid race condition --- Makefile.pre.in | 6 +++++- Misc/NEWS | 4 ++++ setup.py | 12 ++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 57167e6..2a687e5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -586,11 +586,15 @@ pybuilddir.txt: $(BUILDPYTHON) exit 1 ; \ fi +# This is shared by the math and cmath modules +Modules/_math.o: Modules/_math.c Modules/_math.h + $(CC) -c $(CCSHARED) $(PY_CORE_CFLAGS) -o $@ $< + # Build the shared modules # Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for # -s, --silent or --quiet is always the first char. # Under BSD make, MAKEFLAGS might be " -s -v x=y". -sharedmods: $(BUILDPYTHON) pybuilddir.txt +sharedmods: $(BUILDPYTHON) pybuilddir.txt Modules/_math.o @case "$$MAKEFLAGS" in \ *\ -s*|s*) quiet="-q";; \ *) quiet="";; \ diff --git a/Misc/NEWS b/Misc/NEWS index a182887..4532488 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -671,6 +671,10 @@ Build - Issue #24986: It is now possible to build Python on Windows without errors when external libraries are not available. +- Issue #24421: Compile Modules/_math.c once, before building extensions. + Previously it could fail to compile properly if the math and cmath builds + were concurrent. + - Issue #25798: Update OS X 10.5 installer to use OpenSSL 1.0.2e. Windows diff --git a/setup.py b/setup.py index da67731..930b64a 100644 --- a/setup.py +++ b/setup.py @@ -582,13 +582,17 @@ class PyBuildExt(build_ext): # array objects exts.append( Extension('array', ['arraymodule.c']) ) + + shared_math = 'Modules/_math.o' # complex math library functions - exts.append( Extension('cmath', ['cmathmodule.c', '_math.c'], - depends=['_math.h'], + exts.append( Extension('cmath', ['cmathmodule.c'], + extra_objects=[shared_math], + depends=['_math.h', shared_math], libraries=math_libs) ) # math library functions, e.g. sin() - exts.append( Extension('math', ['mathmodule.c', '_math.c'], - depends=['_math.h'], + exts.append( Extension('math', ['mathmodule.c'], + extra_objects=[shared_math], + depends=['_math.h', shared_math], libraries=math_libs) ) # time libraries: librt may be needed for clock_gettime() -- cgit v0.12 From f50215412cdc41b7845a6c0bb040a8f3521a0b84 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 4 Feb 2016 02:46:16 -0800 Subject: Add early-out for the common case where kwds is NULL (gives 1.1% speedup). --- Objects/setobject.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 8cb3f36..c9834a8 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1088,7 +1088,8 @@ frozenset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyObject *iterable = NULL, *result; - if (type == &PyFrozenSet_Type && !_PyArg_NoKeywords("frozenset()", kwds)) + if (kwds != NULL && type == &PyFrozenSet_Type + && !_PyArg_NoKeywords("frozenset()", kwds)) return NULL; if (!PyArg_UnpackTuple(args, type->tp_name, 0, 1, &iterable)) @@ -1130,7 +1131,7 @@ PySet_Fini(void) static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (type == &PySet_Type && !_PyArg_NoKeywords("set()", kwds)) + if (kwds != NULL && type == &PySet_Type && !_PyArg_NoKeywords("set()", kwds)) return NULL; return make_new_set(type, NULL); -- cgit v0.12 From 135d5f49f6da605bb1073f939e826e352a2d655f Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 5 Feb 2016 18:23:08 -0500 Subject: Fix issue 26287: While handling FORMAT_VALUE opcode, the top of stack was being corrupted if an error occurred in PyObject_Format(). --- Lib/test/test_fstring.py | 11 +++++++++++ Python/ceval.c | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index d6f781c..a82dedf 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -692,6 +692,17 @@ f'{a * x()}'""" r"f'{a(4]}'", ]) + def test_errors(self): + # see issue 26287 + self.assertAllRaise(TypeError, 'non-empty', + [r"f'{(lambda: 0):x}'", + r"f'{(0,):x}'", + ]) + self.assertAllRaise(ValueError, 'Unknown format code', + [r"f'{1000:j}'", + r"f'{1000:j}'", + ]) + def test_loop(self): for i in range(1000): self.assertEqual(f'i:{i}', 'i:' + str(i)) diff --git a/Python/ceval.c b/Python/ceval.c index 743a969..b815ccd 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3383,7 +3383,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; fmt_spec = have_fmt_spec ? POP() : NULL; - value = TOP(); + value = POP(); /* See if any conversion is specified. */ switch (which_conversion) { @@ -3426,7 +3426,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) goto error; } - SET_TOP(result); + PUSH(result); DISPATCH(); } -- cgit v0.12 From eb588a1d10e94085c622eb732cfe535e0ebaec8a Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 5 Feb 2016 18:26:20 -0500 Subject: Switch to more idiomatic C code. --- Python/ceval.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b815ccd..8904d7a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3399,10 +3399,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* If there's a conversion function, call it and replace value with that result. Otherwise, just use value, without conversion. */ - if (conv_fn) { + if (conv_fn != NULL) { result = conv_fn(value); Py_DECREF(value); - if (!result) { + if (result == NULL) { Py_XDECREF(fmt_spec); goto error; } @@ -3422,8 +3422,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) result = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_XDECREF(fmt_spec); - if (!result) + if (result == NULL) { goto error; + } } PUSH(result); -- cgit v0.12 From 186c30b7ae2c304b863143809867cfef7eb1bf29 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 5 Feb 2016 19:40:01 -0500 Subject: Issue #26288: Optimize PyLong_AsDouble. --- Misc/NEWS | 1 + Objects/longobject.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index dcccc4a..510748b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -165,6 +165,7 @@ Core and Builtins - Issue #25660: Fix TAB key behaviour in REPL with readline. +- Issue #26288: Optimize PyLong_AsDouble. Library ------- diff --git a/Objects/longobject.c b/Objects/longobject.c index d05de8b..c1edeac 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2769,6 +2769,13 @@ PyLong_AsDouble(PyObject *v) PyErr_SetString(PyExc_TypeError, "an integer is required"); return -1.0; } + if (Py_ABS(Py_SIZE(v)) <= 1) { + /* Fast path; single digit will always fit decimal. + This improves performance of FP/long operations by at + least 20%. This is even visible on macro-benchmarks. + */ + return (double)MEDIUM_VALUE((PyLongObject *)v); + } x = _PyLong_Frexp((PyLongObject *)v, &exponent); if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, -- cgit v0.12 From a0fcaca4e1c29078f578fb96971398c70cf38900 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 6 Feb 2016 12:21:33 -0500 Subject: Issue #26288: Fix comment --- Objects/longobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index c1edeac..f3afb10 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2770,9 +2770,9 @@ PyLong_AsDouble(PyObject *v) return -1.0; } if (Py_ABS(Py_SIZE(v)) <= 1) { - /* Fast path; single digit will always fit decimal. - This improves performance of FP/long operations by at - least 20%. This is even visible on macro-benchmarks. + /* Fast path; single digit long (31 bits) will cast safely + to double. This improves performance of FP/long operations + by 20%. */ return (double)MEDIUM_VALUE((PyLongObject *)v); } -- cgit v0.12 From 503f908090bd22df70ed21e3bb3c2513a1eee969 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 00:02:25 +0200 Subject: Issue #26039: Added zipfile.ZipInfo.from_file() and zipinfo.ZipInfo.is_dir(). Patch by Thomas Kluyver. --- Doc/library/zipfile.rst | 24 ++++++++++++++++ Doc/whatsnew/3.6.rst | 10 +++++++ Lib/test/test_zipfile.py | 15 ++++++++++ Lib/zipfile.py | 75 ++++++++++++++++++++++++++++++------------------ Misc/NEWS | 3 ++ 5 files changed, 99 insertions(+), 28 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index e3c2217..54b8cc5 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -465,6 +465,22 @@ Instances of the :class:`ZipInfo` class are returned by the :meth:`.getinfo` and :meth:`.infolist` methods of :class:`ZipFile` objects. Each object stores information about a single member of the ZIP archive. +There is one classmethod to make a :class:`ZipInfo` instance for a filesystem +file: + +.. classmethod:: ZipInfo.from_file(filename, arcname=None) + + Construct a :class:`ZipInfo` instance for a file on the filesystem, in + preparation for adding it to a zip file. + + *filename* should be the path to a file or directory on the filesystem. + + If *arcname* is specified, it is used as the name within the archive. + If *arcname* is not specified, the name will be the same as *filename*, but + with any drive letter and leading path separators removed. + + .. versionadded:: 3.6 + Instances have the following attributes: @@ -574,3 +590,11 @@ Instances have the following attributes: .. attribute:: ZipInfo.file_size Size of the uncompressed file. + +There is one method: + +.. method:: ZipInfo.is_dir() + + Return ``True`` if the ZipInfo represents a directory. + + .. versionadded:: 3.6 diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 158dad3..b7cc159 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -140,6 +140,16 @@ urllib.robotparser (Contributed by Nikolay Bogoychev in :issue:`16099`.) +zipfile +------- + +A new :meth:`ZipInfo.from_file() ` class method +allow to make :class:`~zipfile.ZipInfo` instance from a filesystem file. +A new :meth:`ZipInfo.is_dir() ` method can be used +to check if the :class:`~zipfile.ZipInfo` instance represents a directory. +(Contributed by Thomas Kluyver in :issue:`26039`.) + + Optimizations ============= diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 2c10821..8589342 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -3,6 +3,7 @@ import io import os import sys import importlib.util +import posixpath import time import struct import zipfile @@ -2071,5 +2072,19 @@ class LzmaUniversalNewlineTests(AbstractUniversalNewlineTests, unittest.TestCase): compression = zipfile.ZIP_LZMA +class ZipInfoTests(unittest.TestCase): + def test_from_file(self): + zi = zipfile.ZipInfo.from_file(__file__) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + + def test_from_dir(self): + dirpath = os.path.dirname(os.path.abspath(__file__)) + zi = zipfile.ZipInfo.from_file(dirpath, 'stdlib_tests') + self.assertEqual(zi.filename, 'stdlib_tests/') + self.assertTrue(zi.is_dir()) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + self.assertEqual(zi.file_size, 0) + if __name__ == "__main__": unittest.main() diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 56a2479..e0598d2 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -371,7 +371,7 @@ class ZipInfo (object): result.append(' filemode=%r' % stat.filemode(hi)) if lo: result.append(' external_attr=%#x' % lo) - isdir = self.filename[-1:] == '/' + isdir = self.is_dir() if not isdir or self.file_size: result.append(' file_size=%r' % self.file_size) if ((not isdir or self.compress_size) and @@ -469,6 +469,41 @@ class ZipInfo (object): extra = extra[ln+4:] + @classmethod + def from_file(cls, filename, arcname=None): + """Construct an appropriate ZipInfo for a file on the filesystem. + + filename should be the path to a file or directory on the filesystem. + + arcname is the name which it will have within the archive (by default, + this will be the same as filename, but without a drive letter and with + leading path separators removed). + """ + st = os.stat(filename) + isdir = stat.S_ISDIR(st.st_mode) + mtime = time.localtime(st.st_mtime) + date_time = mtime[0:6] + # Create ZipInfo instance to store file information + if arcname is None: + arcname = filename + arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) + while arcname[0] in (os.sep, os.altsep): + arcname = arcname[1:] + if isdir: + arcname += '/' + zinfo = cls(arcname, date_time) + zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes + if isdir: + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + zinfo.file_size = st.st_size + + return zinfo + + def is_dir(self): + return self.filename[-1] == '/' + class _ZipDecrypter: """Class to handle decryption of files stored within a ZIP archive. @@ -1389,7 +1424,7 @@ class ZipFile: if upperdirs and not os.path.exists(upperdirs): os.makedirs(upperdirs) - if member.filename[-1] == '/': + if member.is_dir(): if not os.path.isdir(targetpath): os.mkdir(targetpath) return targetpath @@ -1430,29 +1465,17 @@ class ZipFile: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") - st = os.stat(filename) - isdir = stat.S_ISDIR(st.st_mode) - mtime = time.localtime(st.st_mtime) - date_time = mtime[0:6] - # Create ZipInfo instance to store file information - if arcname is None: - arcname = filename - arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) - while arcname[0] in (os.sep, os.altsep): - arcname = arcname[1:] - if isdir: - arcname += '/' - zinfo = ZipInfo(arcname, date_time) - zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes - if isdir: - zinfo.compress_type = ZIP_STORED - elif compress_type is None: - zinfo.compress_type = self.compression + zinfo = ZipInfo.from_file(filename, arcname) + + if zinfo.is_dir(): + zinfo.compress_size = 0 + zinfo.CRC = 0 else: - zinfo.compress_type = compress_type + if compress_type is not None: + zinfo.compress_type = compress_type + else: + zinfo.compress_type = self.compression - zinfo.file_size = st.st_size - zinfo.flag_bits = 0x00 with self._lock: if self._seekable: self.fp.seek(self.start_dir) @@ -1464,11 +1487,7 @@ class ZipFile: self._writecheck(zinfo) self._didModify = True - if isdir: - zinfo.file_size = 0 - zinfo.compress_size = 0 - zinfo.CRC = 0 - zinfo.external_attr |= 0x10 # MS-DOS directory flag + if zinfo.is_dir(): self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) diff --git a/Misc/NEWS b/Misc/NEWS index 510748b..a7c3f58 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,9 @@ Core and Builtins Library ------- +- Issue #26039: Added zipfile.ZipInfo.from_file() and zipinfo.ZipInfo.is_dir(). + Patch by Thomas Kluyver. + - Issue #12923: Reset FancyURLopener's redirect counter even if there is an exception. Based on patches by Brian Brazil and Daniel Rocco. -- cgit v0.12 From 4cd63ef67a3e0974f0c48c550769babd401075e3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 01:22:47 +0200 Subject: Issue #26198: ValueError is now raised instead of TypeError on buffer overflow in parsing "es#" and "et#" format units. SystemError is now raised instead of TypeError on programmical error in parsing format string. --- Lib/test/test_capi.py | 4 ++-- Lib/test/test_getargs2.py | 8 ++++---- Misc/NEWS | 7 +++++++ Python/getargs.c | 9 +++++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 96666d1..1a743fd 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -488,10 +488,10 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, format.encode("ascii"), keywords) when_not_skipped = False - except TypeError as e: + except SystemError as e: s = "argument 1 (impossible)" when_not_skipped = (str(e) == s) - except RuntimeError as e: + except (TypeError, RuntimeError): when_not_skipped = False # test the format unit when skipped diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 6491135..be777af 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -636,10 +636,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) def test_et_hash(self): from _testcapi import getargs_et_hash @@ -662,10 +662,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) def test_u(self): from _testcapi import getargs_u diff --git a/Misc/NEWS b/Misc/NEWS index a7c3f58..7a198d6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -713,6 +713,13 @@ Tools/Demos - Issue #25154: The pyvenv script has been deprecated in favour of `python3 -m venv`. +C API +----- + +- Issue #26198: ValueError is now raised instead of TypeError on buffer + overflow in parsing "es#" and "et#" format units. SystemError is now raised + instead of TypeError on programmical error in parsing format string. + What's New in Python 3.5.1 final? ================================= diff --git a/Python/getargs.c b/Python/getargs.c index 60b7677..be6e375 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -394,7 +394,12 @@ seterror(Py_ssize_t iarg, const char *msg, int *levels, const char *fname, PyOS_snprintf(p, sizeof(buf) - (p - buf), " %.256s", msg); message = buf; } - PyErr_SetString(PyExc_TypeError, message); + if (msg[0] == '(') { + PyErr_SetString(PyExc_SystemError, message); + } + else { + PyErr_SetString(PyExc_TypeError, message); + } } @@ -1129,7 +1134,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { if (size + 1 > BUFFER_LEN) { Py_DECREF(s); - PyErr_Format(PyExc_TypeError, + PyErr_Format(PyExc_ValueError, "encoded string too long " "(%zd, maximum length %zd)", (Py_ssize_t)size, (Py_ssize_t)(BUFFER_LEN-1)); -- cgit v0.12 From d2962f145a4ad73ac5dad8ff9b3b26905d44d682 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 16:39:05 +0200 Subject: Issue #25949: __dict__ for an OrderedDict instance is now created only when needed. --- Lib/test/test_ordered_dict.py | 4 +++- Misc/NEWS | 3 +++ Objects/odictobject.c | 23 +++++++---------------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 633e909..e1564ba 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -298,9 +298,11 @@ class OrderedDictTests: # do not save instance dictionary if not needed pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) + self.assertIsInstance(od.__dict__, dict) self.assertIsNone(od.__reduce__()[2]) od.x = 10 - self.assertIsNotNone(od.__reduce__()[2]) + self.assertEqual(od.__dict__['x'], 10) + self.assertEqual(od.__reduce__()[2], {'x': 10}) def test_pickle_recursive(self): OrderedDict = self.OrderedDict diff --git a/Misc/NEWS b/Misc/NEWS index bb7c57d..e6590b5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,9 @@ Core and Builtins Library ------- +- Issue #25949: __dict__ for an OrderedDict instance is now created only when + needed. + - Issue #25911: Restored support of bytes paths in os.walk() on Windows. - Issue #26045: Add UTF-8 suggestion to error message when posting a diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 1abdd02..dccbb3e 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1424,14 +1424,13 @@ static PyMethodDef odict_methods[] = { * OrderedDict members */ -/* tp_members */ +/* tp_getset */ -static PyMemberDef odict_members[] = { - {"__dict__", T_OBJECT, offsetof(PyODictObject, od_inst_dict), READONLY}, - {0} +static PyGetSetDef odict_getset[] = { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, + {NULL} }; - /* ---------------------------------------------- * OrderedDict type slot methods */ @@ -1653,20 +1652,12 @@ odict_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * odict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *dict; PyODictObject *od; - dict = PyDict_New(); - if (dict == NULL) - return NULL; - od = (PyODictObject *)PyDict_Type.tp_new(type, args, kwds); - if (od == NULL) { - Py_DECREF(dict); + if (od == NULL) return NULL; - } - od->od_inst_dict = dict; /* type constructor fills the memory with zeros (see PyType_GenericAlloc()), there is no need to set them to zero again */ if (_odict_resize(od) < 0) { @@ -1708,8 +1699,8 @@ PyTypeObject PyODict_Type = { (getiterfunc)odict_iter, /* tp_iter */ 0, /* tp_iternext */ odict_methods, /* tp_methods */ - odict_members, /* tp_members */ - 0, /* tp_getset */ + 0, /* tp_members */ + odict_getset, /* tp_getset */ &PyDict_Type, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ -- cgit v0.12 From f089196beb0ad8fb262be42eddd92af3f0fe81c1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 17:15:21 +0100 Subject: Simplify main() of test_ast * Use ast.parse() to get the AST for a statement * Use str%args syntax for format a line Issue #26204. --- Lib/test/test_ast.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 57060d8..f5c4ade 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1064,8 +1064,9 @@ def main(): for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), (eval_tests, "eval")): print(kind+"_results = [") - for s in statements: - print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) print("]") print("main()") raise SystemExit -- cgit v0.12 From 51d8c526d5634fa1f0e4976fd357c5423a792082 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 17:57:02 +0100 Subject: Replace noop constant statement with expression * Constant statements will be ignored and the compiler will emit a SyntaxWarning. * Replace constant statement (ex: "1") with an expression statement (ex: "x=1"). * test_traceback: use context manager on the file. Issue #26204. --- Lib/test/test_inspect.py | 2 +- Lib/test/test_peepholer.py | 20 +++++++++++--------- Lib/test/test_support.py | 3 ++- Lib/test/test_sys_settrace.py | 4 ++-- Lib/test/test_traceback.py | 14 +++++++------- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 422a3d6..98d596e 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -401,7 +401,7 @@ class TestRetrievingSourceCode(GetSourceBase): self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) self.assertEqual(normcase(inspect.getsourcefile(git.abuse)), modfile) fn = "_non_existing_filename_used_for_sourcefile_test.py" - co = compile("None", fn, "exec") + co = compile("x=1", fn, "exec") self.assertEqual(inspect.getsourcefile(co), None) linecache.cache[co.co_filename] = (1, None, "None", co.co_filename) try: diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 41e5091..b033640 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,9 +1,8 @@ import dis import re import sys -from io import StringIO +import textwrap import unittest -from math import copysign from test.bytecode_helper import BytecodeTestCase @@ -30,22 +29,25 @@ class TestTranforms(BytecodeTestCase): def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False - def f(x): - None - None + def f(): + x = None + x = None return x - def g(x): - True + def g(): + x = True return x - def h(x): - False + def h(): + x = False return x + for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + def f(): 'Adding a docstring made this test fail in Py2.5.0' return None + self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index f86ea91..269d9bf 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -230,7 +230,8 @@ class TestSupport(unittest.TestCase): def test_check_syntax_error(self): support.check_syntax_error(self, "def class") - self.assertRaises(AssertionError, support.check_syntax_error, self, "1") + with self.assertRaises(AssertionError): + support.check_syntax_error(self, "x=1") def test_CleanImport(self): import importlib diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index ae8f845..bb83623 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -338,8 +338,8 @@ class TraceTestCase(unittest.TestCase): def test_14_onliner_if(self): def onliners(): - if True: False - else: True + if True: x=False + else: x=True return 0 self.run_and_compare( onliners, diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b7695d6..2f739ba 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -129,12 +129,12 @@ class SyntaxTracebackCases(unittest.TestCase): def do_test(firstlines, message, charset, lineno): # Raise the message in a subprocess, and catch the output try: - output = open(TESTFN, "w", encoding=charset) - output.write("""{0}if 1: - import traceback; - raise RuntimeError('{1}') - """.format(firstlines, message)) - output.close() + with open(TESTFN, "w", encoding=charset) as output: + output.write("""{0}if 1: + import traceback; + raise RuntimeError('{1}') + """.format(firstlines, message)) + process = subprocess.Popen([sys.executable, TESTFN], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = process.communicate() @@ -176,7 +176,7 @@ class SyntaxTracebackCases(unittest.TestCase): do_test(" \t\f\n# coding: {0}\n".format(charset), text, charset, 5) # Issue #18960: coding spec should has no effect - do_test("0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) + do_test("x=0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) def test_print_traceback_at_exit(self): # Issue #22599: Ensure that it is possible to use the traceback module -- cgit v0.12 From a2724095cd55d75497d9cd48d35911599f4dedda Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 18:17:58 +0100 Subject: compiler now ignores constant statements The compile ignores constant statements and emit a SyntaxWarning warning. Don't emit the warning for string statement because triple quoted string is a common syntax for multiline comments. Don't emit the warning on ellipis neither: 'def f(): ...' is a legit syntax for abstract functions. Changes: * test_ast: ignore SyntaxWarning when compiling test statements. Modify test_load_const() to use assignment expressions rather than constant expression. * test_code: add more kinds of constant statements, ignore SyntaxWarning when testing that the compiler removes constant statements. * test_grammar: ignore SyntaxWarning on the statement "1" --- Lib/test/test_ast.py | 33 +++++++++++++-------------- Lib/test/test_code.py | 58 +++++++++++++++++++++++++++++++++--------------- Lib/test/test_grammar.py | 6 ++++- Misc/NEWS | 4 ++++ Python/compile.c | 41 +++++++++++++++++++++++++--------- 5 files changed, 95 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index f5c4ade..dbcd9f7 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3,6 +3,7 @@ import dis import os import sys import unittest +import warnings import weakref from test import support @@ -239,8 +240,10 @@ class AST_Tests(unittest.TestCase): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) - with self.subTest(action="compiling", input=i): - compile(ast_tree, "?", kind) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=SyntaxWarning) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice @@ -1020,27 +1023,23 @@ class ConstantTests(unittest.TestCase): b'bytes', (1, 2, 3)] - code = '\n'.join(map(repr, consts)) - code += '\n...' - - code_consts = [const for const in consts - if (not isinstance(const, (str, int, float, complex)) - or isinstance(const, bool))] - code_consts.append(Ellipsis) - # the compiler adds a final "LOAD_CONST None" - code_consts.append(None) + code = '\n'.join(['x={!r}'.format(const) for const in consts]) + code += '\nx = ...' + consts.extend((Ellipsis, None)) tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), code_consts) + self.assertEqual(self.get_load_const(tree), + consts) # Replace expression nodes with constants - for expr_node, const in zip(tree.body, consts): - assert isinstance(expr_node, ast.Expr) + for assign, const in zip(tree.body, consts): + assert isinstance(assign, ast.Assign), ast.dump(assign) new_node = ast.Constant(value=const) - ast.copy_location(new_node, expr_node.value) - expr_node.value = new_node + ast.copy_location(new_node, assign.value) + assign.value = new_node - self.assertEqual(self.get_load_const(tree), code_consts) + self.assertEqual(self.get_load_const(tree), + consts) def test_literal_eval(self): tree = ast.parse("1 + 2") diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 21b12a5..2cf088c 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -66,24 +66,6 @@ nlocals: 1 flags: 67 consts: ('None',) ->>> def optimize_away(): -... 'doc string' -... 'not a docstring' -... 53 -... 0x53 - ->>> dump(optimize_away.__code__) -name: optimize_away -argcount: 0 -kwonlyargcount: 0 -names: () -varnames: () -cellvars: () -freevars: () -nlocals: 0 -flags: 67 -consts: ("'doc string'", 'None') - >>> def keywordonly_args(a,b,*,k1): ... return a,b,k1 ... @@ -102,8 +84,10 @@ consts: ('None',) """ +import textwrap import unittest import weakref +import warnings from test.support import run_doctest, run_unittest, cpython_only @@ -134,6 +118,44 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) + def dump(self, co): + dump = {} + for attr in ["name", "argcount", "kwonlyargcount", "names", "varnames", + "cellvars", "freevars", "nlocals", "flags"]: + dump[attr] = getattr(co, "co_" + attr) + dump['consts'] = tuple(consts(co.co_consts)) + return dump + + def test_optimize_away(self): + ns = {} + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=SyntaxWarning) + exec(textwrap.dedent(''' + def optimize_away(): + 'doc string' + 'not a docstring' + 53 + 0x53 + b'bytes' + 1.0 + True + False + None + ... + '''), ns) + + self.assertEqual(self.dump(ns['optimize_away'].__code__), + {'name': 'optimize_away', + 'argcount': 0, + 'kwonlyargcount': 0, + 'names': (), + 'varnames': (), + 'cellvars': (), + 'freevars': (), + 'nlocals': 0, + 'flags': 67, + 'consts': ("'doc string'", 'None')}) + class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8f8d71c..2d6f5ed 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -7,6 +7,7 @@ import unittest import sys # testing import * from sys import * +from test import support class TokenTests(unittest.TestCase): @@ -424,8 +425,11 @@ class GrammarTests(unittest.TestCase): # Tested below def test_expr_stmt(self): + msg = 'ignore constant statement' + with support.check_warnings((msg, SyntaxWarning)): + exec("1") + # (exprlist '=')* exprlist - 1 1, 2, 3 x = 1 x = 1, 2, 3 diff --git a/Misc/NEWS b/Misc/NEWS index 0c7d18e..e1875c0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26204: The compiler now ignores constant statements (ex: "def f(): 1") + and emit a SyntaxWarning warning. The warning is not emitted for string and + ellipsis (...) statements. + - Issue #4806: Avoid masking the original TypeError exception when using star (*) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. diff --git a/Python/compile.c b/Python/compile.c index ccb05cf..84b79a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2616,20 +2616,39 @@ compiler_visit_stmt_expr(struct compiler *c, expr_ty value) return 1; } - if (value->kind == Str_kind || value->kind == Num_kind) { - /* ignore strings and numbers */ + switch (value->kind) + { + case Str_kind: + case Ellipsis_kind: + /* Issue #26204: ignore string statement, but don't emit a + * SyntaxWarning. Triple quoted strings is a common syntax for + * multiline comments. + * + * Don't emit warning on "def f(): ..." neither. It's a legit syntax + * for abstract function. */ return 1; - } - if (value->kind == Constant_kind) { - PyObject *cst = value->v.Constant.value; - if (PyUnicode_CheckExact(cst) - || PyLong_CheckExact(cst) - || PyFloat_CheckExact(cst) - || PyComplex_CheckExact(cst)) { - /* ignore strings and numbers */ - return 1; + case Bytes_kind: + case Num_kind: + case NameConstant_kind: + case Constant_kind: + { + PyObject *msg = PyUnicode_FromString("ignore constant statement"); + if (msg == NULL) + return 0; + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, + msg, + c->c_filename, c->u->u_lineno, + NULL, NULL) == -1) { + Py_DECREF(msg); + return 0; } + Py_DECREF(msg); + return 1; + } + + default: + break; } VISIT(c, expr, value); -- cgit v0.12 From 15a3095d64e96d0fe7448270f2c5b0bf22f9c4e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 22:45:06 +0100 Subject: compiler: don't emit SyntaxWarning on const stmt Issue #26204: the compiler doesn't emit SyntaxWarning warnings anymore when constant statements are ignored. --- Lib/test/test_ast.py | 7 ++---- Lib/test/test_code.py | 58 +++++++++++++++--------------------------------- Lib/test/test_grammar.py | 6 +---- Misc/NEWS | 7 +++--- Python/compile.c | 24 ++------------------ 5 files changed, 27 insertions(+), 75 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index dbcd9f7..a025c20 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3,7 +3,6 @@ import dis import os import sys import unittest -import warnings import weakref from test import support @@ -240,10 +239,8 @@ class AST_Tests(unittest.TestCase): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=SyntaxWarning) - with self.subTest(action="compiling", input=i, kind=kind): - compile(ast_tree, "?", kind) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 2cf088c..21b12a5 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -66,6 +66,24 @@ nlocals: 1 flags: 67 consts: ('None',) +>>> def optimize_away(): +... 'doc string' +... 'not a docstring' +... 53 +... 0x53 + +>>> dump(optimize_away.__code__) +name: optimize_away +argcount: 0 +kwonlyargcount: 0 +names: () +varnames: () +cellvars: () +freevars: () +nlocals: 0 +flags: 67 +consts: ("'doc string'", 'None') + >>> def keywordonly_args(a,b,*,k1): ... return a,b,k1 ... @@ -84,10 +102,8 @@ consts: ('None',) """ -import textwrap import unittest import weakref -import warnings from test.support import run_doctest, run_unittest, cpython_only @@ -118,44 +134,6 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) - def dump(self, co): - dump = {} - for attr in ["name", "argcount", "kwonlyargcount", "names", "varnames", - "cellvars", "freevars", "nlocals", "flags"]: - dump[attr] = getattr(co, "co_" + attr) - dump['consts'] = tuple(consts(co.co_consts)) - return dump - - def test_optimize_away(self): - ns = {} - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=SyntaxWarning) - exec(textwrap.dedent(''' - def optimize_away(): - 'doc string' - 'not a docstring' - 53 - 0x53 - b'bytes' - 1.0 - True - False - None - ... - '''), ns) - - self.assertEqual(self.dump(ns['optimize_away'].__code__), - {'name': 'optimize_away', - 'argcount': 0, - 'kwonlyargcount': 0, - 'names': (), - 'varnames': (), - 'cellvars': (), - 'freevars': (), - 'nlocals': 0, - 'flags': 67, - 'consts': ("'doc string'", 'None')}) - class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 2d6f5ed..8f8d71c 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -7,7 +7,6 @@ import unittest import sys # testing import * from sys import * -from test import support class TokenTests(unittest.TestCase): @@ -425,11 +424,8 @@ class GrammarTests(unittest.TestCase): # Tested below def test_expr_stmt(self): - msg = 'ignore constant statement' - with support.check_warnings((msg, SyntaxWarning)): - exec("1") - # (exprlist '=')* exprlist + 1 1, 2, 3 x = 1 x = 1, 2, 3 diff --git a/Misc/NEWS b/Misc/NEWS index e1875c0..d60aeb6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,9 +10,10 @@ Release date: tba Core and Builtins ----------------- -- Issue #26204: The compiler now ignores constant statements (ex: "def f(): 1") - and emit a SyntaxWarning warning. The warning is not emitted for string and - ellipsis (...) statements. +- Issue #26204: The compiler now ignores all constant statements: bytes, str, + int, float, complex, name constants (None, False, True), Ellipsis + and ast.Constant; not only str and int. For example, ``1.0`` is now ignored + in ``def f(): 1.0``. - Issue #4806: Avoid masking the original TypeError exception when using star (*) unpacking in function calls. Based on patch by Hagen Fürstenau and diff --git a/Python/compile.c b/Python/compile.c index 84b79a2..ca1d865 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2619,33 +2619,13 @@ compiler_visit_stmt_expr(struct compiler *c, expr_ty value) switch (value->kind) { case Str_kind: + case Num_kind: case Ellipsis_kind: - /* Issue #26204: ignore string statement, but don't emit a - * SyntaxWarning. Triple quoted strings is a common syntax for - * multiline comments. - * - * Don't emit warning on "def f(): ..." neither. It's a legit syntax - * for abstract function. */ - return 1; - case Bytes_kind: - case Num_kind: case NameConstant_kind: case Constant_kind: - { - PyObject *msg = PyUnicode_FromString("ignore constant statement"); - if (msg == NULL) - return 0; - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, - msg, - c->c_filename, c->u->u_lineno, - NULL, NULL) == -1) { - Py_DECREF(msg); - return 0; - } - Py_DECREF(msg); + /* ignore constant statement */ return 1; - } default: break; -- cgit v0.12 From 38418662e0339b6331e0c7e00ea55f41b28bf38d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 8 Feb 2016 20:34:49 -0800 Subject: Issue #26200: The SETREF macro adds unnecessary work in some cases. --- Modules/_collectionsmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 479b052..0b7a88f 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1218,6 +1218,7 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) static int deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) { + PyObject *old_value; block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; @@ -1246,7 +1247,9 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) b = b->leftlink; } Py_INCREF(v); - Py_SETREF(b->data[i], v); + old_value = b->data[i]; + b->data[i] = v; + Py_DECREF(old_value); return 0; } -- cgit v0.12 From 1fe0d13d1246b5d60a4f4fb8c627babd0e94ab87 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 10:06:36 +0000 Subject: Issue #26243: zlib.compress() keyword argument support by Aviv Palivoda --- Doc/library/zlib.rst | 11 ++++++++--- Doc/whatsnew/3.6.rst | 7 +++++++ Lib/test/test_zlib.py | 4 ++++ Misc/NEWS | 3 +++ Modules/clinic/zlibmodule.c.h | 27 ++++++++++++++------------- Modules/zlibmodule.c | 17 ++++++++--------- 6 files changed, 44 insertions(+), 25 deletions(-) diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 1869bb8..09026cb 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -46,14 +46,19 @@ The available exception and functions in this module are: platforms, use ``adler32(data) & 0xffffffff``. -.. function:: compress(data[, level]) +.. function:: compress(data, level=-1) Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` controlling the level of compression; + *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``6``. + produces the most. ``0`` is no compression. The default value is ``-1`` + (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. + .. versionchanged:: 3.6 + Keyword arguments are now supported. + .. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b7cc159..87199df 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -150,6 +150,13 @@ to check if the :class:`~zipfile.ZipInfo` instance represents a directory. (Contributed by Thomas Kluyver in :issue:`26039`.) +zlib +---- + +The :func:`~zlib.compress` function now accepts keyword arguments. +(Contributed by Aviv Palivoda in :issue:`26243`.) + + Optimizations ============= diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index ecdb5a7..ca30116 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -162,6 +162,10 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): x = zlib.compress(HAMLET_SCENE) self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + def test_keywords(self): + x = zlib.compress(data=HAMLET_SCENE, level=3) + self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 diff --git a/Misc/NEWS b/Misc/NEWS index d308762..6ce7e2d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -175,6 +175,9 @@ Core and Builtins Library ------- +- Issue #26243: Support keyword arguments to zlib.compress(). Patch by Aviv + Palivoda. + - Issue #26117: The os.scandir() iterator now closes file descriptor not only when the iteration is finished, but when it was failed with error. diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 2d75bc9..d92f88b 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -3,38 +3,39 @@ preserve [clinic start generated code]*/ PyDoc_STRVAR(zlib_compress__doc__, -"compress($module, bytes, level=Z_DEFAULT_COMPRESSION, /)\n" +"compress($module, /, data, level=Z_DEFAULT_COMPRESSION)\n" "--\n" "\n" "Returns a bytes object containing compressed data.\n" "\n" -" bytes\n" +" data\n" " Binary data to be compressed.\n" " level\n" " Compression level, in 0-9."); #define ZLIB_COMPRESS_METHODDEF \ - {"compress", (PyCFunction)zlib_compress, METH_VARARGS, zlib_compress__doc__}, + {"compress", (PyCFunction)zlib_compress, METH_VARARGS|METH_KEYWORDS, zlib_compress__doc__}, static PyObject * -zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level); +zlib_compress_impl(PyModuleDef *module, Py_buffer *data, int level); static PyObject * -zlib_compress(PyModuleDef *module, PyObject *args) +zlib_compress(PyModuleDef *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - Py_buffer bytes = {NULL, NULL}; + static char *_keywords[] = {"data", "level", NULL}; + Py_buffer data = {NULL, NULL}; int level = Z_DEFAULT_COMPRESSION; - if (!PyArg_ParseTuple(args, "y*|i:compress", - &bytes, &level)) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|i:compress", _keywords, + &data, &level)) goto exit; - return_value = zlib_compress_impl(module, &bytes, level); + return_value = zlib_compress_impl(module, &data, level); exit: - /* Cleanup for bytes */ - if (bytes.obj) - PyBuffer_Release(&bytes); + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); return return_value; } @@ -439,4 +440,4 @@ exit: #ifndef ZLIB_COMPRESS_COPY_METHODDEF #define ZLIB_COMPRESS_COPY_METHODDEF #endif /* !defined(ZLIB_COMPRESS_COPY_METHODDEF) */ -/*[clinic end generated code: output=cf81e1deae3af0ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3c96b58b923c1273 input=a9049054013a1b77]*/ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 5cdab45..8ddc774 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -137,18 +137,17 @@ PyZlib_Free(voidpf ctx, void *ptr) /*[clinic input] zlib.compress - bytes: Py_buffer + data: Py_buffer Binary data to be compressed. level: int(c_default="Z_DEFAULT_COMPRESSION") = Z_DEFAULT_COMPRESSION - Compression level, in 0-9. - / + Compression level, in 0-9 or -1. Returns a bytes object containing compressed data. [clinic start generated code]*/ static PyObject * -zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level) -/*[clinic end generated code: output=5d7dd4588788efd3 input=be3abe9934bda4b3]*/ +zlib_compress_impl(PyModuleDef *module, Py_buffer *data, int level) +/*[clinic end generated code: output=1b97589132b203b4 input=671c615a4b2267da]*/ { PyObject *ReturnVal = NULL; Byte *input, *output = NULL; @@ -156,13 +155,13 @@ zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level) int err; z_stream zst; - if ((size_t)bytes->len > UINT_MAX) { + if ((size_t)data->len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); goto error; } - input = bytes->buf; - length = (unsigned int)bytes->len; + input = data->buf; + length = (unsigned int)data->len; zst.avail_out = length + length/1000 + 12 + 1; @@ -1323,7 +1322,7 @@ PyDoc_STRVAR(zlib_module_documentation, "zlib library, which is based on GNU zip.\n" "\n" "adler32(string[, start]) -- Compute an Adler-32 checksum.\n" -"compress(string[, level]) -- Compress string, with compression level in 0-9.\n" +"compress(data[, level]) -- Compress data, with compression level 0-9 or -1.\n" "compressobj([level[, ...]]) -- Return a compressor object.\n" "crc32(string[, start]) -- Compute a CRC-32 checksum.\n" "decompress(string,[wbits],[bufsize]) -- Decompresses a compressed string.\n" -- cgit v0.12 From b0cb42dfdb422495fb0c3a4374f2fa010f56a5ac Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 10:45:54 +0000 Subject: Issue 26243: Forgot to update zlib doc strings in Argument Clinic --- Modules/clinic/zlibmodule.c.h | 4 ++-- Modules/zlibmodule.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index d92f88b..222cb67 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -11,7 +11,7 @@ PyDoc_STRVAR(zlib_compress__doc__, " data\n" " Binary data to be compressed.\n" " level\n" -" Compression level, in 0-9."); +" Compression level, in 0-9 or -1."); #define ZLIB_COMPRESS_METHODDEF \ {"compress", (PyCFunction)zlib_compress, METH_VARARGS|METH_KEYWORDS, zlib_compress__doc__}, @@ -440,4 +440,4 @@ exit: #ifndef ZLIB_COMPRESS_COPY_METHODDEF #define ZLIB_COMPRESS_COPY_METHODDEF #endif /* !defined(ZLIB_COMPRESS_COPY_METHODDEF) */ -/*[clinic end generated code: output=3c96b58b923c1273 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e6f3b79e051ecc35 input=a9049054013a1b77]*/ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 8ddc774..db3d8c9 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -147,7 +147,7 @@ Returns a bytes object containing compressed data. static PyObject * zlib_compress_impl(PyModuleDef *module, Py_buffer *data, int level) -/*[clinic end generated code: output=1b97589132b203b4 input=671c615a4b2267da]*/ +/*[clinic end generated code: output=1b97589132b203b4 input=abed30f4fa14e213]*/ { PyObject *ReturnVal = NULL; Byte *input, *output = NULL; -- cgit v0.12 From 7e3a91a5fc056f72c93f015bb9f4f22aef9e72fb Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 04:40:48 +0000 Subject: Issue #26136: Upgrade the generator_stop warning to DeprecationWarning Patch by Anish Shah. --- Doc/whatsnew/3.6.rst | 8 ++++++++ Lib/test/test_contextlib.py | 2 +- Lib/test/test_generators.py | 6 +++--- Lib/test/test_with.py | 4 ++-- Misc/NEWS | 4 ++++ Objects/genobject.c | 2 +- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 87199df..3822d01 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -234,6 +234,14 @@ Deprecated features (Contributed by Rose Ames in :issue:`25791`.) +Deprecated Python behavior +-------------------------- + +* Raising the :exc:`StopIteration` exception inside a generator will now generate a + :exc:`DeprecationWarning`, and will trigger a :exc:`RuntimeError` in Python 3.7. + See :ref:`whatsnew-pep-479` for details. + + Removed ======= diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 30a6377..04fc875 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -89,7 +89,7 @@ class ContextManagerTestCase(unittest.TestCase): def woohoo(): yield try: - with self.assertWarnsRegex(PendingDeprecationWarning, + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with woohoo(): raise stop_exc diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index b92d5ce..0389b48 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -245,11 +245,11 @@ class ExceptionTest(unittest.TestCase): yield with self.assertRaises(StopIteration), \ - self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + self.assertWarnsRegex(DeprecationWarning, "StopIteration"): next(gen()) - with self.assertRaisesRegex(PendingDeprecationWarning, + with self.assertRaisesRegex(DeprecationWarning, "generator .* raised StopIteration"), \ warnings.catch_warnings(): @@ -268,7 +268,7 @@ class ExceptionTest(unittest.TestCase): g = f() self.assertEqual(next(g), 1) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with self.assertRaises(StopIteration): next(g) diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index e8d789b..3815062 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -454,7 +454,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise StopIteration("from with") - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedStopIteration2(self): @@ -482,7 +482,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise next(iter([])) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedGeneratorExit1(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6ce7e2d..7912790 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26136: Upgrade the warning when a generator raises StopIteration + from PendingDeprecationWarning to DeprecationWarning. Patch by Anish + Shah. + - Issue #26204: The compiler now ignores all constant statements: bytes, str, int, float, complex, name constants (None, False, True), Ellipsis and ast.Constant; not only str and int. For example, ``1.0`` is now ignored diff --git a/Objects/genobject.c b/Objects/genobject.c index 81a80b7..8ab7cf1 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -178,7 +178,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) /* Pop the exception before issuing a warning. */ PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_PendingDeprecationWarning, 1, + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "generator '%.50S' raised StopIteration", gen->gi_qualname)) { /* Warning was converted to an error. */ -- cgit v0.12 From 288ed038aa28b9037e862026d743d4010a615f4e Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 05:45:55 +0000 Subject: =?UTF-8?q?Issue=20#26304:=20Additional=20=E2=80=9Callows=20to?= =?UTF-8?q?=E2=80=9D=20fix=20specific=20to=203.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/whatsnew/3.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3822d01..bc5e550 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -144,7 +144,7 @@ zipfile ------- A new :meth:`ZipInfo.from_file() ` class method -allow to make :class:`~zipfile.ZipInfo` instance from a filesystem file. +allows making a :class:`~zipfile.ZipInfo` instance from a filesystem file. A new :meth:`ZipInfo.is_dir() ` method can be used to check if the :class:`~zipfile.ZipInfo` instance represents a directory. (Contributed by Thomas Kluyver in :issue:`26039`.) -- cgit v0.12 From 3fb5612e085bb9fd979057372978c0ad08c29226 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 10 Feb 2016 09:46:56 -0800 Subject: Hopefully clarify the difference between Optional[t] and an optional argument. --- Doc/library/typing.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 41f594e..12b5490 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -286,6 +286,13 @@ The module defines the following classes, functions and decorators: ``Optional[X]`` is equivalent to ``Union[X, type(None)]``. + Note that this is not the same concept as an optional argument, + which is one that has a default. An optional argument with a + default needn't use the ``Optional`` qualifier on its type + annotation (although it is inferred if the default is ``None``). + A mandatory argument may still have an ``Optional`` type if an + explicit value of ``None`` is allowed. + .. class:: Tuple Tuple type; ``Tuple[X, Y]`` is the is the type of a tuple of two items -- cgit v0.12 From 78f55ffc63ba637ffa4a07ca869f451e680b002a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 10 Feb 2016 22:58:18 +0000 Subject: Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. --- Lib/multiprocessing/pool.py | 20 ++++++++++++-------- Lib/test/_test_multiprocessing.py | 24 ++++++++++++++++++++++++ Misc/NEWS | 2 ++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 6d25469..ffdf426 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -638,22 +638,26 @@ class MapResult(ApplyResult): self._number_left = length//chunksize + bool(length % chunksize) def _set(self, i, success_result): + self._number_left -= 1 success, result = success_result - if success: + if success and self._success: self._value[i*self._chunksize:(i+1)*self._chunksize] = result - self._number_left -= 1 if self._number_left == 0: if self._callback: self._callback(self._value) del self._cache[self._job] self._event.set() else: - self._success = False - self._value = result - if self._error_callback: - self._error_callback(self._value) - del self._cache[self._job] - self._event.set() + if not success and self._success: + # only store first exception + self._success = False + self._value = result + if self._number_left == 0: + # only consider the result ready once all jobs are done + if self._error_callback: + self._error_callback(self._value) + del self._cache[self._job] + self._event.set() # # Class whose instances are returned by `Pool.imap()` diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e9120ab..a59d2ba 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1660,6 +1660,10 @@ def sqr(x, wait=0.0): def mul(x, y): return x*y +def raise_large_valuerror(wait): + time.sleep(wait) + raise ValueError("x" * 1024**2) + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1895,6 +1899,26 @@ class _TestPool(BaseTestCase): with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + def test_map_no_failfast(self): + # Issue #23992: the fail-fast behaviour when an exception is raised + # during map() would make Pool.join() deadlock, because a worker + # process would fill the result queue (after the result handler thread + # terminated, hence not draining it anymore). + + t_start = time.time() + + with self.assertRaises(ValueError): + with self.Pool(2) as p: + try: + p.map(raise_large_valuerror, [0, 1]) + finally: + time.sleep(0.5) + p.close() + p.join() + + # check that we indeed waited for all jobs + self.assertGreater(time.time() - t_start, 0.9) + def raising(): raise KeyError("key") diff --git a/Misc/NEWS b/Misc/NEWS index 1ac4ad0..22a392e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -179,6 +179,8 @@ Core and Builtins Library ------- +- Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. + - Issue #26243: Support keyword arguments to zlib.compress(). Patch by Aviv Palivoda. -- cgit v0.12 From a9725f86a984f74e74f09c4808fc8f4b403728b2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 12:41:40 +0200 Subject: Issue #26312: SystemError is now raised in all programming bugs with using PyArg_ParseTupleAndKeywords(). RuntimeError did raised before in some programming bugs. --- Lib/test/test_capi.py | 4 ++-- Misc/NEWS | 4 ++++ Python/getargs.c | 12 ++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 1a743fd..74ec6c5 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -491,7 +491,7 @@ class SkipitemTest(unittest.TestCase): except SystemError as e: s = "argument 1 (impossible)" when_not_skipped = (str(e) == s) - except (TypeError, RuntimeError): + except TypeError: when_not_skipped = False # test the format unit when skipped @@ -500,7 +500,7 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, optional_format.encode("ascii"), keywords) when_skipped = False - except RuntimeError as e: + except SystemError as e: s = "impossible: '{}'".format(format) when_skipped = (str(e) == s) diff --git a/Misc/NEWS b/Misc/NEWS index 22a392e..82b49d2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -741,6 +741,10 @@ Tools/Demos C API ----- +- Issue #26312: SystemError is now raised in all programming bugs with using + PyArg_ParseTupleAndKeywords(). RuntimeError did raised before in some + programming bugs. + - Issue #26198: ValueError is now raised instead of TypeError on buffer overflow in parsing "es#" and "et#" format units. SystemError is now raised instead of TypeError on programmical error in parsing format string. diff --git a/Python/getargs.c b/Python/getargs.c index be6e375..66a0c00 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1512,7 +1512,7 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, keyword = kwlist[i]; if (*format == '|') { if (min != INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_SystemError, "Invalid format string (| specified twice)"); return cleanreturn(0, &freelist); } @@ -1521,14 +1521,14 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, format++; if (max != INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_SystemError, "Invalid format string ($ before |)"); return cleanreturn(0, &freelist); } } if (*format == '$') { if (max != INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_SystemError, "Invalid format string ($ specified twice)"); return cleanreturn(0, &freelist); } @@ -1546,7 +1546,7 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, } } if (IS_END_OF_FORMAT(*format)) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_SystemError, "More keyword list entries (%d) than " "format specifiers (%d)", len, i); return cleanreturn(0, &freelist); @@ -1598,14 +1598,14 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, * keyword args */ msg = skipitem(&format, p_va, flags); if (msg) { - PyErr_Format(PyExc_RuntimeError, "%s: '%s'", msg, + PyErr_Format(PyExc_SystemError, "%s: '%s'", msg, format); return cleanreturn(0, &freelist); } } if (!IS_END_OF_FORMAT(*format) && (*format != '|') && (*format != '$')) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_SystemError, "more argument specifiers than keyword list entries " "(remaining format:'%s')", format); return cleanreturn(0, &freelist); -- cgit v0.12 From 885bdc4946890f4bb80557fab80c3874b2cc4d39 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 13:10:36 +0200 Subject: Issue #25985: sys.version_info is now used instead of sys.version to format short Python version. --- Doc/c-api/intro.rst | 7 ++++--- Doc/includes/test.py | 2 +- Doc/library/sysconfig.rst | 2 +- Lib/distutils/command/bdist_msi.py | 2 +- Lib/distutils/command/bdist_wininst.py | 2 +- Lib/distutils/command/build.py | 4 ++-- Lib/distutils/command/install.py | 4 ++-- Lib/distutils/command/install_egg_info.py | 4 ++-- Lib/distutils/sysconfig.py | 2 +- Lib/distutils/tests/test_build.py | 5 +++-- Lib/importlib/_bootstrap_external.py | 2 +- Lib/pydoc.py | 6 +++--- Lib/site.py | 4 ++-- Lib/sysconfig.py | 8 ++++---- Lib/test/test_importlib/test_windows.py | 2 +- Lib/test/test_site.py | 5 +++-- Lib/test/test_venv.py | 2 +- Lib/urllib/request.py | 2 +- Lib/xmlrpc/client.py | 2 +- Makefile.pre.in | 2 +- Python/importlib_external.h | 31 ++++++++++++++++--------------- Tools/freeze/freeze.py | 2 +- Tools/scripts/nm2def.py | 4 ++-- setup.py | 2 +- 24 files changed, 56 insertions(+), 52 deletions(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 95cbef5..74681d2 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -64,9 +64,10 @@ The header files are typically installed with Python. On Unix, these are located in the directories :file:`{prefix}/include/pythonversion/` and :file:`{exec_prefix}/include/pythonversion/`, where :envvar:`prefix` and :envvar:`exec_prefix` are defined by the corresponding parameters to Python's -:program:`configure` script and *version* is ``sys.version[:3]``. On Windows, -the headers are installed in :file:`{prefix}/include`, where :envvar:`prefix` is -the installation directory specified to the installer. +:program:`configure` script and *version* is +``'%d.%d' % sys.version_info[:2]``. On Windows, the headers are installed +in :file:`{prefix}/include`, where :envvar:`prefix` is the installation +directory specified to the installer. To include the headers, place both directories (if different) on your compiler's search path for includes. Do *not* place the parent directories on the search diff --git a/Doc/includes/test.py b/Doc/includes/test.py index 7ebf46a..9e9d4a6 100644 --- a/Doc/includes/test.py +++ b/Doc/includes/test.py @@ -204,7 +204,7 @@ Test cyclic gc(?) import os import sys from distutils.util import get_platform -PLAT_SPEC = "%s-%s" % (get_platform(), sys.version[0:3]) +PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) src = os.path.join("build", "lib.%s" % PLAT_SPEC) sys.path.append(src) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 535ac54..47f3900 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -163,7 +163,7 @@ Other functions .. function:: get_python_version() Return the ``MAJOR.MINOR`` Python version number as a string. Similar to - ``sys.version[:3]``. + ``'%d.%d' % sys.version_info[:2]``. .. function:: get_platform() diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py index b3cfe9c..f6c21ae 100644 --- a/Lib/distutils/command/bdist_msi.py +++ b/Lib/distutils/command/bdist_msi.py @@ -199,7 +199,7 @@ class bdist_msi(Command): target_version = self.target_version if not target_version: assert self.skip_build, "Should have already checked this" - target_version = sys.version[0:3] + target_version = '%d.%d' % sys.version_info[:2] plat_specifier = ".%s-%s" % (self.plat_name, target_version) build = self.get_finalized_command('build') build.build_lib = os.path.join(build.build_base, diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py index 0c0e2c1..d3e1d3a 100644 --- a/Lib/distutils/command/bdist_wininst.py +++ b/Lib/distutils/command/bdist_wininst.py @@ -141,7 +141,7 @@ class bdist_wininst(Command): target_version = self.target_version if not target_version: assert self.skip_build, "Should have already checked this" - target_version = sys.version[0:3] + target_version = '%d.%d' % sys.version_info[:2] plat_specifier = ".%s-%s" % (self.plat_name, target_version) build = self.get_finalized_command('build') build.build_lib = os.path.join(build.build_base, diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py index 337dd0b..c6f52e6 100644 --- a/Lib/distutils/command/build.py +++ b/Lib/distutils/command/build.py @@ -81,7 +81,7 @@ class build(Command): "--plat-name only supported on Windows (try " "using './configure --help' on your platform)") - plat_specifier = ".%s-%s" % (self.plat_name, sys.version[0:3]) + plat_specifier = ".%s-%d.%d" % (self.plat_name, *sys.version_info[:2]) # Make it so Python 2.x and Python 2.x with --with-pydebug don't # share the same build directories. Doing so confuses the build @@ -114,7 +114,7 @@ class build(Command): 'temp' + plat_specifier) if self.build_scripts is None: self.build_scripts = os.path.join(self.build_base, - 'scripts-' + sys.version[0:3]) + 'scripts-%d.%d' % sys.version_info[:2]) if self.executable is None: self.executable = os.path.normpath(sys.executable) diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 67db007..9474e9c 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -290,8 +290,8 @@ class install(Command): 'dist_version': self.distribution.get_version(), 'dist_fullname': self.distribution.get_fullname(), 'py_version': py_version, - 'py_version_short': py_version[0:3], - 'py_version_nodot': py_version[0] + py_version[2], + 'py_version_short': '%d.%d' % sys.version_info[:2], + 'py_version_nodot': '%d%d' % sys.version_info[:2], 'sys_prefix': prefix, 'prefix': prefix, 'sys_exec_prefix': exec_prefix, diff --git a/Lib/distutils/command/install_egg_info.py b/Lib/distutils/command/install_egg_info.py index c2a7d64..0ddc736 100644 --- a/Lib/distutils/command/install_egg_info.py +++ b/Lib/distutils/command/install_egg_info.py @@ -21,10 +21,10 @@ class install_egg_info(Command): def finalize_options(self): self.set_undefined_options('install_lib',('install_dir','install_dir')) - basename = "%s-%s-py%s.egg-info" % ( + basename = "%s-%s-py%d.%d.egg-info" % ( to_filename(safe_name(self.distribution.get_name())), to_filename(safe_version(self.distribution.get_version())), - sys.version[:3] + *sys.version_info[:2] ) self.target = os.path.join(self.install_dir, basename) self.outputs = [self.target] diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 573724d..d203f8e 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -70,7 +70,7 @@ def get_python_version(): leaving off the patchlevel. Sample return values could be '1.5' or '2.2'. """ - return sys.version[:3] + return '%d.%d' % sys.version_info[:2] def get_python_inc(plat_specific=0, prefix=None): diff --git a/Lib/distutils/tests/test_build.py b/Lib/distutils/tests/test_build.py index 3391f36..b020a5b 100644 --- a/Lib/distutils/tests/test_build.py +++ b/Lib/distutils/tests/test_build.py @@ -27,7 +27,7 @@ class BuildTestCase(support.TempdirManager, # build_platlib is 'build/lib.platform-x.x[-pydebug]' # examples: # build/lib.macosx-10.3-i386-2.7 - plat_spec = '.%s-%s' % (cmd.plat_name, sys.version[0:3]) + plat_spec = '.%s-%d.%d' % (cmd.plat_name, *sys.version_info[:2]) if hasattr(sys, 'gettotalrefcount'): self.assertTrue(cmd.build_platlib.endswith('-pydebug')) plat_spec += '-pydebug' @@ -42,7 +42,8 @@ class BuildTestCase(support.TempdirManager, self.assertEqual(cmd.build_temp, wanted) # build_scripts is build/scripts-x.x - wanted = os.path.join(cmd.build_base, 'scripts-' + sys.version[0:3]) + wanted = os.path.join(cmd.build_base, + 'scripts-%d.%d' % sys.version_info[:2]) self.assertEqual(cmd.build_scripts, wanted) # executable is os.path.normpath(sys.executable) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 71098f1..ddb2456 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -593,7 +593,7 @@ class WindowsRegistryFinder: else: registry_key = cls.REGISTRY_KEY key = registry_key.format(fullname=fullname, - sys_version=sys.version[:3]) + sys_version='%d.%d' % sys.version_info[:2]) try: with cls._open_registry(key) as hkey: filepath = _winreg.QueryValue(hkey, '') diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a73298d..5e5a8ae 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1911,10 +1911,10 @@ has the same effect as typing a particular string at the help> prompt. def intro(self): self.output.write(''' -Welcome to Python %s's help utility! +Welcome to Python {0}'s help utility! If this is your first time using Python, you should definitely check out -the tutorial on the Internet at http://docs.python.org/%s/tutorial/. +the tutorial on the Internet at http://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing Python programs and using Python modules. To quit this help utility and @@ -1924,7 +1924,7 @@ To get a list of available modules, keywords, symbols, or topics, type "modules", "keywords", "symbols", or "topics". Each module also comes with a one-line summary of what it does; to list the modules whose name or summary contain a given string such as "spam", type "modules spam". -''' % tuple([sys.version[:3]]*2)) +'''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): items = list(sorted(items)) diff --git a/Lib/site.py b/Lib/site.py index 13ec8fa..56ba709 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -304,7 +304,7 @@ def getsitepackages(prefixes=None): if os.sep == '/': sitepackages.append(os.path.join(prefix, "lib", - "python" + sys.version[:3], + "python%d.%d" % sys.version_info[:2], "site-packages")) else: sitepackages.append(prefix) @@ -317,7 +317,7 @@ def getsitepackages(prefixes=None): if framework: sitepackages.append( os.path.join("/Library", framework, - sys.version[:3], "site-packages")) + '%d.%d' % sys.version_info[:2], "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 9c34be0..f18b1bc 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -86,8 +86,8 @@ _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', # FIXME don't rely on sys.version here, its format is an implementation detail # of CPython, use sys.version_info or sys.hexversion _PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = sys.version[:3] -_PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2] +_PY_VERSION_SHORT = '%d.%d' % sys.version_info[:2] +_PY_VERSION_SHORT_NO_DOT = '%d%d' % sys.version_info[:2] _PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) @@ -386,7 +386,7 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3]) + pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT) if hasattr(sys, "gettotalrefcount"): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) @@ -518,7 +518,7 @@ def get_config_vars(*args): _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION[0] + _PY_VERSION[2] + _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT _CONFIG_VARS['installed_base'] = _BASE_PREFIX _CONFIG_VARS['base'] = _PREFIX _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index c893bcf..005b685 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -40,7 +40,7 @@ def setup_module(machinery, name, path=None): else: root = machinery.WindowsRegistryFinder.REGISTRY_KEY key = root.format(fullname=name, - sys_version=sys.version[:3]) + sys_version='%d.%d' % sys.version_info[:2]) try: with temp_module(name, "a = 1") as location: subkey = CreateKey(HKEY_CURRENT_USER, key) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 6615080..574e496 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -238,13 +238,14 @@ class HelperFunctionsTests(unittest.TestCase): self.assertEqual(len(dirs), 2) wanted = os.path.join('/Library', sysconfig.get_config_var("PYTHONFRAMEWORK"), - sys.version[:3], + '%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[1], wanted) elif os.sep == '/': # OS X non-framwework builds, Linux, FreeBSD, etc self.assertEqual(len(dirs), 1) - wanted = os.path.join('xoxo', 'lib', 'python' + sys.version[:3], + wanted = os.path.join('xoxo', 'lib', + 'python%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[0], wanted) else: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 28b0f6c..f38f52df 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -51,7 +51,7 @@ class BaseTest(unittest.TestCase): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%s' % sys.version[:3]) + self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ: executable = os.environ['__PYVENV_LAUNCHER__'] diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 4c2b9fe..e3eed16 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -133,7 +133,7 @@ __all__ = [ ] # used in User-Agent header sent -__version__ = sys.version[:3] +__version__ = '%d.%d' % sys.version_info[:2] _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index bf42835..07a4f03 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -151,7 +151,7 @@ def escape(s): return s.replace(">", ">",) # used in User-Agent header sent -__version__ = sys.version[:3] +__version__ = '%d.%d' % sys.version_info[:2] # xmlrpc integer limits MAXINT = 2**31-1 diff --git a/Makefile.pre.in b/Makefile.pre.in index 2a687e5..6514bf8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -568,7 +568,7 @@ $(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) platform: $(BUILDPYTHON) pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print(get_platform()+"-"+sys.version[0:3])' >platform + $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform # Create build directory and generate the sysconfig build-time data there. # pybuilddir.txt contains the name of the build dir and is used for diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 97c4948..3d7aff0 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -888,22 +888,23 @@ const unsigned char _Py_M__importlib_external[] = { 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, 115,116,114,121,99,2,0,0,0,0,0,0,0,6,0,0, - 0,16,0,0,0,67,0,0,0,115,143,0,0,0,124,0, + 0,16,0,0,0,67,0,0,0,115,147,0,0,0,124,0, 0,106,0,0,114,21,0,124,0,0,106,1,0,125,2,0, 110,9,0,124,0,0,106,2,0,125,2,0,124,2,0,106, - 3,0,100,1,0,124,1,0,100,2,0,116,4,0,106,5, - 0,100,0,0,100,3,0,133,2,0,25,131,0,2,125,3, - 0,121,47,0,124,0,0,106,6,0,124,3,0,131,1,0, - 143,25,0,125,4,0,116,7,0,106,8,0,124,4,0,100, - 4,0,131,2,0,125,5,0,87,100,0,0,81,82,88,87, - 110,22,0,4,116,9,0,107,10,0,114,138,0,1,1,1, - 100,0,0,83,89,110,1,0,88,124,5,0,83,41,5,78, - 114,119,0,0,0,90,11,115,121,115,95,118,101,114,115,105, - 111,110,114,80,0,0,0,114,30,0,0,0,41,10,218,11, - 68,69,66,85,71,95,66,85,73,76,68,218,18,82,69,71, - 73,83,84,82,89,95,75,69,89,95,68,69,66,85,71,218, - 12,82,69,71,73,83,84,82,89,95,75,69,89,114,47,0, - 0,0,114,7,0,0,0,218,7,118,101,114,115,105,111,110, + 3,0,100,1,0,124,1,0,100,2,0,100,3,0,116,4, + 0,106,5,0,100,0,0,100,4,0,133,2,0,25,22,131, + 0,2,125,3,0,121,47,0,124,0,0,106,6,0,124,3, + 0,131,1,0,143,25,0,125,4,0,116,7,0,106,8,0, + 124,4,0,100,5,0,131,2,0,125,5,0,87,100,0,0, + 81,82,88,87,110,22,0,4,116,9,0,107,10,0,114,142, + 0,1,1,1,100,0,0,83,89,110,1,0,88,124,5,0, + 83,41,6,78,114,119,0,0,0,90,11,115,121,115,95,118, + 101,114,115,105,111,110,122,5,37,100,46,37,100,114,56,0, + 0,0,114,30,0,0,0,41,10,218,11,68,69,66,85,71, + 95,66,85,73,76,68,218,18,82,69,71,73,83,84,82,89, + 95,75,69,89,95,68,69,66,85,71,218,12,82,69,71,73, + 83,84,82,89,95,75,69,89,114,47,0,0,0,114,7,0, + 0,0,218,12,118,101,114,115,105,111,110,95,105,110,102,111, 114,166,0,0,0,114,163,0,0,0,90,10,81,117,101,114, 121,86,97,108,117,101,114,40,0,0,0,41,6,114,164,0, 0,0,114,119,0,0,0,90,12,114,101,103,105,115,116,114, @@ -911,7 +912,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, 99,104,95,114,101,103,105,115,116,114,121,77,2,0,0,115, - 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, + 22,0,0,0,0,2,9,1,12,2,9,1,15,1,26,1, 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, 114,46,95,115,101,97,114,99,104,95,114,101,103,105,115,116, diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py index c075807..389fffd 100755 --- a/Tools/freeze/freeze.py +++ b/Tools/freeze/freeze.py @@ -218,7 +218,7 @@ def main(): ishome = os.path.exists(os.path.join(prefix, 'Python', 'ceval.c')) # locations derived from options - version = sys.version[:3] + version = '%d.%d' % sys.version_info[:2] flagged_version = version + sys.abiflags if win: extensions_c = 'frozen_extensions.c' diff --git a/Tools/scripts/nm2def.py b/Tools/scripts/nm2def.py index 8f07559..83bbcd7 100755 --- a/Tools/scripts/nm2def.py +++ b/Tools/scripts/nm2def.py @@ -36,8 +36,8 @@ option to produce this format (since it is the original v7 Unix format). """ import os, sys -PYTHONLIB = 'libpython'+sys.version[:3]+'.a' -PC_PYTHONLIB = 'Python'+sys.version[0]+sys.version[2]+'.dll' +PYTHONLIB = 'libpython%d.%d.a' % sys.version_info[:2] +PC_PYTHONLIB = 'Python%d%d.dll' % sys.version_info[:2] NM = 'nm -p -g %s' # For Linux, use "nm -g %s" def symbols(lib=PYTHONLIB,types=('T','C','D')): diff --git a/setup.py b/setup.py index 930b64a..720f568 100644 --- a/setup.py +++ b/setup.py @@ -2226,7 +2226,7 @@ def main(): setup(# PyPI Metadata (PEP 301) name = "Python", version = sys.version.split()[0], - url = "http://www.python.org/%s" % sys.version[:3], + url = "http://www.python.org/%d.%d" % sys.version_info[:2], maintainer = "Guido van Rossum and the Python community", maintainer_email = "python-dev@python.org", description = "A high-level object-oriented programming language", -- cgit v0.12 From ffe96ae10be8a3117fa18c35034fcfc45c3cf7b7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 13:21:30 +0200 Subject: Issue #25994: Added the close() method and the support of the context manager protocol for the os.scandir() iterator. --- Doc/library/os.rst | 27 +++++++++++++-- Doc/whatsnew/3.6.rst | 11 ++++++ Lib/os.py | 94 ++++++++++++++++++++++++++++++--------------------- Lib/test/test_os.py | 52 ++++++++++++++++++++++++++++ Misc/NEWS | 3 ++ Modules/posixmodule.c | 73 +++++++++++++++++++++++++++++++++++---- 6 files changed, 211 insertions(+), 49 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f16a3fb..f064717 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1891,14 +1891,29 @@ features: :attr:`~DirEntry.path` attributes of each :class:`DirEntry` will be of the same type as *path*. + The :func:`scandir` iterator supports the :term:`context manager` protocol + and has the following method: + + .. method:: scandir.close() + + Close the iterator and free acquired resources. + + This is called automatically when the iterator is exhausted or garbage + collected, or when an error happens during iterating. However it + is advisable to call it explicitly or use the :keyword:`with` + statement. + + .. versionadded:: 3.6 + The following example shows a simple use of :func:`scandir` to display all the files (excluding directories) in the given *path* that don't start with ``'.'``. The ``entry.is_file()`` call will generally not make an additional system call:: - for entry in os.scandir(path): - if not entry.name.startswith('.') and entry.is_file(): - print(entry.name) + with os.scandir(path) as it: + for entry in it: + if not entry.name.startswith('.') and entry.is_file(): + print(entry.name) .. note:: @@ -1914,6 +1929,12 @@ features: .. versionadded:: 3.5 + .. versionadded:: 3.6 + Added support for the :term:`context manager` protocol and the + :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither + exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted + in its destructor. + .. class:: DirEntry diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index bc5e550..b7d14f2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -104,6 +104,17 @@ directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) +os +-- + +A new :meth:`~os.scandir.close` method allows explicitly closing a +:func:`~os.scandir` iterator. The :func:`~os.scandir` iterator now +supports the :term:`context manager` protocol. If a :func:`scandir` +iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` +will be emitted in its destructor. +(Contributed by Serhiy Storchaka in :issue:`25994`.) + + pickle ------ diff --git a/Lib/os.py b/Lib/os.py index 674a7d7..c3f674e 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -374,46 +374,47 @@ def walk(top, topdown=True, onerror=None, followlinks=False): onerror(error) return - while True: - try: + with scandir_it: + while True: try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - return - - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + return - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider that the entry is not + # a directory, same behaviour than os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink + nondirs.append(entry.name) - if walk_into: - yield from walk(entry.path, topdown, onerror, followlinks) + if not topdown and is_dir: + # Bottom-up: recurse into sub-directory, but exclude symlinks to + # directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider that the + # entry is not a symbolic link, same behaviour than + # os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + yield from walk(entry.path, topdown, onerror, followlinks) # Yield before recursion if going top down if topdown: @@ -437,15 +438,30 @@ class _DummyDirEntry: def __init__(self, dir, name): self.name = name self.path = path.join(dir, name) + def is_dir(self): return path.isdir(self.path) + def is_symlink(self): return path.islink(self.path) -def _dummy_scandir(dir): +class _dummy_scandir: # listdir-based implementation for bytes patches on Windows - for name in listdir(dir): - yield _DummyDirEntry(dir, name) + def __init__(self, dir): + self.dir = dir + self.it = iter(listdir(dir)) + + def __iter__(self): + return self + + def __next__(self): + return _DummyDirEntry(self.dir, next(self.it)) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.it = iter(()) __all__.append("walk") diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 66a426a..07682f2 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2808,6 +2808,8 @@ class ExportsTests(unittest.TestCase): class TestScandir(unittest.TestCase): + check_no_resource_warning = support.check_no_resource_warning + def setUp(self): self.path = os.path.realpath(support.TESTFN) self.addCleanup(support.rmtree, self.path) @@ -3030,6 +3032,56 @@ class TestScandir(unittest.TestCase): for obj in [1234, 1.234, {}, []]: self.assertRaises(TypeError, os.scandir, obj) + def test_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + iterator.close() + # multiple closes + iterator.close() + with self.check_no_resource_warning(): + del iterator + + def test_context_manager(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + with self.check_no_resource_warning(): + del iterator + + def test_context_manager_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + iterator.close() + + def test_context_manager_exception(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with self.assertRaises(ZeroDivisionError): + with os.scandir(self.path) as iterator: + next(iterator) + 1/0 + with self.check_no_resource_warning(): + del iterator + + def test_resource_warning(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + with self.assertWarns(ResourceWarning): + del iterator + support.gc_collect() + # exhausted iterator + iterator = os.scandir(self.path) + list(iterator) + with self.check_no_resource_warning(): + del iterator + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 405e0b9..92a210f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -179,6 +179,9 @@ Core and Builtins Library ------- +- Issue #25994: Added the close() method and the support of the context manager + protocol for the os.scandir() iterator. + - Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. - Issue #26243: Support keyword arguments to zlib.compress(). Patch by Aviv diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2688cbc..6e0081d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11937,8 +11937,14 @@ typedef struct { #ifdef MS_WINDOWS +static int +ScandirIterator_is_closed(ScandirIterator *iterator) +{ + return iterator->handle == INVALID_HANDLE_VALUE; +} + static void -ScandirIterator_close(ScandirIterator *iterator) +ScandirIterator_closedir(ScandirIterator *iterator) { if (iterator->handle == INVALID_HANDLE_VALUE) return; @@ -11956,7 +11962,7 @@ ScandirIterator_iternext(ScandirIterator *iterator) BOOL success; PyObject *entry; - /* Happens if the iterator is iterated twice */ + /* Happens if the iterator is iterated twice, or closed explicitly */ if (iterator->handle == INVALID_HANDLE_VALUE) return NULL; @@ -11987,14 +11993,20 @@ ScandirIterator_iternext(ScandirIterator *iterator) } /* Error or no more files */ - ScandirIterator_close(iterator); + ScandirIterator_closedir(iterator); return NULL; } #else /* POSIX */ +static int +ScandirIterator_is_closed(ScandirIterator *iterator) +{ + return !iterator->dirp; +} + static void -ScandirIterator_close(ScandirIterator *iterator) +ScandirIterator_closedir(ScandirIterator *iterator) { if (!iterator->dirp) return; @@ -12014,7 +12026,7 @@ ScandirIterator_iternext(ScandirIterator *iterator) int is_dot; PyObject *entry; - /* Happens if the iterator is iterated twice */ + /* Happens if the iterator is iterated twice, or closed explicitly */ if (!iterator->dirp) return NULL; @@ -12051,21 +12063,67 @@ ScandirIterator_iternext(ScandirIterator *iterator) } /* Error or no more files */ - ScandirIterator_close(iterator); + ScandirIterator_closedir(iterator); return NULL; } #endif +static PyObject * +ScandirIterator_close(ScandirIterator *self, PyObject *args) +{ + ScandirIterator_closedir(self); + Py_RETURN_NONE; +} + +static PyObject * +ScandirIterator_enter(PyObject *self, PyObject *args) +{ + Py_INCREF(self); + return self; +} + +static PyObject * +ScandirIterator_exit(ScandirIterator *self, PyObject *args) +{ + ScandirIterator_closedir(self); + Py_RETURN_NONE; +} + static void ScandirIterator_dealloc(ScandirIterator *iterator) { - ScandirIterator_close(iterator); + if (!ScandirIterator_is_closed(iterator)) { + PyObject *exc, *val, *tb; + Py_ssize_t old_refcount = Py_REFCNT(iterator); + /* Py_INCREF/Py_DECREF cannot be used, because the refcount is + * likely zero, Py_DECREF would call again the destructor. + */ + ++Py_REFCNT(iterator); + PyErr_Fetch(&exc, &val, &tb); + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed scandir iterator %R", iterator)) { + /* Spurious errors can appear at shutdown */ + if (PyErr_ExceptionMatches(PyExc_Warning)) + PyErr_WriteUnraisable((PyObject *) iterator); + } + PyErr_Restore(exc, val, tb); + Py_REFCNT(iterator) = old_refcount; + + ScandirIterator_closedir(iterator); + } Py_XDECREF(iterator->path.object); path_cleanup(&iterator->path); Py_TYPE(iterator)->tp_free((PyObject *)iterator); } +static PyMethodDef ScandirIterator_methods[] = { + {"__enter__", (PyCFunction)ScandirIterator_enter, METH_NOARGS}, + {"__exit__", (PyCFunction)ScandirIterator_exit, METH_VARARGS}, + {"close", (PyCFunction)ScandirIterator_close, METH_NOARGS}, + {NULL} +}; + static PyTypeObject ScandirIteratorType = { PyVarObject_HEAD_INIT(NULL, 0) MODNAME ".ScandirIterator", /* tp_name */ @@ -12095,6 +12153,7 @@ static PyTypeObject ScandirIteratorType = { 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)ScandirIterator_iternext, /* tp_iternext */ + ScandirIterator_methods, /* tp_methods */ }; static PyObject * -- cgit v0.12 From 7c90a82a01aed150eb589b1c7035352e11cf8429 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 13:31:00 +0200 Subject: Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. Different solution from 3.5. --- Lib/os.py | 6 +++++- Misc/NEWS | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index c3f674e..0ea7b62 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -356,6 +356,7 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirs = [] nondirs = [] + walk_dirs = [] # We may not have read permission for top, in which case we can't # get a list of the files the directory contains. os.walk @@ -414,7 +415,7 @@ def walk(top, topdown=True, onerror=None, followlinks=False): walk_into = not is_symlink if walk_into: - yield from walk(entry.path, topdown, onerror, followlinks) + walk_dirs.append(entry.path) # Yield before recursion if going top down if topdown: @@ -431,6 +432,9 @@ def walk(top, topdown=True, onerror=None, followlinks=False): if followlinks or not islink(new_path): yield from walk(new_path, topdown, onerror, followlinks) else: + # Recurse into sub-directories + for new_path in walk_dirs: + yield from walk(new_path, topdown, onerror, followlinks) # Yield after recursion if going bottom up yield top, dirs, nondirs diff --git a/Misc/NEWS b/Misc/NEWS index 92a210f..0114a86 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -179,6 +179,8 @@ Core and Builtins Library ------- +- Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. + - Issue #25994: Added the close() method and the support of the context manager protocol for the os.scandir() iterator. -- cgit v0.12 From e0b23095ee03a11d09f38cbc689307dc5c93afda Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 11 Feb 2016 10:26:27 -0500 Subject: Issues #26289 and #26315: Optimize floor/modulo div for single-digit longs Microbenchmarks show 2-2.5x improvement. Built-in 'divmod' function is now also ~10% faster. -m timeit -s "x=22331" "x//2;x//-3;x//4;x//5;x//-6;x//7;x//8;x//-99;x//100;" with patch: 0.321 without patch: 0.633 -m timeit -s "x=22331" "x%2;x%3;x%-4;x%5;x%6;x%-7;x%8;x%99;x%-100;" with patch: 0.224 without patch: 0.66 Big thanks to Serhiy Storchaka, Mark Dickinson and Victor Stinner for thorow code reviews and algorithms improvements. --- Lib/test/test_long.py | 33 +++++++++++++++++++++ Misc/NEWS | 4 +++ Objects/longobject.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 62e69a9..5b8b4dc 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -689,6 +689,20 @@ class LongTest(unittest.TestCase): self.assertRaises(OverflowError, int, float('-inf')) self.assertRaises(ValueError, int, float('nan')) + def test_mod_division(self): + with self.assertRaises(ZeroDivisionError): + _ = 1 % 0 + + self.assertEqual(13 % 10, 3) + self.assertEqual(-13 % 10, 7) + self.assertEqual(13 % -10, -7) + self.assertEqual(-13 % -10, -3) + + self.assertEqual(12 % 4, 0) + self.assertEqual(-12 % 4, 0) + self.assertEqual(12 % -4, 0) + self.assertEqual(-12 % -4, 0) + def test_true_division(self): huge = 1 << 40000 mhuge = -huge @@ -723,6 +737,25 @@ class LongTest(unittest.TestCase): for zero in ["huge / 0", "mhuge / 0"]: self.assertRaises(ZeroDivisionError, eval, zero, namespace) + def test_floordiv(self): + with self.assertRaises(ZeroDivisionError): + _ = 1 // 0 + + self.assertEqual(2 // 3, 0) + self.assertEqual(2 // -3, -1) + self.assertEqual(-2 // 3, -1) + self.assertEqual(-2 // -3, 0) + + self.assertEqual(-11 // -3, 3) + self.assertEqual(-11 // 3, -4) + self.assertEqual(11 // -3, -4) + self.assertEqual(11 // 3, 3) + + self.assertEqual(-12 // -3, 4) + self.assertEqual(-12 // 3, -4) + self.assertEqual(12 // -3, -4) + self.assertEqual(12 // 3, 4) + def check_truediv(self, a, b, skip_small=True): """Verify that the result of a/b is correctly rounded, by comparing it with a pure Python implementation of correctly diff --git a/Misc/NEWS b/Misc/NEWS index 0114a86..d51c3f3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -176,6 +176,10 @@ Core and Builtins - Issue #26288: Optimize PyLong_AsDouble. +- Issues #26289 and #26315: Optimize floor and modulo division for + single-digit longs. Microbenchmarks show 2-2.5x improvement. Built-in + 'divmod' function is now also ~10% faster. + Library ------- diff --git a/Objects/longobject.c b/Objects/longobject.c index f3afb10..3f9837f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3502,6 +3502,52 @@ long_mul(PyLongObject *a, PyLongObject *b) return (PyObject *)z; } +/* Fast modulo division for single-digit longs. */ +static PyObject * +fast_mod(PyLongObject *a, PyLongObject *b) +{ + sdigit left = a->ob_digit[0]; + sdigit right = b->ob_digit[0]; + sdigit mod; + + assert(Py_ABS(Py_SIZE(a)) == 1); + assert(Py_ABS(Py_SIZE(b)) == 1); + + if (Py_SIZE(a) == Py_SIZE(b)) { + /* 'a' and 'b' have the same sign. */ + mod = left % right; + } + else { + /* Either 'a' or 'b' is negative. */ + mod = right - 1 - (left - 1) % right; + } + + return PyLong_FromLong(mod * Py_SIZE(b)); +} + +/* Fast floor division for single-digit longs. */ +static PyObject * +fast_floor_div(PyLongObject *a, PyLongObject *b) +{ + sdigit left = a->ob_digit[0]; + sdigit right = b->ob_digit[0]; + sdigit div; + + assert(Py_ABS(Py_SIZE(a)) == 1); + assert(Py_ABS(Py_SIZE(b)) == 1); + + if (Py_SIZE(a) == Py_SIZE(b)) { + /* 'a' and 'b' have the same sign. */ + div = left / right; + } + else { + /* Either 'a' or 'b' is negative. */ + div = -1 - (left - 1) / right; + } + + return PyLong_FromLong(div); +} + /* The / and % operators are now defined in terms of divmod(). The expression a mod b has the value a - b*floor(a/b). The long_divrem function gives the remainder after division of @@ -3529,6 +3575,30 @@ l_divmod(PyLongObject *v, PyLongObject *w, { PyLongObject *div, *mod; + if (Py_ABS(Py_SIZE(v)) == 1 && Py_ABS(Py_SIZE(w)) == 1) { + /* Fast path for single-digit longs */ + div = NULL; + if (pdiv != NULL) { + div = (PyLongObject *)fast_floor_div(v, w); + if (div == NULL) { + return -1; + } + } + if (pmod != NULL) { + mod = (PyLongObject *)fast_mod(v, w); + if (mod == NULL) { + Py_XDECREF(div); + return -1; + } + *pmod = mod; + } + if (pdiv != NULL) { + /* We only want to set `*pdiv` when `*pmod` is + set successfully. */ + *pdiv = div; + } + return 0; + } if (long_divrem(v, w, &div, &mod) < 0) return -1; if ((Py_SIZE(mod) < 0 && Py_SIZE(w) > 0) || @@ -3573,6 +3643,11 @@ long_div(PyObject *a, PyObject *b) PyLongObject *div; CHECK_BINOP(a, b); + + if (Py_ABS(Py_SIZE(a)) == 1 && Py_ABS(Py_SIZE(b)) == 1) { + return fast_floor_div((PyLongObject*)a, (PyLongObject*)b); + } + if (l_divmod((PyLongObject*)a, (PyLongObject*)b, &div, NULL) < 0) div = NULL; return (PyObject *)div; @@ -3848,6 +3923,10 @@ long_mod(PyObject *a, PyObject *b) CHECK_BINOP(a, b); + if (Py_ABS(Py_SIZE(a)) == 1 && Py_ABS(Py_SIZE(b)) == 1) { + return fast_mod((PyLongObject*)a, (PyLongObject*)b); + } + if (l_divmod((PyLongObject*)a, (PyLongObject*)b, NULL, &mod) < 0) mod = NULL; return (PyObject *)mod; -- cgit v0.12 From bc1ee460dcbe5811ebb10a6cacd8d3670846f039 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 13 Feb 2016 00:41:37 +0000 Subject: Issue #25179: Documentation for formatted string literals aka f-strings Some of the inspiration and wording is taken from the text of PEP 498 by Eric V. Smith, and the existing str.format() documentation. --- Doc/faq/programming.rst | 3 +- Doc/library/datetime.rst | 9 ++-- Doc/library/enum.rst | 3 +- Doc/library/stdtypes.rst | 15 +++--- Doc/library/string.rst | 10 ++-- Doc/reference/datamodel.rst | 5 +- Doc/reference/lexical_analysis.rst | 108 ++++++++++++++++++++++++++++++++++++- Doc/tutorial/inputoutput.rst | 3 +- Doc/tutorial/introduction.rst | 3 ++ Doc/whatsnew/3.6.rst | 20 ++++++- 10 files changed, 158 insertions(+), 21 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 9fba9fe..7b529a1 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -839,7 +839,8 @@ How do I convert a number to a string? To convert, e.g., the number 144 to the string '144', use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`formatstrings` section, e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections, +e.g. ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1f3edd2..a822842 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -605,7 +605,8 @@ Instance methods: .. method:: date.__format__(format) Same as :meth:`.date.strftime`. This makes it possible to specify a format - string for a :class:`.date` object when using :meth:`str.format`. For a + string for a :class:`.date` object in :ref:`formatted string + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1180,7 +1181,8 @@ Instance methods: .. method:: datetime.__format__(format) Same as :meth:`.datetime.strftime`. This makes it possible to specify a format - string for a :class:`.datetime` object when using :meth:`str.format`. For a + string for a :class:`.datetime` object in :ref:`formatted string + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1425,7 +1427,8 @@ Instance methods: .. method:: time.__format__(format) Same as :meth:`.time.strftime`. This makes it possible to specify a format string - for a :class:`.time` object when using :meth:`str.format`. For a + for a :class:`.time` object in :ref:`formatted string + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 333adfc..b3691ca 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -558,7 +558,8 @@ Some rules: 4. %-style formatting: `%s` and `%r` call the :class:`Enum` class's :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :meth:`str.format` (or :func:`format`) will use the mixed-in +5. :ref:`Formatted string literals `, :meth:`str.format`, + and :func:`format` will use the mixed-in type's :meth:`__format__`. If the :class:`Enum` class's :func:`str` or :func:`repr` is desired, use the `!s` or `!r` format codes. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index e491fd2..4aebcb8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1450,8 +1450,8 @@ multiple fragments. For more information on the ``str`` class and its methods, see :ref:`textseq` and the :ref:`string-methods` section below. To output - formatted strings, see the :ref:`formatstrings` section. In addition, - see the :ref:`stringservices` section. + formatted strings, see the :ref:`f-strings` and :ref:`formatstrings` + sections. In addition, see the :ref:`stringservices` section. .. index:: @@ -2053,8 +2053,8 @@ expression support in the :mod:`re` module). .. index:: single: formatting, string (%) single: interpolation, string (%) - single: string; formatting - single: string; interpolation + single: string; formatting, printf + single: string; interpolation, printf single: printf-style formatting single: sprintf-style formatting single: % formatting @@ -2064,9 +2064,10 @@ expression support in the :mod:`re` module). The formatting operations described here exhibit a variety of quirks that lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :meth:`str.format` interface - helps avoid these errors, and also provides a generally more powerful, - flexible and extensible approach to formatting text. + dictionaries correctly). Using the newer :ref:`formatted + string literals ` or the :meth:`str.format` interface + helps avoid these errors. These alternatives also provide more powerful, + flexible and extensible approaches to formatting text. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 5b917d9..cda8e86 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -188,7 +188,9 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, -subclasses can define their own format string syntax). +subclasses can define their own format string syntax). The syntax is +related to that of :ref:`formatted string literals `, but +there are differences. Format strings contain "replacement fields" surrounded by curly braces ``{}``. Anything that is not contained in braces is considered literal text, which is @@ -283,7 +285,8 @@ Format Specification Mini-Language "Format specifications" are used within replacement fields contained within a format string to define how individual values are presented (see -:ref:`formatstrings`). They can also be passed directly to the built-in +:ref:`formatstrings` and :ref:`f-strings`). +They can also be passed directly to the built-in :func:`format` function. Each formattable type may define how the format specification is to be interpreted. @@ -308,7 +311,8 @@ The general form of a *standard format specifier* is: If a valid *align* value is specified, it can be preceded by a *fill* character that can be any character and defaults to a space if omitted. It is not possible to use a literal curly brace ("``{``" or "``}``") as -the *fill* character when using the :meth:`str.format` +the *fill* character in a :ref:`formatted string literal +` or when using the :meth:`str.format` method. However, it is possible to insert a curly brace with a nested replacement field. This limitation doesn't affect the :func:`format` function. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 764c491..56ec0d0 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1234,8 +1234,9 @@ Basic customization .. method:: object.__format__(self, format_spec) - Called by the :func:`format` built-in function (and by extension, the - :meth:`str.format` method of class :class:`str`) to produce a "formatted" + Called by the :func:`format` built-in function, + and by extension, evaluation of :ref:`formatted string literals + ` and the :meth:`str.format` method, to produce a "formatted" string representation of an object. The ``format_spec`` argument is a string that contains a description of the formatting options desired. The interpretation of the ``format_spec`` argument is up to the type diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 8ad975f..aad5b0c 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -405,7 +405,8 @@ String literals are described by the following lexical definitions: .. productionlist:: stringliteral: [`stringprefix`](`shortstring` | `longstring`) - stringprefix: "r" | "u" | "R" | "U" + stringprefix: "r" | "u" | "R" | "U" | "f" | "F" + : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" shortstring: "'" `shortstringitem`* "'" | '"' `shortstringitem`* '"' longstring: "'''" `longstringitem`* "'''" | '"""' `longstringitem`* '"""' shortstringitem: `shortstringchar` | `stringescapeseq` @@ -464,6 +465,11 @@ is not supported. to simplify the maintenance of dual Python 2.x and 3.x codebases. See :pep:`414` for more information. +A string literal with ``'f'`` or ``'F'`` in its prefix is a +:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be +combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw +formatted strings are possible, but formatted bytes literals are not. + In triple-quoted literals, unescaped newlines and quotes are allowed (and are retained), except that three unescaped quotes in a row terminate the literal. (A "quote" is the character used to open the literal, i.e. either ``'`` or ``"``.) @@ -584,7 +590,105 @@ comments to parts of strings, for example:: Note that this feature is defined at the syntactical level, but implemented at compile time. The '+' operator must be used to concatenate string expressions at run time. Also note that literal concatenation can use different quoting -styles for each component (even mixing raw strings and triple quoted strings). +styles for each component (even mixing raw strings and triple quoted strings), +and formatted string literals may be concatenated with plain string literals. + + +.. index:: + single: formatted string literal + single: interpolated string literal + single: string; formatted literal + single: string; interpolated literal + single: f-string +.. _f-strings: + +Formatted string literals +------------------------- + +.. versionadded:: 3.6 + +A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal +that is prefixed with ``'f'`` or ``'F'``. These strings may contain +replacement fields, which are expressions delimited by curly braces ``{}``. +While other string literals always have a constant value, formatted strings +are really expressions evaluated at run time. + +Escape sequences are decoded like in ordinary string literals (except when +a literal is also marked as a raw string). After decoding, the grammar +for the contents of the string is: + +.. productionlist:: + f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* + replacement_field: "{" `f_expression` ["!" `conversion`] [":" `format_spec`] "}" + f_expression: `conditional_expression` ("," `conditional_expression`)* [","] + : | `yield_expression` + conversion: "s" | "r" | "a" + format_spec: (`literal_char` | NULL | `replacement_field`)* + literal_char: + +The parts of the string outside curly braces are treated literally, +except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced +with the corresponding single curly brace. A single opening curly +bracket ``'{'`` marks a replacement field, which starts with a +Python expression. After the expression, there may be a conversion field, +introduced by an exclamation point ``'!'``. A format specifier may also +be appended, introduced by a colon ``':'``. A replacement field ends +with a closing curly bracket ``'}'``. + +Expressions in formatted string literals are treated like regular +Python expressions surrounded by parentheses, with a few exceptions. +An empty expression is not allowed, and a :keyword:`lambda` expression +must be surrounded by explicit parentheses. Replacement expressions +can contain line breaks (e.g. in triple-quoted strings), but they +cannot contain comments. Each expression is evaluated in the context +where the formatted string literal appears, in order from left to right. + +If a conversion is specified, the result of evaluating the expression +is converted before formatting. Conversion ``'!s'`` calls :func:`str` on +the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. + +The result is then formatted using the :func:`format` protocol. The +format specifier is passed to the :meth:`__format__` method of the +expression or conversion result. An empty string is passed when the +format specifier is omitted. The formatted result is then included in +the final value of the whole string. + +Top-level format specifiers may include nested replacement fields. +These nested fields may include their own conversion fields and +format specifiers, but may not include more deeply-nested replacement fields. + +Formatted string literals may be concatenated, but replacement fields +cannot be split across literals. + +Some examples of formatted string literals:: + + >>> name = "Fred" + >>> f"He said his name is {name!r}." + "He said his name is 'Fred'." + >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r + "He said his name is 'Fred'." + >>> width = 10 + >>> precision = 4 + >>> value = decimal.Decimal("12.34567") + >>> f"result: {value:{width}.{precision}}" # nested fields + 'result: 12.35' + +A consequence of sharing the same syntax as regular string literals is +that characters in the replacement fields must not conflict with the +quoting used in the outer formatted string literal. Also, escape +sequences normally apply to the outer formatted string literal, +rather than inner string literals:: + + f"abc {a["x"]} def" # error: outer string literal ended prematurely + f"abc {a[\"x\"]} def" # workaround: escape the inner quotes + f"abc {a['x']} def" # workaround: use different quoting + + f"newline: {ord('\n')}" # error: literal line break in inner string + f"newline: {ord('\\n')}" # workaround: double escaping + fr"newline: {ord('\n')}" # workaround: raw outer string + +See also :pep:`498` for the proposal that added formatted string literals, +and :meth:`str.format`, which uses a related format string mechanism. .. _numbers: diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 5314fed..f2171f4 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -25,7 +25,8 @@ first way is to do all the string handling yourself; using string slicing and concatenation operations you can create any layout you can imagine. The string type has some methods that perform useful operations for padding strings to a given column width; these will be discussed shortly. The second -way is to use the :meth:`str.format` method. +way is to use :ref:`formatted string literals `, or the +:meth:`str.format` method. The :mod:`string` module contains a :class:`~string.Template` class which offers yet another way to substitute values into strings. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 8758f38..87f0fa5 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -352,6 +352,9 @@ The built-in function :func:`len` returns the length of a string:: Strings support a large number of methods for basic transformations and searching. + :ref:`f-strings` + String literals that have embedded expressions. + :ref:`formatstrings` Information about string formatting with :meth:`str.format`. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b7d14f2..8b879de 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -62,7 +62,7 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.6. Brevity is key. -* None yet. +* PEP 498: :ref:`Formatted string literals ` .. PEP-sized items next. @@ -80,6 +80,24 @@ Summary -- Release highlights PEP written by Carl Meyer +.. _whatsnew-fstrings: + +PEP 498: Formatted string literals +---------------------------------- + +Formatted string literals are a new kind of string literal, prefixed +with ``'f'``. They are similar to the format strings accepted by +:meth:`str.format`. They contain replacement fields surrounded by +curly braces. The replacement fields are expressions, which are +evaluated at run time, and then formatted using the :func:`format` protocol. + + >>> name = "Fred" + >>> f"He said his name is {name}." + 'He said his name is Fred.' + +See :pep:`498` and the main documentation at :ref:`f-strings`. + + Other Language Changes ====================== -- cgit v0.12 From b6f17f5210843295ae87c779596acbe6835b36f0 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 14 Feb 2016 01:41:35 -0800 Subject: The return type of a rich comparison is an int --- Modules/_bisectmodule.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 02b55d1..22ddbf2 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -12,7 +12,8 @@ static Py_ssize_t internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t hi) { PyObject *litem; - Py_ssize_t mid, res; + Py_ssize_t mid; + int res; if (lo < 0) { PyErr_SetString(PyExc_ValueError, "lo must be non-negative"); @@ -115,7 +116,8 @@ static Py_ssize_t internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t hi) { PyObject *litem; - Py_ssize_t mid, res; + Py_ssize_t mid; + int res; if (lo < 0) { PyErr_SetString(PyExc_ValueError, "lo must be non-negative"); -- cgit v0.12 From 40383c8f10c6a34aee6f513dc30b2005951cc30b Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 15 Feb 2016 17:50:33 +0100 Subject: Minor clarification in tutorial. --- Doc/tutorial/controlflow.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 813c828..8453796 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -78,6 +78,9 @@ slice notation makes this especially convenient:: >>> words ['defenestrate', 'cat', 'window', 'defenestrate'] +With ``for w in words:``, the example would attempt to create an infinite list, +inserting ``defenestrate`` over and over again. + .. _tut-range: -- cgit v0.12 From 4cbab346dfcea1f221bfb0811fabed8dfb435e7e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 20 Feb 2016 18:45:56 -0800 Subject: Issue #26397: Update an importlib example to use util.module_from_spec() instead of create_module() --- Doc/library/importlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 4d429bb..17a65aa 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1389,7 +1389,7 @@ Python 3.6 and newer for other parts of the code). break else: raise ImportError(f'No module named {absolute_name!r}') - module = spec.loader.create_module(spec) + module = util.module_from_spec(spec) spec.loader.exec_module(module) sys.modules[absolute_name] = module if path is not None: -- cgit v0.12 From 86a8be00ed5266550a3d97a3c065f7a22018b6a6 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 20 Feb 2016 18:47:09 -0800 Subject: Fix a name in an example --- Doc/library/importlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 17a65aa..2bb586c3 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1389,7 +1389,7 @@ Python 3.6 and newer for other parts of the code). break else: raise ImportError(f'No module named {absolute_name!r}') - module = util.module_from_spec(spec) + module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) sys.modules[absolute_name] = module if path is not None: -- cgit v0.12 From d9108d1253eef9f43c60e0ed5f1ec5603c1fba6c Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 21 Feb 2016 08:49:56 +0000 Subject: Issue #23430: Stop socketserver from catching SystemExit etc from handlers Also make handle_error() consistently output to stderr, and fix the documentation. --- Doc/library/socketserver.rst | 6 ++- Doc/whatsnew/3.6.rst | 11 +++++- Lib/socketserver.py | 31 +++++++++------ Lib/test/test_socketserver.py | 92 +++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 6 +++ 5 files changed, 131 insertions(+), 15 deletions(-) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index f0cfc80..aaaa61e 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -304,7 +304,11 @@ Server Objects This function is called if the :meth:`~BaseRequestHandler.handle` method of a :attr:`RequestHandlerClass` instance raises an exception. The default action is to print the traceback to - standard output and continue handling further requests. + standard error and continue handling further requests. + + .. versionchanged:: 3.6 + Now only called for exceptions derived from the :exc:`Exception` + class. .. method:: handle_timeout() diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8b879de..5c02b7d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -334,7 +334,16 @@ Changes in the Python API * When a relative import is performed and no parent package is known, then :exc:`ImportError` will be raised. Previously, :exc:`SystemError` could be - raised. (Contribute by Brett Cannon in :issue:`18018`.) + raised. (Contributed by Brett Cannon in :issue:`18018`.) + +* Servers based on the :mod:`socketserver` module, including those + defined in :mod:`http.server`, :mod:`xmlrpc.server` and + :mod:`wsgiref.simple_server`, now only catch exceptions derived + from :exc:`Exception`. Therefore if a request handler raises + an exception like :exc:`SystemExit` or :exc:`KeyboardInterrupt`, + :meth:`~socketserver.BaseServer.handle_error` is no longer called, and + the exception will stop a single-threaded server. (Contributed by + Martin Panter in :issue:`23430`.) Changes in the C API diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 1524d16..64689991 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -132,6 +132,7 @@ import socket import selectors import os import errno +import sys try: import threading except ImportError: @@ -316,9 +317,12 @@ class BaseServer: if self.verify_request(request, client_address): try: self.process_request(request, client_address) - except: + except Exception: self.handle_error(request, client_address) self.shutdown_request(request) + except: + self.shutdown_request(request) + raise else: self.shutdown_request(request) @@ -372,12 +376,12 @@ class BaseServer: The default is to print a traceback and continue. """ - print('-'*40) - print('Exception happened during processing of request from', end=' ') - print(client_address) + print('-'*40, file=sys.stderr) + print('Exception happened during processing of request from', + client_address, file=sys.stderr) import traceback - traceback.print_exc() # XXX But this goes to stderr! - print('-'*40) + traceback.print_exc() + print('-'*40, file=sys.stderr) class TCPServer(BaseServer): @@ -601,16 +605,17 @@ class ForkingMixIn: else: # Child process. # This must never return, hence os._exit()! + status = 1 try: self.finish_request(request, client_address) - self.shutdown_request(request) - os._exit(0) - except: + status = 0 + except Exception: + self.handle_error(request, client_address) + finally: try: - self.handle_error(request, client_address) self.shutdown_request(request) finally: - os._exit(1) + os._exit(status) class ThreadingMixIn: @@ -628,9 +633,9 @@ class ThreadingMixIn: """ try: self.finish_request(request, client_address) - self.shutdown_request(request) - except: + except Exception: self.handle_error(request, client_address) + finally: self.shutdown_request(request) def process_request(self, request, client_address): diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index dc23210..bff0ff4 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -58,6 +58,7 @@ if HAVE_UNIX_SOCKETS: @contextlib.contextmanager def simple_subprocess(testcase): + """Tests that a custom child process is not waited on (Issue 1540386)""" pid = os.fork() if pid == 0: # Don't raise an exception; it would be caught by the test harness. @@ -281,6 +282,97 @@ class SocketServerTest(unittest.TestCase): socketserver.StreamRequestHandler) +class ErrorHandlerTest(unittest.TestCase): + """Test that the servers pass normal exceptions from the handler to + handle_error(), and that exiting exceptions like SystemExit and + KeyboardInterrupt are not passed.""" + + def tearDown(self): + test.support.unlink(test.support.TESTFN) + + def test_sync_handled(self): + BaseErrorTestServer(ValueError) + self.check_result(handled=True) + + def test_sync_not_handled(self): + with self.assertRaises(SystemExit): + BaseErrorTestServer(SystemExit) + self.check_result(handled=False) + + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_threading_handled(self): + ThreadingErrorTestServer(ValueError) + self.check_result(handled=True) + + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_threading_not_handled(self): + ThreadingErrorTestServer(SystemExit) + self.check_result(handled=False) + + @requires_forking + def test_forking_handled(self): + ForkingErrorTestServer(ValueError) + self.check_result(handled=True) + + @requires_forking + def test_forking_not_handled(self): + ForkingErrorTestServer(SystemExit) + self.check_result(handled=False) + + def check_result(self, handled): + with open(test.support.TESTFN) as log: + expected = 'Handler called\n' + 'Error handled\n' * handled + self.assertEqual(log.read(), expected) + + +class BaseErrorTestServer(socketserver.TCPServer): + def __init__(self, exception): + self.exception = exception + super().__init__((HOST, 0), BadHandler) + with socket.create_connection(self.server_address): + pass + try: + self.handle_request() + finally: + self.server_close() + self.wait_done() + + def handle_error(self, request, client_address): + with open(test.support.TESTFN, 'a') as log: + log.write('Error handled\n') + + def wait_done(self): + pass + + +class BadHandler(socketserver.BaseRequestHandler): + def handle(self): + with open(test.support.TESTFN, 'a') as log: + log.write('Handler called\n') + raise self.server.exception('Test error') + + +class ThreadingErrorTestServer(socketserver.ThreadingMixIn, + BaseErrorTestServer): + def __init__(self, *pos, **kw): + self.done = threading.Event() + super().__init__(*pos, **kw) + + def shutdown_request(self, *pos, **kw): + super().shutdown_request(*pos, **kw) + self.done.set() + + def wait_done(self): + self.done.wait() + + +class ForkingErrorTestServer(socketserver.ForkingMixIn, BaseErrorTestServer): + def wait_done(self): + [child] = self.active_children + os.waitpid(child, 0) + self.active_children.clear() + + class MiscTestCase(unittest.TestCase): def test_all(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6c1e83f..4043e01 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -196,6 +196,12 @@ Issue #26186: Remove an invalid type check in importlib.util.LazyLoader. the connected socket) when verify_request() returns false. Patch by Aviv Palivoda. +- Issue #23430: Change the socketserver module to only catch exceptions + raised from a request handler that are derived from Exception (instead of + BaseException). Therefore SystemExit and KeyboardInterrupt no longer + trigger the handle_error() method, and will now to stop a single-threaded + server. + - Issue #25939: On Windows open the cert store readonly in ssl.enum_certificates. - Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. -- cgit v0.12 From 62bc32181901061fbadfc7e68c518871f6d8faea Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 25 Feb 2016 00:25:45 -0800 Subject: Fix out-of-date comment --- Lib/collections/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 9a26a93..e5497b3 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -842,7 +842,7 @@ class Counter(dict): ######################################################################## -### ChainMap (helper for configparser and string.Template) +### ChainMap ######################################################################## class ChainMap(MutableMapping): -- cgit v0.12 From 202803a0c00f00e62ad613fff119d7a5432f342c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 25 Feb 2016 22:34:45 -0800 Subject: c_encoding can never be NULL, which allows some code simplification --- Python/ast.c | 90 +++++++++++++++++++++++++++++------------------------------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index d19546a..c342609 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4002,55 +4002,52 @@ decode_unicode(struct compiling *c, const char *s, size_t len, const char *encod char *p; const char *end; - if (encoding == NULL) { - u = NULL; - } else { - /* check for integer overflow */ - if (len > PY_SIZE_MAX / 6) - return NULL; - /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 - "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ - u = PyBytes_FromStringAndSize((char *)NULL, len * 6); - if (u == NULL) - return NULL; - p = buf = PyBytes_AsString(u); - end = s + len; - while (s < end) { - if (*s == '\\') { - *p++ = *s++; - if (*s & 0x80) { - strcpy(p, "u005c"); - p += 5; - } + /* check for integer overflow */ + if (len > PY_SIZE_MAX / 6) + return NULL; + /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 + "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ + u = PyBytes_FromStringAndSize((char *)NULL, len * 6); + if (u == NULL) + return NULL; + p = buf = PyBytes_AsString(u); + end = s + len; + while (s < end) { + if (*s == '\\') { + *p++ = *s++; + if (*s & 0x80) { + strcpy(p, "u005c"); + p += 5; } - if (*s & 0x80) { /* XXX inefficient */ - PyObject *w; - int kind; - void *data; - Py_ssize_t len, i; - w = decode_utf8(c, &s, end); - if (w == NULL) { - Py_DECREF(u); - return NULL; - } - kind = PyUnicode_KIND(w); - data = PyUnicode_DATA(w); - len = PyUnicode_GET_LENGTH(w); - for (i = 0; i < len; i++) { - Py_UCS4 chr = PyUnicode_READ(kind, data, i); - sprintf(p, "\\U%08x", chr); - p += 10; - } - /* Should be impossible to overflow */ - assert(p - buf <= Py_SIZE(u)); - Py_DECREF(w); - } else { - *p++ = *s++; + } + if (*s & 0x80) { /* XXX inefficient */ + PyObject *w; + int kind; + void *data; + Py_ssize_t len, i; + w = decode_utf8(c, &s, end); + if (w == NULL) { + Py_DECREF(u); + return NULL; } + kind = PyUnicode_KIND(w); + data = PyUnicode_DATA(w); + len = PyUnicode_GET_LENGTH(w); + for (i = 0; i < len; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + sprintf(p, "\\U%08x", chr); + p += 10; + } + /* Should be impossible to overflow */ + assert(p - buf <= Py_SIZE(u)); + Py_DECREF(w); + } else { + *p++ = *s++; } - len = p - buf; - s = buf; } + len = p - buf; + s = buf; + v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); Py_XDECREF(u); return v; @@ -4994,8 +4991,7 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) } } } - need_encoding = (!*bytesmode && c->c_encoding != NULL && - strcmp(c->c_encoding, "utf-8") != 0); + need_encoding = !*bytesmode && strcmp(c->c_encoding, "utf-8") != 0; if (rawmode || strchr(s, '\\') == NULL) { if (need_encoding) { PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL); -- cgit v0.12 From 768921cf3360822dc252d7e295e94960afa9feee Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 25 Feb 2016 23:13:53 -0800 Subject: rewrite parsestr() so it's comprehensible; remove dead code --- Python/ast.c | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index c342609..ba6b4ba 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3995,7 +3995,7 @@ decode_utf8(struct compiling *c, const char **sPtr, const char *end) } static PyObject * -decode_unicode(struct compiling *c, const char *s, size_t len, const char *encoding) +decode_unicode_with_escapes(struct compiling *c, const char *s, size_t len) { PyObject *v, *u; char *buf; @@ -4921,7 +4921,6 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) const char *s = STR(n); int quote = Py_CHARMASK(*s); int rawmode = 0; - int need_encoding; if (Py_ISALPHA(quote)) { while (!*bytesmode || !rawmode) { if (quote == 'b' || quote == 'B') { @@ -4977,11 +4976,10 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) return NULL; } } - if (!*bytesmode && !rawmode) { - return decode_unicode(c, s, len, c->c_encoding); - } + /* Avoid invoking escape decoding routines if possible. */ + rawmode = rawmode || strchr(s, '\\') == NULL; if (*bytesmode) { - /* Disallow non-ascii characters (but not escapes) */ + /* Disallow non-ASCII characters. */ const char *ch; for (ch = s; *ch; ch++) { if (Py_CHARMASK(*ch) >= 0x80) { @@ -4990,26 +4988,16 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) return NULL; } } - } - need_encoding = !*bytesmode && strcmp(c->c_encoding, "utf-8") != 0; - if (rawmode || strchr(s, '\\') == NULL) { - if (need_encoding) { - PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL); - if (u == NULL || !*bytesmode) - return u; - v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL); - Py_DECREF(u); - return v; - } else if (*bytesmode) { + if (rawmode) return PyBytes_FromStringAndSize(s, len); - } else if (strcmp(c->c_encoding, "utf-8") == 0) { - return PyUnicode_FromStringAndSize(s, len); - } else { - return PyUnicode_DecodeLatin1(s, len, NULL); - } + else + return PyBytes_DecodeEscape(s, len, NULL, /* ignored */ 0, NULL); + } else { + if (rawmode) + return PyUnicode_DecodeUTF8Stateful(s, len, NULL, NULL); + else + return decode_unicode_with_escapes(c, s, len); } - return PyBytes_DecodeEscape(s, len, NULL, 1, - need_encoding ? c->c_encoding : NULL); } /* Accepts a STRING+ atom, and produces an expr_ty node. Run through -- cgit v0.12 From 9d66d4af06ad15226b271988cd7f5d0c681924af Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 25 Feb 2016 23:25:14 -0800 Subject: remove unused c_encoding struct member --- Python/ast.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index ba6b4ba..ad771cf 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -574,7 +574,6 @@ PyAST_Validate(mod_ty mod) /* Data structure used internally */ struct compiling { - char *c_encoding; /* source encoding */ PyArena *c_arena; /* Arena for allocating memory. */ PyObject *c_filename; /* filename */ PyObject *c_normalize; /* Normalization function from unicodedata. */ @@ -761,23 +760,11 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, c.c_arena = arena; /* borrowed reference */ c.c_filename = filename; - c.c_normalize = c.c_normalize_args = NULL; - if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) { - c.c_encoding = "utf-8"; - if (TYPE(n) == encoding_decl) { -#if 0 - ast_error(c, n, "encoding declaration in Unicode string"); - goto out; -#endif - n = CHILD(n, 0); - } - } else if (TYPE(n) == encoding_decl) { - c.c_encoding = STR(n); + c.c_normalize = NULL; + c.c_normalize_args = NULL; + + if (TYPE(n) == encoding_decl) n = CHILD(n, 0); - } else { - /* PEP 3120 */ - c.c_encoding = "utf-8"; - } k = 0; switch (TYPE(n)) { -- cgit v0.12 From fc6f2efd13a6b667e48cdf1f5cf68d34fcb0fa74 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 27 Feb 2016 02:19:22 +0100 Subject: compile.c: inline compiler_use_new_block() * Inline compiler_use_new_block() function into its only callee, compiler_enter_scope() * Remove unused NEW_BLOCK() macro --- Python/compile.c | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index ca1d865..568bdfe 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -171,7 +171,6 @@ static int compiler_addop(struct compiler *, int); static int compiler_addop_o(struct compiler *, int, PyObject *, PyObject *); static int compiler_addop_i(struct compiler *, int, Py_ssize_t); static int compiler_addop_j(struct compiler *, int, basicblock *, int); -static basicblock *compiler_use_new_block(struct compiler *); static int compiler_error(struct compiler *, const char *); static int compiler_nameop(struct compiler *, identifier, expr_context_ty); @@ -523,6 +522,7 @@ compiler_enter_scope(struct compiler *c, identifier name, int scope_type, void *key, int lineno) { struct compiler_unit *u; + basicblock *block; u = (struct compiler_unit *)PyObject_Malloc(sizeof( struct compiler_unit)); @@ -620,8 +620,11 @@ compiler_enter_scope(struct compiler *c, identifier name, c->u = u; c->c_nestlevel++; - if (compiler_use_new_block(c) == NULL) + + block = compiler_new_block(c); + if (block == NULL) return 0; + c->u->u_curblock = block; if (u->u_scope_type != COMPILER_SCOPE_MODULE) { if (!compiler_set_qualname(c)) @@ -756,16 +759,6 @@ compiler_new_block(struct compiler *c) } static basicblock * -compiler_use_new_block(struct compiler *c) -{ - basicblock *block = compiler_new_block(c); - if (block == NULL) - return NULL; - c->u->u_curblock = block; - return block; -} - -static basicblock * compiler_next_block(struct compiler *c) { basicblock *block = compiler_new_block(c); @@ -1208,22 +1201,12 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) return 1; } -/* The distinction between NEW_BLOCK and NEXT_BLOCK is subtle. (I'd - like to find better names.) NEW_BLOCK() creates a new block and sets - it as the current block. NEXT_BLOCK() also creates an implicit jump - from the current block to the new block. -*/ +/* NEXT_BLOCK() creates an implicit jump from the current block + to the new block. -/* The returns inside these macros make it impossible to decref objects - created in the local function. Local objects should use the arena. + The returns inside this macro make it impossible to decref objects + created in the local function. Local objects should use the arena. */ - - -#define NEW_BLOCK(C) { \ - if (compiler_use_new_block((C)) == NULL) \ - return 0; \ -} - #define NEXT_BLOCK(C) { \ if (compiler_next_block((C)) == NULL) \ return 0; \ -- cgit v0.12 From 2ad474ba5e8ac06b7edae3cd2e15ab93212dd01b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 1 Mar 2016 23:34:47 +0100 Subject: Update assertion in compiler_addop_i() In practice, bytecode instruction arguments are unsigned. Update the assertion to make it more explicit that argument must be greater or equal than 0. Rewrite also the comment. --- Python/compile.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 568bdfe..e86b293 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1163,10 +1163,14 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg) struct instr *i; int off; - /* Integer arguments are limit to 16-bit. There is an extension for 32-bit - integer arguments. */ - assert((-2147483647-1) <= oparg); - assert(oparg <= 2147483647); + /* oparg value is unsigned, but a signed C int is usually used to store + it in the C code (like Python/ceval.c). + + Limit to 32-bit signed C int (rather than INT_MAX) for portability. + + The argument of a concrete bytecode instruction is limited to 16-bit. + EXTENDED_ARG is used for 32-bit arguments. */ + assert(0 <= oparg && oparg <= 2147483647); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) -- cgit v0.12 From 589106b20683b95aac1fe29cf9db5aed632f1cf3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 2 Mar 2016 00:06:21 -0800 Subject: Put block length computations in a more logical order. --- Modules/_collectionsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 0b7a88f..309dfd2 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -589,8 +589,8 @@ deque_clear(dequeobject *deque) /* Now the old size, leftblock, and leftindex are disconnected from the empty deque and we can use them to decref the pointers. */ - itemptr = &leftblock->data[leftindex]; m = (BLOCKLEN - leftindex > n) ? n : BLOCKLEN - leftindex; + itemptr = &leftblock->data[leftindex]; limit = &leftblock->data[leftindex + m]; n -= m; while (1) { @@ -600,8 +600,8 @@ deque_clear(dequeobject *deque) CHECK_NOT_END(leftblock->rightlink); prevblock = leftblock; leftblock = leftblock->rightlink; - itemptr = leftblock->data; m = (n > BLOCKLEN) ? BLOCKLEN : n; + itemptr = leftblock->data; limit = &leftblock->data[m]; n -= m; freeblock(prevblock); -- cgit v0.12 From 6f86a3308a089e80be2562316dec142d65c85451 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 2 Mar 2016 00:30:58 -0800 Subject: Factor-out common subexpression. --- Modules/_collectionsmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 309dfd2..7dcd7e7 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -572,9 +572,9 @@ deque_clear(dequeobject *deque) } /* Remember the old size, leftblock, and leftindex */ + n = Py_SIZE(deque); leftblock = deque->leftblock; leftindex = deque->leftindex; - n = Py_SIZE(deque); /* Set the deque to be empty using the newly allocated block */ MARK_END(b->leftlink); @@ -591,7 +591,7 @@ deque_clear(dequeobject *deque) */ m = (BLOCKLEN - leftindex > n) ? n : BLOCKLEN - leftindex; itemptr = &leftblock->data[leftindex]; - limit = &leftblock->data[leftindex + m]; + limit = itemptr + m; n -= m; while (1) { if (itemptr == limit) { @@ -602,7 +602,7 @@ deque_clear(dequeobject *deque) leftblock = leftblock->rightlink; m = (n > BLOCKLEN) ? BLOCKLEN : n; itemptr = leftblock->data; - limit = &leftblock->data[m]; + limit = itemptr + m; n -= m; freeblock(prevblock); } -- cgit v0.12 From 3ebaea005dc6ab4a06fb585c23321b9bb0398b90 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Mar 2016 10:43:45 -0500 Subject: Sync selectors.py with upstream asyncio --- Lib/selectors.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/selectors.py b/Lib/selectors.py index ecd8632..d8769e3 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -49,11 +49,12 @@ SelectorKey.__doc__ = """SelectorKey(fileobj, fd, events, data) Object used to associate a file object to its backing file descriptor, selected event mask, and attached data. """ -SelectorKey.fileobj.__doc__ = 'File object registered.' -SelectorKey.fd.__doc__ = 'Underlying file descriptor.' -SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' -SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. -For example, this could be used to store a per-client session ID.''') +if sys.version_info >= (3, 5): + SelectorKey.fileobj.__doc__ = 'File object registered.' + SelectorKey.fd.__doc__ = 'Underlying file descriptor.' + SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' + SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. + For example, this could be used to store a per-client session ID.''') class _SelectorMapping(Mapping): """Mapping of file objects to selector keys.""" -- cgit v0.12 From 6282e656e994a8722f39db6aea99aad2a826dee6 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 2 Mar 2016 19:30:18 +0200 Subject: Issue #26335: Make mmap.write() return the number of bytes written like other write methods. Patch by Jakub Stasiak. --- Doc/library/mmap.rst | 9 +++++++-- Lib/test/test_mmap.py | 8 ++++++++ Misc/NEWS | 3 +++ Modules/mmapmodule.c | 5 +++-- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 33baf2b..fb24728 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -263,13 +263,18 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. method:: write(bytes) Write the bytes in *bytes* into memory at the current position of the - file pointer; the file position is updated to point after the bytes that - were written. If the mmap was created with :const:`ACCESS_READ`, then + file pointer and return the number of bytes written (never less than + ``len(bytes)``, since if the write fails, a :exc:`ValueError` will be + raised). The file position is updated to point after the bytes that + were written. If the mmap was created with :const:`ACCESS_READ`, then writing to it will raise a :exc:`TypeError` exception. .. versionchanged:: 3.5 Writable :term:`bytes-like object` is now accepted. + .. versionchanged:: 3.6 + The number of bytes written is now returned. + .. method:: write_byte(byte) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 0f25742..be2e65a 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -713,6 +713,14 @@ class MmapTests(unittest.TestCase): gc_collect() self.assertIs(wr(), None) + def test_write_returning_the_number_of_bytes_written(self): + mm = mmap.mmap(-1, 16) + self.assertEqual(mm.write(b""), 0) + self.assertEqual(mm.write(b"x"), 1) + self.assertEqual(mm.write(b"yz"), 2) + self.assertEqual(mm.write(b"python"), 6) + + class LargeMmapTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS b/Misc/NEWS index 8770705..eea0b33 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -198,6 +198,9 @@ Core and Builtins Library ------- +- Issue #26335: Make mmap.write() return the number of bytes written like + other write methods. Patch by Jakub Stasiak. + - Issue #26457: Fixed the subnets() methods in IP network classes for the case when resulting prefix length is equal to maximal prefix length. Based on patch by Xiang Zhang. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index bb98a99..50cec24 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -389,6 +389,7 @@ mmap_write_method(mmap_object *self, PyObject *args) { Py_buffer data; + PyObject *result; CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "y*:write", &data)) @@ -406,9 +407,9 @@ mmap_write_method(mmap_object *self, } memcpy(self->data + self->pos, data.buf, data.len); self->pos = self->pos + data.len; + result = PyLong_FromSsize_t(data.len); PyBuffer_Release(&data); - Py_INCREF(Py_None); - return Py_None; + return result; } static PyObject * -- cgit v0.12 From d79d5b1a50347b06397e160440e6bb9dd5a8ed43 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 4 Mar 2016 09:55:07 -0800 Subject: More logicial order. Move space saving step to just before it is used. --- Modules/_collectionsmodule.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 7dcd7e7..19a86d1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -386,6 +386,13 @@ deque_extend(dequeobject *deque, PyObject *iterable) return result; } + it = PyObject_GetIter(iterable); + if (it == NULL) + return NULL; + + if (maxlen == 0) + return consume_iterator(it); + /* Space saving heuristic. Start filling from the left */ if (Py_SIZE(deque) == 0) { assert(deque->leftblock == deque->rightblock); @@ -394,13 +401,6 @@ deque_extend(dequeobject *deque, PyObject *iterable) deque->rightindex = 0; } - it = PyObject_GetIter(iterable); - if (it == NULL) - return NULL; - - if (maxlen == 0) - return consume_iterator(it); - iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque_append_internal(deque, item, maxlen) < 0) { @@ -433,6 +433,13 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) return result; } + it = PyObject_GetIter(iterable); + if (it == NULL) + return NULL; + + if (maxlen == 0) + return consume_iterator(it); + /* Space saving heuristic. Start filling from the right */ if (Py_SIZE(deque) == 0) { assert(deque->leftblock == deque->rightblock); @@ -441,13 +448,6 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) deque->rightindex = BLOCKLEN - 2; } - it = PyObject_GetIter(iterable); - if (it == NULL) - return NULL; - - if (maxlen == 0) - return consume_iterator(it); - iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque_appendleft_internal(deque, item, maxlen) < 0) { -- cgit v0.12 From 841b930a41be90c6c05fb4bab8392f50e01d86b7 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 5 Mar 2016 14:05:45 +0200 Subject: Issue #17940: Remove redundant code from _Section.format_help() Output of func(*args) stored in the next line: item_help = join([func(*args) for func, args in self.items]) _Section.items only used by HelpFormatter._add_item() and it looks like it doesn't have any side effects. Patch by Yogesh Chaudhari. --- Lib/argparse.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index cc53841..209b4e9 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -210,8 +210,6 @@ class HelpFormatter(object): if self.parent is not None: self.formatter._indent() join = self.formatter._join_parts - for func, args in self.items: - func(*args) item_help = join([func(*args) for func, args in self.items]) if self.parent is not None: self.formatter._dedent() -- cgit v0.12 From a0d416f0d186b5ee6ea6a898418871157a55b4f6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Mar 2016 08:55:21 +0200 Subject: Issue #26482: Allowed pickling recursive dequeues. --- Lib/test/test_deque.py | 68 +++++++++++++++++++++++--------------------- Misc/NEWS | 2 ++ Modules/_collectionsmodule.c | 36 +++++++++++------------ 3 files changed, 56 insertions(+), 50 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index e2c6b6c..634d71f 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -622,20 +622,22 @@ class TestBasic(unittest.TestCase): self.assertEqual(list(d), list(e)) def test_pickle(self): - d = deque(range(200)) - for i in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(d, i) - e = pickle.loads(s) - self.assertNotEqual(id(d), id(e)) - self.assertEqual(list(d), list(e)) - -## def test_pickle_recursive(self): -## d = deque('abc') -## d.append(d) -## for i in range(pickle.HIGHEST_PROTOCOL + 1): -## e = pickle.loads(pickle.dumps(d, i)) -## self.assertNotEqual(id(d), id(e)) -## self.assertEqual(id(e), id(e[-1])) + for d in deque(range(200)), deque(range(200), 100): + for i in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(d, i) + e = pickle.loads(s) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(list(e), list(d)) + self.assertEqual(e.maxlen, d.maxlen) + + def test_pickle_recursive(self): + for d in deque('abc'), deque('abc', 3): + d.append(d) + for i in range(pickle.HIGHEST_PROTOCOL + 1): + e = pickle.loads(pickle.dumps(d, i)) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(id(e[-1]), id(e)) + self.assertEqual(e.maxlen, d.maxlen) def test_iterator_pickle(self): data = deque(range(200)) @@ -827,24 +829,26 @@ class TestSubclass(unittest.TestCase): self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) -## def test_pickle(self): -## d = Deque('abc') -## d.append(d) -## -## e = pickle.loads(pickle.dumps(d)) -## self.assertNotEqual(id(d), id(e)) -## self.assertEqual(type(d), type(e)) -## dd = d.pop() -## ee = e.pop() -## self.assertEqual(id(e), id(ee)) -## self.assertEqual(d, e) -## -## d.x = d -## e = pickle.loads(pickle.dumps(d)) -## self.assertEqual(id(e), id(e.x)) -## -## d = DequeWithBadIter('abc') -## self.assertRaises(TypeError, pickle.dumps, d) + def test_pickle_recursive(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for d in Deque('abc'), Deque('abc', 3): + d.append(d) + + e = pickle.loads(pickle.dumps(d, proto)) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(type(e), type(d)) + self.assertEqual(e.maxlen, d.maxlen) + dd = d.pop() + ee = e.pop() + self.assertEqual(id(ee), id(e)) + self.assertEqual(e, d) + + d.x = d + e = pickle.loads(pickle.dumps(d, proto)) + self.assertEqual(id(e.x), id(e)) + + for d in DequeWithBadIter('abc'), DequeWithBadIter('abc', 2): + self.assertRaises(TypeError, pickle.dumps, d, proto) def test_weakref(self): d = deque('gallahad') diff --git a/Misc/NEWS b/Misc/NEWS index 3eee944..4ce8bf6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,8 @@ Core and Builtins Library ------- +- Issue #26482: Allowed pickling recursive dequeues. + - Issue #26335: Make mmap.write() return the number of bytes written like other write methods. Patch by Jakub Stasiak. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 19a86d1..d57f1ba 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1296,31 +1296,31 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) static PyObject * deque_reduce(dequeobject *deque) { - PyObject *dict, *result, *aslist; + PyObject *dict, *it; _Py_IDENTIFIER(__dict__); dict = _PyObject_GetAttrId((PyObject *)deque, &PyId___dict__); - if (dict == NULL) + if (dict == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } PyErr_Clear(); - aslist = PySequence_List((PyObject *)deque); - if (aslist == NULL) { - Py_XDECREF(dict); + dict = Py_None; + Py_INCREF(dict); + } + + it = PyObject_GetIter((PyObject *)deque); + if (it == NULL) { + Py_DECREF(dict); return NULL; } - if (dict == NULL) { - if (deque->maxlen < 0) - result = Py_BuildValue("O(O)", Py_TYPE(deque), aslist); - else - result = Py_BuildValue("O(On)", Py_TYPE(deque), aslist, deque->maxlen); - } else { - if (deque->maxlen < 0) - result = Py_BuildValue("O(OO)O", Py_TYPE(deque), aslist, Py_None, dict); - else - result = Py_BuildValue("O(On)O", Py_TYPE(deque), aslist, deque->maxlen, dict); + + if (deque->maxlen < 0) { + return Py_BuildValue("O()NN", Py_TYPE(deque), dict, it); + } + else { + return Py_BuildValue("O(()n)NN", Py_TYPE(deque), deque->maxlen, dict, it); } - Py_XDECREF(dict); - Py_DECREF(aslist); - return result; } PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); -- cgit v0.12 From 818e18dd94611a2f0f4b0015661488ef0cd867dc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Mar 2016 14:56:57 +0200 Subject: Issue #26167: Minimized overhead in copy.copy() and copy.deepcopy(). Optimized copying and deepcopying bytearrays, NotImplemented, slices, short lists, tuples, dicts, sets. --- Lib/copy.py | 115 +++++++++++++++++++++----------------------------- Lib/test/test_copy.py | 55 +++++++++++++++++++++--- Misc/NEWS | 4 ++ 3 files changed, 102 insertions(+), 72 deletions(-) diff --git a/Lib/copy.py b/Lib/copy.py index 972b94a..f86040a 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -51,7 +51,6 @@ __getstate__() and __setstate__(). See the documentation for module import types import weakref from copyreg import dispatch_table -import builtins class Error(Exception): pass @@ -102,37 +101,33 @@ def copy(x): else: raise Error("un(shallow)copyable object of type %s" % cls) - return _reconstruct(x, rv, 0) + if isinstance(rv, str): + return x + return _reconstruct(x, None, *rv) _copy_dispatch = d = {} def _copy_immutable(x): return x -for t in (type(None), int, float, bool, str, tuple, - bytes, frozenset, type, range, - types.BuiltinFunctionType, type(Ellipsis), +for t in (type(None), int, float, bool, complex, str, tuple, + bytes, frozenset, type, range, slice, + types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented), types.FunctionType, weakref.ref): d[t] = _copy_immutable t = getattr(types, "CodeType", None) if t is not None: d[t] = _copy_immutable -for name in ("complex", "unicode"): - t = getattr(builtins, name, None) - if t is not None: - d[t] = _copy_immutable - -def _copy_with_constructor(x): - return type(x)(x) -for t in (list, dict, set): - d[t] = _copy_with_constructor - -def _copy_with_copy_method(x): - return x.copy() + +d[list] = list.copy +d[dict] = dict.copy +d[set] = set.copy +d[bytearray] = bytearray.copy + if PyStringMap is not None: - d[PyStringMap] = _copy_with_copy_method + d[PyStringMap] = PyStringMap.copy -del d +del d, t def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. @@ -179,7 +174,10 @@ def deepcopy(x, memo=None, _nil=[]): else: raise Error( "un(deep)copyable object of type %s" % cls) - y = _reconstruct(x, rv, 1, memo) + if isinstance(rv, str): + y = x + else: + y = _reconstruct(x, memo, *rv) # If is its own copy, don't memoize. if y is not x: @@ -193,13 +191,11 @@ def _deepcopy_atomic(x, memo): return x d[type(None)] = _deepcopy_atomic d[type(Ellipsis)] = _deepcopy_atomic +d[type(NotImplemented)] = _deepcopy_atomic d[int] = _deepcopy_atomic d[float] = _deepcopy_atomic d[bool] = _deepcopy_atomic -try: - d[complex] = _deepcopy_atomic -except NameError: - pass +d[complex] = _deepcopy_atomic d[bytes] = _deepcopy_atomic d[str] = _deepcopy_atomic try: @@ -211,15 +207,16 @@ d[types.BuiltinFunctionType] = _deepcopy_atomic d[types.FunctionType] = _deepcopy_atomic d[weakref.ref] = _deepcopy_atomic -def _deepcopy_list(x, memo): +def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] memo[id(x)] = y + append = y.append for a in x: - y.append(deepcopy(a, memo)) + append(deepcopy(a, memo)) return y d[list] = _deepcopy_list -def _deepcopy_tuple(x, memo): +def _deepcopy_tuple(x, memo, deepcopy=deepcopy): y = [deepcopy(a, memo) for a in x] # We're not going to put the tuple in the memo, but it's still important we # check for it, in case the tuple contains recursive mutable structures. @@ -236,7 +233,7 @@ def _deepcopy_tuple(x, memo): return y d[tuple] = _deepcopy_tuple -def _deepcopy_dict(x, memo): +def _deepcopy_dict(x, memo, deepcopy=deepcopy): y = {} memo[id(x)] = y for key, value in x.items(): @@ -248,7 +245,9 @@ if PyStringMap is not None: def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) -_deepcopy_dispatch[types.MethodType] = _deepcopy_method +d[types.MethodType] = _deepcopy_method + +del d def _keep_alive(x, memo): """Keeps a reference to the object x in the memo. @@ -266,31 +265,15 @@ def _keep_alive(x, memo): # aha, this is the first one :-) memo[id(memo)]=[x] -def _reconstruct(x, info, deep, memo=None): - if isinstance(info, str): - return x - assert isinstance(info, tuple) - if memo is None: - memo = {} - n = len(info) - assert n in (2, 3, 4, 5) - callable, args = info[:2] - if n > 2: - state = info[2] - else: - state = None - if n > 3: - listiter = info[3] - else: - listiter = None - if n > 4: - dictiter = info[4] - else: - dictiter = None +def _reconstruct(x, memo, func, args, + state=None, listiter=None, dictiter=None, + deepcopy=deepcopy): + deep = memo is not None + if deep and args: + args = (deepcopy(arg, memo) for arg in args) + y = func(*args) if deep: - args = deepcopy(args, memo) - y = callable(*args) - memo[id(x)] = y + memo[id(x)] = y if state is not None: if deep: @@ -309,22 +292,22 @@ def _reconstruct(x, info, deep, memo=None): setattr(y, key, value) if listiter is not None: - for item in listiter: - if deep: + if deep: + for item in listiter: item = deepcopy(item, memo) - y.append(item) + y.append(item) + else: + for item in listiter: + y.append(item) if dictiter is not None: - for key, value in dictiter: - if deep: + if deep: + for key, value in dictiter: key = deepcopy(key, memo) value = deepcopy(value, memo) - y[key] = value + y[key] = value + else: + for key, value in dictiter: + y[key] = value return y -del d - -del types - -# Helper for instance creation without calling __init__ -class _EmptyClass: - pass +del types, weakref, PyStringMap diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 7912c7c..45a6920 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -95,24 +95,67 @@ class TestCopy(unittest.TestCase): pass class WithMetaclass(metaclass=abc.ABCMeta): pass - tests = [None, 42, 2**100, 3.14, True, False, 1j, + tests = [None, ..., NotImplemented, + 42, 2**100, 3.14, True, False, 1j, "hello", "hello\u1234", f.__code__, - b"world", bytes(range(256)), - NewStyle, range(10), Classic, max, WithMetaclass] + b"world", bytes(range(256)), range(10), slice(1, 10, 2), + NewStyle, Classic, max, WithMetaclass] for x in tests: self.assertIs(copy.copy(x), x) def test_copy_list(self): x = [1, 2, 3] - self.assertEqual(copy.copy(x), x) + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = [] + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) def test_copy_tuple(self): x = (1, 2, 3) - self.assertEqual(copy.copy(x), x) + self.assertIs(copy.copy(x), x) + x = () + self.assertIs(copy.copy(x), x) + x = (1, 2, 3, []) + self.assertIs(copy.copy(x), x) def test_copy_dict(self): x = {"foo": 1, "bar": 2} - self.assertEqual(copy.copy(x), x) + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = {} + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + + def test_copy_set(self): + x = {1, 2, 3} + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = set() + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + + def test_copy_frozenset(self): + x = frozenset({1, 2, 3}) + self.assertIs(copy.copy(x), x) + x = frozenset() + self.assertIs(copy.copy(x), x) + + def test_copy_bytearray(self): + x = bytearray(b'abc') + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = bytearray() + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) def test_copy_inst_vanilla(self): class C: diff --git a/Misc/NEWS b/Misc/NEWS index 3e10fe4..026db17 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,10 @@ Core and Builtins Library ------- +- Issue #26167: Minimized overhead in copy.copy() and copy.deepcopy(). + Optimized copying and deepcopying bytearrays, NotImplemented, slices, + short lists, tuples, dicts, sets. + - Issue #25718: Fixed pickling and copying the accumulate() iterator with total is None. -- cgit v0.12 From a2998a63c8951ab8254b96860fe64dcaa215eb1f Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 6 Mar 2016 14:58:43 -0500 Subject: Closes #19475: Added timespec to the datetime.isoformat() method. Added an optional argument timespec to the datetime isoformat() method to choose the precision of the time component. Original patch by Alessandro Cucci. --- Doc/library/datetime.rst | 70 ++++++++++++++++++++++++-- Lib/datetime.py | 51 +++++++++++++------ Lib/test/datetimetester.py | 48 +++++++++++++++--- Misc/ACKS | 1 + Misc/NEWS | 3 ++ Modules/_datetimemodule.c | 123 ++++++++++++++++++++++++++++++++++----------- 6 files changed, 244 insertions(+), 52 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index a822842..d73e97b 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1134,7 +1134,7 @@ Instance methods: ``self.date().isocalendar()``. -.. method:: datetime.isoformat(sep='T') +.. method:: datetime.isoformat(sep='T', timespec='auto') Return a string representing the date and time in ISO 8601 format, YYYY-MM-DDTHH:MM:SS.mmmmmm or, if :attr:`microsecond` is 0, @@ -1155,6 +1155,37 @@ Instance methods: >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' + The optional argument *timespec* specifies the number of additional + components of the time to include (the default is ``'auto'``). + It can be one of the following: + + - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, + same as ``'microseconds'`` otherwise. + - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` + in HH:MM:SS format. + - ``'milliseconds'``: Include full time, but truncate fractional second + part to milliseconds. HH:MM:SS.sss format. + - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. + + .. note:: + + Excluded time components are truncated, not rounded. + + :exc:`ValueError` will be raised on an invalid *timespec* argument. + + + >>> from datetime import datetime + >>> datetime.now().isoformat(timespec='minutes') + '2002-12-25T00:00' + >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) + >>> dt.isoformat(timespec='microseconds') + '2015-01-01T12:30:59.000000' + + .. versionadded:: 3.6 + Added the *timespec* argument. + .. method:: datetime.__str__() @@ -1404,13 +1435,46 @@ Instance methods: aware :class:`.time`, without conversion of the time data. -.. method:: time.isoformat() +.. method:: time.isoformat(timespec='auto') Return a string representing the time in ISO 8601 format, HH:MM:SS.mmmmmm or, if - self.microsecond is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a + :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a 6-character string is appended, giving the UTC offset in (signed) hours and minutes: HH:MM:SS.mmmmmm+HH:MM or, if self.microsecond is 0, HH:MM:SS+HH:MM + The optional argument *timespec* specifies the number of additional + components of the time to include (the default is ``'auto'``). + It can be one of the following: + + - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, + same as ``'microseconds'`` otherwise. + - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` + in HH:MM:SS format. + - ``'milliseconds'``: Include full time, but truncate fractional second + part to milliseconds. HH:MM:SS.sss format. + - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. + + .. note:: + + Excluded time components are truncated, not rounded. + + :exc:`ValueError` will be raised on an invalid *timespec* argument. + + + >>> from datetime import time + >>> time(hours=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') + '12:34' + >>> dt = time(hours=12, minute=34, second=56, microsecond=0) + >>> dt.isoformat(timespec='microseconds') + '12:34:56.000000' + >>> dt.isoformat(timespec='auto') + '12:34:56' + + .. versionadded:: 3.6 + Added the *timespec* argument. + .. method:: time.__str__() diff --git a/Lib/datetime.py b/Lib/datetime.py index 3206923..d4b7fc4 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -152,12 +152,26 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): dnum = _days_before_month(y, m) + d return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) -def _format_time(hh, mm, ss, us): - # Skip trailing microseconds when us==0. - result = "%02d:%02d:%02d" % (hh, mm, ss) - if us: - result += ".%06d" % us - return result +def _format_time(hh, mm, ss, us, timespec='auto'): + specs = { + 'hours': '{:02d}', + 'minutes': '{:02d}:{:02d}', + 'seconds': '{:02d}:{:02d}:{:02d}', + 'milliseconds': '{:02d}:{:02d}:{:02d}.{:03d}', + 'microseconds': '{:02d}:{:02d}:{:02d}.{:06d}' + } + + if timespec == 'auto': + # Skip trailing microseconds when us==0. + timespec = 'microseconds' if us else 'seconds' + elif timespec == 'milliseconds': + us //= 1000 + try: + fmt = specs[timespec] + except KeyError: + raise ValueError('Unknown timespec value') + else: + return fmt.format(hh, mm, ss, us) # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): @@ -1194,14 +1208,17 @@ class time: s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" return s - def isoformat(self): + def isoformat(self, timespec='auto'): """Return the time formatted according to ISO. - This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if - self.microsecond == 0. + The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional + part is omitted if self.microsecond == 0. + + The optional argument timespec specifies the number of additional + terms of the time to include. """ s = _format_time(self._hour, self._minute, self._second, - self._microsecond) + self._microsecond, timespec) tz = self._tzstr() if tz: s += tz @@ -1550,21 +1567,25 @@ class datetime(date): self._hour, self._minute, self._second, self._year) - def isoformat(self, sep='T'): + def isoformat(self, sep='T', timespec='auto'): """Return the time formatted according to ISO. - This is 'YYYY-MM-DD HH:MM:SS.mmmmmm', or 'YYYY-MM-DD HH:MM:SS' if - self.microsecond == 0. + The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. + By default, the fractional part is omitted if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving - 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM' or 'YYYY-MM-DD HH:MM:SS+HH:MM'. + giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. + + The optional argument timespec specifies the number of additional + terms of the time to include. """ s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + _format_time(self._hour, self._minute, self._second, - self._microsecond)) + self._microsecond, timespec)) + off = self.utcoffset() if off is not None: if off.days < 0: diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 68c18bd..b49942a 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1556,13 +1556,32 @@ class TestDateTime(TestDate): self.assertEqual(dt, dt2) def test_isoformat(self): - t = self.theclass(2, 3, 2, 4, 5, 1, 123) - self.assertEqual(t.isoformat(), "0002-03-02T04:05:01.000123") - self.assertEqual(t.isoformat('T'), "0002-03-02T04:05:01.000123") - self.assertEqual(t.isoformat(' '), "0002-03-02 04:05:01.000123") - self.assertEqual(t.isoformat('\x00'), "0002-03-02\x0004:05:01.000123") + t = self.theclass(1, 2, 3, 4, 5, 1, 123) + self.assertEqual(t.isoformat(), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat('T'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(' '), "0001-02-03 04:05:01.000123") + self.assertEqual(t.isoformat('\x00'), "0001-02-03\x0004:05:01.000123") + self.assertEqual(t.isoformat(timespec='hours'), "0001-02-03T04") + self.assertEqual(t.isoformat(timespec='minutes'), "0001-02-03T04:05") + self.assertEqual(t.isoformat(timespec='seconds'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(sep=' ', timespec='minutes'), "0001-02-03 04:05") + self.assertRaises(ValueError, t.isoformat, timespec='foo') # str is ISO format with the separator forced to a blank. - self.assertEqual(str(t), "0002-03-02 04:05:01.000123") + self.assertEqual(str(t), "0001-02-03 04:05:01.000123") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500, tzinfo=timezone.utc) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999+00:00") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999") + + t = self.theclass(1, 2, 3, 4, 5, 1) + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000000") t = self.theclass(2, 3, 2) self.assertEqual(t.isoformat(), "0002-03-02T00:00:00") @@ -2322,6 +2341,23 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): self.assertEqual(t.isoformat(), "00:00:00.100000") self.assertEqual(t.isoformat(), str(t)) + t = self.theclass(hour=12, minute=34, second=56, microsecond=123456) + self.assertEqual(t.isoformat(timespec='hours'), "12") + self.assertEqual(t.isoformat(timespec='minutes'), "12:34") + self.assertEqual(t.isoformat(timespec='seconds'), "12:34:56") + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.123") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.123456") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56.123456") + self.assertRaises(ValueError, t.isoformat, timespec='monkey') + + t = self.theclass(hour=12, minute=34, second=56, microsecond=999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.999") + + t = self.theclass(hour=12, minute=34, second=56, microsecond=0) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.000000") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56") + def test_1653736(self): # verify it doesn't accept extra keyword arguments t = self.theclass(second=1) diff --git a/Misc/ACKS b/Misc/ACKS index 5e6fee7..adc6848 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -309,6 +309,7 @@ Laura Creighton Simon Cross Felipe Cruz Drew Csillag +Alessandro Cucci Joaquin Cuenca Abela John Cugini Tom Culliton diff --git a/Misc/NEWS b/Misc/NEWS index 9c94db3..d90422d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,9 @@ Core and Builtins Library ------- +- Issue #19475: Added an optional argument timespec to the datetime + isoformat() method to choose the precision of the time component. + - Issue #2202: Fix UnboundLocalError in AbstractDigestAuthHandler.get_algorithm_impls. Initial patch by Mathieu Dupuy. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 4c8f3f6..347a9d9 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3608,23 +3608,56 @@ time_str(PyDateTime_Time *self) } static PyObject * -time_isoformat(PyDateTime_Time *self, PyObject *unused) +time_isoformat(PyDateTime_Time *self, PyObject *args, PyObject *kw) { char buf[100]; + char *timespec = NULL; + static char *keywords[] = {"timespec", NULL}; PyObject *result; int us = TIME_GET_MICROSECOND(self); + static char *specs[][2] = { + {"hours", "%02d"}, + {"minutes", "%02d:%02d"}, + {"seconds", "%02d:%02d:%02d"}, + {"milliseconds", "%02d:%02d:%02d.%03d"}, + {"microseconds", "%02d:%02d:%02d.%06d"}, + }; + size_t given_spec; - if (us) - result = PyUnicode_FromFormat("%02d:%02d:%02d.%06d", - TIME_GET_HOUR(self), - TIME_GET_MINUTE(self), - TIME_GET_SECOND(self), - us); - else - result = PyUnicode_FromFormat("%02d:%02d:%02d", - TIME_GET_HOUR(self), - TIME_GET_MINUTE(self), - TIME_GET_SECOND(self)); + if (!PyArg_ParseTupleAndKeywords(args, kw, "|s:isoformat", keywords, ×pec)) + return NULL; + + if (timespec == NULL || strcmp(timespec, "auto") == 0) { + if (us == 0) { + /* seconds */ + given_spec = 2; + } + else { + /* microseconds */ + given_spec = 4; + } + } + else { + for (given_spec = 0; given_spec < Py_ARRAY_LENGTH(specs); given_spec++) { + if (strcmp(timespec, specs[given_spec][0]) == 0) { + if (given_spec == 3) { + /* milliseconds */ + us = us / 1000; + } + break; + } + } + } + + if (given_spec == Py_ARRAY_LENGTH(specs)) { + PyErr_Format(PyExc_ValueError, "Unknown timespec value"); + return NULL; + } + else { + result = PyUnicode_FromFormat(specs[given_spec][1], + TIME_GET_HOUR(self), TIME_GET_MINUTE(self), + TIME_GET_SECOND(self), us); + } if (result == NULL || !HASTZINFO(self) || self->tzinfo == Py_None) return result; @@ -3845,9 +3878,10 @@ time_reduce(PyDateTime_Time *self, PyObject *arg) static PyMethodDef time_methods[] = { - {"isoformat", (PyCFunction)time_isoformat, METH_NOARGS, - PyDoc_STR("Return string in ISO 8601 format, HH:MM:SS[.mmmmmm]" - "[+HH:MM].")}, + {"isoformat", (PyCFunction)time_isoformat, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" + "[+HH:MM].\n\n" + "timespec specifies what components of the time to include.\n")}, {"strftime", (PyCFunction)time_strftime, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("format -> strftime() style string.")}, @@ -4476,25 +4510,55 @@ static PyObject * datetime_isoformat(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) { int sep = 'T'; - static char *keywords[] = {"sep", NULL}; + char *timespec = NULL; + static char *keywords[] = {"sep", "timespec", NULL}; char buffer[100]; - PyObject *result; + PyObject *result = NULL; int us = DATE_GET_MICROSECOND(self); + static char *specs[][2] = { + {"hours", "%04d-%02d-%02d%c%02d"}, + {"minutes", "%04d-%02d-%02d%c%02d:%02d"}, + {"seconds", "%04d-%02d-%02d%c%02d:%02d:%02d"}, + {"milliseconds", "%04d-%02d-%02d%c%02d:%02d:%02d.%03d"}, + {"microseconds", "%04d-%02d-%02d%c%02d:%02d:%02d.%06d"}, + }; + size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|C:isoformat", keywords, &sep)) + if (!PyArg_ParseTupleAndKeywords(args, kw, "|Cs:isoformat", keywords, &sep, ×pec)) return NULL; - if (us) - result = PyUnicode_FromFormat("%04d-%02d-%02d%c%02d:%02d:%02d.%06d", + + if (timespec == NULL || strcmp(timespec, "auto") == 0) { + if (us == 0) { + /* seconds */ + given_spec = 2; + } + else { + /* microseconds */ + given_spec = 4; + } + } + else { + for (given_spec = 0; given_spec < Py_ARRAY_LENGTH(specs); given_spec++) { + if (strcmp(timespec, specs[given_spec][0]) == 0) { + if (given_spec == 3) { + us = us / 1000; + } + break; + } + } + } + + if (given_spec == Py_ARRAY_LENGTH(specs)) { + PyErr_Format(PyExc_ValueError, "Unknown timespec value"); + return NULL; + } + else { + result = PyUnicode_FromFormat(specs[given_spec][1], GET_YEAR(self), GET_MONTH(self), GET_DAY(self), (int)sep, DATE_GET_HOUR(self), DATE_GET_MINUTE(self), DATE_GET_SECOND(self), us); - else - result = PyUnicode_FromFormat("%04d-%02d-%02d%c%02d:%02d:%02d", - GET_YEAR(self), GET_MONTH(self), - GET_DAY(self), (int)sep, - DATE_GET_HOUR(self), DATE_GET_MINUTE(self), - DATE_GET_SECOND(self)); + } if (!result || !HASTZINFO(self)) return result; @@ -5028,9 +5092,12 @@ static PyMethodDef datetime_methods[] = { {"isoformat", (PyCFunction)datetime_isoformat, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("[sep] -> string in ISO 8601 format, " - "YYYY-MM-DDTHH:MM:SS[.mmmmmm][+HH:MM].\n\n" + "YYYY-MM-DDT[HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\n" "sep is used to separate the year from the time, and " - "defaults to 'T'.")}, + "defaults to 'T'.\n" + "timespec specifies what components of the time to include" + " (allowed values are 'auto', 'hours', 'minutes', 'seconds'," + " 'milliseconds', and 'microseconds').\n")}, {"utcoffset", (PyCFunction)datetime_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, -- cgit v0.12 From b9f3114d42601b07c9830af8b18330bff475bcfc Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 10 Mar 2016 01:06:23 +0000 Subject: Issue #21042: Return full path in ctypes.util.find_library() on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Tamás Bence Gedai. --- Doc/library/ctypes.rst | 11 +++++++---- Lib/ctypes/test/test_find.py | 40 ++++++++++++++++++++++++---------------- Lib/ctypes/util.py | 4 ++-- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3b0c956..58e1ea3 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1252,15 +1252,15 @@ The exact functionality is system dependent. On Linux, :func:`find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, and ``objdump``) to find the library file. It -returns the filename of the library file. Here are some examples:: +returns the absolute path of the library file. Here are some examples:: >>> from ctypes.util import find_library >>> find_library("m") - 'libm.so.6' + '/lib/x86_64-linux-gnu/libm.so.6' >>> find_library("c") - 'libc.so.6' + '/lib/x86_64-linux-gnu/libc.so.6' >>> find_library("bz2") - 'libbz2.so.1.0' + '/lib/x86_64-linux-gnu/libbz2.so.1.0' >>> On OS X, :func:`find_library` tries several predefined naming schemes and paths @@ -1829,6 +1829,9 @@ Utility functions The exact functionality is system dependent. + .. versionchanged:: 3.6 + On Linux it returns an absolute path. + .. function:: find_msvcrt() :module: ctypes.util diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py index e6bc19d..1845bb0 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/ctypes/test/test_find.py @@ -9,39 +9,39 @@ from ctypes.util import find_library class Test_OpenGL_libs(unittest.TestCase): @classmethod def setUpClass(cls): - lib_gl = lib_glu = lib_gle = None + cls.lib_gl = cls.lib_glu = cls.lib_gle = None if sys.platform == "win32": - lib_gl = find_library("OpenGL32") - lib_glu = find_library("Glu32") + cls.lib_gl = find_library("OpenGL32") + cls.lib_glu = find_library("Glu32") elif sys.platform == "darwin": - lib_gl = lib_glu = find_library("OpenGL") + cls.lib_gl = cls.lib_glu = find_library("OpenGL") else: - lib_gl = find_library("GL") - lib_glu = find_library("GLU") - lib_gle = find_library("gle") + cls.lib_gl = find_library("GL") + cls.lib_glu = find_library("GLU") + cls.lib_gle = find_library("gle") ## print, for debugging if test.support.verbose: print("OpenGL libraries:") - for item in (("GL", lib_gl), - ("GLU", lib_glu), - ("gle", lib_gle)): + for item in (("GL", cls.lib_gl), + ("GLU", cls.lib_glu), + ("gle", cls.lib_gle)): print("\t", item) cls.gl = cls.glu = cls.gle = None - if lib_gl: + if cls.lib_gl: try: - cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) + cls.gl = CDLL(cls.lib_gl, mode=RTLD_GLOBAL) except OSError: pass - if lib_glu: + if cls.lib_glu: try: - cls.glu = CDLL(lib_glu, RTLD_GLOBAL) + cls.glu = CDLL(cls.lib_glu, RTLD_GLOBAL) except OSError: pass - if lib_gle: + if cls.lib_gle: try: - cls.gle = CDLL(lib_gle) + cls.gle = CDLL(cls.lib_gle) except OSError: pass @@ -64,6 +64,14 @@ class Test_OpenGL_libs(unittest.TestCase): self.skipTest('lib_gle not available') self.gle.gleGetJoinStyle + def test_abspath(self): + if self.lib_gl: + self.assertTrue(os.path.isabs(self.lib_gl)) + if self.lib_glu: + self.assertTrue(os.path.isabs(self.lib_glu)) + if self.lib_gle: + self.assertTrue(os.path.isabs(self.lib_gle)) + # On platforms where the default shared library suffix is '.so', # at least some libraries can be loaded as attributes of the cdll # object, since ctypes now tries loading the lib again diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 9e74ccd..d8e3bfa 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -221,8 +221,8 @@ elif os.name == "posix": abi_type = mach_map.get(machine, 'libc6') # XXX assuming GLIBC's ldconfig (with option -p) - regex = os.fsencode( - '\s+(lib%s\.[^\s]+)\s+$%s' % (re.escape(name), abi_type)) + regex = r'lib%s\.[^\s]+\s\(%s(?:,\s.*)?$\s=>\s(.*)' + regex = os.fsencode(regex % (re.escape(name), abi_type)) try: with subprocess.Popen(['/sbin/ldconfig', '-p'], stdin=subprocess.DEVNULL, diff --git a/Misc/ACKS b/Misc/ACKS index 6d8fcf8..0a67f39 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -487,6 +487,7 @@ Matthieu Gautier Stephen M. Gava Xavier de Gaye Harry Henry Gebel +Tamás Bence Gedai Marius Gedminas Jan-Philip Gehrcke Thomas Gellekum diff --git a/Misc/NEWS b/Misc/NEWS index 205891b..d678a69 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,9 @@ Core and Builtins Library ------- +- Issue #21042: Make ctypes.util.find_library() return the full path on + Linux, similar to other platforms. Patch by Tamás Bence Gedai. + - Issue #15068: Got rid of excessive buffering in fileinput. The bufsize parameter is now deprecated and ignored. -- cgit v0.12 From 2c2a4e63d794eb55e9163322ea11b9765e9e0db5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 11 Mar 2016 22:17:48 +0100 Subject: Add Mock.assert_called() Issue #26323: Add assert_called() and assert_called_once() methods to unittest.mock.Mock. --- Doc/library/unittest.mock.rst | 28 ++++++++++++++++++++++++++++ Doc/whatsnew/3.6.rst | 12 ++++++++++++ Lib/unittest/mock.py | 18 ++++++++++++++++++ Lib/unittest/test/testmock/testmock.py | 21 +++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 6 files changed, 83 insertions(+) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 9a51194..c4dc4ed 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -259,6 +259,34 @@ the *new_callable* argument to :func:`patch`. used to set attributes on the mock after it is created. See the :meth:`configure_mock` method for details. + .. method:: assert_called(*args, **kwargs) + + Assert that the mock was called at least once. + + >>> mock = Mock() + >>> mock.method() + + >>> mock.method.assert_called() + + .. versionadded:: 3.6 + + .. method:: assert_called_once(*args, **kwargs) + + Assert that the mock was called exactly once. + + >>> mock = Mock() + >>> mock.method() + + >>> mock.method.assert_called_once() + >>> mock.method() + + >>> mock.method.assert_called_once() + Traceback (most recent call last): + ... + AssertionError: Expected 'method' to have been called once. Called 2 times. + + .. versionadded:: 3.6 + .. method:: assert_called_with(*args, **kwargs) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 5c02b7d..3afe2d4 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -161,6 +161,18 @@ telnetlib Stéphane Wirtel in :issue:`25485`). +unittest.mock +------------- + +The :class:`~unittest.mock.Mock` class has the following improvements: + +* Two new methods, :meth:`Mock.assert_called() + ` and :meth:`Mock.assert_called_once() + ` to check if the mock object + was called. + (Contributed by Amit Saha in :issue:`26323`.) + + urllib.robotparser ------------------ diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 21f49fa..50ff949 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -772,6 +772,24 @@ class NonCallableMock(Base): (self._mock_name or 'mock', self.call_count)) raise AssertionError(msg) + def assert_called(_mock_self): + """assert that the mock was called at least once + """ + self = _mock_self + if self.call_count == 0: + msg = ("Expected '%s' to have been called." % + self._mock_name or 'mock') + raise AssertionError(msg) + + def assert_called_once(_mock_self): + """assert that the mock was called only once. + """ + self = _mock_self + if not self.call_count == 1: + msg = ("Expected '%s' to have been called once. Called %s times." % + (self._mock_name or 'mock', self.call_count)) + raise AssertionError(msg) + def assert_called_with(_mock_self, *args, **kwargs): """assert that the mock was called with the specified arguments. diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py index 2a6069f..b2d9acb 100644 --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/unittest/test/testmock/testmock.py @@ -1222,6 +1222,27 @@ class MockTest(unittest.TestCase): with self.assertRaises(AssertionError): m.hello.assert_not_called() + def test_assert_called(self): + m = Mock() + with self.assertRaises(AssertionError): + m.hello.assert_called() + m.hello() + m.hello.assert_called() + + m.hello() + m.hello.assert_called() + + def test_assert_called_once(self): + m = Mock() + with self.assertRaises(AssertionError): + m.hello.assert_called_once() + m.hello() + m.hello.assert_called_once() + + m.hello() + with self.assertRaises(AssertionError): + m.hello.assert_called_once() + #Issue21256 printout of keyword args should be in deterministic order def test_sorted_call_signature(self): m = Mock() diff --git a/Misc/ACKS b/Misc/ACKS index 0a67f39..a19e113 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1267,6 +1267,7 @@ Bernt Røskar Brenna Constantina S. Patrick Sabin Sébastien Sablé +Amit Saha Suman Saha Hajime Saitou George Sakkis diff --git a/Misc/NEWS b/Misc/NEWS index 45d8f2e..1e87de8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,9 @@ Core and Builtins Library ------- +- Issue #26323: Add Mock.assert_called() and Mock.assert_called_once() + methods to unittest.mock. Patch written by Amit Saha. + - Issue #20589: Invoking Path.owner() and Path.group() on Windows now raise NotImplementedError instead of ImportError. -- cgit v0.12 From 474ebbbe50ea3247ad120324874b8b32d5fa550d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 11 Mar 2016 22:36:14 +0100 Subject: Always test datetime.strftime("%4Y") Issue #13305: Always test datetime.datetime.strftime("%4Y") for years < 1900. --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b49942a..0be1af4 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1422,8 +1422,8 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): if d.strftime("%Y") != '%04d' % y: # Year 42 returns '42', not padded self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + # '0042' is obtained anyway + self.assertEqual(d.strftime("%4Y"), '%04d' % y) def test_replace(self): cls = self.theclass -- cgit v0.12 From 15a83e845fa2421428f9d06c6fe6e85fde59d22c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2016 08:16:48 +0100 Subject: Revert change 291d47954618 Revert: "Always test datetime.strftime("%4Y") Issue #13305: Always test datetime.datetime.strftime("%4Y") for years < 1900." In fact, strftime("%4Y") fails on most platforms. --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 0be1af4..b49942a 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1422,8 +1422,8 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): if d.strftime("%Y") != '%04d' % y: # Year 42 returns '42', not padded self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + # '0042' is obtained anyway + self.assertEqual(d.strftime("%4Y"), '%04d' % y) def test_replace(self): cls = self.theclass -- cgit v0.12 From 04bc5b9e4864a91f4dbea74ce2d88f606aa578f3 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 14 Mar 2016 06:06:03 +0200 Subject: Issue #747320: Use email.utils.formatdate() to avoid code duplication in BaseHTTPRequestHandler Initial patch by karlcow. --- Lib/http/server.py | 8 ++------ Lib/test/test_httpservers.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index e1b71ab..5e91826 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -87,6 +87,7 @@ __all__ = [ "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", ] +import email.utils import html import http.client import io @@ -566,12 +567,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """Return the current date and time formatted for a message header.""" if timestamp is None: timestamp = time.time() - year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp) - s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( - self.weekdayname[wd], - day, self.monthname[month], year, - hh, mm, ss) - return s + return email.utils.formatdate(timestamp, usegmt=True) def log_date_time_string(self): """Return the current time formatted for logging.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index d4c8eab..5d44825 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -17,6 +17,7 @@ import urllib.parse import html import http.client import tempfile +import time from io import BytesIO import unittest @@ -873,6 +874,19 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.handler.handle() self.assertRaises(StopIteration, next, close_values) + def test_date_time_string(self): + now = time.time() + # this is the old code that formats the timestamp + year, month, day, hh, mm, ss, wd, y, z = time.gmtime(now) + expected = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + self.handler.weekdayname[wd], + day, + self.handler.monthname[month], + year, hh, mm, ss + ) + self.assertEqual(self.handler.date_time_string(timestamp=now), expected) + + class SimpleHTTPRequestHandlerTestCase(unittest.TestCase): """ Test url parsing """ def setUp(self): -- cgit v0.12 From 34be807ca4dfecc5b0a9e577a48535e738dce32b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 12:04:26 +0100 Subject: Add PYTHONMALLOC env var Issue #26516: * Add PYTHONMALLOC environment variable to set the Python memory allocators and/or install debug hooks. * PyMem_SetupDebugHooks() can now also be used on Python compiled in release mode. * The PYTHONMALLOCSTATS environment variable can now also be used on Python compiled in release mode. It now has no effect if set to an empty string. * In debug mode, debug hooks are now also installed on Python memory allocators when Python is configured without pymalloc. --- Doc/c-api/memory.rst | 38 ++++++--- Doc/using/cmdline.rst | 51 +++++++++++-- Doc/whatsnew/3.6.rst | 31 ++++++++ Include/pymem.h | 9 +++ Lib/test/test_capi.py | 59 ++++++++++++++ Misc/NEWS | 13 ++++ Misc/README.valgrind | 3 + Modules/_testcapimodule.c | 29 +++++++ Modules/main.c | 25 ++++-- Objects/obmalloc.c | 191 ++++++++++++++++++++++++++++++++-------------- Programs/python.c | 9 +++ Python/pylifecycle.c | 9 ++- Python/sysmodule.c | 6 +- 13 files changed, 383 insertions(+), 90 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 290ef09..fe1cd5f 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -85,9 +85,12 @@ for the I/O buffer escapes completely the Python memory manager. .. seealso:: + The :envvar:`PYTHONMALLOC` environment variable can be used to configure + the memory allocators used by Python. + The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print - memory allocation statistics every time a new object arena is created, and - on shutdown. + statistics of the :ref:`pymalloc memory allocator ` every time a + new pymalloc object arena is created, and on shutdown. Raw Memory Interface @@ -343,25 +346,36 @@ Customize Memory Allocators - detect write before the start of the buffer (buffer underflow) - detect write after the end of the buffer (buffer overflow) - The function does nothing if Python is not compiled is debug mode. + These hooks are installed by default if Python is compiled in debug + mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install + debug hooks on a Python compiled in release mode. + + .. versionchanged:: 3.6 + This function now also works on Python compiled in release mode. + +.. _pymalloc: -Customize PyObject Arena Allocator -================================== +The pymalloc allocator +====================== -Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This -allocator is optimized for small objects with a short lifetime. It uses memory -mappings called "arenas" with a fixed size of 256 KB. It falls back to -:c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger -than 512 bytes. *pymalloc* is the default allocator used by -:c:func:`PyObject_Malloc`. +Python has a *pymalloc* allocator optimized for small objects (smaller or equal +to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" +with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and +:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. -The default arena allocator uses the following functions: +*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_OBJ` domain +(:c:func:`PyObject_Malloc` & cie). + +The arena allocator uses the following functions: * :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, * :c:func:`mmap` and :c:func:`munmap` if available, * :c:func:`malloc` and :c:func:`free` otherwise. +Customize pymalloc Arena Allocator +---------------------------------- + .. versionadded:: 3.4 .. c:type:: PyObjectArenaAllocator diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index ec744a3..684ccb6 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -621,6 +621,51 @@ conflict. .. versionadded:: 3.4 +.. envvar:: PYTHONMALLOC + + Set the Python memory allocators and/or install debug hooks. + + Set the family of memory allocators used by Python: + + * ``malloc``: use the :c:func:`malloc` function of the C library + for all Python memory allocators (:c:func:`PyMem_RawMalloc`, + :c:func:`PyMem_Malloc`, :c:func:`PyObject_Malloc` & cie). + * ``pymalloc``: :c:func:`PyObject_Malloc`, :c:func:`PyObject_Calloc` and + :c:func:`PyObject_Realloc` use the :ref:`pymalloc allocator `. + Other Python memory allocators (:c:func:`PyMem_RawMalloc`, + :c:func:`PyMem_Malloc` & cie) use :c:func:`malloc`. + + Install debug hooks: + + * ``debug``: install debug hooks on top of the default memory allocator + * ``malloc_debug``: same than ``malloc`` but also install debug hooks + * ``pymalloc_debug``: same than ``malloc`` but also install debug hooks + + See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python + memory allocators. + + .. note:: + ``pymalloc`` and ``pymalloc_debug`` are not available if Python is + configured without ``pymalloc`` support. + + .. versionadded:: 3.6 + + +.. envvar:: PYTHONMALLOCSTATS + + If set to a non-empty string, Python will print statistics of the + :ref:`pymalloc memory allocator ` every time a new pymalloc object + arena is created, and on shutdown. + + This variable is ignored if the :envvar:`PYTHONMALLOC` environment variable + is used to force the :c:func:`malloc` allocator of the C library, or if + Python is configured without ``pymalloc`` support. + + .. versionchanged:: 3.6 + This variable can now also be used on Python compiled in release mode. + It now has no effect if set to an empty string. + + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ @@ -636,9 +681,3 @@ if Python was configured with the ``--with-pydebug`` build option. If set, Python will dump objects and reference counts still alive after shutting down the interpreter. - - -.. envvar:: PYTHONMALLOCSTATS - - If set, Python will print memory allocation statistics every time a new - object arena is created, and on shutdown. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3afe2d4..588826b 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -80,6 +80,9 @@ Summary -- Release highlights PEP written by Carl Meyer +New Features +============ + .. _whatsnew-fstrings: PEP 498: Formatted string literals @@ -98,6 +101,34 @@ evaluated at run time, and then formatted using the :func:`format` protocol. See :pep:`498` and the main documentation at :ref:`f-strings`. +PYTHONMALLOC environment variable +--------------------------------- + +The new :envvar:`PYTHONMALLOC` environment variable allows to set the Python +memory allocators and/or install debug hooks. + +It is now possible to install debug hooks on Python memory allocators on Python +compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: + +* Newly allocated memory is filled with the byte ``0xCB`` +* Freed memory is filled with the byte ``0xDB`` +* Detect violations of Python memory allocator API. For example, + :c:func:`PyObject_Free` called on a memory block allocated by + :c:func:`PyMem_Malloc`. +* Detect write before the start of the buffer (buffer underflow) +* Detect write after the end of the buffer (buffer overflow) + +See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python +memory allocators. + +It is now also possible to force the usage of the :c:func:`malloc` allocator of +the C library for all Python memory allocations using ``PYTHONMALLOC=malloc``. +It helps to use external memory debuggers like Valgrind on a Python compiled in +release mode. + +(Contributed by Victor Stinner in :issue:`26516`.) + + Other Language Changes ====================== diff --git a/Include/pymem.h b/Include/pymem.h index 043db64..b1f06ef 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -16,8 +16,17 @@ PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_RawFree(void *ptr); + +/* Configure the Python memory allocators. Pass NULL to use default + allocators. */ +PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt); + +#ifdef WITH_PYMALLOC +PyAPI_FUNC(int) _PyMem_PymallocEnabled(void); #endif +#endif /* !Py_LIMITED_API */ + /* BEWARE: diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 74ec6c5..d56d702 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -6,6 +6,7 @@ import pickle import random import subprocess import sys +import sysconfig import textwrap import time import unittest @@ -521,6 +522,7 @@ class SkipitemTest(unittest.TestCase): self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, (), {}, b'', [42]) + @unittest.skipUnless(threading, 'Threading required for this test.') class TestThreadState(unittest.TestCase): @@ -545,6 +547,7 @@ class TestThreadState(unittest.TestCase): t.start() t.join() + class Test_testcapi(unittest.TestCase): def test__testcapi(self): for name in dir(_testcapi): @@ -553,5 +556,61 @@ class Test_testcapi(unittest.TestCase): test = getattr(_testcapi, name) test() + +class MallocTests(unittest.TestCase): + ENV = 'debug' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure('-c', code, PYTHONMALLOC=self.ENV) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + r" 16 bytes originally requested\n" + r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail=0x[0-9a-f]+ are not all FORBIDDENBYTE $0x[0-9a-f]{2}$:\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfb\n" + r" at tail\+2: 0xfb\n" + r" at tail\+3: 0xfb\n" + r" at tail\+4: 0xfb\n" + r" at tail\+5: 0xfb\n" + r" at tail\+6: 0xfb\n" + r" at tail\+7: 0xfb\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb\n" + r"Fatal Python error: bad trailing pad byte") + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + r" 16 bytes originally requested\n" + r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail=0x[0-9a-f]+ are FORBIDDENBYTE, as expected.\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: .*\n" + r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") + self.assertRegex(out, regex) + + +class MallocDebugTests(MallocTests): + ENV = 'malloc_debug' + + +@unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, + 'need pymalloc') +class PymallocDebugTests(MallocTests): + ENV = 'pymalloc_debug' + + +@unittest.skipUnless(Py_DEBUG, 'need Py_DEBUG') +class DefaultMallocDebugTests(MallocTests): + ENV = '' + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 1c894a4..852be06 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,19 @@ Release date: tba Core and Builtins ----------------- +- Issue #26516: Add :envvar`PYTHONMALLOC` environment variable to set the + Python memory allocators and/or install debug hooks. + +- Issue #26516: The :c:func`PyMem_SetupDebugHooks` function can now also be + used on Python compiled in release mode. + +- Issue #26516: The :envvar:`PYTHONMALLOCSTATS` environment variable can now + also be used on Python compiled in release mode. It now has no effect if + set to an empty string. + +- Issue #26516: In debug mode, debug hooks are now also installed on Python + memory allocators when Python is configured without pymalloc. + - Issue #26464: Fix str.translate() when string is ASCII and first replacements removes character, but next replacement uses a non-ASCII character or a string longer than 1 character. Regression introduced in Python 3.5.0. diff --git a/Misc/README.valgrind b/Misc/README.valgrind index b5a9a32..908f137 100644 --- a/Misc/README.valgrind +++ b/Misc/README.valgrind @@ -2,6 +2,9 @@ This document describes some caveats about the use of Valgrind with Python. Valgrind is used periodically by Python developers to try to ensure there are no memory leaks or invalid memory reads/writes. +UPDATE: Python 3.6 now supports PYTHONMALLOC=malloc environment variable which +can be used to force the usage of the malloc() allocator of the C library. + If you don't want to read about the details of using Valgrind, there are still two things you must do to suppress the warnings. First, you must use a suppressions file. One is supplied in diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a21a584..babffc4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3616,6 +3616,33 @@ get_recursion_depth(PyObject *self, PyObject *args) return PyLong_FromLong(tstate->recursion_depth - 1); } +static PyObject* +pymem_buffer_overflow(PyObject *self, PyObject *args) +{ + char *buffer; + + /* Deliberate buffer overflow to check that PyMem_Free() detects + the overflow when debug hooks are installed. */ + buffer = PyMem_Malloc(16); + buffer[16] = 'x'; + PyMem_Free(buffer); + + Py_RETURN_NONE; +} + +static PyObject* +pymem_api_misuse(PyObject *self, PyObject *args) +{ + char *buffer; + + /* Deliberate misusage of Python allocators: + allococate with PyMem but release with PyMem_Raw. */ + buffer = PyMem_Malloc(16); + PyMem_RawFree(buffer); + + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3798,6 +3825,8 @@ static PyMethodDef TestMethods[] = { {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, + {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, + {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/main.c b/Modules/main.c index ee129a5..b6dcdd0 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -93,14 +93,15 @@ static const char usage_5[] = " The default module search path uses %s.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n\ -"; -static const char usage_6[] = "\ -PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n\ - to seed the hashes of str, bytes and datetime objects. It can also be\n\ - set to an integer in the range [0,4294967295] to get hash values with a\n\ - predictable seed.\n\ -"; +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n"; +static const char usage_6[] = +"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str, bytes and datetime objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash values with a\n" +" predictable seed.\n" +"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" +" hooks.\n"; static int usage(int exitcode, const wchar_t* program) @@ -341,6 +342,7 @@ Py_Main(int argc, wchar_t **argv) int help = 0; int version = 0; int saw_unbuffered_flag = 0; + char *opt; PyCompilerFlags cf; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -365,6 +367,13 @@ Py_Main(int argc, wchar_t **argv) } } + opt = Py_GETENV("PYTHONMALLOC"); + if (_PyMem_SetupAllocators(opt) < 0) { + fprintf(stderr, + "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt); + exit(1); + } + Py_HashRandomizationFlag = 1; _PyRandom_Init(); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7cc889f..e4bd8ac 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,7 +2,19 @@ /* Python's malloc wrappers (see pymem.h) */ -#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ +/* + * Basic types + * I don't care if these are defined in or elsewhere. Axiom. + */ +#undef uchar +#define uchar unsigned char /* assuming == 8 bits */ + +#undef uint +#define uint unsigned int /* assuming >= 16 bits */ + +#undef uptr +#define uptr Py_uintptr_t + /* Forward declaration */ static void* _PyMem_DebugMalloc(void *ctx, size_t size); static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize); @@ -11,7 +23,6 @@ static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size); static void _PyObject_DebugDumpAddress(const void *p); static void _PyMem_DebugCheckAddress(char api_id, const void *p); -#endif #if defined(__has_feature) /* Clang */ #if __has_feature(address_sanitizer) /* is ASAN enabled? */ @@ -147,7 +158,6 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size) #endif #define PYMEM_FUNCS PYRAW_FUNCS -#ifdef PYMALLOC_DEBUG typedef struct { /* We tag each block with an API ID in order to tag API violations */ char api_id; @@ -164,10 +174,9 @@ static struct { }; #define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree -#endif static PyMemAllocatorEx _PyMem_Raw = { -#ifdef PYMALLOC_DEBUG +#ifdef Py_DEBUG &_PyMem_Debug.raw, PYDBG_FUNCS #else NULL, PYRAW_FUNCS @@ -175,7 +184,7 @@ static PyMemAllocatorEx _PyMem_Raw = { }; static PyMemAllocatorEx _PyMem = { -#ifdef PYMALLOC_DEBUG +#ifdef Py_DEBUG &_PyMem_Debug.mem, PYDBG_FUNCS #else NULL, PYMEM_FUNCS @@ -183,13 +192,71 @@ static PyMemAllocatorEx _PyMem = { }; static PyMemAllocatorEx _PyObject = { -#ifdef PYMALLOC_DEBUG +#ifdef Py_DEBUG &_PyMem_Debug.obj, PYDBG_FUNCS #else NULL, PYOBJ_FUNCS #endif }; +int +_PyMem_SetupAllocators(const char *opt) +{ + if (opt == NULL || *opt == '\0') { + /* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line + options): use default allocators */ +#ifdef Py_DEBUG +# ifdef WITH_PYMALLOC + opt = "pymalloc_debug"; +# else + opt = "malloc_debug"; +# endif +#else + /* !Py_DEBUG */ +# ifdef WITH_PYMALLOC + opt = "pymalloc"; +# else + opt = "malloc"; +# endif +#endif + } + + if (strcmp(opt, "debug") == 0) { + PyMem_SetupDebugHooks(); + } + else if (strcmp(opt, "malloc") == 0 || strcmp(opt, "malloc_debug") == 0) + { + PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS}; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + + if (strcmp(opt, "malloc_debug") == 0) + PyMem_SetupDebugHooks(); + } +#ifdef WITH_PYMALLOC + else if (strcmp(opt, "pymalloc") == 0 + || strcmp(opt, "pymalloc_debug") == 0) + { + PyMemAllocatorEx mem_alloc = {NULL, PYRAW_FUNCS}; + PyMemAllocatorEx obj_alloc = {NULL, PYOBJ_FUNCS}; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &mem_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &mem_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &obj_alloc); + + if (strcmp(opt, "pymalloc_debug") == 0) + PyMem_SetupDebugHooks(); + } +#endif + else { + /* unknown allocator */ + return -1; + } + return 0; +} + #undef PYRAW_FUNCS #undef PYMEM_FUNCS #undef PYOBJ_FUNCS @@ -205,12 +272,34 @@ static PyObjectArenaAllocator _PyObject_Arena = {NULL, #endif }; +static int +_PyMem_DebugEnabled(void) +{ + return (_PyObject.malloc == _PyMem_DebugMalloc); +} + +#ifdef WITH_PYMALLOC +int +_PyMem_PymallocEnabled(void) +{ + if (_PyMem_DebugEnabled()) { + return (_PyMem_Debug.obj.alloc.malloc == _PyObject_Malloc); + } + else { + return (_PyObject.malloc == _PyObject_Malloc); + } +} +#endif + void PyMem_SetupDebugHooks(void) { -#ifdef PYMALLOC_DEBUG PyMemAllocatorEx alloc; + /* hooks already installed */ + if (_PyMem_DebugEnabled()) + return; + alloc.malloc = _PyMem_DebugMalloc; alloc.calloc = _PyMem_DebugCalloc; alloc.realloc = _PyMem_DebugRealloc; @@ -233,7 +322,6 @@ PyMem_SetupDebugHooks(void) PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); } -#endif } void @@ -264,7 +352,6 @@ PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break; /* ignore unknown domain */ } - } void @@ -642,22 +729,6 @@ static int running_on_valgrind = -1; #define SIMPLELOCK_LOCK(lock) /* acquire released lock */ #define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ -/* - * Basic types - * I don't care if these are defined in or elsewhere. Axiom. - */ -#undef uchar -#define uchar unsigned char /* assuming == 8 bits */ - -#undef uint -#define uint unsigned int /* assuming >= 16 bits */ - -#undef ulong -#define ulong unsigned long /* assuming >= 32 bits */ - -#undef uptr -#define uptr Py_uintptr_t - /* When you say memory, my mind reasons in terms of (pointers to) blocks */ typedef uchar block; @@ -949,11 +1020,15 @@ new_arena(void) struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ void *address; + static int debug_stats = -1; -#ifdef PYMALLOC_DEBUG - if (Py_GETENV("PYTHONMALLOCSTATS")) + if (debug_stats == -1) { + char *opt = Py_GETENV("PYTHONMALLOCSTATS"); + debug_stats = (opt != NULL && *opt != '\0'); + } + if (debug_stats) _PyObject_DebugMallocStats(stderr); -#endif + if (unused_arena_objects == NULL) { uint i; uint numarenas; @@ -1709,7 +1784,7 @@ _Py_GetAllocatedBlocks(void) #endif /* WITH_PYMALLOC */ -#ifdef PYMALLOC_DEBUG + /*==========================================================================*/ /* A x-platform debugging allocator. This doesn't manage memory directly, * it wraps a real allocator, adding extra debugging info to the memory blocks. @@ -1767,31 +1842,6 @@ write_size_t(void *p, size_t n) } } -#ifdef Py_DEBUG -/* Is target in the list? The list is traversed via the nextpool pointers. - * The list may be NULL-terminated, or circular. Return 1 if target is in - * list, else 0. - */ -static int -pool_is_in_list(const poolp target, poolp list) -{ - poolp origlist = list; - assert(target != NULL); - if (list == NULL) - return 0; - do { - if (target == list) - return 1; - list = list->nextpool; - } while (list != NULL && list != origlist); - return 0; -} - -#else -#define pool_is_in_list(X, Y) 1 - -#endif /* Py_DEBUG */ - /* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -2106,7 +2156,6 @@ _PyObject_DebugDumpAddress(const void *p) } } -#endif /* PYMALLOC_DEBUG */ static size_t printone(FILE *out, const char* msg, size_t value) @@ -2158,8 +2207,30 @@ _PyDebugAllocatorStats(FILE *out, (void)printone(out, buf2, num_blocks * sizeof_block); } + #ifdef WITH_PYMALLOC +#ifdef Py_DEBUG +/* Is target in the list? The list is traversed via the nextpool pointers. + * The list may be NULL-terminated, or circular. Return 1 if target is in + * list, else 0. + */ +static int +pool_is_in_list(const poolp target, poolp list) +{ + poolp origlist = list; + assert(target != NULL); + if (list == NULL) + return 0; + do { + if (target == list) + return 1; + list = list->nextpool; + } while (list != NULL && list != origlist); + return 0; +} +#endif + /* Print summary info to "out" about the state of pymalloc's structures. * In Py_DEBUG mode, also perform some expensive internal consistency * checks. @@ -2233,7 +2304,9 @@ _PyObject_DebugMallocStats(FILE *out) if (p->ref.count == 0) { /* currently unused */ +#ifdef Py_DEBUG assert(pool_is_in_list(p, arenas[i].freepools)); +#endif continue; } ++numpools[sz]; @@ -2273,9 +2346,8 @@ _PyObject_DebugMallocStats(FILE *out) quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } fputc('\n', out); -#ifdef PYMALLOC_DEBUG - (void)printone(out, "# times object malloc called", serialno); -#endif + if (_PyMem_DebugEnabled()) + (void)printone(out, "# times object malloc called", serialno); (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); (void)printone(out, "# arenas highwater mark", narenas_highwater); @@ -2303,6 +2375,7 @@ _PyObject_DebugMallocStats(FILE *out) #endif /* #ifdef WITH_PYMALLOC */ + #ifdef Py_USING_MEMORY_DEBUGGER /* Make this function last so gcc won't inline it since the definition is * after the reference. diff --git a/Programs/python.c b/Programs/python.c index 37b10b8..a7afbc7 100644 --- a/Programs/python.c +++ b/Programs/python.c @@ -24,6 +24,9 @@ main(int argc, char **argv) int i, res; char *oldloc; + /* Force malloc() allocator to bootstrap Python */ + (void)_PyMem_SetupAllocators("malloc"); + argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); if (!argv_copy || !argv_copy2) { @@ -62,7 +65,13 @@ main(int argc, char **argv) setlocale(LC_ALL, oldloc); PyMem_RawFree(oldloc); + res = Py_Main(argc, argv_copy); + + /* Force again malloc() allocator to release memory blocks allocated + before Py_Main() */ + (void)_PyMem_SetupAllocators("malloc"); + for (i = 0; i < argc; i++) { PyMem_RawFree(argv_copy2[i]); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e9db7f6..715a547 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -702,9 +702,12 @@ Py_FinalizeEx(void) if (Py_GETENV("PYTHONDUMPREFS")) _Py_PrintReferenceAddresses(stderr); #endif /* Py_TRACE_REFS */ -#ifdef PYMALLOC_DEBUG - if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(stderr); +#ifdef WITH_PYMALLOC + if (_PyMem_PymallocEnabled()) { + char *opt = Py_GETENV("PYTHONMALLOCSTATS"); + if (opt != NULL && *opt != '\0') + _PyObject_DebugMallocStats(stderr); + } #endif call_ll_exitfuncs(); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 702e8f0..c5b4ac1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1151,8 +1151,10 @@ static PyObject * sys_debugmallocstats(PyObject *self, PyObject *args) { #ifdef WITH_PYMALLOC - _PyObject_DebugMallocStats(stderr); - fputc('\n', stderr); + if (_PyMem_PymallocEnabled()) { + _PyObject_DebugMallocStats(stderr); + fputc('\n', stderr); + } #endif _PyObject_DebugTypeStats(stderr); -- cgit v0.12 From 791da1cc264574f8f3e44570d0fce293f755fdf3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 16:53:12 +0100 Subject: Fix Py_FatalError() if called without the GIL Issue #26558: If Py_FatalError() is called without the GIL, don't try to print the current exception, nor try to flush stdout and stderr: only dump the traceback of Python threads. --- Python/pylifecycle.c | 84 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 715a547..aaf5811 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1267,31 +1267,62 @@ initstdio(void) } +static void +_Py_FatalError_DumpTracebacks(int fd) +{ + PyThreadState *tstate; + +#ifdef WITH_THREAD + /* PyGILState_GetThisThreadState() works even if the GIL was released */ + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_GET(); +#endif + if (tstate == NULL) { + /* _Py_DumpTracebackThreads() requires the thread state to display + * frames */ + return; + } + + fputc('\n', stderr); + fflush(stderr); + + /* display the current Python stack */ + _Py_DumpTracebackThreads(fd, tstate->interp, tstate); +} + /* Print the current exception (if an exception is set) with its traceback, - * or display the current Python stack. - * - * Don't call PyErr_PrintEx() and the except hook, because Py_FatalError() is - * called on catastrophic cases. */ + or display the current Python stack. -static void -_Py_PrintFatalError(int fd) + Don't call PyErr_PrintEx() and the except hook, because Py_FatalError() is + called on catastrophic cases. + + Return 1 if the traceback was displayed, 0 otherwise. */ + +static int +_Py_FatalError_PrintExc(int fd) { PyObject *ferr, *res; PyObject *exception, *v, *tb; int has_tb; - PyThreadState *tstate; + + if (PyThreadState_GET() == NULL) { + /* The GIL is released: trying to acquire it is likely to deadlock, + just give up. */ + return 0; + } PyErr_Fetch(&exception, &v, &tb); if (exception == NULL) { /* No current exception */ - goto display_stack; + return 0; } ferr = _PySys_GetObjectId(&PyId_stderr); if (ferr == NULL || ferr == Py_None) { /* sys.stderr is not set yet or set to None, no need to try to display the exception */ - goto display_stack; + return 0; } PyErr_NormalizeException(&exception, &v, &tb); @@ -1302,7 +1333,7 @@ _Py_PrintFatalError(int fd) PyException_SetTraceback(v, tb); if (exception == NULL) { /* PyErr_NormalizeException() failed */ - goto display_stack; + return 0; } has_tb = (tb != Py_None); @@ -1318,28 +1349,9 @@ _Py_PrintFatalError(int fd) else Py_DECREF(res); - if (has_tb) - return; - -display_stack: -#ifdef WITH_THREAD - /* PyGILState_GetThisThreadState() works even if the GIL was released */ - tstate = PyGILState_GetThisThreadState(); -#else - tstate = PyThreadState_GET(); -#endif - if (tstate == NULL) { - /* _Py_DumpTracebackThreads() requires the thread state to display - * frames */ - return; - } - - fputc('\n', stderr); - fflush(stderr); - - /* display the current Python stack */ - _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + return has_tb; } + /* Print fatal error message and abort */ void @@ -1365,10 +1377,14 @@ Py_FatalError(const char *msg) /* Print the exception (if an exception is set) with its traceback, * or display the current Python stack. */ - _Py_PrintFatalError(fd); + if (!_Py_FatalError_PrintExc(fd)) + _Py_FatalError_DumpTracebacks(fd); - /* Flush sys.stdout and sys.stderr */ - flush_std_files(); + /* Check if the current Python thread hold the GIL */ + if (PyThreadState_GET() != NULL) { + /* Flush sys.stdout and sys.stderr */ + flush_std_files(); + } /* The main purpose of faulthandler is to display the traceback. We already * did our best to display it. So faulthandler can now be disabled. -- cgit v0.12 From d222653f8f39733f65ea93bfeab7748185ba4ea8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 17:01:32 +0100 Subject: Issue #26558: Remove useless check in tracemalloc The first instruction of tracemalloc_add_trace() is traceback_new() which already checks the GIL. --- Modules/_tracemalloc.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 226a473..d62e682 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -439,10 +439,6 @@ tracemalloc_add_trace(void *ptr, size_t size) trace_t trace; int res; -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - traceback = traceback_new(); if (traceback == NULL) return -1; -- cgit v0.12 From a1bc28a91de9bfe88fc16cab400cd50dd1a27467 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 17:10:36 +0100 Subject: Issue #26516: Fix test_capi on Windows Pointers are formatted differently. --- Lib/test/test_capi.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index d56d702..6a066f1 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -559,6 +559,8 @@ class Test_testcapi(unittest.TestCase): class MallocTests(unittest.TestCase): ENV = 'debug' + # '0x04c06e0' or '04C06E0' + PTR_REGEX = r'(?:0x[0-9a-f]+|[0-9A-F]+)' def check(self, code): with support.SuppressCrashReport(): @@ -568,10 +570,10 @@ class MallocTests(unittest.TestCase): def test_buffer_overflow(self): out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') - regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail=0x[0-9a-f]+ are not all FORBIDDENBYTE $0x[0-9a-f]{2}$:\n" + r" The 8 pad bytes at tail={ptr} are not all FORBIDDENBYTE $0x[0-9a-f]{{2}}$:\n" r" at tail\+0: 0x78 \*\*\* OUCH\n" r" at tail\+1: 0xfb\n" r" at tail\+2: 0xfb\n" @@ -583,17 +585,19 @@ class MallocTests(unittest.TestCase): r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb\n" r"Fatal Python error: bad trailing pad byte") + regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) def test_api_misuse(self): out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') - regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail=0x[0-9a-f]+ are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: .*\n" r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") + regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) -- cgit v0.12 From b3adb1adebc8090e8b838a7b921a4efef1adcf4c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 17:40:09 +0100 Subject: Issue #26516: Fix test_capi on 32-bit system On 32-bit system, only 4 bytes after dumped for the tail. --- Lib/test/test_capi.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6a066f1..5761686 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -4,6 +4,7 @@ import os import pickle import random +import re import subprocess import sys import sysconfig @@ -572,30 +573,27 @@ class MallocTests(unittest.TestCase): out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" - r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail={ptr} are not all FORBIDDENBYTE $0x[0-9a-f]{{2}}$:\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE $0x[0-9a-f]{{2}}$:\n" r" at tail\+0: 0x78 \*\*\* OUCH\n" r" at tail\+1: 0xfb\n" r" at tail\+2: 0xfb\n" - r" at tail\+3: 0xfb\n" - r" at tail\+4: 0xfb\n" - r" at tail\+5: 0xfb\n" - r" at tail\+6: 0xfb\n" - r" at tail\+7: 0xfb\n" + r" .*\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" - r" Data at p: cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb\n" + r" Data at p: cb cb cb .*\n" r"Fatal Python error: bad trailing pad byte") regex = regex.format(ptr=self.PTR_REGEX) + regex = re.compile(regex, flags=re.DOTALL) self.assertRegex(out, regex) def test_api_misuse(self): out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" - r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" - r" Data at p: .*\n" + r" Data at p: cb cb cb .*\n" r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) -- cgit v0.12 From c44f70770bf629469a5a179b643e53dfeca884ad Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 18:07:53 +0100 Subject: posix_getcwd(): limit to INT_MAX on Windows It's more to fix a conversion warning during compilation, I don't think that Windows support current working directory larger than 2 GB ... --- Modules/posixmodule.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6e0081d..7e89878 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3320,12 +3320,22 @@ posix_getcwd(int use_bytes) Py_BEGIN_ALLOW_THREADS do { buflen += chunk; +#ifdef MS_WINDOWS + if (buflen > INT_MAX) { + PyErr_NoMemory(); + break; + } +#endif tmpbuf = PyMem_RawRealloc(buf, buflen); if (tmpbuf == NULL) break; buf = tmpbuf; +#ifdef MS_WINDOWS + cwd = getcwd(buf, (int)buflen); +#else cwd = getcwd(buf, buflen); +#endif } while (cwd == NULL && errno == ERANGE); Py_END_ALLOW_THREADS -- cgit v0.12 From 21b47117ac061f0ba08219aa1a6d81a4a5a9c483 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 18:09:39 +0100 Subject: _pickle: Fix load_counted_tuple(), use Py_ssize_t for size Fix a warning on Windows 64-bit. --- Modules/_pickle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index bb3330a..5c3530b 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5056,7 +5056,7 @@ load_counted_binunicode(UnpicklerObject *self, int nbytes) } static int -load_counted_tuple(UnpicklerObject *self, int len) +load_counted_tuple(UnpicklerObject *self, Py_ssize_t len) { PyObject *tuple; -- cgit v0.12 From 08572f68a980576e883e4eba1227b59782d5192e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 21:55:43 +0100 Subject: Issue #26516: Fix test_capi on AIX Fix regex for parse a pointer address. --- Lib/test/test_capi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 5761686..6c940ef 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -561,7 +561,7 @@ class Test_testcapi(unittest.TestCase): class MallocTests(unittest.TestCase): ENV = 'debug' # '0x04c06e0' or '04C06E0' - PTR_REGEX = r'(?:0x[0-9a-f]+|[0-9A-F]+)' + PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' def check(self, code): with support.SuppressCrashReport(): -- cgit v0.12 From 8a1be61849341528c866924cf69d378ce7790889 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 22:07:55 +0100 Subject: Add more checks on the GIL Issue #10915, #15751, #26558: * PyGILState_Check() now returns 1 (success) before the creation of the GIL and after the destruction of the GIL. It allows to use the function early in Python initialization and late in Python finalization. * Add a flag to disable PyGILState_Check(). Disable PyGILState_Check() when Py_NewInterpreter() is called * Add assert(PyGILState_Check()) to: _Py_dup(), _Py_fstat(), _Py_read() and _Py_write() --- Include/pystate.h | 4 ++++ Parser/pgenmain.c | 8 ++++++++ Python/fileutils.c | 16 ++++++++++++++++ Python/pylifecycle.c | 5 +++++ Python/pystate.c | 24 +++++++++++++++++++----- 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index b50b16b..550c332 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -247,6 +247,10 @@ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); * currently holds the GIL, 0 otherwise */ #ifndef Py_LIMITED_API +/* Issue #26558: Flag to disable PyGILState_Check(). + If set, PyGILState_Check() always return 1. */ +PyAPI_DATA(int) _PyGILState_check_enabled; + PyAPI_FUNC(int) PyGILState_Check(void); #endif diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 3ca4afe..671a0cb 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -37,6 +37,14 @@ Py_Exit(int sts) exit(sts); } +#ifdef WITH_THREAD +/* Needed by obmalloc.c */ +int PyGILState_Check(void) +{ + return 1; +} +#endif + int main(int argc, char **argv) { diff --git a/Python/fileutils.c b/Python/fileutils.c index 06d632a..a710c99 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -683,6 +683,10 @@ _Py_fstat(int fd, struct _Py_stat_struct *status) { int res; +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + Py_BEGIN_ALLOW_THREADS res = _Py_fstat_noraise(fd, status); Py_END_ALLOW_THREADS @@ -1164,6 +1168,10 @@ _Py_read(int fd, void *buf, size_t count) int err; int async_err = 0; +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + /* _Py_read() must not be called with an exception set, otherwise the * caller may think that read() was interrupted by a signal and the signal * handler raised an exception. */ @@ -1319,6 +1327,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) Py_ssize_t _Py_write(int fd, const void *buf, size_t count) { +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + /* _Py_write() must not be called with an exception set, otherwise the * caller may think that write() was interrupted by a signal and the signal * handler raised an exception. */ @@ -1468,6 +1480,10 @@ _Py_dup(int fd) DWORD ftype; #endif +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (!_PyVerify_fd(fd)) { PyErr_SetFromErrno(PyExc_OSError); return -1; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index aaf5811..4fc6a15 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -692,6 +692,7 @@ Py_FinalizeEx(void) /* Delete current thread. After this, many C API calls become crashy. */ PyThreadState_Swap(NULL); + PyInterpreterState_Delete(interp); #ifdef Py_TRACE_REFS @@ -743,6 +744,10 @@ Py_NewInterpreter(void) if (!initialized) Py_FatalError("Py_NewInterpreter: call Py_Initialize first"); + /* Issue #10915, #15751: The GIL API doesn't work with multiple + interpreters: disable PyGILState_Check(). */ + _PyGILState_check_enabled = 0; + interp = PyInterpreterState_New(); if (interp == NULL) return NULL; diff --git a/Python/pystate.c b/Python/pystate.c index 853e5c7..e8026c5 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -34,6 +34,8 @@ to avoid the expense of doing their own locking). extern "C" { #endif +int _PyGILState_check_enabled = 1; + #ifdef WITH_THREAD #include "pythread.h" static PyThread_type_lock head_mutex = NULL; /* Protects interp->tstate_head */ @@ -45,7 +47,7 @@ static PyThread_type_lock head_mutex = NULL; /* Protects interp->tstate_head */ GILState implementation */ static PyInterpreterState *autoInterpreterState = NULL; -static int autoTLSkey = 0; +static int autoTLSkey = -1; #else #define HEAD_INIT() /* Nothing */ #define HEAD_LOCK() /* Nothing */ @@ -449,10 +451,10 @@ PyThreadState_DeleteCurrent() if (tstate == NULL) Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); - SET_TSTATE(NULL); + tstate_delete_common(tstate); if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) PyThread_delete_key_value(autoTLSkey); - tstate_delete_common(tstate); + SET_TSTATE(NULL); PyEval_ReleaseLock(); } #endif /* WITH_THREAD */ @@ -716,6 +718,7 @@ void _PyGILState_Fini(void) { PyThread_delete_key(autoTLSkey); + autoTLSkey = -1; autoInterpreterState = NULL; } @@ -784,8 +787,19 @@ PyGILState_GetThisThreadState(void) int PyGILState_Check(void) { - PyThreadState *tstate = GET_TSTATE(); - return tstate && (tstate == PyGILState_GetThisThreadState()); + PyThreadState *tstate; + + if (!_PyGILState_check_enabled) + return 1; + + if (autoTLSkey == -1) + return 1; + + tstate = GET_TSTATE(); + if (tstate == NULL) + return 0; + + return (tstate == PyGILState_GetThisThreadState()); } PyGILState_STATE -- cgit v0.12 From c4aec3628b6effcf205d70ce7d80f69847954b66 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 22:26:53 +0100 Subject: Check the GIL in PyObject_Malloc() Issue #26558: The debug hook of PyObject_Malloc() now checks that the GIL is held when the function is called. --- Doc/c-api/memory.rst | 9 +++-- Doc/whatsnew/3.6.rst | 3 ++ Lib/test/test_capi.py | 30 +++++++++++----- Misc/NEWS | 4 +++ Modules/_testcapimodule.c | 15 ++++++++ Objects/obmalloc.c | 92 +++++++++++++++++++++++++++++++++++------------ 6 files changed, 119 insertions(+), 34 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index fe1cd5f..25867d9 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -341,10 +341,13 @@ Customize Memory Allocators Newly allocated memory is filled with the byte ``0xCB``, freed memory is filled with the byte ``0xDB``. Additional checks: - - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer + - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer allocated by :c:func:`PyMem_Malloc` - - detect write before the start of the buffer (buffer underflow) - - detect write after the end of the buffer (buffer overflow) + - Detect write before the start of the buffer (buffer underflow) + - Detect write after the end of the buffer (buffer overflow) + - Check that the :term:`GIL ` is held when + allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: + :c:func:`PyObject_Malloc`) are called These hooks are installed by default if Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 588826b..b644a5c 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -117,6 +117,9 @@ compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: :c:func:`PyMem_Malloc`. * Detect write before the start of the buffer (buffer underflow) * Detect write after the end of the buffer (buffer overflow) +* Check that the :term:`GIL ` is held when allocator + functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: + :c:func:`PyObject_Malloc`) are called See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6c940ef..1de19a0 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -441,6 +441,7 @@ class EmbeddingTests(unittest.TestCase): self.maxDiff = None self.assertEqual(out.strip(), expected_output) + class SkipitemTest(unittest.TestCase): def test_skipitem(self): @@ -558,14 +559,15 @@ class Test_testcapi(unittest.TestCase): test() -class MallocTests(unittest.TestCase): - ENV = 'debug' +class PyMemDebugTests(unittest.TestCase): + PYTHONMALLOC = 'debug' # '0x04c06e0' or '04C06E0' PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' def check(self, code): with support.SuppressCrashReport(): - out = assert_python_failure('-c', code, PYTHONMALLOC=self.ENV) + out = assert_python_failure('-c', code, + PYTHONMALLOC=self.PYTHONMALLOC) stderr = out.err return stderr.decode('ascii', 'replace') @@ -598,20 +600,30 @@ class MallocTests(unittest.TestCase): regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) + def test_pyobject_malloc_without_gil(self): + # Calling PyObject_Malloc() without holding the GIL must raise an + # error in debug mode. + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + out = self.check(code) + expected = ('Fatal Python error: Python memory allocator called ' + 'without holding the GIL') + self.assertIn(expected, out) + -class MallocDebugTests(MallocTests): - ENV = 'malloc_debug' +class PyMemMallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'malloc_debug' @unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, 'need pymalloc') -class PymallocDebugTests(MallocTests): - ENV = 'pymalloc_debug' +class PyMemPymallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'pymalloc_debug' @unittest.skipUnless(Py_DEBUG, 'need Py_DEBUG') -class DefaultMallocDebugTests(MallocTests): - ENV = '' +class PyMemDefaultTests(PyMemDebugTests): + # test default allocator of Python compiled in debug mode + PYTHONMALLOC = '' if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index 852be06..5233a43 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26558: The debug hooks on Python memory allocator + :c:func:`PyObject_Malloc` now detect when functions are called without + holding the GIL. + - Issue #26516: Add :envvar`PYTHONMALLOC` environment variable to set the Python memory allocators and/or install debug hooks. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index babffc4..b3d8818 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3643,6 +3643,20 @@ pymem_api_misuse(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject* +pyobject_malloc_without_gil(PyObject *self, PyObject *args) +{ + char *buffer; + + Py_BEGIN_ALLOW_THREADS + buffer = PyObject_Malloc(10); + Py_END_ALLOW_THREADS + + PyObject_Free(buffer); + + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3827,6 +3841,7 @@ static PyMethodDef TestMethods[] = { {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, + {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index e4bd8ac..f526b1f 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -16,10 +16,15 @@ #define uptr Py_uintptr_t /* Forward declaration */ +static void* _PyMem_DebugRawMalloc(void *ctx, size_t size); +static void* _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize); +static void* _PyMem_DebugRawRealloc(void *ctx, void *ptr, size_t size); +static void _PyMem_DebugRawFree(void *ctx, void *p); + static void* _PyMem_DebugMalloc(void *ctx, size_t size); static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize); -static void _PyMem_DebugFree(void *ctx, void *p); static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size); +static void _PyMem_DebugFree(void *ctx, void *p); static void _PyObject_DebugDumpAddress(const void *p); static void _PyMem_DebugCheckAddress(char api_id, const void *p); @@ -173,11 +178,14 @@ static struct { {'o', {NULL, PYOBJ_FUNCS}} }; -#define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree +#define PYRAWDBG_FUNCS \ + _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree +#define PYDBG_FUNCS \ + _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree static PyMemAllocatorEx _PyMem_Raw = { #ifdef Py_DEBUG - &_PyMem_Debug.raw, PYDBG_FUNCS + &_PyMem_Debug.raw, PYRAWDBG_FUNCS #else NULL, PYRAW_FUNCS #endif @@ -185,7 +193,7 @@ static PyMemAllocatorEx _PyMem_Raw = { static PyMemAllocatorEx _PyMem = { #ifdef Py_DEBUG - &_PyMem_Debug.mem, PYDBG_FUNCS + &_PyMem_Debug.mem, PYRAWDBG_FUNCS #else NULL, PYMEM_FUNCS #endif @@ -260,6 +268,7 @@ _PyMem_SetupAllocators(const char *opt) #undef PYRAW_FUNCS #undef PYMEM_FUNCS #undef PYOBJ_FUNCS +#undef PYRAWDBG_FUNCS #undef PYDBG_FUNCS static PyObjectArenaAllocator _PyObject_Arena = {NULL, @@ -296,27 +305,28 @@ PyMem_SetupDebugHooks(void) { PyMemAllocatorEx alloc; - /* hooks already installed */ - if (_PyMem_DebugEnabled()) - return; - - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; + alloc.malloc = _PyMem_DebugRawMalloc; + alloc.calloc = _PyMem_DebugRawCalloc; + alloc.realloc = _PyMem_DebugRawRealloc; + alloc.free = _PyMem_DebugRawFree; - if (_PyMem_Raw.malloc != _PyMem_DebugMalloc) { + if (_PyMem_Raw.malloc != _PyMem_DebugRawMalloc) { alloc.ctx = &_PyMem_Debug.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); } - if (_PyMem.malloc != _PyMem_DebugMalloc) { + if (_PyMem.malloc != _PyMem_DebugRawMalloc) { alloc.ctx = &_PyMem_Debug.mem; PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); } + alloc.malloc = _PyMem_DebugMalloc; + alloc.calloc = _PyMem_DebugCalloc; + alloc.realloc = _PyMem_DebugRealloc; + alloc.free = _PyMem_DebugFree; + if (_PyObject.malloc != _PyMem_DebugMalloc) { alloc.ctx = &_PyMem_Debug.obj; PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); @@ -1869,7 +1879,7 @@ p[2*S+n+S: 2*S+n+2*S] */ static void * -_PyMem_DebugAlloc(int use_calloc, void *ctx, size_t nbytes) +_PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) { debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *p; /* base address of malloc'ed block */ @@ -1906,18 +1916,18 @@ _PyMem_DebugAlloc(int use_calloc, void *ctx, size_t nbytes) } static void * -_PyMem_DebugMalloc(void *ctx, size_t nbytes) +_PyMem_DebugRawMalloc(void *ctx, size_t nbytes) { - return _PyMem_DebugAlloc(0, ctx, nbytes); + return _PyMem_DebugRawAlloc(0, ctx, nbytes); } static void * -_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) +_PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize) { size_t nbytes; assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; - return _PyMem_DebugAlloc(1, ctx, nbytes); + return _PyMem_DebugRawAlloc(1, ctx, nbytes); } /* The debug free first checks the 2*SST bytes on each end for sanity (in @@ -1926,7 +1936,7 @@ _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) Then calls the underlying free. */ static void -_PyMem_DebugFree(void *ctx, void *p) +_PyMem_DebugRawFree(void *ctx, void *p) { debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *q = (uchar *)p - 2*SST; /* address returned from malloc */ @@ -1943,7 +1953,7 @@ _PyMem_DebugFree(void *ctx, void *p) } static void * -_PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes) +_PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) { debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *q = (uchar *)p, *oldq; @@ -1953,7 +1963,7 @@ _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes) int i; if (p == NULL) - return _PyMem_DebugAlloc(0, ctx, nbytes); + return _PyMem_DebugRawAlloc(0, ctx, nbytes); _PyMem_DebugCheckAddress(api->api_id, p); bumpserialno(); @@ -1996,6 +2006,44 @@ _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes) return q; } +static void +_PyMem_DebugCheckGIL(void) +{ +#ifdef WITH_THREAD + if (!PyGILState_Check()) + Py_FatalError("Python memory allocator called " + "without holding the GIL"); +#endif +} + +static void * +_PyMem_DebugMalloc(void *ctx, size_t nbytes) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawMalloc(ctx, nbytes); +} + +static void * +_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawCalloc(ctx, nelem, elsize); +} + +static void +_PyMem_DebugFree(void *ctx, void *ptr) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawFree(ctx, ptr); +} + +static void * +_PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawRealloc(ctx, ptr, nbytes); +} + /* Check the forbidden bytes on both ends of the memory allocated for p. * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress, * and call Py_FatalError to kill the program. -- cgit v0.12 From 32eb840a42ec0e131daac48d43aa35290e72571e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 11:12:35 +0100 Subject: Issue #26566: Rewrite test_signal.InterProcessSignalTests * Add Lib/test/signalinterproctester.py * Don't disable the garbage collector anymore * Don't use os.fork() with a subprocess to not inherit existing signal handlers or threads: start from a fresh process * Don't use UNIX kill command to send a signal but Python os.kill() * Use a timeout of 10 seconds to wait for the signal instead of 1 second * Always use signal.pause(), instead of time.wait(1), to wait for a signal * Use context manager on subprocess.Popen * remove code to retry on EINTR: it's no more needed since the PEP 475 * remove unused function exit_subprocess() * Cleanup the code --- Lib/test/signalinterproctester.py | 84 +++++++++++++++++++ Lib/test/test_signal.py | 171 ++------------------------------------ 2 files changed, 93 insertions(+), 162 deletions(-) create mode 100644 Lib/test/signalinterproctester.py diff --git a/Lib/test/signalinterproctester.py b/Lib/test/signalinterproctester.py new file mode 100644 index 0000000..d3ae170 --- /dev/null +++ b/Lib/test/signalinterproctester.py @@ -0,0 +1,84 @@ +import os +import signal +import subprocess +import sys +import time +import unittest + + +class SIGUSR1Exception(Exception): + pass + + +class InterProcessSignalTests(unittest.TestCase): + def setUp(self): + self.got_signals = {'SIGHUP': 0, 'SIGUSR1': 0, 'SIGALRM': 0} + + def sighup_handler(self, signum, frame): + self.got_signals['SIGHUP'] += 1 + + def sigusr1_handler(self, signum, frame): + self.got_signals['SIGUSR1'] += 1 + raise SIGUSR1Exception + + def wait_signal(self, child, signame, exc_class=None): + try: + if child is not None: + # This wait should be interrupted by exc_class + # (if set) + child.wait() + + timeout = 10.0 + deadline = time.monotonic() + timeout + + while time.monotonic() < deadline: + if self.got_signals[signame]: + return + signal.pause() + except BaseException as exc: + if exc_class is not None and isinstance(exc, exc_class): + # got the expected exception + return + raise + + self.fail('signal %s not received after %s seconds' + % (signame, timeout)) + + def subprocess_send_signal(self, pid, signame): + code = 'import os, signal; os.kill(%s, signal.%s)' % (pid, signame) + args = [sys.executable, '-I', '-c', code] + return subprocess.Popen(args) + + def test_interprocess_signal(self): + # Install handlers. This function runs in a sub-process, so we + # don't worry about re-setting the default handlers. + signal.signal(signal.SIGHUP, self.sighup_handler) + signal.signal(signal.SIGUSR1, self.sigusr1_handler) + signal.signal(signal.SIGUSR2, signal.SIG_IGN) + signal.signal(signal.SIGALRM, signal.default_int_handler) + + # Let the sub-processes know who to send signals to. + pid = str(os.getpid()) + + with self.subprocess_send_signal(pid, "SIGHUP") as child: + self.wait_signal(child, 'SIGHUP') + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 0, + 'SIGALRM': 0}) + + with self.subprocess_send_signal(pid, "SIGUSR1") as child: + self.wait_signal(child, 'SIGUSR1', SIGUSR1Exception) + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 1, + 'SIGALRM': 0}) + + with self.subprocess_send_signal(pid, "SIGUSR2") as child: + # Nothing should happen: SIGUSR2 is ignored + child.wait() + + signal.alarm(1) + self.wait_signal(None, 'SIGALRM', KeyboardInterrupt) + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 1, + 'SIGALRM': 0}) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 1b80ff0..ab42ed7 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -22,29 +22,6 @@ except ImportError: _testcapi = None -class HandlerBCalled(Exception): - pass - - -def exit_subprocess(): - """Use os._exit(0) to exit the current subprocess. - - Otherwise, the test catches the SystemExit and continues executing - in parallel with the original test, so you wind up with an - exponential number of tests running concurrently. - """ - os._exit(0) - - -def ignoring_eintr(__func, *args, **kwargs): - try: - return __func(*args, **kwargs) - except OSError as e: - if e.errno != errno.EINTR: - raise - return None - - class GenericTests(unittest.TestCase): @unittest.skipIf(threading is None, "test needs threading module") @@ -63,145 +40,6 @@ class GenericTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") -class InterProcessSignalTests(unittest.TestCase): - MAX_DURATION = 20 # Entire test should last at most 20 sec. - - def setUp(self): - self.using_gc = gc.isenabled() - gc.disable() - - def tearDown(self): - if self.using_gc: - gc.enable() - - def format_frame(self, frame, limit=None): - return ''.join(traceback.format_stack(frame, limit=limit)) - - def handlerA(self, signum, frame): - self.a_called = True - - def handlerB(self, signum, frame): - self.b_called = True - raise HandlerBCalled(signum, self.format_frame(frame)) - - def wait(self, child): - """Wait for child to finish, ignoring EINTR.""" - while True: - try: - child.wait() - return - except OSError as e: - if e.errno != errno.EINTR: - raise - - def run_test(self): - # Install handlers. This function runs in a sub-process, so we - # don't worry about re-setting the default handlers. - signal.signal(signal.SIGHUP, self.handlerA) - signal.signal(signal.SIGUSR1, self.handlerB) - signal.signal(signal.SIGUSR2, signal.SIG_IGN) - signal.signal(signal.SIGALRM, signal.default_int_handler) - - # Variables the signals will modify: - self.a_called = False - self.b_called = False - - # Let the sub-processes know who to send signals to. - pid = os.getpid() - - child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) - if child: - self.wait(child) - if not self.a_called: - time.sleep(1) # Give the signal time to be delivered. - self.assertTrue(self.a_called) - self.assertFalse(self.b_called) - self.a_called = False - - # Make sure the signal isn't delivered while the previous - # Popen object is being destroyed, because __del__ swallows - # exceptions. - del child - try: - child = subprocess.Popen(['kill', '-USR1', str(pid)]) - # This wait should be interrupted by the signal's exception. - self.wait(child) - time.sleep(1) # Give the signal time to be delivered. - self.fail('HandlerBCalled exception not raised') - except HandlerBCalled: - self.assertTrue(self.b_called) - self.assertFalse(self.a_called) - - child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) - if child: - self.wait(child) # Nothing should happen. - - try: - signal.alarm(1) - # The race condition in pause doesn't matter in this case, - # since alarm is going to raise a KeyboardException, which - # will skip the call. - signal.pause() - # But if another signal arrives before the alarm, pause - # may return early. - time.sleep(1) - except KeyboardInterrupt: - pass - except: - self.fail("Some other exception woke us from pause: %s" % - traceback.format_exc()) - else: - self.fail("pause returned of its own accord, and the signal" - " didn't arrive after another second.") - - # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform=='freebsd6', - 'inter process signals not reliable (do not mix well with threading) ' - 'on freebsd6') - def test_main(self): - # This function spawns a child process to insulate the main - # test-running process from all the signals. It then - # communicates with that child process over a pipe and - # re-raises information about any exceptions the child - # raises. The real work happens in self.run_test(). - os_done_r, os_done_w = os.pipe() - with closing(os.fdopen(os_done_r, 'rb')) as done_r, \ - closing(os.fdopen(os_done_w, 'wb')) as done_w: - child = os.fork() - if child == 0: - # In the child process; run the test and report results - # through the pipe. - try: - done_r.close() - # Have to close done_w again here because - # exit_subprocess() will skip the enclosing with block. - with closing(done_w): - try: - self.run_test() - except: - pickle.dump(traceback.format_exc(), done_w) - else: - pickle.dump(None, done_w) - except: - print('Uh oh, raised from pickle.') - traceback.print_exc() - finally: - exit_subprocess() - - done_w.close() - # Block for up to MAX_DURATION seconds for the test to finish. - r, w, x = select.select([done_r], [], [], self.MAX_DURATION) - if done_r in r: - tb = pickle.load(done_r) - if tb: - self.fail(tb) - else: - os.kill(child, signal.SIGKILL) - self.fail('Test deadlocked after %d seconds.' % - self.MAX_DURATION) - - -@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass @@ -224,6 +62,15 @@ class PosixTests(unittest.TestCase): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + # Issue 3864, unknown if this affects earlier versions of freebsd also + @unittest.skipIf(sys.platform=='freebsd6', + 'inter process signals not reliable (do not mix well with threading) ' + 'on freebsd6') + def test_interprocess_signal(self): + dirname = os.path.dirname(__file__) + script = os.path.join(dirname, 'signalinterproctester.py') + assert_python_ok(script) + @unittest.skipUnless(sys.platform == "win32", "Windows specific") class WindowsSignalTests(unittest.TestCase): -- cgit v0.12 From 7105e9f3de801ce50e24fcdfbdd450858e28a0fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 14:28:04 +0100 Subject: _tracemalloc: filename cannot be NULL --- Modules/_tracemalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index d62e682..2eb4dca 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -65,6 +65,8 @@ __attribute__((packed)) _declspec(align(4)) #endif { + /* filename cannot be NULL: "" is used if the Python frame + filename is NULL */ PyObject *filename; int lineno; } frame_t; @@ -987,8 +989,6 @@ frame_to_pyobject(frame_t *frame) if (frame_obj == NULL) return NULL; - if (frame->filename == NULL) - frame->filename = Py_None; Py_INCREF(frame->filename); PyTuple_SET_ITEM(frame_obj, 0, frame->filename); -- cgit v0.12 From 89e7cdcb9c124a01c5c3a69e086a0e8b26e65545 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 21:49:37 +0100 Subject: Enhance and rewrite traceback dump C functions Issue #26564: * Expose _Py_DumpASCII() and _Py_DumpDecimal() in traceback.h * Change the type of the second _Py_DumpASCII() parameter from int to unsigned long * Rewrite _Py_DumpDecimal() and dump_hexadecimal() to write directly characters in the expected order, avoid the need of reversing the string. * dump_hexadecimal() limits width to the size of the buffer * _Py_DumpASCII() does nothing if the object is not a Unicode string * dump_frame() wrtites "???" as the line number if the line number is negative --- Include/traceback.h | 18 +++++++++ Python/traceback.c | 109 +++++++++++++++++++++++++++------------------------- 2 files changed, 74 insertions(+), 53 deletions(-) diff --git a/Include/traceback.h b/Include/traceback.h index 891000c..7c630e3 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -67,6 +67,24 @@ PyAPI_DATA(const char*) _Py_DumpTracebackThreads( PyThreadState *current_thread); +#ifndef Py_LIMITED_API + +/* Write a Unicode object into the file descriptor fd. Encode the string to + ASCII using the backslashreplace error handler. + + Do nothing if text is not a Unicode object. The function accepts Unicode + string which is not ready (PyUnicode_WCHAR_KIND). + + This function is signal safe. */ +PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text); + +/* Format an integer as decimal into the file descriptor fd. + + This function is signal safe. */ +PyAPI_FUNC(void) _Py_DumpDecimal(int fd, unsigned long value); + +#endif /* !Py_LIMITED_API */ + #ifdef __cplusplus } #endif diff --git a/Python/traceback.c b/Python/traceback.c index 941d1cb..a40dbd1 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -479,40 +479,26 @@ PyTraceBack_Print(PyObject *v, PyObject *f) This function is signal safe. */ -static void -reverse_string(char *text, const size_t len) -{ - char tmp; - size_t i, j; - if (len == 0) - return; - for (i=0, j=len-1; i < j; i++, j--) { - tmp = text[i]; - text[i] = text[j]; - text[j] = tmp; - } -} - -/* Format an integer in range [0; 999999] to decimal, - and write it into the file fd. - - This function is signal safe. */ - -static void -dump_decimal(int fd, int value) +void +_Py_DumpDecimal(int fd, unsigned long value) { - char buffer[7]; - int len; - if (value < 0 || 999999 < value) - return; - len = 0; + /* maximum number of characters required for output of %lld or %p. + We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits, + plus 1 for the null byte. 53/22 is an upper bound for log10(256). */ + char buffer[1 + (sizeof(unsigned long)*53-1) / 22 + 1]; + char *ptr, *end; + + end = &buffer[Py_ARRAY_LENGTH(buffer) - 1]; + ptr = end; + *ptr = '\0'; do { - buffer[len] = '0' + (value % 10); + --ptr; + assert(ptr >= buffer); + *ptr = '0' + (value % 10); value /= 10; - len++; } while (value); - reverse_string(buffer, len); - _Py_write_noraise(fd, buffer, len); + + _Py_write_noraise(fd, ptr, end - ptr); } /* Format an integer in range [0; 0xffffffff] to hexadecimal of 'width' digits, @@ -521,26 +507,29 @@ dump_decimal(int fd, int value) This function is signal safe. */ static void -dump_hexadecimal(int fd, unsigned long value, int width) +dump_hexadecimal(int fd, unsigned long value, Py_ssize_t width) { - int len; - char buffer[sizeof(unsigned long) * 2 + 1]; - len = 0; + Py_ssize_t size = sizeof(unsigned long) * 2; + char buffer[size + 1], *ptr, *end; + + if (width > size) + width = size; + + end = &buffer[Py_ARRAY_LENGTH(buffer) - 1]; + ptr = end; + *ptr = '\0'; do { - buffer[len] = Py_hexdigits[value & 15]; + --ptr; + assert(ptr >= buffer); + *ptr = Py_hexdigits[value & 15]; value >>= 4; - len++; - } while (len < width || value); - reverse_string(buffer, len); - _Py_write_noraise(fd, buffer, len); -} - -/* Write an unicode object into the file fd using ascii+backslashreplace. + } while ((end - ptr) < width || value); - This function is signal safe. */ + _Py_write_noraise(fd, ptr, end - ptr); +} -static void -dump_ascii(int fd, PyObject *text) +void +_Py_DumpASCII(int fd, PyObject *text) { PyASCIIObject *ascii = (PyASCIIObject *)text; Py_ssize_t i, size; @@ -550,6 +539,9 @@ dump_ascii(int fd, PyObject *text) wchar_t *wstr = NULL; Py_UCS4 ch; + if (!PyUnicode_Check(text)) + return; + size = ascii->length; kind = ascii->state.kind; if (ascii->state.compact) { @@ -574,8 +566,9 @@ dump_ascii(int fd, PyObject *text) size = MAX_STRING_LENGTH; truncated = 1; } - else + else { truncated = 0; + } for (i=0; i < size; i++) { if (kind != PyUnicode_WCHAR_KIND) @@ -600,8 +593,9 @@ dump_ascii(int fd, PyObject *text) dump_hexadecimal(fd, ch, 8); } } - if (truncated) + if (truncated) { PUTS(fd, "..."); + } } /* Write a frame into the file fd: "File "xxx", line xxx in xxx". @@ -620,7 +614,7 @@ dump_frame(int fd, PyFrameObject *frame) && PyUnicode_Check(code->co_filename)) { PUTS(fd, "\""); - dump_ascii(fd, code->co_filename); + _Py_DumpASCII(fd, code->co_filename); PUTS(fd, "\""); } else { PUTS(fd, "???"); @@ -629,14 +623,21 @@ dump_frame(int fd, PyFrameObject *frame) /* PyFrame_GetLineNumber() was introduced in Python 2.7.0 and 3.2.0 */ lineno = PyCode_Addr2Line(code, frame->f_lasti); PUTS(fd, ", line "); - dump_decimal(fd, lineno); + if (lineno >= 0) { + _Py_DumpDecimal(fd, (unsigned long)lineno); + } + else { + PUTS(fd, "???"); + } PUTS(fd, " in "); if (code != NULL && code->co_name != NULL - && PyUnicode_Check(code->co_name)) - dump_ascii(fd, code->co_name); - else + && PyUnicode_Check(code->co_name)) { + _Py_DumpASCII(fd, code->co_name); + } + else { PUTS(fd, "???"); + } PUTS(fd, "\n"); } @@ -692,7 +693,9 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) PUTS(fd, "Current thread 0x"); else PUTS(fd, "Thread 0x"); - dump_hexadecimal(fd, (unsigned long)tstate->thread_id, sizeof(unsigned long)*2); + dump_hexadecimal(fd, + (unsigned long)tstate->thread_id, + sizeof(unsigned long) * 2); PUTS(fd, " (most recent call first):\n"); } -- cgit v0.12 From 0611c26a58a937dace420691e797fbea21db619b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 22:22:13 +0100 Subject: On memory error, dump the memory block traceback Issue #26564: _PyObject_DebugDumpAddress() now dumps the traceback where a memory block was allocated on memory block. Use the tracemalloc module to get the traceback. --- Doc/c-api/memory.rst | 7 +++++ Doc/whatsnew/3.6.rst | 43 +++++++++++++++++++++++++++++- Misc/NEWS | 4 +++ Modules/_tracemalloc.c | 67 +++++++++++++++++++++++++++++++++++++++-------- Modules/hashtable.c | 6 ++--- Objects/bytearrayobject.c | 1 + Objects/obmalloc.c | 9 +++++++ Parser/pgenmain.c | 8 +++--- 8 files changed, 126 insertions(+), 19 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 25867d9..843ccac 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -349,12 +349,19 @@ Customize Memory Allocators allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: :c:func:`PyObject_Malloc`) are called + On error, the debug hooks use the :mod:`tracemalloc` module to get the + traceback where a memory block was allocated. The traceback is only + displayed if :mod:`tracemalloc` is tracing Python memory allocations and the + memory block was traced. + These hooks are installed by default if Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install debug hooks on a Python compiled in release mode. .. versionchanged:: 3.6 This function now also works on Python compiled in release mode. + On error, the debug hooks now use :mod:`tracemalloc` to get the traceback + where a memory block was allocated. .. _pymalloc: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b644a5c..443e46a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -129,7 +129,48 @@ the C library for all Python memory allocations using ``PYTHONMALLOC=malloc``. It helps to use external memory debuggers like Valgrind on a Python compiled in release mode. -(Contributed by Victor Stinner in :issue:`26516`.) +On error, the debug hooks on Python memory allocators now use the +:mod:`tracemalloc` module to get the traceback where a memory block was +allocated. + +Example of fatal error on buffer overflow using +``python3.6 -X tracemalloc=5`` (store 5 frames in traces):: + + Debug memory block at address p=0x7fbcd41666f8: API 'o' + 4 bytes originally requested + The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected. + The 8 pad bytes at tail=0x7fbcd41666fc are not all FORBIDDENBYTE (0xfb): + at tail+0: 0x02 *** OUCH + at tail+1: 0xfb + at tail+2: 0xfb + at tail+3: 0xfb + at tail+4: 0xfb + at tail+5: 0xfb + at tail+6: 0xfb + at tail+7: 0xfb + The block was made by call #1233329 to debug malloc/realloc. + Data at p: 1a 2b 30 00 + + Memory block allocated at (most recent call first): + File "test/test_bytes.py", line 323 + File "unittest/case.py", line 600 + File "unittest/case.py", line 648 + File "unittest/suite.py", line 122 + File "unittest/suite.py", line 84 + + Fatal Python error: bad trailing pad byte + + Current thread 0x00007fbcdbd32700 (most recent call first): + File "test/test_bytes.py", line 323 in test_hex + File "unittest/case.py", line 600 in run + File "unittest/case.py", line 648 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + ... + +(Contributed by Victor Stinner in :issue:`26516` and :issue:`26564`.) Other Language Changes diff --git a/Misc/NEWS b/Misc/NEWS index 45769ba..cbb1a51 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26564: On error, the debug hooks on Python memory allocators now use + the :mod:`tracemalloc` module to get the traceback where a memory block was + allocated. + - Issue #26558: The debug hooks on Python memory allocator :c:func:`PyObject_Malloc` now detect when functions are called without holding the GIL. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 1940af4..5752904 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1161,6 +1161,25 @@ finally: return get_traces.list; } +static traceback_t* +tracemalloc_get_traceback(const void *ptr) +{ + trace_t trace; + int found; + + if (!tracemalloc_config.tracing) + return NULL; + + TABLES_LOCK(); + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); + + if (!found) + return NULL; + + return trace.traceback; +} + PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, "_get_object_traceback(obj)\n" "\n" @@ -1175,11 +1194,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) { PyTypeObject *type; void *ptr; - trace_t trace; - int found; - - if (!tracemalloc_config.tracing) - Py_RETURN_NONE; + traceback_t *traceback; type = Py_TYPE(obj); if (PyType_IS_GC(type)) @@ -1187,16 +1202,46 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); - TABLES_UNLOCK(); - - if (!found) + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) Py_RETURN_NONE; - return traceback_to_pyobject(trace.traceback, NULL); + return traceback_to_pyobject(traceback, NULL); +} + +#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) + +static void +_PyMem_DumpFrame(int fd, frame_t * frame) +{ + PUTS(fd, " File \""); + _Py_DumpASCII(fd, frame->filename); + PUTS(fd, "\", line "); + _Py_DumpDecimal(fd, frame->lineno); + PUTS(fd, "\n"); } +/* Dump the traceback where a memory block was allocated into file descriptor + fd. The function may block on TABLES_LOCK() but it is unlikely. */ +void +_PyMem_DumpTraceback(int fd, const void *ptr) +{ + traceback_t *traceback; + int i; + + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) + return; + + PUTS(fd, "Memory block allocated at (most recent call first):\n"); + for (i=0; i < traceback->nframe; i++) { + _PyMem_DumpFrame(fd, &traceback->frames[i]); + } + PUTS(fd, "\n"); +} + +#undef PUTS + PyDoc_STRVAR(tracemalloc_start_doc, "start(nframe: int=1)\n" "\n" diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 133f313..7de154b 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -486,9 +486,9 @@ _Py_hashtable_copy(_Py_hashtable_t *src) void *data, *new_data; dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, - src->hash_func, src->compare_func, - src->copy_data_func, src->free_data_func, - src->get_data_size_func, &src->alloc); + src->hash_func, src->compare_func, + src->copy_data_func, src->free_data_func, + src->get_data_size_func, &src->alloc); if (dst == NULL) return NULL; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 9e8ba39..67ae7d9 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2820,6 +2820,7 @@ bytearray_hex(PyBytesObject *self) { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); + PyByteArray_AS_STRING(self)[arglen+1] = 2; return _Py_strhex(argbuf, arglen); } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f526b1f..8812f59 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,5 +1,10 @@ #include "Python.h" + +/* Defined in tracemalloc.c */ +extern void _PyMem_DumpTraceback(int fd, const void *ptr); + + /* Python's malloc wrappers (see pymem.h) */ /* @@ -2202,6 +2207,10 @@ _PyObject_DebugDumpAddress(const void *p) } fputc('\n', stderr); } + fputc('\n', stderr); + + fflush(stderr); + _PyMem_DumpTraceback(fileno(stderr), p); } diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 671a0cb..d5a13fe 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -38,11 +38,11 @@ Py_Exit(int sts) } #ifdef WITH_THREAD -/* Needed by obmalloc.c */ +/* Functions needed by obmalloc.c */ int PyGILState_Check(void) -{ - return 1; -} +{ return 1; } +void _PyMem_DumpTraceback(int fd, const void *ptr) +{} #endif int -- cgit v0.12 From ffcf1a54d39fcb2b9f9830b64708c423c2c4a18d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 22:49:40 +0100 Subject: Oops, revert unwanted change used to create an example Issue #26564. --- Objects/bytearrayobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 67ae7d9..9e8ba39 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2820,7 +2820,6 @@ bytearray_hex(PyBytesObject *self) { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); - PyByteArray_AS_STRING(self)[arglen+1] = 2; return _Py_strhex(argbuf, arglen); } -- cgit v0.12 From 6453e9ed0add5f60d150692cef4596702dcc3290 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 23:36:28 +0100 Subject: Issue #26564: Fix test_capi --- Lib/test/test_capi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 1de19a0..8e6245b 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -583,6 +583,7 @@ class PyMemDebugTests(unittest.TestCase): r" .*\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: cb cb cb .*\n" + r"\n" r"Fatal Python error: bad trailing pad byte") regex = regex.format(ptr=self.PTR_REGEX) regex = re.compile(regex, flags=re.DOTALL) @@ -596,6 +597,7 @@ class PyMemDebugTests(unittest.TestCase): r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: cb cb cb .*\n" + r"\n" r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) -- cgit v0.12 From 82f04e2dfdd2bdfef5bd94f8d6615ac0a1f620b4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 23:08:44 +0100 Subject: regrtest: Fix module.__path__ Issue #26538: libregrtest: Fix setup_tests() to keep module.__path__ type (_NamespacePath), don't convert to a list. Add _NamespacePath.__setitem__() method to importlib._bootstrap_external. --- Lib/importlib/_bootstrap_external.py | 3 + Lib/test/libregrtest/setup.py | 4 +- Python/importlib_external.h | 1871 +++++++++++++++++----------------- 3 files changed, 945 insertions(+), 933 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index ddb2456..5d63a1f 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -971,6 +971,9 @@ class _NamespacePath: def __iter__(self): return iter(self._recalculate()) + def __setitem__(self, index, path): + self._path[index] = path + def __len__(self): return len(self._recalculate()) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 6e05c7e..de52bb5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -41,8 +41,8 @@ def setup_tests(ns): # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) - for path in module.__path__] + for index, path in enumerate(module.__path__): + module.__path__[index] = os.path.abspath(path) if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 3d7aff0..840e9c4 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1637,7 +1637,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,0,0,115,20,0,0,0,12,6,6,2,12,4,12,4, 12,3,12,8,12,6,12,6,12,4,12,4,114,218,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0, - 0,0,64,0,0,0,115,130,0,0,0,101,0,0,90,1, + 0,0,64,0,0,0,115,142,0,0,0,101,0,0,90,1, 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, @@ -1645,939 +1645,948 @@ const unsigned char _Py_M__importlib_external[] = { 100,11,0,132,0,0,90,8,0,100,12,0,100,13,0,132, 0,0,90,9,0,100,14,0,100,15,0,132,0,0,90,10, 0,100,16,0,100,17,0,132,0,0,90,11,0,100,18,0, - 100,19,0,132,0,0,90,12,0,100,20,0,83,41,21,218, - 14,95,78,97,109,101,115,112,97,99,101,80,97,116,104,97, - 38,1,0,0,82,101,112,114,101,115,101,110,116,115,32,97, - 32,110,97,109,101,115,112,97,99,101,32,112,97,99,107,97, - 103,101,39,115,32,112,97,116,104,46,32,32,73,116,32,117, - 115,101,115,32,116,104,101,32,109,111,100,117,108,101,32,110, - 97,109,101,10,32,32,32,32,116,111,32,102,105,110,100,32, - 105,116,115,32,112,97,114,101,110,116,32,109,111,100,117,108, - 101,44,32,97,110,100,32,102,114,111,109,32,116,104,101,114, - 101,32,105,116,32,108,111,111,107,115,32,117,112,32,116,104, - 101,32,112,97,114,101,110,116,39,115,10,32,32,32,32,95, - 95,112,97,116,104,95,95,46,32,32,87,104,101,110,32,116, - 104,105,115,32,99,104,97,110,103,101,115,44,32,116,104,101, - 32,109,111,100,117,108,101,39,115,32,111,119,110,32,112,97, - 116,104,32,105,115,32,114,101,99,111,109,112,117,116,101,100, - 44,10,32,32,32,32,117,115,105,110,103,32,112,97,116,104, - 95,102,105,110,100,101,114,46,32,32,70,111,114,32,116,111, - 112,45,108,101,118,101,108,32,109,111,100,117,108,101,115,44, - 32,116,104,101,32,112,97,114,101,110,116,32,109,111,100,117, - 108,101,39,115,32,112,97,116,104,10,32,32,32,32,105,115, - 32,115,121,115,46,112,97,116,104,46,99,4,0,0,0,0, - 0,0,0,4,0,0,0,2,0,0,0,67,0,0,0,115, - 52,0,0,0,124,1,0,124,0,0,95,0,0,124,2,0, - 124,0,0,95,1,0,116,2,0,124,0,0,106,3,0,131, - 0,0,131,1,0,124,0,0,95,4,0,124,3,0,124,0, - 0,95,5,0,100,0,0,83,41,1,78,41,6,218,5,95, - 110,97,109,101,218,5,95,112,97,116,104,114,93,0,0,0, - 218,16,95,103,101,116,95,112,97,114,101,110,116,95,112,97, - 116,104,218,17,95,108,97,115,116,95,112,97,114,101,110,116, - 95,112,97,116,104,218,12,95,112,97,116,104,95,102,105,110, - 100,101,114,41,4,114,100,0,0,0,114,98,0,0,0,114, - 35,0,0,0,218,11,112,97,116,104,95,102,105,110,100,101, - 114,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,170,3,0,0,115,8,0,0,0,0,1, - 9,1,9,1,21,1,122,23,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,99, - 1,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, - 67,0,0,0,115,53,0,0,0,124,0,0,106,0,0,106, - 1,0,100,1,0,131,1,0,92,3,0,125,1,0,125,2, - 0,125,3,0,124,2,0,100,2,0,107,2,0,114,43,0, - 100,6,0,83,124,1,0,100,5,0,102,2,0,83,41,7, - 122,62,82,101,116,117,114,110,115,32,97,32,116,117,112,108, - 101,32,111,102,32,40,112,97,114,101,110,116,45,109,111,100, - 117,108,101,45,110,97,109,101,44,32,112,97,114,101,110,116, - 45,112,97,116,104,45,97,116,116,114,45,110,97,109,101,41, - 114,58,0,0,0,114,30,0,0,0,114,7,0,0,0,114, - 35,0,0,0,90,8,95,95,112,97,116,104,95,95,41,2, - 122,3,115,121,115,122,4,112,97,116,104,41,2,114,225,0, - 0,0,114,32,0,0,0,41,4,114,100,0,0,0,114,216, - 0,0,0,218,3,100,111,116,90,2,109,101,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,23,95,102,105, - 110,100,95,112,97,114,101,110,116,95,112,97,116,104,95,110, - 97,109,101,115,176,3,0,0,115,8,0,0,0,0,2,27, - 1,12,2,4,3,122,38,95,78,97,109,101,115,112,97,99, - 101,80,97,116,104,46,95,102,105,110,100,95,112,97,114,101, - 110,116,95,112,97,116,104,95,110,97,109,101,115,99,1,0, - 0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0, - 0,0,115,38,0,0,0,124,0,0,106,0,0,131,0,0, - 92,2,0,125,1,0,125,2,0,116,1,0,116,2,0,106, - 3,0,124,1,0,25,124,2,0,131,2,0,83,41,1,78, - 41,4,114,232,0,0,0,114,110,0,0,0,114,7,0,0, - 0,218,7,109,111,100,117,108,101,115,41,3,114,100,0,0, - 0,90,18,112,97,114,101,110,116,95,109,111,100,117,108,101, - 95,110,97,109,101,90,14,112,97,116,104,95,97,116,116,114, - 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,227,0,0,0,186,3,0,0,115,4,0, - 0,0,0,1,18,1,122,31,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,46,95,103,101,116,95,112,97,114,101, - 110,116,95,112,97,116,104,99,1,0,0,0,0,0,0,0, - 3,0,0,0,3,0,0,0,67,0,0,0,115,118,0,0, - 0,116,0,0,124,0,0,106,1,0,131,0,0,131,1,0, - 125,1,0,124,1,0,124,0,0,106,2,0,107,3,0,114, - 111,0,124,0,0,106,3,0,124,0,0,106,4,0,124,1, - 0,131,2,0,125,2,0,124,2,0,100,0,0,107,9,0, - 114,102,0,124,2,0,106,5,0,100,0,0,107,8,0,114, - 102,0,124,2,0,106,6,0,114,102,0,124,2,0,106,6, - 0,124,0,0,95,7,0,124,1,0,124,0,0,95,2,0, - 124,0,0,106,7,0,83,41,1,78,41,8,114,93,0,0, - 0,114,227,0,0,0,114,228,0,0,0,114,229,0,0,0, - 114,225,0,0,0,114,120,0,0,0,114,150,0,0,0,114, - 226,0,0,0,41,3,114,100,0,0,0,90,11,112,97,114, - 101,110,116,95,112,97,116,104,114,158,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,114, - 101,99,97,108,99,117,108,97,116,101,190,3,0,0,115,16, - 0,0,0,0,2,18,1,15,1,21,3,27,1,9,1,12, - 1,9,1,122,27,95,78,97,109,101,115,112,97,99,101,80, - 97,116,104,46,95,114,101,99,97,108,99,117,108,97,116,101, + 100,19,0,132,0,0,90,12,0,100,20,0,100,21,0,132, + 0,0,90,13,0,100,22,0,83,41,23,218,14,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,97,38,1,0,0, + 82,101,112,114,101,115,101,110,116,115,32,97,32,110,97,109, + 101,115,112,97,99,101,32,112,97,99,107,97,103,101,39,115, + 32,112,97,116,104,46,32,32,73,116,32,117,115,101,115,32, + 116,104,101,32,109,111,100,117,108,101,32,110,97,109,101,10, + 32,32,32,32,116,111,32,102,105,110,100,32,105,116,115,32, + 112,97,114,101,110,116,32,109,111,100,117,108,101,44,32,97, + 110,100,32,102,114,111,109,32,116,104,101,114,101,32,105,116, + 32,108,111,111,107,115,32,117,112,32,116,104,101,32,112,97, + 114,101,110,116,39,115,10,32,32,32,32,95,95,112,97,116, + 104,95,95,46,32,32,87,104,101,110,32,116,104,105,115,32, + 99,104,97,110,103,101,115,44,32,116,104,101,32,109,111,100, + 117,108,101,39,115,32,111,119,110,32,112,97,116,104,32,105, + 115,32,114,101,99,111,109,112,117,116,101,100,44,10,32,32, + 32,32,117,115,105,110,103,32,112,97,116,104,95,102,105,110, + 100,101,114,46,32,32,70,111,114,32,116,111,112,45,108,101, + 118,101,108,32,109,111,100,117,108,101,115,44,32,116,104,101, + 32,112,97,114,101,110,116,32,109,111,100,117,108,101,39,115, + 32,112,97,116,104,10,32,32,32,32,105,115,32,115,121,115, + 46,112,97,116,104,46,99,4,0,0,0,0,0,0,0,4, + 0,0,0,2,0,0,0,67,0,0,0,115,52,0,0,0, + 124,1,0,124,0,0,95,0,0,124,2,0,124,0,0,95, + 1,0,116,2,0,124,0,0,106,3,0,131,0,0,131,1, + 0,124,0,0,95,4,0,124,3,0,124,0,0,95,5,0, + 100,0,0,83,41,1,78,41,6,218,5,95,110,97,109,101, + 218,5,95,112,97,116,104,114,93,0,0,0,218,16,95,103, + 101,116,95,112,97,114,101,110,116,95,112,97,116,104,218,17, + 95,108,97,115,116,95,112,97,114,101,110,116,95,112,97,116, + 104,218,12,95,112,97,116,104,95,102,105,110,100,101,114,41, + 4,114,100,0,0,0,114,98,0,0,0,114,35,0,0,0, + 218,11,112,97,116,104,95,102,105,110,100,101,114,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,179,0,0, + 0,170,3,0,0,115,8,0,0,0,0,1,9,1,9,1, + 21,1,122,23,95,78,97,109,101,115,112,97,99,101,80,97, + 116,104,46,95,95,105,110,105,116,95,95,99,1,0,0,0, + 0,0,0,0,4,0,0,0,3,0,0,0,67,0,0,0, + 115,53,0,0,0,124,0,0,106,0,0,106,1,0,100,1, + 0,131,1,0,92,3,0,125,1,0,125,2,0,125,3,0, + 124,2,0,100,2,0,107,2,0,114,43,0,100,6,0,83, + 124,1,0,100,5,0,102,2,0,83,41,7,122,62,82,101, + 116,117,114,110,115,32,97,32,116,117,112,108,101,32,111,102, + 32,40,112,97,114,101,110,116,45,109,111,100,117,108,101,45, + 110,97,109,101,44,32,112,97,114,101,110,116,45,112,97,116, + 104,45,97,116,116,114,45,110,97,109,101,41,114,58,0,0, + 0,114,30,0,0,0,114,7,0,0,0,114,35,0,0,0, + 90,8,95,95,112,97,116,104,95,95,41,2,122,3,115,121, + 115,122,4,112,97,116,104,41,2,114,225,0,0,0,114,32, + 0,0,0,41,4,114,100,0,0,0,114,216,0,0,0,218, + 3,100,111,116,90,2,109,101,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,23,95,102,105,110,100,95,112, + 97,114,101,110,116,95,112,97,116,104,95,110,97,109,101,115, + 176,3,0,0,115,8,0,0,0,0,2,27,1,12,2,4, + 3,122,38,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,95,102,105,110,100,95,112,97,114,101,110,116,95,112, + 97,116,104,95,110,97,109,101,115,99,1,0,0,0,0,0, + 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,38, + 0,0,0,124,0,0,106,0,0,131,0,0,92,2,0,125, + 1,0,125,2,0,116,1,0,116,2,0,106,3,0,124,1, + 0,25,124,2,0,131,2,0,83,41,1,78,41,4,114,232, + 0,0,0,114,110,0,0,0,114,7,0,0,0,218,7,109, + 111,100,117,108,101,115,41,3,114,100,0,0,0,90,18,112, + 97,114,101,110,116,95,109,111,100,117,108,101,95,110,97,109, + 101,90,14,112,97,116,104,95,97,116,116,114,95,110,97,109, + 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,227,0,0,0,186,3,0,0,115,4,0,0,0,0,1, + 18,1,122,31,95,78,97,109,101,115,112,97,99,101,80,97, + 116,104,46,95,103,101,116,95,112,97,114,101,110,116,95,112, + 97,116,104,99,1,0,0,0,0,0,0,0,3,0,0,0, + 3,0,0,0,67,0,0,0,115,118,0,0,0,116,0,0, + 124,0,0,106,1,0,131,0,0,131,1,0,125,1,0,124, + 1,0,124,0,0,106,2,0,107,3,0,114,111,0,124,0, + 0,106,3,0,124,0,0,106,4,0,124,1,0,131,2,0, + 125,2,0,124,2,0,100,0,0,107,9,0,114,102,0,124, + 2,0,106,5,0,100,0,0,107,8,0,114,102,0,124,2, + 0,106,6,0,114,102,0,124,2,0,106,6,0,124,0,0, + 95,7,0,124,1,0,124,0,0,95,2,0,124,0,0,106, + 7,0,83,41,1,78,41,8,114,93,0,0,0,114,227,0, + 0,0,114,228,0,0,0,114,229,0,0,0,114,225,0,0, + 0,114,120,0,0,0,114,150,0,0,0,114,226,0,0,0, + 41,3,114,100,0,0,0,90,11,112,97,114,101,110,116,95, + 112,97,116,104,114,158,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,12,95,114,101,99,97,108, + 99,117,108,97,116,101,190,3,0,0,115,16,0,0,0,0, + 2,18,1,15,1,21,3,27,1,9,1,12,1,9,1,122, + 27,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, + 95,114,101,99,97,108,99,117,108,97,116,101,99,1,0,0, + 0,0,0,0,0,1,0,0,0,2,0,0,0,67,0,0, + 0,115,16,0,0,0,116,0,0,124,0,0,106,1,0,131, + 0,0,131,1,0,83,41,1,78,41,2,218,4,105,116,101, + 114,114,234,0,0,0,41,1,114,100,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,8,95,95, + 105,116,101,114,95,95,203,3,0,0,115,2,0,0,0,0, + 1,122,23,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,95,95,105,116,101,114,95,95,99,3,0,0,0,0, + 0,0,0,3,0,0,0,3,0,0,0,67,0,0,0,115, + 17,0,0,0,124,2,0,124,0,0,106,0,0,124,1,0, + 60,100,0,0,83,41,1,78,41,1,114,226,0,0,0,41, + 3,114,100,0,0,0,218,5,105,110,100,101,120,114,35,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,11,95,95,115,101,116,105,116,101,109,95,95,206,3, + 0,0,115,2,0,0,0,0,1,122,26,95,78,97,109,101, + 115,112,97,99,101,80,97,116,104,46,95,95,115,101,116,105, + 116,101,109,95,95,99,1,0,0,0,0,0,0,0,1,0, + 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,116, + 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, + 1,78,41,2,114,31,0,0,0,114,234,0,0,0,41,1, + 114,100,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,218,7,95,95,108,101,110,95,95,209,3,0, + 0,115,2,0,0,0,0,1,122,22,95,78,97,109,101,115, + 112,97,99,101,80,97,116,104,46,95,95,108,101,110,95,95, 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, - 0,67,0,0,0,115,16,0,0,0,116,0,0,124,0,0, - 106,1,0,131,0,0,131,1,0,83,41,1,78,41,2,218, - 4,105,116,101,114,114,234,0,0,0,41,1,114,100,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,8,95,95,105,116,101,114,95,95,203,3,0,0,115,2, - 0,0,0,0,1,122,23,95,78,97,109,101,115,112,97,99, - 101,80,97,116,104,46,95,95,105,116,101,114,95,95,99,1, - 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, - 0,0,0,115,16,0,0,0,116,0,0,124,0,0,106,1, - 0,131,0,0,131,1,0,83,41,1,78,41,2,114,31,0, - 0,0,114,234,0,0,0,41,1,114,100,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,218,7,95, - 95,108,101,110,95,95,206,3,0,0,115,2,0,0,0,0, - 1,122,22,95,78,97,109,101,115,112,97,99,101,80,97,116, - 104,46,95,95,108,101,110,95,95,99,1,0,0,0,0,0, - 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, - 0,0,0,100,1,0,106,0,0,124,0,0,106,1,0,131, - 1,0,83,41,2,78,122,20,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,40,123,33,114,125,41,41,2,114,47, - 0,0,0,114,226,0,0,0,41,1,114,100,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,95,114,101,112,114,95,95,209,3,0,0,115,2,0,0, - 0,0,1,122,23,95,78,97,109,101,115,112,97,99,101,80, - 97,116,104,46,95,95,114,101,112,114,95,95,99,2,0,0, - 0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,0, - 0,115,16,0,0,0,124,1,0,124,0,0,106,0,0,131, - 0,0,107,6,0,83,41,1,78,41,1,114,234,0,0,0, - 41,2,114,100,0,0,0,218,4,105,116,101,109,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,95, - 99,111,110,116,97,105,110,115,95,95,212,3,0,0,115,2, - 0,0,0,0,1,122,27,95,78,97,109,101,115,112,97,99, - 101,80,97,116,104,46,95,95,99,111,110,116,97,105,110,115, - 95,95,99,2,0,0,0,0,0,0,0,2,0,0,0,2, - 0,0,0,67,0,0,0,115,20,0,0,0,124,0,0,106, - 0,0,106,1,0,124,1,0,131,1,0,1,100,0,0,83, - 41,1,78,41,2,114,226,0,0,0,114,157,0,0,0,41, - 2,114,100,0,0,0,114,239,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,215, - 3,0,0,115,2,0,0,0,0,1,122,21,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,97,112,112,101,110, - 100,78,41,13,114,105,0,0,0,114,104,0,0,0,114,106, - 0,0,0,114,107,0,0,0,114,179,0,0,0,114,232,0, - 0,0,114,227,0,0,0,114,234,0,0,0,114,236,0,0, - 0,114,237,0,0,0,114,238,0,0,0,114,240,0,0,0, - 114,157,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,224,0,0,0,163,3, - 0,0,115,20,0,0,0,12,5,6,2,12,6,12,10,12, - 4,12,13,12,3,12,3,12,3,12,3,114,224,0,0,0, - 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, - 0,64,0,0,0,115,118,0,0,0,101,0,0,90,1,0, - 100,0,0,90,2,0,100,1,0,100,2,0,132,0,0,90, - 3,0,101,4,0,100,3,0,100,4,0,132,0,0,131,1, - 0,90,5,0,100,5,0,100,6,0,132,0,0,90,6,0, - 100,7,0,100,8,0,132,0,0,90,7,0,100,9,0,100, - 10,0,132,0,0,90,8,0,100,11,0,100,12,0,132,0, - 0,90,9,0,100,13,0,100,14,0,132,0,0,90,10,0, - 100,15,0,100,16,0,132,0,0,90,11,0,100,17,0,83, - 41,18,218,16,95,78,97,109,101,115,112,97,99,101,76,111, - 97,100,101,114,99,4,0,0,0,0,0,0,0,4,0,0, - 0,4,0,0,0,67,0,0,0,115,25,0,0,0,116,0, - 0,124,1,0,124,2,0,124,3,0,131,3,0,124,0,0, - 95,1,0,100,0,0,83,41,1,78,41,2,114,224,0,0, - 0,114,226,0,0,0,41,4,114,100,0,0,0,114,98,0, - 0,0,114,35,0,0,0,114,230,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,179,0,0,0, - 221,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, - 109,101,115,112,97,99,101,76,111,97,100,101,114,46,95,95, - 105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,2, - 0,0,0,2,0,0,0,67,0,0,0,115,16,0,0,0, - 100,1,0,106,0,0,124,1,0,106,1,0,131,1,0,83, - 41,2,122,115,82,101,116,117,114,110,32,114,101,112,114,32, - 102,111,114,32,116,104,101,32,109,111,100,117,108,101,46,10, - 10,32,32,32,32,32,32,32,32,84,104,101,32,109,101,116, - 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, - 100,46,32,32,84,104,101,32,105,109,112,111,114,116,32,109, - 97,99,104,105,110,101,114,121,32,100,111,101,115,32,116,104, - 101,32,106,111,98,32,105,116,115,101,108,102,46,10,10,32, - 32,32,32,32,32,32,32,122,25,60,109,111,100,117,108,101, - 32,123,33,114,125,32,40,110,97,109,101,115,112,97,99,101, - 41,62,41,2,114,47,0,0,0,114,105,0,0,0,41,2, - 114,164,0,0,0,114,184,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,11,109,111,100,117,108, - 101,95,114,101,112,114,224,3,0,0,115,2,0,0,0,0, - 7,122,28,95,78,97,109,101,115,112,97,99,101,76,111,97, - 100,101,114,46,109,111,100,117,108,101,95,114,101,112,114,99, - 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, - 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,78, - 84,114,4,0,0,0,41,2,114,100,0,0,0,114,119,0, + 0,67,0,0,0,115,16,0,0,0,100,1,0,106,0,0, + 124,0,0,106,1,0,131,1,0,83,41,2,78,122,20,95, + 78,97,109,101,115,112,97,99,101,80,97,116,104,40,123,33, + 114,125,41,41,2,114,47,0,0,0,114,226,0,0,0,41, + 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,8,95,95,114,101,112,114,95,95,212, + 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,95,114,101,112, + 114,95,95,99,2,0,0,0,0,0,0,0,2,0,0,0, + 2,0,0,0,67,0,0,0,115,16,0,0,0,124,1,0, + 124,0,0,106,0,0,131,0,0,107,6,0,83,41,1,78, + 41,1,114,234,0,0,0,41,2,114,100,0,0,0,218,4, + 105,116,101,109,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,12,95,95,99,111,110,116,97,105,110,115,95, + 95,215,3,0,0,115,2,0,0,0,0,1,122,27,95,78, + 97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,99, + 111,110,116,97,105,110,115,95,95,99,2,0,0,0,0,0, + 0,0,2,0,0,0,2,0,0,0,67,0,0,0,115,20, + 0,0,0,124,0,0,106,0,0,106,1,0,124,1,0,131, + 1,0,1,100,0,0,83,41,1,78,41,2,114,226,0,0, + 0,114,157,0,0,0,41,2,114,100,0,0,0,114,241,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,153,0,0,0,233,3,0,0,115,2,0,0,0,0, - 1,122,27,95,78,97,109,101,115,112,97,99,101,76,111,97, - 100,101,114,46,105,115,95,112,97,99,107,97,103,101,99,2, - 0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67, - 0,0,0,115,4,0,0,0,100,1,0,83,41,2,78,114, - 30,0,0,0,114,4,0,0,0,41,2,114,100,0,0,0, - 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,236,3,0,0,115,2,0, - 0,0,0,1,122,27,95,78,97,109,101,115,112,97,99,101, - 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, - 101,99,2,0,0,0,0,0,0,0,2,0,0,0,6,0, - 0,0,67,0,0,0,115,22,0,0,0,116,0,0,100,1, - 0,100,2,0,100,3,0,100,4,0,100,5,0,131,3,1, - 83,41,6,78,114,30,0,0,0,122,8,60,115,116,114,105, - 110,103,62,114,183,0,0,0,114,198,0,0,0,84,41,1, - 114,199,0,0,0,41,2,114,100,0,0,0,114,119,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,181,0,0,0,239,3,0,0,115,2,0,0,0,0,1, - 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, - 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,1,0,83,41,2,122,42,85,115,101, - 32,100,101,102,97,117,108,116,32,115,101,109,97,110,116,105, - 99,115,32,102,111,114,32,109,111,100,117,108,101,32,99,114, - 101,97,116,105,111,110,46,78,114,4,0,0,0,41,2,114, - 100,0,0,0,114,158,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,242,3,0, - 0,115,0,0,0,0,122,30,95,78,97,109,101,115,112,97, - 99,101,76,111,97,100,101,114,46,99,114,101,97,116,101,95, - 109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,2, - 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, - 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,100, + 0,114,157,0,0,0,218,3,0,0,115,2,0,0,0,0, + 1,122,21,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,97,112,112,101,110,100,78,41,14,114,105,0,0,0, + 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, + 179,0,0,0,114,232,0,0,0,114,227,0,0,0,114,234, + 0,0,0,114,236,0,0,0,114,238,0,0,0,114,239,0, + 0,0,114,240,0,0,0,114,242,0,0,0,114,157,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,224,0,0,0,163,3,0,0,115,22, + 0,0,0,12,5,6,2,12,6,12,10,12,4,12,13,12, + 3,12,3,12,3,12,3,12,3,114,224,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, + 0,0,0,115,118,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,100,2,0,132,0,0,90,3,0, + 101,4,0,100,3,0,100,4,0,132,0,0,131,1,0,90, + 5,0,100,5,0,100,6,0,132,0,0,90,6,0,100,7, + 0,100,8,0,132,0,0,90,7,0,100,9,0,100,10,0, + 132,0,0,90,8,0,100,11,0,100,12,0,132,0,0,90, + 9,0,100,13,0,100,14,0,132,0,0,90,10,0,100,15, + 0,100,16,0,132,0,0,90,11,0,100,17,0,83,41,18, + 218,16,95,78,97,109,101,115,112,97,99,101,76,111,97,100, + 101,114,99,4,0,0,0,0,0,0,0,4,0,0,0,4, + 0,0,0,67,0,0,0,115,25,0,0,0,116,0,0,124, + 1,0,124,2,0,124,3,0,131,3,0,124,0,0,95,1, + 0,100,0,0,83,41,1,78,41,2,114,224,0,0,0,114, + 226,0,0,0,41,4,114,100,0,0,0,114,98,0,0,0, + 114,35,0,0,0,114,230,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,179,0,0,0,224,3, + 0,0,115,2,0,0,0,0,1,122,25,95,78,97,109,101, + 115,112,97,99,101,76,111,97,100,101,114,46,95,95,105,110, + 105,116,95,95,99,2,0,0,0,0,0,0,0,2,0,0, + 0,2,0,0,0,67,0,0,0,115,16,0,0,0,100,1, + 0,106,0,0,124,1,0,106,1,0,131,1,0,83,41,2, + 122,115,82,101,116,117,114,110,32,114,101,112,114,32,102,111, + 114,32,116,104,101,32,109,111,100,117,108,101,46,10,10,32, + 32,32,32,32,32,32,32,84,104,101,32,109,101,116,104,111, + 100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46, + 32,32,84,104,101,32,105,109,112,111,114,116,32,109,97,99, + 104,105,110,101,114,121,32,100,111,101,115,32,116,104,101,32, + 106,111,98,32,105,116,115,101,108,102,46,10,10,32,32,32, + 32,32,32,32,32,122,25,60,109,111,100,117,108,101,32,123, + 33,114,125,32,40,110,97,109,101,115,112,97,99,101,41,62, + 41,2,114,47,0,0,0,114,105,0,0,0,41,2,114,164, 0,0,0,114,184,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,185,0,0,0,245,3,0,0, - 115,2,0,0,0,0,1,122,28,95,78,97,109,101,115,112, - 97,99,101,76,111,97,100,101,114,46,101,120,101,99,95,109, - 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,35,0,0,0,116, - 0,0,106,1,0,100,1,0,124,0,0,106,2,0,131,2, - 0,1,116,0,0,106,3,0,124,0,0,124,1,0,131,2, - 0,83,41,2,122,98,76,111,97,100,32,97,32,110,97,109, - 101,115,112,97,99,101,32,109,111,100,117,108,101,46,10,10, - 32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116, - 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, - 100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100, - 117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10, - 32,32,32,32,32,32,32,32,122,38,110,97,109,101,115,112, - 97,99,101,32,109,111,100,117,108,101,32,108,111,97,100,101, - 100,32,119,105,116,104,32,112,97,116,104,32,123,33,114,125, - 41,4,114,114,0,0,0,114,129,0,0,0,114,226,0,0, - 0,114,186,0,0,0,41,2,114,100,0,0,0,114,119,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,187,0,0,0,248,3,0,0,115,6,0,0,0,0, - 7,9,1,10,1,122,28,95,78,97,109,101,115,112,97,99, - 101,76,111,97,100,101,114,46,108,111,97,100,95,109,111,100, - 117,108,101,78,41,12,114,105,0,0,0,114,104,0,0,0, - 114,106,0,0,0,114,179,0,0,0,114,177,0,0,0,114, - 242,0,0,0,114,153,0,0,0,114,196,0,0,0,114,181, - 0,0,0,114,180,0,0,0,114,185,0,0,0,114,187,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,241,0,0,0,220,3,0,0,115, - 16,0,0,0,12,1,12,3,18,9,12,3,12,3,12,3, - 12,3,12,3,114,241,0,0,0,99,0,0,0,0,0,0, - 0,0,0,0,0,0,5,0,0,0,64,0,0,0,115,160, - 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, - 1,0,90,3,0,101,4,0,100,2,0,100,3,0,132,0, - 0,131,1,0,90,5,0,101,4,0,100,4,0,100,5,0, - 132,0,0,131,1,0,90,6,0,101,4,0,100,6,0,100, - 7,0,132,0,0,131,1,0,90,7,0,101,4,0,100,8, - 0,100,9,0,132,0,0,131,1,0,90,8,0,101,4,0, - 100,10,0,100,11,0,100,12,0,132,1,0,131,1,0,90, - 9,0,101,4,0,100,10,0,100,10,0,100,13,0,100,14, - 0,132,2,0,131,1,0,90,10,0,101,4,0,100,10,0, - 100,15,0,100,16,0,132,1,0,131,1,0,90,11,0,100, - 10,0,83,41,17,218,10,80,97,116,104,70,105,110,100,101, - 114,122,62,77,101,116,97,32,112,97,116,104,32,102,105,110, - 100,101,114,32,102,111,114,32,115,121,115,46,112,97,116,104, - 32,97,110,100,32,112,97,99,107,97,103,101,32,95,95,112, - 97,116,104,95,95,32,97,116,116,114,105,98,117,116,101,115, - 46,99,1,0,0,0,0,0,0,0,2,0,0,0,4,0, - 0,0,67,0,0,0,115,55,0,0,0,120,48,0,116,0, - 0,106,1,0,106,2,0,131,0,0,68,93,31,0,125,1, - 0,116,3,0,124,1,0,100,1,0,131,2,0,114,16,0, - 124,1,0,106,4,0,131,0,0,1,113,16,0,87,100,2, - 0,83,41,3,122,125,67,97,108,108,32,116,104,101,32,105, - 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, - 40,41,32,109,101,116,104,111,100,32,111,110,32,97,108,108, - 32,112,97,116,104,32,101,110,116,114,121,32,102,105,110,100, - 101,114,115,10,32,32,32,32,32,32,32,32,115,116,111,114, - 101,100,32,105,110,32,115,121,115,46,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,115,32,40, - 119,104,101,114,101,32,105,109,112,108,101,109,101,110,116,101, - 100,41,46,218,17,105,110,118,97,108,105,100,97,116,101,95, - 99,97,99,104,101,115,78,41,5,114,7,0,0,0,218,19, - 112,97,116,104,95,105,109,112,111,114,116,101,114,95,99,97, - 99,104,101,218,6,118,97,108,117,101,115,114,108,0,0,0, - 114,244,0,0,0,41,2,114,164,0,0,0,218,6,102,105, - 110,100,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,244,0,0,0,10,4,0,0,115,6,0,0, - 0,0,4,22,1,15,1,122,28,80,97,116,104,70,105,110, - 100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99, - 97,99,104,101,115,99,2,0,0,0,0,0,0,0,3,0, - 0,0,12,0,0,0,67,0,0,0,115,107,0,0,0,116, - 0,0,106,1,0,100,1,0,107,9,0,114,41,0,116,0, - 0,106,1,0,12,114,41,0,116,2,0,106,3,0,100,2, - 0,116,4,0,131,2,0,1,120,59,0,116,0,0,106,1, - 0,68,93,44,0,125,2,0,121,14,0,124,2,0,124,1, - 0,131,1,0,83,87,113,51,0,4,116,5,0,107,10,0, - 114,94,0,1,1,1,119,51,0,89,113,51,0,88,113,51, - 0,87,100,1,0,83,100,1,0,83,41,3,122,113,83,101, - 97,114,99,104,32,115,101,113,117,101,110,99,101,32,111,102, - 32,104,111,111,107,115,32,102,111,114,32,97,32,102,105,110, - 100,101,114,32,102,111,114,32,39,112,97,116,104,39,46,10, - 10,32,32,32,32,32,32,32,32,73,102,32,39,104,111,111, - 107,115,39,32,105,115,32,102,97,108,115,101,32,116,104,101, - 110,32,117,115,101,32,115,121,115,46,112,97,116,104,95,104, - 111,111,107,115,46,10,10,32,32,32,32,32,32,32,32,78, - 122,23,115,121,115,46,112,97,116,104,95,104,111,111,107,115, - 32,105,115,32,101,109,112,116,121,41,6,114,7,0,0,0, - 218,10,112,97,116,104,95,104,111,111,107,115,114,60,0,0, - 0,114,61,0,0,0,114,118,0,0,0,114,99,0,0,0, - 41,3,114,164,0,0,0,114,35,0,0,0,90,4,104,111, - 111,107,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,11,95,112,97,116,104,95,104,111,111,107,115,18,4, - 0,0,115,16,0,0,0,0,7,25,1,16,1,16,1,3, - 1,14,1,13,1,12,2,122,22,80,97,116,104,70,105,110, - 100,101,114,46,95,112,97,116,104,95,104,111,111,107,115,99, - 2,0,0,0,0,0,0,0,3,0,0,0,19,0,0,0, - 67,0,0,0,115,123,0,0,0,124,1,0,100,1,0,107, - 2,0,114,53,0,121,16,0,116,0,0,106,1,0,131,0, - 0,125,1,0,87,110,22,0,4,116,2,0,107,10,0,114, - 52,0,1,1,1,100,2,0,83,89,110,1,0,88,121,17, - 0,116,3,0,106,4,0,124,1,0,25,125,2,0,87,110, - 46,0,4,116,5,0,107,10,0,114,118,0,1,1,1,124, - 0,0,106,6,0,124,1,0,131,1,0,125,2,0,124,2, - 0,116,3,0,106,4,0,124,1,0,60,89,110,1,0,88, - 124,2,0,83,41,3,122,210,71,101,116,32,116,104,101,32, - 102,105,110,100,101,114,32,102,111,114,32,116,104,101,32,112, - 97,116,104,32,101,110,116,114,121,32,102,114,111,109,32,115, - 121,115,46,112,97,116,104,95,105,109,112,111,114,116,101,114, - 95,99,97,99,104,101,46,10,10,32,32,32,32,32,32,32, - 32,73,102,32,116,104,101,32,112,97,116,104,32,101,110,116, - 114,121,32,105,115,32,110,111,116,32,105,110,32,116,104,101, - 32,99,97,99,104,101,44,32,102,105,110,100,32,116,104,101, - 32,97,112,112,114,111,112,114,105,97,116,101,32,102,105,110, - 100,101,114,10,32,32,32,32,32,32,32,32,97,110,100,32, - 99,97,99,104,101,32,105,116,46,32,73,102,32,110,111,32, - 102,105,110,100,101,114,32,105,115,32,97,118,97,105,108,97, - 98,108,101,44,32,115,116,111,114,101,32,78,111,110,101,46, - 10,10,32,32,32,32,32,32,32,32,114,30,0,0,0,78, - 41,7,114,3,0,0,0,114,45,0,0,0,218,17,70,105, - 108,101,78,111,116,70,111,117,110,100,69,114,114,111,114,114, - 7,0,0,0,114,245,0,0,0,114,131,0,0,0,114,249, - 0,0,0,41,3,114,164,0,0,0,114,35,0,0,0,114, - 247,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,20,95,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,35,4,0,0,115,22,0, - 0,0,0,8,12,1,3,1,16,1,13,3,9,1,3,1, - 17,1,13,1,15,1,18,1,122,31,80,97,116,104,70,105, - 110,100,101,114,46,95,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,99,3,0,0,0,0,0, - 0,0,6,0,0,0,3,0,0,0,67,0,0,0,115,119, - 0,0,0,116,0,0,124,2,0,100,1,0,131,2,0,114, - 39,0,124,2,0,106,1,0,124,1,0,131,1,0,92,2, - 0,125,3,0,125,4,0,110,21,0,124,2,0,106,2,0, - 124,1,0,131,1,0,125,3,0,103,0,0,125,4,0,124, - 3,0,100,0,0,107,9,0,114,88,0,116,3,0,106,4, - 0,124,1,0,124,3,0,131,2,0,83,116,3,0,106,5, - 0,124,1,0,100,0,0,131,2,0,125,5,0,124,4,0, - 124,5,0,95,6,0,124,5,0,83,41,2,78,114,117,0, - 0,0,41,7,114,108,0,0,0,114,117,0,0,0,114,176, - 0,0,0,114,114,0,0,0,114,173,0,0,0,114,154,0, - 0,0,114,150,0,0,0,41,6,114,164,0,0,0,114,119, - 0,0,0,114,247,0,0,0,114,120,0,0,0,114,121,0, + 0,0,114,5,0,0,0,218,11,109,111,100,117,108,101,95, + 114,101,112,114,227,3,0,0,115,2,0,0,0,0,7,122, + 28,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, + 114,46,109,111,100,117,108,101,95,114,101,112,114,99,2,0, + 0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,0, + 0,0,115,4,0,0,0,100,1,0,83,41,2,78,84,114, + 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, + 153,0,0,0,236,3,0,0,115,2,0,0,0,0,1,122, + 27,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, + 114,46,105,115,95,112,97,99,107,97,103,101,99,2,0,0, + 0,0,0,0,0,2,0,0,0,1,0,0,0,67,0,0, + 0,115,4,0,0,0,100,1,0,83,41,2,78,114,30,0, + 0,0,114,4,0,0,0,41,2,114,100,0,0,0,114,119, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,114,196,0,0,0,239,3,0,0,115,2,0,0,0, + 0,1,122,27,95,78,97,109,101,115,112,97,99,101,76,111, + 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, + 2,0,0,0,0,0,0,0,2,0,0,0,6,0,0,0, + 67,0,0,0,115,22,0,0,0,116,0,0,100,1,0,100, + 2,0,100,3,0,100,4,0,100,5,0,131,3,1,83,41, + 6,78,114,30,0,0,0,122,8,60,115,116,114,105,110,103, + 62,114,183,0,0,0,114,198,0,0,0,84,41,1,114,199, + 0,0,0,41,2,114,100,0,0,0,114,119,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,181, + 0,0,0,242,3,0,0,115,2,0,0,0,0,1,122,25, + 95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,114, + 46,103,101,116,95,99,111,100,101,99,2,0,0,0,0,0, + 0,0,2,0,0,0,1,0,0,0,67,0,0,0,115,4, + 0,0,0,100,1,0,83,41,2,122,42,85,115,101,32,100, + 101,102,97,117,108,116,32,115,101,109,97,110,116,105,99,115, + 32,102,111,114,32,109,111,100,117,108,101,32,99,114,101,97, + 116,105,111,110,46,78,114,4,0,0,0,41,2,114,100,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,16,95,108,101,103,97,99,121,95, - 103,101,116,95,115,112,101,99,57,4,0,0,115,18,0,0, - 0,0,4,15,1,24,2,15,1,6,1,12,1,16,1,18, - 1,9,1,122,27,80,97,116,104,70,105,110,100,101,114,46, - 95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,99, - 78,99,4,0,0,0,0,0,0,0,9,0,0,0,5,0, - 0,0,67,0,0,0,115,243,0,0,0,103,0,0,125,4, - 0,120,230,0,124,2,0,68,93,191,0,125,5,0,116,0, - 0,124,5,0,116,1,0,116,2,0,102,2,0,131,2,0, - 115,43,0,113,13,0,124,0,0,106,3,0,124,5,0,131, - 1,0,125,6,0,124,6,0,100,1,0,107,9,0,114,13, - 0,116,4,0,124,6,0,100,2,0,131,2,0,114,106,0, - 124,6,0,106,5,0,124,1,0,124,3,0,131,2,0,125, - 7,0,110,18,0,124,0,0,106,6,0,124,1,0,124,6, - 0,131,2,0,125,7,0,124,7,0,100,1,0,107,8,0, - 114,139,0,113,13,0,124,7,0,106,7,0,100,1,0,107, - 9,0,114,158,0,124,7,0,83,124,7,0,106,8,0,125, - 8,0,124,8,0,100,1,0,107,8,0,114,191,0,116,9, - 0,100,3,0,131,1,0,130,1,0,124,4,0,106,10,0, - 124,8,0,131,1,0,1,113,13,0,87,116,11,0,106,12, - 0,124,1,0,100,1,0,131,2,0,125,7,0,124,4,0, - 124,7,0,95,8,0,124,7,0,83,100,1,0,83,41,4, - 122,63,70,105,110,100,32,116,104,101,32,108,111,97,100,101, - 114,32,111,114,32,110,97,109,101,115,112,97,99,101,95,112, - 97,116,104,32,102,111,114,32,116,104,105,115,32,109,111,100, - 117,108,101,47,112,97,99,107,97,103,101,32,110,97,109,101, - 46,78,114,175,0,0,0,122,19,115,112,101,99,32,109,105, - 115,115,105,110,103,32,108,111,97,100,101,114,41,13,114,137, - 0,0,0,114,69,0,0,0,218,5,98,121,116,101,115,114, - 251,0,0,0,114,108,0,0,0,114,175,0,0,0,114,252, - 0,0,0,114,120,0,0,0,114,150,0,0,0,114,99,0, - 0,0,114,143,0,0,0,114,114,0,0,0,114,154,0,0, - 0,41,9,114,164,0,0,0,114,119,0,0,0,114,35,0, - 0,0,114,174,0,0,0,218,14,110,97,109,101,115,112,97, - 99,101,95,112,97,116,104,90,5,101,110,116,114,121,114,247, - 0,0,0,114,158,0,0,0,114,121,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,9,95,103, - 101,116,95,115,112,101,99,72,4,0,0,115,40,0,0,0, - 0,5,6,1,13,1,21,1,3,1,15,1,12,1,15,1, - 21,2,18,1,12,1,3,1,15,1,4,1,9,1,12,1, - 12,5,17,2,18,1,9,1,122,20,80,97,116,104,70,105, - 110,100,101,114,46,95,103,101,116,95,115,112,101,99,99,4, - 0,0,0,0,0,0,0,6,0,0,0,4,0,0,0,67, - 0,0,0,115,140,0,0,0,124,2,0,100,1,0,107,8, - 0,114,21,0,116,0,0,106,1,0,125,2,0,124,0,0, - 106,2,0,124,1,0,124,2,0,124,3,0,131,3,0,125, - 4,0,124,4,0,100,1,0,107,8,0,114,58,0,100,1, - 0,83,124,4,0,106,3,0,100,1,0,107,8,0,114,132, - 0,124,4,0,106,4,0,125,5,0,124,5,0,114,125,0, - 100,2,0,124,4,0,95,5,0,116,6,0,124,1,0,124, - 5,0,124,0,0,106,2,0,131,3,0,124,4,0,95,4, - 0,124,4,0,83,100,1,0,83,110,4,0,124,4,0,83, - 100,1,0,83,41,3,122,98,102,105,110,100,32,116,104,101, - 32,109,111,100,117,108,101,32,111,110,32,115,121,115,46,112, - 97,116,104,32,111,114,32,39,112,97,116,104,39,32,98,97, - 115,101,100,32,111,110,32,115,121,115,46,112,97,116,104,95, - 104,111,111,107,115,32,97,110,100,10,32,32,32,32,32,32, - 32,32,115,121,115,46,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,46,78,90,9,110,97,109, - 101,115,112,97,99,101,41,7,114,7,0,0,0,114,35,0, - 0,0,114,255,0,0,0,114,120,0,0,0,114,150,0,0, - 0,114,152,0,0,0,114,224,0,0,0,41,6,114,164,0, - 0,0,114,119,0,0,0,114,35,0,0,0,114,174,0,0, - 0,114,158,0,0,0,114,254,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,104, - 4,0,0,115,26,0,0,0,0,4,12,1,9,1,21,1, - 12,1,4,1,15,1,9,1,6,3,9,1,24,1,4,2, - 7,2,122,20,80,97,116,104,70,105,110,100,101,114,46,102, - 105,110,100,95,115,112,101,99,99,3,0,0,0,0,0,0, - 0,4,0,0,0,3,0,0,0,67,0,0,0,115,41,0, - 0,0,124,0,0,106,0,0,124,1,0,124,2,0,131,2, - 0,125,3,0,124,3,0,100,1,0,107,8,0,114,34,0, - 100,1,0,83,124,3,0,106,1,0,83,41,2,122,170,102, - 105,110,100,32,116,104,101,32,109,111,100,117,108,101,32,111, - 110,32,115,121,115,46,112,97,116,104,32,111,114,32,39,112, - 97,116,104,39,32,98,97,115,101,100,32,111,110,32,115,121, - 115,46,112,97,116,104,95,104,111,111,107,115,32,97,110,100, - 10,32,32,32,32,32,32,32,32,115,121,115,46,112,97,116, - 104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101, - 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, - 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, - 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, - 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, - 10,32,32,32,32,32,32,32,32,78,41,2,114,175,0,0, - 0,114,120,0,0,0,41,4,114,164,0,0,0,114,119,0, - 0,0,114,35,0,0,0,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,176,0,0,0, - 126,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, - 1,122,22,80,97,116,104,70,105,110,100,101,114,46,102,105, - 110,100,95,109,111,100,117,108,101,41,12,114,105,0,0,0, - 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, - 177,0,0,0,114,244,0,0,0,114,249,0,0,0,114,251, - 0,0,0,114,252,0,0,0,114,255,0,0,0,114,175,0, - 0,0,114,176,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,243,0,0,0, - 6,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, - 17,18,22,18,15,3,1,18,31,3,1,21,21,3,1,114, - 243,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, - 0,3,0,0,0,64,0,0,0,115,133,0,0,0,101,0, - 0,90,1,0,100,0,0,90,2,0,100,1,0,90,3,0, - 100,2,0,100,3,0,132,0,0,90,4,0,100,4,0,100, - 5,0,132,0,0,90,5,0,101,6,0,90,7,0,100,6, - 0,100,7,0,132,0,0,90,8,0,100,8,0,100,9,0, - 132,0,0,90,9,0,100,10,0,100,11,0,100,12,0,132, - 1,0,90,10,0,100,13,0,100,14,0,132,0,0,90,11, - 0,101,12,0,100,15,0,100,16,0,132,0,0,131,1,0, - 90,13,0,100,17,0,100,18,0,132,0,0,90,14,0,100, - 10,0,83,41,19,218,10,70,105,108,101,70,105,110,100,101, - 114,122,172,70,105,108,101,45,98,97,115,101,100,32,102,105, - 110,100,101,114,46,10,10,32,32,32,32,73,110,116,101,114, - 97,99,116,105,111,110,115,32,119,105,116,104,32,116,104,101, - 32,102,105,108,101,32,115,121,115,116,101,109,32,97,114,101, - 32,99,97,99,104,101,100,32,102,111,114,32,112,101,114,102, - 111,114,109,97,110,99,101,44,32,98,101,105,110,103,10,32, - 32,32,32,114,101,102,114,101,115,104,101,100,32,119,104,101, - 110,32,116,104,101,32,100,105,114,101,99,116,111,114,121,32, - 116,104,101,32,102,105,110,100,101,114,32,105,115,32,104,97, - 110,100,108,105,110,103,32,104,97,115,32,98,101,101,110,32, - 109,111,100,105,102,105,101,100,46,10,10,32,32,32,32,99, - 2,0,0,0,0,0,0,0,5,0,0,0,5,0,0,0, - 7,0,0,0,115,122,0,0,0,103,0,0,125,3,0,120, - 52,0,124,2,0,68,93,44,0,92,2,0,137,0,0,125, - 4,0,124,3,0,106,0,0,135,0,0,102,1,0,100,1, - 0,100,2,0,134,0,0,124,4,0,68,131,1,0,131,1, - 0,1,113,13,0,87,124,3,0,124,0,0,95,1,0,124, - 1,0,112,79,0,100,3,0,124,0,0,95,2,0,100,6, - 0,124,0,0,95,3,0,116,4,0,131,0,0,124,0,0, - 95,5,0,116,4,0,131,0,0,124,0,0,95,6,0,100, - 5,0,83,41,7,122,154,73,110,105,116,105,97,108,105,122, - 101,32,119,105,116,104,32,116,104,101,32,112,97,116,104,32, - 116,111,32,115,101,97,114,99,104,32,111,110,32,97,110,100, - 32,97,32,118,97,114,105,97,98,108,101,32,110,117,109,98, - 101,114,32,111,102,10,32,32,32,32,32,32,32,32,50,45, - 116,117,112,108,101,115,32,99,111,110,116,97,105,110,105,110, - 103,32,116,104,101,32,108,111,97,100,101,114,32,97,110,100, - 32,116,104,101,32,102,105,108,101,32,115,117,102,102,105,120, - 101,115,32,116,104,101,32,108,111,97,100,101,114,10,32,32, - 32,32,32,32,32,32,114,101,99,111,103,110,105,122,101,115, - 46,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, - 0,0,51,0,0,0,115,27,0,0,0,124,0,0,93,17, - 0,125,1,0,124,1,0,136,0,0,102,2,0,86,1,113, - 3,0,100,0,0,83,41,1,78,114,4,0,0,0,41,2, - 114,22,0,0,0,114,219,0,0,0,41,1,114,120,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 155,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, - 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, - 46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120, - 112,114,62,114,58,0,0,0,114,29,0,0,0,78,114,87, - 0,0,0,41,7,114,143,0,0,0,218,8,95,108,111,97, - 100,101,114,115,114,35,0,0,0,218,11,95,112,97,116,104, - 95,109,116,105,109,101,218,3,115,101,116,218,11,95,112,97, - 116,104,95,99,97,99,104,101,218,19,95,114,101,108,97,120, - 101,100,95,112,97,116,104,95,99,97,99,104,101,41,5,114, - 100,0,0,0,114,35,0,0,0,218,14,108,111,97,100,101, - 114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101, - 114,115,114,160,0,0,0,114,4,0,0,0,41,1,114,120, - 0,0,0,114,5,0,0,0,114,179,0,0,0,149,4,0, - 0,115,16,0,0,0,0,4,6,1,19,1,36,1,9,2, - 15,1,9,1,12,1,122,19,70,105,108,101,70,105,110,100, - 101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0, - 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, - 115,13,0,0,0,100,3,0,124,0,0,95,0,0,100,2, - 0,83,41,4,122,31,73,110,118,97,108,105,100,97,116,101, - 32,116,104,101,32,100,105,114,101,99,116,111,114,121,32,109, - 116,105,109,101,46,114,29,0,0,0,78,114,87,0,0,0, - 41,1,114,2,1,0,0,41,1,114,100,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,244,0, - 0,0,163,4,0,0,115,2,0,0,0,0,2,122,28,70, - 105,108,101,70,105,110,100,101,114,46,105,110,118,97,108,105, - 100,97,116,101,95,99,97,99,104,101,115,99,2,0,0,0, - 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, - 115,59,0,0,0,124,0,0,106,0,0,124,1,0,131,1, - 0,125,2,0,124,2,0,100,1,0,107,8,0,114,37,0, - 100,1,0,103,0,0,102,2,0,83,124,2,0,106,1,0, - 124,2,0,106,2,0,112,55,0,103,0,0,102,2,0,83, - 41,2,122,197,84,114,121,32,116,111,32,102,105,110,100,32, - 97,32,108,111,97,100,101,114,32,102,111,114,32,116,104,101, - 32,115,112,101,99,105,102,105,101,100,32,109,111,100,117,108, - 101,44,32,111,114,32,116,104,101,32,110,97,109,101,115,112, - 97,99,101,10,32,32,32,32,32,32,32,32,112,97,99,107, - 97,103,101,32,112,111,114,116,105,111,110,115,46,32,82,101, - 116,117,114,110,115,32,40,108,111,97,100,101,114,44,32,108, - 105,115,116,45,111,102,45,112,111,114,116,105,111,110,115,41, - 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, - 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, - 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, - 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, - 10,32,32,32,32,32,32,32,32,78,41,3,114,175,0,0, - 0,114,120,0,0,0,114,150,0,0,0,41,3,114,100,0, - 0,0,114,119,0,0,0,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,117,0,0,0, - 169,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, - 1,122,22,70,105,108,101,70,105,110,100,101,114,46,102,105, - 110,100,95,108,111,97,100,101,114,99,6,0,0,0,0,0, - 0,0,7,0,0,0,7,0,0,0,67,0,0,0,115,40, - 0,0,0,124,1,0,124,2,0,124,3,0,131,2,0,125, - 6,0,116,0,0,124,2,0,124,3,0,100,1,0,124,6, - 0,100,2,0,124,4,0,131,2,2,83,41,3,78,114,120, - 0,0,0,114,150,0,0,0,41,1,114,161,0,0,0,41, - 7,114,100,0,0,0,114,159,0,0,0,114,119,0,0,0, - 114,35,0,0,0,90,4,115,109,115,108,114,174,0,0,0, - 114,120,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,255,0,0,0,181,4,0,0,115,6,0, - 0,0,0,1,15,1,18,1,122,20,70,105,108,101,70,105, - 110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99, - 3,0,0,0,0,0,0,0,14,0,0,0,15,0,0,0, - 67,0,0,0,115,228,1,0,0,100,1,0,125,3,0,124, - 1,0,106,0,0,100,2,0,131,1,0,100,3,0,25,125, - 4,0,121,34,0,116,1,0,124,0,0,106,2,0,112,49, - 0,116,3,0,106,4,0,131,0,0,131,1,0,106,5,0, - 125,5,0,87,110,24,0,4,116,6,0,107,10,0,114,85, - 0,1,1,1,100,10,0,125,5,0,89,110,1,0,88,124, - 5,0,124,0,0,106,7,0,107,3,0,114,120,0,124,0, - 0,106,8,0,131,0,0,1,124,5,0,124,0,0,95,7, - 0,116,9,0,131,0,0,114,153,0,124,0,0,106,10,0, - 125,6,0,124,4,0,106,11,0,131,0,0,125,7,0,110, - 15,0,124,0,0,106,12,0,125,6,0,124,4,0,125,7, - 0,124,7,0,124,6,0,107,6,0,114,45,1,116,13,0, - 124,0,0,106,2,0,124,4,0,131,2,0,125,8,0,120, - 100,0,124,0,0,106,14,0,68,93,77,0,92,2,0,125, - 9,0,125,10,0,100,5,0,124,9,0,23,125,11,0,116, - 13,0,124,8,0,124,11,0,131,2,0,125,12,0,116,15, - 0,124,12,0,131,1,0,114,208,0,124,0,0,106,16,0, - 124,10,0,124,1,0,124,12,0,124,8,0,103,1,0,124, - 2,0,131,5,0,83,113,208,0,87,116,17,0,124,8,0, - 131,1,0,125,3,0,120,120,0,124,0,0,106,14,0,68, - 93,109,0,92,2,0,125,9,0,125,10,0,116,13,0,124, - 0,0,106,2,0,124,4,0,124,9,0,23,131,2,0,125, - 12,0,116,18,0,106,19,0,100,6,0,124,12,0,100,7, - 0,100,3,0,131,2,1,1,124,7,0,124,9,0,23,124, - 6,0,107,6,0,114,55,1,116,15,0,124,12,0,131,1, - 0,114,55,1,124,0,0,106,16,0,124,10,0,124,1,0, - 124,12,0,100,8,0,124,2,0,131,5,0,83,113,55,1, - 87,124,3,0,114,224,1,116,18,0,106,19,0,100,9,0, - 124,8,0,131,2,0,1,116,18,0,106,20,0,124,1,0, - 100,8,0,131,2,0,125,13,0,124,8,0,103,1,0,124, - 13,0,95,21,0,124,13,0,83,100,8,0,83,41,11,122, - 125,84,114,121,32,116,111,32,102,105,110,100,32,97,32,108, - 111,97,100,101,114,32,102,111,114,32,116,104,101,32,115,112, - 101,99,105,102,105,101,100,32,109,111,100,117,108,101,44,32, - 111,114,32,116,104,101,32,110,97,109,101,115,112,97,99,101, - 10,32,32,32,32,32,32,32,32,112,97,99,107,97,103,101, - 32,112,111,114,116,105,111,110,115,46,32,82,101,116,117,114, - 110,115,32,40,108,111,97,100,101,114,44,32,108,105,115,116, - 45,111,102,45,112,111,114,116,105,111,110,115,41,46,70,114, - 58,0,0,0,114,56,0,0,0,114,29,0,0,0,114,179, - 0,0,0,122,9,116,114,121,105,110,103,32,123,125,90,9, - 118,101,114,98,111,115,105,116,121,78,122,25,112,111,115,115, - 105,98,108,101,32,110,97,109,101,115,112,97,99,101,32,102, - 111,114,32,123,125,114,87,0,0,0,41,22,114,32,0,0, - 0,114,39,0,0,0,114,35,0,0,0,114,3,0,0,0, - 114,45,0,0,0,114,213,0,0,0,114,40,0,0,0,114, - 2,1,0,0,218,11,95,102,105,108,108,95,99,97,99,104, - 101,114,6,0,0,0,114,5,1,0,0,114,88,0,0,0, - 114,4,1,0,0,114,28,0,0,0,114,1,1,0,0,114, - 44,0,0,0,114,255,0,0,0,114,46,0,0,0,114,114, - 0,0,0,114,129,0,0,0,114,154,0,0,0,114,150,0, - 0,0,41,14,114,100,0,0,0,114,119,0,0,0,114,174, - 0,0,0,90,12,105,115,95,110,97,109,101,115,112,97,99, - 101,90,11,116,97,105,108,95,109,111,100,117,108,101,114,126, - 0,0,0,90,5,99,97,99,104,101,90,12,99,97,99,104, - 101,95,109,111,100,117,108,101,90,9,98,97,115,101,95,112, - 97,116,104,114,219,0,0,0,114,159,0,0,0,90,13,105, - 110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,117, - 108,108,95,112,97,116,104,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,175,0,0,0, - 186,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, - 1,34,1,13,1,11,1,15,1,10,1,9,2,9,1,9, - 1,15,2,9,1,6,2,12,1,18,1,22,1,10,1,15, - 1,12,1,32,4,12,2,22,1,22,1,22,1,16,1,12, - 1,15,1,14,1,6,1,16,1,18,1,12,1,4,1,122, - 20,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100, - 95,115,112,101,99,99,1,0,0,0,0,0,0,0,9,0, - 0,0,13,0,0,0,67,0,0,0,115,11,1,0,0,124, - 0,0,106,0,0,125,1,0,121,31,0,116,1,0,106,2, - 0,124,1,0,112,33,0,116,1,0,106,3,0,131,0,0, - 131,1,0,125,2,0,87,110,33,0,4,116,4,0,116,5, - 0,116,6,0,102,3,0,107,10,0,114,75,0,1,1,1, - 103,0,0,125,2,0,89,110,1,0,88,116,7,0,106,8, - 0,106,9,0,100,1,0,131,1,0,115,112,0,116,10,0, - 124,2,0,131,1,0,124,0,0,95,11,0,110,111,0,116, - 10,0,131,0,0,125,3,0,120,90,0,124,2,0,68,93, - 82,0,125,4,0,124,4,0,106,12,0,100,2,0,131,1, - 0,92,3,0,125,5,0,125,6,0,125,7,0,124,6,0, - 114,191,0,100,3,0,106,13,0,124,5,0,124,7,0,106, - 14,0,131,0,0,131,2,0,125,8,0,110,6,0,124,5, - 0,125,8,0,124,3,0,106,15,0,124,8,0,131,1,0, - 1,113,128,0,87,124,3,0,124,0,0,95,11,0,116,7, - 0,106,8,0,106,9,0,116,16,0,131,1,0,114,7,1, - 100,4,0,100,5,0,132,0,0,124,2,0,68,131,1,0, - 124,0,0,95,17,0,100,6,0,83,41,7,122,68,70,105, - 108,108,32,116,104,101,32,99,97,99,104,101,32,111,102,32, - 112,111,116,101,110,116,105,97,108,32,109,111,100,117,108,101, - 115,32,97,110,100,32,112,97,99,107,97,103,101,115,32,102, - 111,114,32,116,104,105,115,32,100,105,114,101,99,116,111,114, - 121,46,114,0,0,0,0,114,58,0,0,0,122,5,123,125, - 46,123,125,99,1,0,0,0,0,0,0,0,2,0,0,0, - 3,0,0,0,83,0,0,0,115,28,0,0,0,104,0,0, - 124,0,0,93,18,0,125,1,0,124,1,0,106,0,0,131, - 0,0,146,2,0,113,6,0,83,114,4,0,0,0,41,1, - 114,88,0,0,0,41,2,114,22,0,0,0,90,2,102,110, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,250, - 9,60,115,101,116,99,111,109,112,62,5,5,0,0,115,2, - 0,0,0,9,0,122,41,70,105,108,101,70,105,110,100,101, - 114,46,95,102,105,108,108,95,99,97,99,104,101,46,60,108, - 111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,62, - 78,41,18,114,35,0,0,0,114,3,0,0,0,90,7,108, - 105,115,116,100,105,114,114,45,0,0,0,114,250,0,0,0, - 218,15,80,101,114,109,105,115,115,105,111,110,69,114,114,111, - 114,218,18,78,111,116,65,68,105,114,101,99,116,111,114,121, - 69,114,114,111,114,114,7,0,0,0,114,8,0,0,0,114, - 9,0,0,0,114,3,1,0,0,114,4,1,0,0,114,83, - 0,0,0,114,47,0,0,0,114,88,0,0,0,218,3,97, - 100,100,114,10,0,0,0,114,5,1,0,0,41,9,114,100, - 0,0,0,114,35,0,0,0,90,8,99,111,110,116,101,110, - 116,115,90,21,108,111,119,101,114,95,115,117,102,102,105,120, - 95,99,111,110,116,101,110,116,115,114,239,0,0,0,114,98, - 0,0,0,114,231,0,0,0,114,219,0,0,0,90,8,110, - 101,119,95,110,97,109,101,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,7,1,0,0,232,4,0,0,115, - 34,0,0,0,0,2,9,1,3,1,31,1,22,3,11,3, - 18,1,18,7,9,1,13,1,24,1,6,1,27,2,6,1, - 17,1,9,1,18,1,122,22,70,105,108,101,70,105,110,100, - 101,114,46,95,102,105,108,108,95,99,97,99,104,101,99,1, - 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,7, - 0,0,0,115,25,0,0,0,135,0,0,135,1,0,102,2, - 0,100,1,0,100,2,0,134,0,0,125,2,0,124,2,0, - 83,41,3,97,20,1,0,0,65,32,99,108,97,115,115,32, - 109,101,116,104,111,100,32,119,104,105,99,104,32,114,101,116, - 117,114,110,115,32,97,32,99,108,111,115,117,114,101,32,116, - 111,32,117,115,101,32,111,110,32,115,121,115,46,112,97,116, - 104,95,104,111,111,107,10,32,32,32,32,32,32,32,32,119, - 104,105,99,104,32,119,105,108,108,32,114,101,116,117,114,110, - 32,97,110,32,105,110,115,116,97,110,99,101,32,117,115,105, - 110,103,32,116,104,101,32,115,112,101,99,105,102,105,101,100, - 32,108,111,97,100,101,114,115,32,97,110,100,32,116,104,101, - 32,112,97,116,104,10,32,32,32,32,32,32,32,32,99,97, - 108,108,101,100,32,111,110,32,116,104,101,32,99,108,111,115, - 117,114,101,46,10,10,32,32,32,32,32,32,32,32,73,102, - 32,116,104,101,32,112,97,116,104,32,99,97,108,108,101,100, - 32,111,110,32,116,104,101,32,99,108,111,115,117,114,101,32, - 105,115,32,110,111,116,32,97,32,100,105,114,101,99,116,111, - 114,121,44,32,73,109,112,111,114,116,69,114,114,111,114,32, - 105,115,10,32,32,32,32,32,32,32,32,114,97,105,115,101, - 100,46,10,10,32,32,32,32,32,32,32,32,99,1,0,0, - 0,0,0,0,0,1,0,0,0,4,0,0,0,19,0,0, - 0,115,43,0,0,0,116,0,0,124,0,0,131,1,0,115, - 30,0,116,1,0,100,1,0,100,2,0,124,0,0,131,1, - 1,130,1,0,136,0,0,124,0,0,136,1,0,140,1,0, - 83,41,3,122,45,80,97,116,104,32,104,111,111,107,32,102, - 111,114,32,105,109,112,111,114,116,108,105,98,46,109,97,99, - 104,105,110,101,114,121,46,70,105,108,101,70,105,110,100,101, - 114,46,122,30,111,110,108,121,32,100,105,114,101,99,116,111, - 114,105,101,115,32,97,114,101,32,115,117,112,112,111,114,116, - 101,100,114,35,0,0,0,41,2,114,46,0,0,0,114,99, - 0,0,0,41,1,114,35,0,0,0,41,2,114,164,0,0, - 0,114,6,1,0,0,114,4,0,0,0,114,5,0,0,0, - 218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95, - 70,105,108,101,70,105,110,100,101,114,17,5,0,0,115,6, - 0,0,0,0,2,12,1,18,1,122,54,70,105,108,101,70, - 105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46, - 60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111, - 111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101, - 114,114,4,0,0,0,41,3,114,164,0,0,0,114,6,1, - 0,0,114,12,1,0,0,114,4,0,0,0,41,2,114,164, - 0,0,0,114,6,1,0,0,114,5,0,0,0,218,9,112, - 97,116,104,95,104,111,111,107,7,5,0,0,115,4,0,0, - 0,0,10,21,6,122,20,70,105,108,101,70,105,110,100,101, - 114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,0, - 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, - 115,16,0,0,0,100,1,0,106,0,0,124,0,0,106,1, - 0,131,1,0,83,41,2,78,122,16,70,105,108,101,70,105, - 110,100,101,114,40,123,33,114,125,41,41,2,114,47,0,0, - 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,238,0,0, - 0,25,5,0,0,115,2,0,0,0,0,1,122,19,70,105, - 108,101,70,105,110,100,101,114,46,95,95,114,101,112,114,95, - 95,41,15,114,105,0,0,0,114,104,0,0,0,114,106,0, - 0,0,114,107,0,0,0,114,179,0,0,0,114,244,0,0, - 0,114,123,0,0,0,114,176,0,0,0,114,117,0,0,0, - 114,255,0,0,0,114,175,0,0,0,114,7,1,0,0,114, - 177,0,0,0,114,13,1,0,0,114,238,0,0,0,114,4, + 0,114,5,0,0,0,114,180,0,0,0,245,3,0,0,115, + 0,0,0,0,122,30,95,78,97,109,101,115,112,97,99,101, + 76,111,97,100,101,114,46,99,114,101,97,116,101,95,109,111, + 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, + 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,0, + 0,83,41,1,78,114,4,0,0,0,41,2,114,100,0,0, + 0,114,184,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,185,0,0,0,248,3,0,0,115,2, + 0,0,0,0,1,122,28,95,78,97,109,101,115,112,97,99, + 101,76,111,97,100,101,114,46,101,120,101,99,95,109,111,100, + 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 3,0,0,0,67,0,0,0,115,35,0,0,0,116,0,0, + 106,1,0,100,1,0,124,0,0,106,2,0,131,2,0,1, + 116,0,0,106,3,0,124,0,0,124,1,0,131,2,0,83, + 41,2,122,98,76,111,97,100,32,97,32,110,97,109,101,115, + 112,97,99,101,32,109,111,100,117,108,101,46,10,10,32,32, + 32,32,32,32,32,32,84,104,105,115,32,109,101,116,104,111, + 100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46, + 32,32,85,115,101,32,101,120,101,99,95,109,111,100,117,108, + 101,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32, + 32,32,32,32,32,32,122,38,110,97,109,101,115,112,97,99, + 101,32,109,111,100,117,108,101,32,108,111,97,100,101,100,32, + 119,105,116,104,32,112,97,116,104,32,123,33,114,125,41,4, + 114,114,0,0,0,114,129,0,0,0,114,226,0,0,0,114, + 186,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, + 187,0,0,0,251,3,0,0,115,6,0,0,0,0,7,9, + 1,10,1,122,28,95,78,97,109,101,115,112,97,99,101,76, + 111,97,100,101,114,46,108,111,97,100,95,109,111,100,117,108, + 101,78,41,12,114,105,0,0,0,114,104,0,0,0,114,106, + 0,0,0,114,179,0,0,0,114,177,0,0,0,114,244,0, + 0,0,114,153,0,0,0,114,196,0,0,0,114,181,0,0, + 0,114,180,0,0,0,114,185,0,0,0,114,187,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,243,0,0,0,223,3,0,0,115,16,0, + 0,0,12,1,12,3,18,9,12,3,12,3,12,3,12,3, + 12,3,114,243,0,0,0,99,0,0,0,0,0,0,0,0, + 0,0,0,0,5,0,0,0,64,0,0,0,115,160,0,0, + 0,101,0,0,90,1,0,100,0,0,90,2,0,100,1,0, + 90,3,0,101,4,0,100,2,0,100,3,0,132,0,0,131, + 1,0,90,5,0,101,4,0,100,4,0,100,5,0,132,0, + 0,131,1,0,90,6,0,101,4,0,100,6,0,100,7,0, + 132,0,0,131,1,0,90,7,0,101,4,0,100,8,0,100, + 9,0,132,0,0,131,1,0,90,8,0,101,4,0,100,10, + 0,100,11,0,100,12,0,132,1,0,131,1,0,90,9,0, + 101,4,0,100,10,0,100,10,0,100,13,0,100,14,0,132, + 2,0,131,1,0,90,10,0,101,4,0,100,10,0,100,15, + 0,100,16,0,132,1,0,131,1,0,90,11,0,100,10,0, + 83,41,17,218,10,80,97,116,104,70,105,110,100,101,114,122, + 62,77,101,116,97,32,112,97,116,104,32,102,105,110,100,101, + 114,32,102,111,114,32,115,121,115,46,112,97,116,104,32,97, + 110,100,32,112,97,99,107,97,103,101,32,95,95,112,97,116, + 104,95,95,32,97,116,116,114,105,98,117,116,101,115,46,99, + 1,0,0,0,0,0,0,0,2,0,0,0,4,0,0,0, + 67,0,0,0,115,55,0,0,0,120,48,0,116,0,0,106, + 1,0,106,2,0,131,0,0,68,93,31,0,125,1,0,116, + 3,0,124,1,0,100,1,0,131,2,0,114,16,0,124,1, + 0,106,4,0,131,0,0,1,113,16,0,87,100,2,0,83, + 41,3,122,125,67,97,108,108,32,116,104,101,32,105,110,118, + 97,108,105,100,97,116,101,95,99,97,99,104,101,115,40,41, + 32,109,101,116,104,111,100,32,111,110,32,97,108,108,32,112, + 97,116,104,32,101,110,116,114,121,32,102,105,110,100,101,114, + 115,10,32,32,32,32,32,32,32,32,115,116,111,114,101,100, + 32,105,110,32,115,121,115,46,112,97,116,104,95,105,109,112, + 111,114,116,101,114,95,99,97,99,104,101,115,32,40,119,104, + 101,114,101,32,105,109,112,108,101,109,101,110,116,101,100,41, + 46,218,17,105,110,118,97,108,105,100,97,116,101,95,99,97, + 99,104,101,115,78,41,5,114,7,0,0,0,218,19,112,97, + 116,104,95,105,109,112,111,114,116,101,114,95,99,97,99,104, + 101,218,6,118,97,108,117,101,115,114,108,0,0,0,114,246, + 0,0,0,41,2,114,164,0,0,0,218,6,102,105,110,100, + 101,114,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,114,246,0,0,0,13,4,0,0,115,6,0,0,0,0, + 4,22,1,15,1,122,28,80,97,116,104,70,105,110,100,101, + 114,46,105,110,118,97,108,105,100,97,116,101,95,99,97,99, + 104,101,115,99,2,0,0,0,0,0,0,0,3,0,0,0, + 12,0,0,0,67,0,0,0,115,107,0,0,0,116,0,0, + 106,1,0,100,1,0,107,9,0,114,41,0,116,0,0,106, + 1,0,12,114,41,0,116,2,0,106,3,0,100,2,0,116, + 4,0,131,2,0,1,120,59,0,116,0,0,106,1,0,68, + 93,44,0,125,2,0,121,14,0,124,2,0,124,1,0,131, + 1,0,83,87,113,51,0,4,116,5,0,107,10,0,114,94, + 0,1,1,1,119,51,0,89,113,51,0,88,113,51,0,87, + 100,1,0,83,100,1,0,83,41,3,122,113,83,101,97,114, + 99,104,32,115,101,113,117,101,110,99,101,32,111,102,32,104, + 111,111,107,115,32,102,111,114,32,97,32,102,105,110,100,101, + 114,32,102,111,114,32,39,112,97,116,104,39,46,10,10,32, + 32,32,32,32,32,32,32,73,102,32,39,104,111,111,107,115, + 39,32,105,115,32,102,97,108,115,101,32,116,104,101,110,32, + 117,115,101,32,115,121,115,46,112,97,116,104,95,104,111,111, + 107,115,46,10,10,32,32,32,32,32,32,32,32,78,122,23, + 115,121,115,46,112,97,116,104,95,104,111,111,107,115,32,105, + 115,32,101,109,112,116,121,41,6,114,7,0,0,0,218,10, + 112,97,116,104,95,104,111,111,107,115,114,60,0,0,0,114, + 61,0,0,0,114,118,0,0,0,114,99,0,0,0,41,3, + 114,164,0,0,0,114,35,0,0,0,90,4,104,111,111,107, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,218, + 11,95,112,97,116,104,95,104,111,111,107,115,21,4,0,0, + 115,16,0,0,0,0,7,25,1,16,1,16,1,3,1,14, + 1,13,1,12,2,122,22,80,97,116,104,70,105,110,100,101, + 114,46,95,112,97,116,104,95,104,111,111,107,115,99,2,0, + 0,0,0,0,0,0,3,0,0,0,19,0,0,0,67,0, + 0,0,115,123,0,0,0,124,1,0,100,1,0,107,2,0, + 114,53,0,121,16,0,116,0,0,106,1,0,131,0,0,125, + 1,0,87,110,22,0,4,116,2,0,107,10,0,114,52,0, + 1,1,1,100,2,0,83,89,110,1,0,88,121,17,0,116, + 3,0,106,4,0,124,1,0,25,125,2,0,87,110,46,0, + 4,116,5,0,107,10,0,114,118,0,1,1,1,124,0,0, + 106,6,0,124,1,0,131,1,0,125,2,0,124,2,0,116, + 3,0,106,4,0,124,1,0,60,89,110,1,0,88,124,2, + 0,83,41,3,122,210,71,101,116,32,116,104,101,32,102,105, + 110,100,101,114,32,102,111,114,32,116,104,101,32,112,97,116, + 104,32,101,110,116,114,121,32,102,114,111,109,32,115,121,115, + 46,112,97,116,104,95,105,109,112,111,114,116,101,114,95,99, + 97,99,104,101,46,10,10,32,32,32,32,32,32,32,32,73, + 102,32,116,104,101,32,112,97,116,104,32,101,110,116,114,121, + 32,105,115,32,110,111,116,32,105,110,32,116,104,101,32,99, + 97,99,104,101,44,32,102,105,110,100,32,116,104,101,32,97, + 112,112,114,111,112,114,105,97,116,101,32,102,105,110,100,101, + 114,10,32,32,32,32,32,32,32,32,97,110,100,32,99,97, + 99,104,101,32,105,116,46,32,73,102,32,110,111,32,102,105, + 110,100,101,114,32,105,115,32,97,118,97,105,108,97,98,108, + 101,44,32,115,116,111,114,101,32,78,111,110,101,46,10,10, + 32,32,32,32,32,32,32,32,114,30,0,0,0,78,41,7, + 114,3,0,0,0,114,45,0,0,0,218,17,70,105,108,101, + 78,111,116,70,111,117,110,100,69,114,114,111,114,114,7,0, + 0,0,114,247,0,0,0,114,131,0,0,0,114,251,0,0, + 0,41,3,114,164,0,0,0,114,35,0,0,0,114,249,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,20,95,112,97,116,104,95,105,109,112,111,114,116,101, + 114,95,99,97,99,104,101,38,4,0,0,115,22,0,0,0, + 0,8,12,1,3,1,16,1,13,3,9,1,3,1,17,1, + 13,1,15,1,18,1,122,31,80,97,116,104,70,105,110,100, + 101,114,46,95,112,97,116,104,95,105,109,112,111,114,116,101, + 114,95,99,97,99,104,101,99,3,0,0,0,0,0,0,0, + 6,0,0,0,3,0,0,0,67,0,0,0,115,119,0,0, + 0,116,0,0,124,2,0,100,1,0,131,2,0,114,39,0, + 124,2,0,106,1,0,124,1,0,131,1,0,92,2,0,125, + 3,0,125,4,0,110,21,0,124,2,0,106,2,0,124,1, + 0,131,1,0,125,3,0,103,0,0,125,4,0,124,3,0, + 100,0,0,107,9,0,114,88,0,116,3,0,106,4,0,124, + 1,0,124,3,0,131,2,0,83,116,3,0,106,5,0,124, + 1,0,100,0,0,131,2,0,125,5,0,124,4,0,124,5, + 0,95,6,0,124,5,0,83,41,2,78,114,117,0,0,0, + 41,7,114,108,0,0,0,114,117,0,0,0,114,176,0,0, + 0,114,114,0,0,0,114,173,0,0,0,114,154,0,0,0, + 114,150,0,0,0,41,6,114,164,0,0,0,114,119,0,0, + 0,114,249,0,0,0,114,120,0,0,0,114,121,0,0,0, + 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,218,16,95,108,101,103,97,99,121,95,103,101, + 116,95,115,112,101,99,60,4,0,0,115,18,0,0,0,0, + 4,15,1,24,2,15,1,6,1,12,1,16,1,18,1,9, + 1,122,27,80,97,116,104,70,105,110,100,101,114,46,95,108, + 101,103,97,99,121,95,103,101,116,95,115,112,101,99,78,99, + 4,0,0,0,0,0,0,0,9,0,0,0,5,0,0,0, + 67,0,0,0,115,243,0,0,0,103,0,0,125,4,0,120, + 230,0,124,2,0,68,93,191,0,125,5,0,116,0,0,124, + 5,0,116,1,0,116,2,0,102,2,0,131,2,0,115,43, + 0,113,13,0,124,0,0,106,3,0,124,5,0,131,1,0, + 125,6,0,124,6,0,100,1,0,107,9,0,114,13,0,116, + 4,0,124,6,0,100,2,0,131,2,0,114,106,0,124,6, + 0,106,5,0,124,1,0,124,3,0,131,2,0,125,7,0, + 110,18,0,124,0,0,106,6,0,124,1,0,124,6,0,131, + 2,0,125,7,0,124,7,0,100,1,0,107,8,0,114,139, + 0,113,13,0,124,7,0,106,7,0,100,1,0,107,9,0, + 114,158,0,124,7,0,83,124,7,0,106,8,0,125,8,0, + 124,8,0,100,1,0,107,8,0,114,191,0,116,9,0,100, + 3,0,131,1,0,130,1,0,124,4,0,106,10,0,124,8, + 0,131,1,0,1,113,13,0,87,116,11,0,106,12,0,124, + 1,0,100,1,0,131,2,0,125,7,0,124,4,0,124,7, + 0,95,8,0,124,7,0,83,100,1,0,83,41,4,122,63, + 70,105,110,100,32,116,104,101,32,108,111,97,100,101,114,32, + 111,114,32,110,97,109,101,115,112,97,99,101,95,112,97,116, + 104,32,102,111,114,32,116,104,105,115,32,109,111,100,117,108, + 101,47,112,97,99,107,97,103,101,32,110,97,109,101,46,78, + 114,175,0,0,0,122,19,115,112,101,99,32,109,105,115,115, + 105,110,103,32,108,111,97,100,101,114,41,13,114,137,0,0, + 0,114,69,0,0,0,218,5,98,121,116,101,115,114,253,0, + 0,0,114,108,0,0,0,114,175,0,0,0,114,254,0,0, + 0,114,120,0,0,0,114,150,0,0,0,114,99,0,0,0, + 114,143,0,0,0,114,114,0,0,0,114,154,0,0,0,41, + 9,114,164,0,0,0,114,119,0,0,0,114,35,0,0,0, + 114,174,0,0,0,218,14,110,97,109,101,115,112,97,99,101, + 95,112,97,116,104,90,5,101,110,116,114,121,114,249,0,0, + 0,114,158,0,0,0,114,121,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,9,95,103,101,116, + 95,115,112,101,99,75,4,0,0,115,40,0,0,0,0,5, + 6,1,13,1,21,1,3,1,15,1,12,1,15,1,21,2, + 18,1,12,1,3,1,15,1,4,1,9,1,12,1,12,5, + 17,2,18,1,9,1,122,20,80,97,116,104,70,105,110,100, + 101,114,46,95,103,101,116,95,115,112,101,99,99,4,0,0, + 0,0,0,0,0,6,0,0,0,4,0,0,0,67,0,0, + 0,115,140,0,0,0,124,2,0,100,1,0,107,8,0,114, + 21,0,116,0,0,106,1,0,125,2,0,124,0,0,106,2, + 0,124,1,0,124,2,0,124,3,0,131,3,0,125,4,0, + 124,4,0,100,1,0,107,8,0,114,58,0,100,1,0,83, + 124,4,0,106,3,0,100,1,0,107,8,0,114,132,0,124, + 4,0,106,4,0,125,5,0,124,5,0,114,125,0,100,2, + 0,124,4,0,95,5,0,116,6,0,124,1,0,124,5,0, + 124,0,0,106,2,0,131,3,0,124,4,0,95,4,0,124, + 4,0,83,100,1,0,83,110,4,0,124,4,0,83,100,1, + 0,83,41,3,122,98,102,105,110,100,32,116,104,101,32,109, + 111,100,117,108,101,32,111,110,32,115,121,115,46,112,97,116, + 104,32,111,114,32,39,112,97,116,104,39,32,98,97,115,101, + 100,32,111,110,32,115,121,115,46,112,97,116,104,95,104,111, + 111,107,115,32,97,110,100,10,32,32,32,32,32,32,32,32, + 115,121,115,46,112,97,116,104,95,105,109,112,111,114,116,101, + 114,95,99,97,99,104,101,46,78,90,9,110,97,109,101,115, + 112,97,99,101,41,7,114,7,0,0,0,114,35,0,0,0, + 114,1,1,0,0,114,120,0,0,0,114,150,0,0,0,114, + 152,0,0,0,114,224,0,0,0,41,6,114,164,0,0,0, + 114,119,0,0,0,114,35,0,0,0,114,174,0,0,0,114, + 158,0,0,0,114,0,1,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,175,0,0,0,107,4,0, + 0,115,26,0,0,0,0,4,12,1,9,1,21,1,12,1, + 4,1,15,1,9,1,6,3,9,1,24,1,4,2,7,2, + 122,20,80,97,116,104,70,105,110,100,101,114,46,102,105,110, + 100,95,115,112,101,99,99,3,0,0,0,0,0,0,0,4, + 0,0,0,3,0,0,0,67,0,0,0,115,41,0,0,0, + 124,0,0,106,0,0,124,1,0,124,2,0,131,2,0,125, + 3,0,124,3,0,100,1,0,107,8,0,114,34,0,100,1, + 0,83,124,3,0,106,1,0,83,41,2,122,170,102,105,110, + 100,32,116,104,101,32,109,111,100,117,108,101,32,111,110,32, + 115,121,115,46,112,97,116,104,32,111,114,32,39,112,97,116, + 104,39,32,98,97,115,101,100,32,111,110,32,115,121,115,46, + 112,97,116,104,95,104,111,111,107,115,32,97,110,100,10,32, + 32,32,32,32,32,32,32,115,121,115,46,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,46,10, + 10,32,32,32,32,32,32,32,32,84,104,105,115,32,109,101, + 116,104,111,100,32,105,115,32,100,101,112,114,101,99,97,116, + 101,100,46,32,32,85,115,101,32,102,105,110,100,95,115,112, + 101,99,40,41,32,105,110,115,116,101,97,100,46,10,10,32, + 32,32,32,32,32,32,32,78,41,2,114,175,0,0,0,114, + 120,0,0,0,41,4,114,164,0,0,0,114,119,0,0,0, + 114,35,0,0,0,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,176,0,0,0,129,4, + 0,0,115,8,0,0,0,0,8,18,1,12,1,4,1,122, + 22,80,97,116,104,70,105,110,100,101,114,46,102,105,110,100, + 95,109,111,100,117,108,101,41,12,114,105,0,0,0,114,104, + 0,0,0,114,106,0,0,0,114,107,0,0,0,114,177,0, + 0,0,114,246,0,0,0,114,251,0,0,0,114,253,0,0, + 0,114,254,0,0,0,114,1,1,0,0,114,175,0,0,0, + 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,245,0,0,0,9,4, + 0,0,115,22,0,0,0,12,2,6,2,18,8,18,17,18, + 22,18,15,3,1,18,31,3,1,21,21,3,1,114,245,0, + 0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,3, + 0,0,0,64,0,0,0,115,133,0,0,0,101,0,0,90, + 1,0,100,0,0,90,2,0,100,1,0,90,3,0,100,2, + 0,100,3,0,132,0,0,90,4,0,100,4,0,100,5,0, + 132,0,0,90,5,0,101,6,0,90,7,0,100,6,0,100, + 7,0,132,0,0,90,8,0,100,8,0,100,9,0,132,0, + 0,90,9,0,100,10,0,100,11,0,100,12,0,132,1,0, + 90,10,0,100,13,0,100,14,0,132,0,0,90,11,0,101, + 12,0,100,15,0,100,16,0,132,0,0,131,1,0,90,13, + 0,100,17,0,100,18,0,132,0,0,90,14,0,100,10,0, + 83,41,19,218,10,70,105,108,101,70,105,110,100,101,114,122, + 172,70,105,108,101,45,98,97,115,101,100,32,102,105,110,100, + 101,114,46,10,10,32,32,32,32,73,110,116,101,114,97,99, + 116,105,111,110,115,32,119,105,116,104,32,116,104,101,32,102, + 105,108,101,32,115,121,115,116,101,109,32,97,114,101,32,99, + 97,99,104,101,100,32,102,111,114,32,112,101,114,102,111,114, + 109,97,110,99,101,44,32,98,101,105,110,103,10,32,32,32, + 32,114,101,102,114,101,115,104,101,100,32,119,104,101,110,32, + 116,104,101,32,100,105,114,101,99,116,111,114,121,32,116,104, + 101,32,102,105,110,100,101,114,32,105,115,32,104,97,110,100, + 108,105,110,103,32,104,97,115,32,98,101,101,110,32,109,111, + 100,105,102,105,101,100,46,10,10,32,32,32,32,99,2,0, + 0,0,0,0,0,0,5,0,0,0,5,0,0,0,7,0, + 0,0,115,122,0,0,0,103,0,0,125,3,0,120,52,0, + 124,2,0,68,93,44,0,92,2,0,137,0,0,125,4,0, + 124,3,0,106,0,0,135,0,0,102,1,0,100,1,0,100, + 2,0,134,0,0,124,4,0,68,131,1,0,131,1,0,1, + 113,13,0,87,124,3,0,124,0,0,95,1,0,124,1,0, + 112,79,0,100,3,0,124,0,0,95,2,0,100,6,0,124, + 0,0,95,3,0,116,4,0,131,0,0,124,0,0,95,5, + 0,116,4,0,131,0,0,124,0,0,95,6,0,100,5,0, + 83,41,7,122,154,73,110,105,116,105,97,108,105,122,101,32, + 119,105,116,104,32,116,104,101,32,112,97,116,104,32,116,111, + 32,115,101,97,114,99,104,32,111,110,32,97,110,100,32,97, + 32,118,97,114,105,97,98,108,101,32,110,117,109,98,101,114, + 32,111,102,10,32,32,32,32,32,32,32,32,50,45,116,117, + 112,108,101,115,32,99,111,110,116,97,105,110,105,110,103,32, + 116,104,101,32,108,111,97,100,101,114,32,97,110,100,32,116, + 104,101,32,102,105,108,101,32,115,117,102,102,105,120,101,115, + 32,116,104,101,32,108,111,97,100,101,114,10,32,32,32,32, + 32,32,32,32,114,101,99,111,103,110,105,122,101,115,46,99, + 1,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, + 51,0,0,0,115,27,0,0,0,124,0,0,93,17,0,125, + 1,0,124,1,0,136,0,0,102,2,0,86,1,113,3,0, + 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,22, + 0,0,0,114,219,0,0,0,41,1,114,120,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,221,0,0,0,158,4, + 0,0,115,2,0,0,0,6,0,122,38,70,105,108,101,70, + 105,110,100,101,114,46,95,95,105,110,105,116,95,95,46,60, + 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, + 62,114,58,0,0,0,114,29,0,0,0,78,114,87,0,0, + 0,41,7,114,143,0,0,0,218,8,95,108,111,97,100,101, + 114,115,114,35,0,0,0,218,11,95,112,97,116,104,95,109, + 116,105,109,101,218,3,115,101,116,218,11,95,112,97,116,104, + 95,99,97,99,104,101,218,19,95,114,101,108,97,120,101,100, + 95,112,97,116,104,95,99,97,99,104,101,41,5,114,100,0, + 0,0,114,35,0,0,0,218,14,108,111,97,100,101,114,95, + 100,101,116,97,105,108,115,90,7,108,111,97,100,101,114,115, + 114,160,0,0,0,114,4,0,0,0,41,1,114,120,0,0, + 0,114,5,0,0,0,114,179,0,0,0,152,4,0,0,115, + 16,0,0,0,0,4,6,1,19,1,36,1,9,2,15,1, + 9,1,12,1,122,19,70,105,108,101,70,105,110,100,101,114, + 46,95,95,105,110,105,116,95,95,99,1,0,0,0,0,0, + 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,13, + 0,0,0,100,3,0,124,0,0,95,0,0,100,2,0,83, + 41,4,122,31,73,110,118,97,108,105,100,97,116,101,32,116, + 104,101,32,100,105,114,101,99,116,111,114,121,32,109,116,105, + 109,101,46,114,29,0,0,0,78,114,87,0,0,0,41,1, + 114,4,1,0,0,41,1,114,100,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,246,0,0,0, + 166,4,0,0,115,2,0,0,0,0,2,122,28,70,105,108, + 101,70,105,110,100,101,114,46,105,110,118,97,108,105,100,97, + 116,101,95,99,97,99,104,101,115,99,2,0,0,0,0,0, + 0,0,3,0,0,0,2,0,0,0,67,0,0,0,115,59, + 0,0,0,124,0,0,106,0,0,124,1,0,131,1,0,125, + 2,0,124,2,0,100,1,0,107,8,0,114,37,0,100,1, + 0,103,0,0,102,2,0,83,124,2,0,106,1,0,124,2, + 0,106,2,0,112,55,0,103,0,0,102,2,0,83,41,2, + 122,197,84,114,121,32,116,111,32,102,105,110,100,32,97,32, + 108,111,97,100,101,114,32,102,111,114,32,116,104,101,32,115, + 112,101,99,105,102,105,101,100,32,109,111,100,117,108,101,44, + 32,111,114,32,116,104,101,32,110,97,109,101,115,112,97,99, + 101,10,32,32,32,32,32,32,32,32,112,97,99,107,97,103, + 101,32,112,111,114,116,105,111,110,115,46,32,82,101,116,117, + 114,110,115,32,40,108,111,97,100,101,114,44,32,108,105,115, + 116,45,111,102,45,112,111,114,116,105,111,110,115,41,46,10, + 10,32,32,32,32,32,32,32,32,84,104,105,115,32,109,101, + 116,104,111,100,32,105,115,32,100,101,112,114,101,99,97,116, + 101,100,46,32,32,85,115,101,32,102,105,110,100,95,115,112, + 101,99,40,41,32,105,110,115,116,101,97,100,46,10,10,32, + 32,32,32,32,32,32,32,78,41,3,114,175,0,0,0,114, + 120,0,0,0,114,150,0,0,0,41,3,114,100,0,0,0, + 114,119,0,0,0,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,117,0,0,0,172,4, + 0,0,115,8,0,0,0,0,7,15,1,12,1,10,1,122, + 22,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100, + 95,108,111,97,100,101,114,99,6,0,0,0,0,0,0,0, + 7,0,0,0,7,0,0,0,67,0,0,0,115,40,0,0, + 0,124,1,0,124,2,0,124,3,0,131,2,0,125,6,0, + 116,0,0,124,2,0,124,3,0,100,1,0,124,6,0,100, + 2,0,124,4,0,131,2,2,83,41,3,78,114,120,0,0, + 0,114,150,0,0,0,41,1,114,161,0,0,0,41,7,114, + 100,0,0,0,114,159,0,0,0,114,119,0,0,0,114,35, + 0,0,0,90,4,115,109,115,108,114,174,0,0,0,114,120, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,0,1,0,0,140,4,0,0,115,20,0,0,0, - 12,7,6,2,12,14,12,4,6,2,12,12,12,5,15,46, - 12,31,18,18,114,0,1,0,0,99,4,0,0,0,0,0, - 0,0,6,0,0,0,11,0,0,0,67,0,0,0,115,195, - 0,0,0,124,0,0,106,0,0,100,1,0,131,1,0,125, - 4,0,124,0,0,106,0,0,100,2,0,131,1,0,125,5, - 0,124,4,0,115,99,0,124,5,0,114,54,0,124,5,0, - 106,1,0,125,4,0,110,45,0,124,2,0,124,3,0,107, - 2,0,114,84,0,116,2,0,124,1,0,124,2,0,131,2, - 0,125,4,0,110,15,0,116,3,0,124,1,0,124,2,0, - 131,2,0,125,4,0,124,5,0,115,126,0,116,4,0,124, - 1,0,124,2,0,100,3,0,124,4,0,131,2,1,125,5, - 0,121,44,0,124,5,0,124,0,0,100,2,0,60,124,4, - 0,124,0,0,100,1,0,60,124,2,0,124,0,0,100,4, - 0,60,124,3,0,124,0,0,100,5,0,60,87,110,18,0, - 4,116,5,0,107,10,0,114,190,0,1,1,1,89,110,1, - 0,88,100,0,0,83,41,6,78,218,10,95,95,108,111,97, - 100,101,114,95,95,218,8,95,95,115,112,101,99,95,95,114, - 120,0,0,0,90,8,95,95,102,105,108,101,95,95,90,10, - 95,95,99,97,99,104,101,100,95,95,41,6,218,3,103,101, - 116,114,120,0,0,0,114,217,0,0,0,114,212,0,0,0, - 114,161,0,0,0,218,9,69,120,99,101,112,116,105,111,110, - 41,6,90,2,110,115,114,98,0,0,0,90,8,112,97,116, - 104,110,97,109,101,90,9,99,112,97,116,104,110,97,109,101, - 114,120,0,0,0,114,158,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,14,95,102,105,120,95, - 117,112,95,109,111,100,117,108,101,31,5,0,0,115,34,0, - 0,0,0,2,15,1,15,1,6,1,6,1,12,1,12,1, - 18,2,15,1,6,1,21,1,3,1,10,1,10,1,10,1, - 14,1,13,2,114,18,1,0,0,99,0,0,0,0,0,0, - 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,55, - 0,0,0,116,0,0,116,1,0,106,2,0,131,0,0,102, - 2,0,125,0,0,116,3,0,116,4,0,102,2,0,125,1, - 0,116,5,0,116,6,0,102,2,0,125,2,0,124,0,0, - 124,1,0,124,2,0,103,3,0,83,41,1,122,95,82,101, - 116,117,114,110,115,32,97,32,108,105,115,116,32,111,102,32, - 102,105,108,101,45,98,97,115,101,100,32,109,111,100,117,108, - 101,32,108,111,97,100,101,114,115,46,10,10,32,32,32,32, - 69,97,99,104,32,105,116,101,109,32,105,115,32,97,32,116, - 117,112,108,101,32,40,108,111,97,100,101,114,44,32,115,117, - 102,102,105,120,101,115,41,46,10,32,32,32,32,41,7,114, - 218,0,0,0,114,139,0,0,0,218,18,101,120,116,101,110, - 115,105,111,110,95,115,117,102,102,105,120,101,115,114,212,0, - 0,0,114,84,0,0,0,114,217,0,0,0,114,74,0,0, - 0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,90, - 6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,100, - 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,155,0,0,0,54,5,0,0,115,8,0,0,0,0,5, - 18,1,12,1,12,1,114,155,0,0,0,99,1,0,0,0, - 0,0,0,0,12,0,0,0,12,0,0,0,67,0,0,0, - 115,70,2,0,0,124,0,0,97,0,0,116,0,0,106,1, - 0,97,1,0,116,0,0,106,2,0,97,2,0,116,1,0, - 106,3,0,116,4,0,25,125,1,0,120,76,0,100,26,0, - 68,93,68,0,125,2,0,124,2,0,116,1,0,106,3,0, - 107,7,0,114,83,0,116,0,0,106,5,0,124,2,0,131, - 1,0,125,3,0,110,13,0,116,1,0,106,3,0,124,2, - 0,25,125,3,0,116,6,0,124,1,0,124,2,0,124,3, - 0,131,3,0,1,113,44,0,87,100,5,0,100,6,0,103, - 1,0,102,2,0,100,7,0,100,8,0,100,6,0,103,2, - 0,102,2,0,102,2,0,125,4,0,120,149,0,124,4,0, - 68,93,129,0,92,2,0,125,5,0,125,6,0,116,7,0, - 100,9,0,100,10,0,132,0,0,124,6,0,68,131,1,0, - 131,1,0,115,199,0,116,8,0,130,1,0,124,6,0,100, - 11,0,25,125,7,0,124,5,0,116,1,0,106,3,0,107, - 6,0,114,241,0,116,1,0,106,3,0,124,5,0,25,125, - 8,0,80,113,156,0,121,20,0,116,0,0,106,5,0,124, - 5,0,131,1,0,125,8,0,80,87,113,156,0,4,116,9, - 0,107,10,0,114,28,1,1,1,1,119,156,0,89,113,156, - 0,88,113,156,0,87,116,9,0,100,12,0,131,1,0,130, - 1,0,116,6,0,124,1,0,100,13,0,124,8,0,131,3, - 0,1,116,6,0,124,1,0,100,14,0,124,7,0,131,3, - 0,1,116,6,0,124,1,0,100,15,0,100,16,0,106,10, - 0,124,6,0,131,1,0,131,3,0,1,121,19,0,116,0, - 0,106,5,0,100,17,0,131,1,0,125,9,0,87,110,24, - 0,4,116,9,0,107,10,0,114,147,1,1,1,1,100,18, - 0,125,9,0,89,110,1,0,88,116,6,0,124,1,0,100, - 17,0,124,9,0,131,3,0,1,116,0,0,106,5,0,100, - 19,0,131,1,0,125,10,0,116,6,0,124,1,0,100,19, - 0,124,10,0,131,3,0,1,124,5,0,100,7,0,107,2, - 0,114,238,1,116,0,0,106,5,0,100,20,0,131,1,0, - 125,11,0,116,6,0,124,1,0,100,21,0,124,11,0,131, - 3,0,1,116,6,0,124,1,0,100,22,0,116,11,0,131, - 0,0,131,3,0,1,116,12,0,106,13,0,116,2,0,106, - 14,0,131,0,0,131,1,0,1,124,5,0,100,7,0,107, - 2,0,114,66,2,116,15,0,106,16,0,100,23,0,131,1, - 0,1,100,24,0,116,12,0,107,6,0,114,66,2,100,25, - 0,116,17,0,95,18,0,100,18,0,83,41,27,122,205,83, - 101,116,117,112,32,116,104,101,32,112,97,116,104,45,98,97, - 115,101,100,32,105,109,112,111,114,116,101,114,115,32,102,111, - 114,32,105,109,112,111,114,116,108,105,98,32,98,121,32,105, - 109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,10, - 32,32,32,32,98,117,105,108,116,45,105,110,32,109,111,100, - 117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,105, - 110,103,32,116,104,101,109,32,105,110,116,111,32,116,104,101, - 32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,99, - 101,46,10,10,32,32,32,32,79,116,104,101,114,32,99,111, - 109,112,111,110,101,110,116,115,32,97,114,101,32,101,120,116, - 114,97,99,116,101,100,32,102,114,111,109,32,116,104,101,32, - 99,111,114,101,32,98,111,111,116,115,116,114,97,112,32,109, - 111,100,117,108,101,46,10,10,32,32,32,32,114,49,0,0, - 0,114,60,0,0,0,218,8,98,117,105,108,116,105,110,115, - 114,136,0,0,0,90,5,112,111,115,105,120,250,1,47,218, - 2,110,116,250,1,92,99,1,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,115,0,0,0,115,33,0,0,0, - 124,0,0,93,23,0,125,1,0,116,0,0,124,1,0,131, - 1,0,100,0,0,107,2,0,86,1,113,3,0,100,1,0, - 83,41,2,114,29,0,0,0,78,41,1,114,31,0,0,0, - 41,2,114,22,0,0,0,114,77,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 90,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, - 116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,101, - 110,101,120,112,114,62,114,59,0,0,0,122,30,105,109,112, - 111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,32, - 112,111,115,105,120,32,111,114,32,110,116,114,3,0,0,0, - 114,25,0,0,0,114,21,0,0,0,114,30,0,0,0,90, - 7,95,116,104,114,101,97,100,78,90,8,95,119,101,97,107, - 114,101,102,90,6,119,105,110,114,101,103,114,163,0,0,0, - 114,6,0,0,0,122,4,46,112,121,119,122,6,95,100,46, - 112,121,100,84,41,4,122,3,95,105,111,122,9,95,119,97, - 114,110,105,110,103,115,122,8,98,117,105,108,116,105,110,115, - 122,7,109,97,114,115,104,97,108,41,19,114,114,0,0,0, - 114,7,0,0,0,114,139,0,0,0,114,233,0,0,0,114, - 105,0,0,0,90,18,95,98,117,105,108,116,105,110,95,102, - 114,111,109,95,110,97,109,101,114,109,0,0,0,218,3,97, - 108,108,218,14,65,115,115,101,114,116,105,111,110,69,114,114, - 111,114,114,99,0,0,0,114,26,0,0,0,114,11,0,0, - 0,114,223,0,0,0,114,143,0,0,0,114,19,1,0,0, - 114,84,0,0,0,114,157,0,0,0,114,162,0,0,0,114, - 167,0,0,0,41,12,218,17,95,98,111,111,116,115,116,114, - 97,112,95,109,111,100,117,108,101,90,11,115,101,108,102,95, - 109,111,100,117,108,101,90,12,98,117,105,108,116,105,110,95, - 110,97,109,101,90,14,98,117,105,108,116,105,110,95,109,111, - 100,117,108,101,90,10,111,115,95,100,101,116,97,105,108,115, - 90,10,98,117,105,108,116,105,110,95,111,115,114,21,0,0, - 0,114,25,0,0,0,90,9,111,115,95,109,111,100,117,108, - 101,90,13,116,104,114,101,97,100,95,109,111,100,117,108,101, - 90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,101, - 90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, - 95,115,101,116,117,112,65,5,0,0,115,82,0,0,0,0, - 8,6,1,9,1,9,3,13,1,13,1,15,1,18,2,13, - 1,20,3,33,1,19,2,31,1,10,1,15,1,13,1,4, - 2,3,1,15,1,5,1,13,1,12,2,12,1,16,1,16, - 1,25,3,3,1,19,1,13,2,11,1,16,3,15,1,16, - 3,12,1,15,1,16,3,19,1,19,1,12,1,13,1,12, - 1,114,27,1,0,0,99,1,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,67,0,0,0,115,116,0,0,0, - 116,0,0,124,0,0,131,1,0,1,116,1,0,131,0,0, - 125,1,0,116,2,0,106,3,0,106,4,0,116,5,0,106, - 6,0,124,1,0,140,0,0,103,1,0,131,1,0,1,116, - 7,0,106,8,0,100,1,0,107,2,0,114,78,0,116,2, - 0,106,9,0,106,10,0,116,11,0,131,1,0,1,116,2, - 0,106,9,0,106,10,0,116,12,0,131,1,0,1,116,5, - 0,124,0,0,95,5,0,116,13,0,124,0,0,95,13,0, - 100,2,0,83,41,3,122,41,73,110,115,116,97,108,108,32, - 116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,105, - 109,112,111,114,116,32,99,111,109,112,111,110,101,110,116,115, - 46,114,22,1,0,0,78,41,14,114,27,1,0,0,114,155, - 0,0,0,114,7,0,0,0,114,248,0,0,0,114,143,0, - 0,0,114,0,1,0,0,114,13,1,0,0,114,3,0,0, - 0,114,105,0,0,0,218,9,109,101,116,97,95,112,97,116, - 104,114,157,0,0,0,114,162,0,0,0,114,243,0,0,0, - 114,212,0,0,0,41,2,114,26,1,0,0,90,17,115,117, - 112,112,111,114,116,101,100,95,108,111,97,100,101,114,115,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,105,110,115,116,97,108,108,133,5,0,0,115,16,0,0, - 0,0,2,10,1,9,1,28,1,15,1,16,1,16,4,9, - 1,114,29,1,0,0,41,3,122,3,119,105,110,114,1,0, - 0,0,114,2,0,0,0,41,56,114,107,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,17,0,0,0,114,19,0, - 0,0,114,28,0,0,0,114,38,0,0,0,114,39,0,0, - 0,114,43,0,0,0,114,44,0,0,0,114,46,0,0,0, - 114,55,0,0,0,218,4,116,121,112,101,218,8,95,95,99, - 111,100,101,95,95,114,138,0,0,0,114,15,0,0,0,114, - 128,0,0,0,114,14,0,0,0,114,18,0,0,0,90,17, - 95,82,65,87,95,77,65,71,73,67,95,78,85,77,66,69, - 82,114,73,0,0,0,114,72,0,0,0,114,84,0,0,0, - 114,74,0,0,0,90,23,68,69,66,85,71,95,66,89,84, - 69,67,79,68,69,95,83,85,70,70,73,88,69,83,90,27, - 79,80,84,73,77,73,90,69,68,95,66,89,84,69,67,79, - 68,69,95,83,85,70,70,73,88,69,83,114,79,0,0,0, - 114,85,0,0,0,114,91,0,0,0,114,95,0,0,0,114, - 97,0,0,0,114,116,0,0,0,114,123,0,0,0,114,135, - 0,0,0,114,141,0,0,0,114,144,0,0,0,114,149,0, - 0,0,218,6,111,98,106,101,99,116,114,156,0,0,0,114, - 161,0,0,0,114,162,0,0,0,114,178,0,0,0,114,188, - 0,0,0,114,204,0,0,0,114,212,0,0,0,114,217,0, - 0,0,114,223,0,0,0,114,218,0,0,0,114,224,0,0, - 0,114,241,0,0,0,114,243,0,0,0,114,0,1,0,0, - 114,18,1,0,0,114,155,0,0,0,114,27,1,0,0,114, - 29,1,0,0,114,4,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,218,8,60,109,111,100,117,108, - 101,62,8,0,0,0,115,102,0,0,0,6,17,6,3,12, - 12,12,5,12,5,12,6,12,12,12,10,12,9,12,5,12, - 7,15,22,15,112,22,1,18,2,6,1,6,2,9,2,9, - 2,10,2,21,44,12,33,12,19,12,12,12,12,12,28,12, - 17,21,55,21,12,18,10,12,14,9,3,12,1,15,65,19, - 64,19,29,22,110,19,41,25,45,25,16,6,3,25,53,19, - 57,19,42,19,127,0,7,19,127,0,20,15,23,12,11,12, - 68, + 0,0,114,1,1,0,0,184,4,0,0,115,6,0,0,0, + 0,1,15,1,18,1,122,20,70,105,108,101,70,105,110,100, + 101,114,46,95,103,101,116,95,115,112,101,99,78,99,3,0, + 0,0,0,0,0,0,14,0,0,0,15,0,0,0,67,0, + 0,0,115,228,1,0,0,100,1,0,125,3,0,124,1,0, + 106,0,0,100,2,0,131,1,0,100,3,0,25,125,4,0, + 121,34,0,116,1,0,124,0,0,106,2,0,112,49,0,116, + 3,0,106,4,0,131,0,0,131,1,0,106,5,0,125,5, + 0,87,110,24,0,4,116,6,0,107,10,0,114,85,0,1, + 1,1,100,10,0,125,5,0,89,110,1,0,88,124,5,0, + 124,0,0,106,7,0,107,3,0,114,120,0,124,0,0,106, + 8,0,131,0,0,1,124,5,0,124,0,0,95,7,0,116, + 9,0,131,0,0,114,153,0,124,0,0,106,10,0,125,6, + 0,124,4,0,106,11,0,131,0,0,125,7,0,110,15,0, + 124,0,0,106,12,0,125,6,0,124,4,0,125,7,0,124, + 7,0,124,6,0,107,6,0,114,45,1,116,13,0,124,0, + 0,106,2,0,124,4,0,131,2,0,125,8,0,120,100,0, + 124,0,0,106,14,0,68,93,77,0,92,2,0,125,9,0, + 125,10,0,100,5,0,124,9,0,23,125,11,0,116,13,0, + 124,8,0,124,11,0,131,2,0,125,12,0,116,15,0,124, + 12,0,131,1,0,114,208,0,124,0,0,106,16,0,124,10, + 0,124,1,0,124,12,0,124,8,0,103,1,0,124,2,0, + 131,5,0,83,113,208,0,87,116,17,0,124,8,0,131,1, + 0,125,3,0,120,120,0,124,0,0,106,14,0,68,93,109, + 0,92,2,0,125,9,0,125,10,0,116,13,0,124,0,0, + 106,2,0,124,4,0,124,9,0,23,131,2,0,125,12,0, + 116,18,0,106,19,0,100,6,0,124,12,0,100,7,0,100, + 3,0,131,2,1,1,124,7,0,124,9,0,23,124,6,0, + 107,6,0,114,55,1,116,15,0,124,12,0,131,1,0,114, + 55,1,124,0,0,106,16,0,124,10,0,124,1,0,124,12, + 0,100,8,0,124,2,0,131,5,0,83,113,55,1,87,124, + 3,0,114,224,1,116,18,0,106,19,0,100,9,0,124,8, + 0,131,2,0,1,116,18,0,106,20,0,124,1,0,100,8, + 0,131,2,0,125,13,0,124,8,0,103,1,0,124,13,0, + 95,21,0,124,13,0,83,100,8,0,83,41,11,122,125,84, + 114,121,32,116,111,32,102,105,110,100,32,97,32,108,111,97, + 100,101,114,32,102,111,114,32,116,104,101,32,115,112,101,99, + 105,102,105,101,100,32,109,111,100,117,108,101,44,32,111,114, + 32,116,104,101,32,110,97,109,101,115,112,97,99,101,10,32, + 32,32,32,32,32,32,32,112,97,99,107,97,103,101,32,112, + 111,114,116,105,111,110,115,46,32,82,101,116,117,114,110,115, + 32,40,108,111,97,100,101,114,44,32,108,105,115,116,45,111, + 102,45,112,111,114,116,105,111,110,115,41,46,70,114,58,0, + 0,0,114,56,0,0,0,114,29,0,0,0,114,179,0,0, + 0,122,9,116,114,121,105,110,103,32,123,125,90,9,118,101, + 114,98,111,115,105,116,121,78,122,25,112,111,115,115,105,98, + 108,101,32,110,97,109,101,115,112,97,99,101,32,102,111,114, + 32,123,125,114,87,0,0,0,41,22,114,32,0,0,0,114, + 39,0,0,0,114,35,0,0,0,114,3,0,0,0,114,45, + 0,0,0,114,213,0,0,0,114,40,0,0,0,114,4,1, + 0,0,218,11,95,102,105,108,108,95,99,97,99,104,101,114, + 6,0,0,0,114,7,1,0,0,114,88,0,0,0,114,6, + 1,0,0,114,28,0,0,0,114,3,1,0,0,114,44,0, + 0,0,114,1,1,0,0,114,46,0,0,0,114,114,0,0, + 0,114,129,0,0,0,114,154,0,0,0,114,150,0,0,0, + 41,14,114,100,0,0,0,114,119,0,0,0,114,174,0,0, + 0,90,12,105,115,95,110,97,109,101,115,112,97,99,101,90, + 11,116,97,105,108,95,109,111,100,117,108,101,114,126,0,0, + 0,90,5,99,97,99,104,101,90,12,99,97,99,104,101,95, + 109,111,100,117,108,101,90,9,98,97,115,101,95,112,97,116, + 104,114,219,0,0,0,114,159,0,0,0,90,13,105,110,105, + 116,95,102,105,108,101,110,97,109,101,90,9,102,117,108,108, + 95,112,97,116,104,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,175,0,0,0,189,4, + 0,0,115,70,0,0,0,0,3,6,1,19,1,3,1,34, + 1,13,1,11,1,15,1,10,1,9,2,9,1,9,1,15, + 2,9,1,6,2,12,1,18,1,22,1,10,1,15,1,12, + 1,32,4,12,2,22,1,22,1,22,1,16,1,12,1,15, + 1,14,1,6,1,16,1,18,1,12,1,4,1,122,20,70, + 105,108,101,70,105,110,100,101,114,46,102,105,110,100,95,115, + 112,101,99,99,1,0,0,0,0,0,0,0,9,0,0,0, + 13,0,0,0,67,0,0,0,115,11,1,0,0,124,0,0, + 106,0,0,125,1,0,121,31,0,116,1,0,106,2,0,124, + 1,0,112,33,0,116,1,0,106,3,0,131,0,0,131,1, + 0,125,2,0,87,110,33,0,4,116,4,0,116,5,0,116, + 6,0,102,3,0,107,10,0,114,75,0,1,1,1,103,0, + 0,125,2,0,89,110,1,0,88,116,7,0,106,8,0,106, + 9,0,100,1,0,131,1,0,115,112,0,116,10,0,124,2, + 0,131,1,0,124,0,0,95,11,0,110,111,0,116,10,0, + 131,0,0,125,3,0,120,90,0,124,2,0,68,93,82,0, + 125,4,0,124,4,0,106,12,0,100,2,0,131,1,0,92, + 3,0,125,5,0,125,6,0,125,7,0,124,6,0,114,191, + 0,100,3,0,106,13,0,124,5,0,124,7,0,106,14,0, + 131,0,0,131,2,0,125,8,0,110,6,0,124,5,0,125, + 8,0,124,3,0,106,15,0,124,8,0,131,1,0,1,113, + 128,0,87,124,3,0,124,0,0,95,11,0,116,7,0,106, + 8,0,106,9,0,116,16,0,131,1,0,114,7,1,100,4, + 0,100,5,0,132,0,0,124,2,0,68,131,1,0,124,0, + 0,95,17,0,100,6,0,83,41,7,122,68,70,105,108,108, + 32,116,104,101,32,99,97,99,104,101,32,111,102,32,112,111, + 116,101,110,116,105,97,108,32,109,111,100,117,108,101,115,32, + 97,110,100,32,112,97,99,107,97,103,101,115,32,102,111,114, + 32,116,104,105,115,32,100,105,114,101,99,116,111,114,121,46, + 114,0,0,0,0,114,58,0,0,0,122,5,123,125,46,123, + 125,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, + 0,0,83,0,0,0,115,28,0,0,0,104,0,0,124,0, + 0,93,18,0,125,1,0,124,1,0,106,0,0,131,0,0, + 146,2,0,113,6,0,83,114,4,0,0,0,41,1,114,88, + 0,0,0,41,2,114,22,0,0,0,90,2,102,110,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,250,9,60, + 115,101,116,99,111,109,112,62,8,5,0,0,115,2,0,0, + 0,9,0,122,41,70,105,108,101,70,105,110,100,101,114,46, + 95,102,105,108,108,95,99,97,99,104,101,46,60,108,111,99, + 97,108,115,62,46,60,115,101,116,99,111,109,112,62,78,41, + 18,114,35,0,0,0,114,3,0,0,0,90,7,108,105,115, + 116,100,105,114,114,45,0,0,0,114,252,0,0,0,218,15, + 80,101,114,109,105,115,115,105,111,110,69,114,114,111,114,218, + 18,78,111,116,65,68,105,114,101,99,116,111,114,121,69,114, + 114,111,114,114,7,0,0,0,114,8,0,0,0,114,9,0, + 0,0,114,5,1,0,0,114,6,1,0,0,114,83,0,0, + 0,114,47,0,0,0,114,88,0,0,0,218,3,97,100,100, + 114,10,0,0,0,114,7,1,0,0,41,9,114,100,0,0, + 0,114,35,0,0,0,90,8,99,111,110,116,101,110,116,115, + 90,21,108,111,119,101,114,95,115,117,102,102,105,120,95,99, + 111,110,116,101,110,116,115,114,241,0,0,0,114,98,0,0, + 0,114,231,0,0,0,114,219,0,0,0,90,8,110,101,119, + 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,9,1,0,0,235,4,0,0,115,34,0, + 0,0,0,2,9,1,3,1,31,1,22,3,11,3,18,1, + 18,7,9,1,13,1,24,1,6,1,27,2,6,1,17,1, + 9,1,18,1,122,22,70,105,108,101,70,105,110,100,101,114, + 46,95,102,105,108,108,95,99,97,99,104,101,99,1,0,0, + 0,0,0,0,0,3,0,0,0,3,0,0,0,7,0,0, + 0,115,25,0,0,0,135,0,0,135,1,0,102,2,0,100, + 1,0,100,2,0,134,0,0,125,2,0,124,2,0,83,41, + 3,97,20,1,0,0,65,32,99,108,97,115,115,32,109,101, + 116,104,111,100,32,119,104,105,99,104,32,114,101,116,117,114, + 110,115,32,97,32,99,108,111,115,117,114,101,32,116,111,32, + 117,115,101,32,111,110,32,115,121,115,46,112,97,116,104,95, + 104,111,111,107,10,32,32,32,32,32,32,32,32,119,104,105, + 99,104,32,119,105,108,108,32,114,101,116,117,114,110,32,97, + 110,32,105,110,115,116,97,110,99,101,32,117,115,105,110,103, + 32,116,104,101,32,115,112,101,99,105,102,105,101,100,32,108, + 111,97,100,101,114,115,32,97,110,100,32,116,104,101,32,112, + 97,116,104,10,32,32,32,32,32,32,32,32,99,97,108,108, + 101,100,32,111,110,32,116,104,101,32,99,108,111,115,117,114, + 101,46,10,10,32,32,32,32,32,32,32,32,73,102,32,116, + 104,101,32,112,97,116,104,32,99,97,108,108,101,100,32,111, + 110,32,116,104,101,32,99,108,111,115,117,114,101,32,105,115, + 32,110,111,116,32,97,32,100,105,114,101,99,116,111,114,121, + 44,32,73,109,112,111,114,116,69,114,114,111,114,32,105,115, + 10,32,32,32,32,32,32,32,32,114,97,105,115,101,100,46, + 10,10,32,32,32,32,32,32,32,32,99,1,0,0,0,0, + 0,0,0,1,0,0,0,4,0,0,0,19,0,0,0,115, + 43,0,0,0,116,0,0,124,0,0,131,1,0,115,30,0, + 116,1,0,100,1,0,100,2,0,124,0,0,131,1,1,130, + 1,0,136,0,0,124,0,0,136,1,0,140,1,0,83,41, + 3,122,45,80,97,116,104,32,104,111,111,107,32,102,111,114, + 32,105,109,112,111,114,116,108,105,98,46,109,97,99,104,105, + 110,101,114,121,46,70,105,108,101,70,105,110,100,101,114,46, + 122,30,111,110,108,121,32,100,105,114,101,99,116,111,114,105, + 101,115,32,97,114,101,32,115,117,112,112,111,114,116,101,100, + 114,35,0,0,0,41,2,114,46,0,0,0,114,99,0,0, + 0,41,1,114,35,0,0,0,41,2,114,164,0,0,0,114, + 8,1,0,0,114,4,0,0,0,114,5,0,0,0,218,24, + 112,97,116,104,95,104,111,111,107,95,102,111,114,95,70,105, + 108,101,70,105,110,100,101,114,20,5,0,0,115,6,0,0, + 0,0,2,12,1,18,1,122,54,70,105,108,101,70,105,110, + 100,101,114,46,112,97,116,104,95,104,111,111,107,46,60,108, + 111,99,97,108,115,62,46,112,97,116,104,95,104,111,111,107, + 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,114, + 4,0,0,0,41,3,114,164,0,0,0,114,8,1,0,0, + 114,14,1,0,0,114,4,0,0,0,41,2,114,164,0,0, + 0,114,8,1,0,0,114,5,0,0,0,218,9,112,97,116, + 104,95,104,111,111,107,10,5,0,0,115,4,0,0,0,0, + 10,21,6,122,20,70,105,108,101,70,105,110,100,101,114,46, + 112,97,116,104,95,104,111,111,107,99,1,0,0,0,0,0, + 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, + 0,0,0,100,1,0,106,0,0,124,0,0,106,1,0,131, + 1,0,83,41,2,78,122,16,70,105,108,101,70,105,110,100, + 101,114,40,123,33,114,125,41,41,2,114,47,0,0,0,114, + 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,240,0,0,0,28, + 5,0,0,115,2,0,0,0,0,1,122,19,70,105,108,101, + 70,105,110,100,101,114,46,95,95,114,101,112,114,95,95,41, + 15,114,105,0,0,0,114,104,0,0,0,114,106,0,0,0, + 114,107,0,0,0,114,179,0,0,0,114,246,0,0,0,114, + 123,0,0,0,114,176,0,0,0,114,117,0,0,0,114,1, + 1,0,0,114,175,0,0,0,114,9,1,0,0,114,177,0, + 0,0,114,15,1,0,0,114,240,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,2,1,0,0,143,4,0,0,115,20,0,0,0,12,7, + 6,2,12,14,12,4,6,2,12,12,12,5,15,46,12,31, + 18,18,114,2,1,0,0,99,4,0,0,0,0,0,0,0, + 6,0,0,0,11,0,0,0,67,0,0,0,115,195,0,0, + 0,124,0,0,106,0,0,100,1,0,131,1,0,125,4,0, + 124,0,0,106,0,0,100,2,0,131,1,0,125,5,0,124, + 4,0,115,99,0,124,5,0,114,54,0,124,5,0,106,1, + 0,125,4,0,110,45,0,124,2,0,124,3,0,107,2,0, + 114,84,0,116,2,0,124,1,0,124,2,0,131,2,0,125, + 4,0,110,15,0,116,3,0,124,1,0,124,2,0,131,2, + 0,125,4,0,124,5,0,115,126,0,116,4,0,124,1,0, + 124,2,0,100,3,0,124,4,0,131,2,1,125,5,0,121, + 44,0,124,5,0,124,0,0,100,2,0,60,124,4,0,124, + 0,0,100,1,0,60,124,2,0,124,0,0,100,4,0,60, + 124,3,0,124,0,0,100,5,0,60,87,110,18,0,4,116, + 5,0,107,10,0,114,190,0,1,1,1,89,110,1,0,88, + 100,0,0,83,41,6,78,218,10,95,95,108,111,97,100,101, + 114,95,95,218,8,95,95,115,112,101,99,95,95,114,120,0, + 0,0,90,8,95,95,102,105,108,101,95,95,90,10,95,95, + 99,97,99,104,101,100,95,95,41,6,218,3,103,101,116,114, + 120,0,0,0,114,217,0,0,0,114,212,0,0,0,114,161, + 0,0,0,218,9,69,120,99,101,112,116,105,111,110,41,6, + 90,2,110,115,114,98,0,0,0,90,8,112,97,116,104,110, + 97,109,101,90,9,99,112,97,116,104,110,97,109,101,114,120, + 0,0,0,114,158,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,14,95,102,105,120,95,117,112, + 95,109,111,100,117,108,101,34,5,0,0,115,34,0,0,0, + 0,2,15,1,15,1,6,1,6,1,12,1,12,1,18,2, + 15,1,6,1,21,1,3,1,10,1,10,1,10,1,14,1, + 13,2,114,20,1,0,0,99,0,0,0,0,0,0,0,0, + 3,0,0,0,3,0,0,0,67,0,0,0,115,55,0,0, + 0,116,0,0,116,1,0,106,2,0,131,0,0,102,2,0, + 125,0,0,116,3,0,116,4,0,102,2,0,125,1,0,116, + 5,0,116,6,0,102,2,0,125,2,0,124,0,0,124,1, + 0,124,2,0,103,3,0,83,41,1,122,95,82,101,116,117, + 114,110,115,32,97,32,108,105,115,116,32,111,102,32,102,105, + 108,101,45,98,97,115,101,100,32,109,111,100,117,108,101,32, + 108,111,97,100,101,114,115,46,10,10,32,32,32,32,69,97, + 99,104,32,105,116,101,109,32,105,115,32,97,32,116,117,112, + 108,101,32,40,108,111,97,100,101,114,44,32,115,117,102,102, + 105,120,101,115,41,46,10,32,32,32,32,41,7,114,218,0, + 0,0,114,139,0,0,0,218,18,101,120,116,101,110,115,105, + 111,110,95,115,117,102,102,105,120,101,115,114,212,0,0,0, + 114,84,0,0,0,114,217,0,0,0,114,74,0,0,0,41, + 3,90,10,101,120,116,101,110,115,105,111,110,115,90,6,115, + 111,117,114,99,101,90,8,98,121,116,101,99,111,100,101,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,155, + 0,0,0,57,5,0,0,115,8,0,0,0,0,5,18,1, + 12,1,12,1,114,155,0,0,0,99,1,0,0,0,0,0, + 0,0,12,0,0,0,12,0,0,0,67,0,0,0,115,70, + 2,0,0,124,0,0,97,0,0,116,0,0,106,1,0,97, + 1,0,116,0,0,106,2,0,97,2,0,116,1,0,106,3, + 0,116,4,0,25,125,1,0,120,76,0,100,26,0,68,93, + 68,0,125,2,0,124,2,0,116,1,0,106,3,0,107,7, + 0,114,83,0,116,0,0,106,5,0,124,2,0,131,1,0, + 125,3,0,110,13,0,116,1,0,106,3,0,124,2,0,25, + 125,3,0,116,6,0,124,1,0,124,2,0,124,3,0,131, + 3,0,1,113,44,0,87,100,5,0,100,6,0,103,1,0, + 102,2,0,100,7,0,100,8,0,100,6,0,103,2,0,102, + 2,0,102,2,0,125,4,0,120,149,0,124,4,0,68,93, + 129,0,92,2,0,125,5,0,125,6,0,116,7,0,100,9, + 0,100,10,0,132,0,0,124,6,0,68,131,1,0,131,1, + 0,115,199,0,116,8,0,130,1,0,124,6,0,100,11,0, + 25,125,7,0,124,5,0,116,1,0,106,3,0,107,6,0, + 114,241,0,116,1,0,106,3,0,124,5,0,25,125,8,0, + 80,113,156,0,121,20,0,116,0,0,106,5,0,124,5,0, + 131,1,0,125,8,0,80,87,113,156,0,4,116,9,0,107, + 10,0,114,28,1,1,1,1,119,156,0,89,113,156,0,88, + 113,156,0,87,116,9,0,100,12,0,131,1,0,130,1,0, + 116,6,0,124,1,0,100,13,0,124,8,0,131,3,0,1, + 116,6,0,124,1,0,100,14,0,124,7,0,131,3,0,1, + 116,6,0,124,1,0,100,15,0,100,16,0,106,10,0,124, + 6,0,131,1,0,131,3,0,1,121,19,0,116,0,0,106, + 5,0,100,17,0,131,1,0,125,9,0,87,110,24,0,4, + 116,9,0,107,10,0,114,147,1,1,1,1,100,18,0,125, + 9,0,89,110,1,0,88,116,6,0,124,1,0,100,17,0, + 124,9,0,131,3,0,1,116,0,0,106,5,0,100,19,0, + 131,1,0,125,10,0,116,6,0,124,1,0,100,19,0,124, + 10,0,131,3,0,1,124,5,0,100,7,0,107,2,0,114, + 238,1,116,0,0,106,5,0,100,20,0,131,1,0,125,11, + 0,116,6,0,124,1,0,100,21,0,124,11,0,131,3,0, + 1,116,6,0,124,1,0,100,22,0,116,11,0,131,0,0, + 131,3,0,1,116,12,0,106,13,0,116,2,0,106,14,0, + 131,0,0,131,1,0,1,124,5,0,100,7,0,107,2,0, + 114,66,2,116,15,0,106,16,0,100,23,0,131,1,0,1, + 100,24,0,116,12,0,107,6,0,114,66,2,100,25,0,116, + 17,0,95,18,0,100,18,0,83,41,27,122,205,83,101,116, + 117,112,32,116,104,101,32,112,97,116,104,45,98,97,115,101, + 100,32,105,109,112,111,114,116,101,114,115,32,102,111,114,32, + 105,109,112,111,114,116,108,105,98,32,98,121,32,105,109,112, + 111,114,116,105,110,103,32,110,101,101,100,101,100,10,32,32, + 32,32,98,117,105,108,116,45,105,110,32,109,111,100,117,108, + 101,115,32,97,110,100,32,105,110,106,101,99,116,105,110,103, + 32,116,104,101,109,32,105,110,116,111,32,116,104,101,32,103, + 108,111,98,97,108,32,110,97,109,101,115,112,97,99,101,46, + 10,10,32,32,32,32,79,116,104,101,114,32,99,111,109,112, + 111,110,101,110,116,115,32,97,114,101,32,101,120,116,114,97, + 99,116,101,100,32,102,114,111,109,32,116,104,101,32,99,111, + 114,101,32,98,111,111,116,115,116,114,97,112,32,109,111,100, + 117,108,101,46,10,10,32,32,32,32,114,49,0,0,0,114, + 60,0,0,0,218,8,98,117,105,108,116,105,110,115,114,136, + 0,0,0,90,5,112,111,115,105,120,250,1,47,218,2,110, + 116,250,1,92,99,1,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,115,0,0,0,115,33,0,0,0,124,0, + 0,93,23,0,125,1,0,116,0,0,124,1,0,131,1,0, + 100,0,0,107,2,0,86,1,113,3,0,100,1,0,83,41, + 2,114,29,0,0,0,78,41,1,114,31,0,0,0,41,2, + 114,22,0,0,0,114,77,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,221,0,0,0,93,5, + 0,0,115,2,0,0,0,6,0,122,25,95,115,101,116,117, + 112,46,60,108,111,99,97,108,115,62,46,60,103,101,110,101, + 120,112,114,62,114,59,0,0,0,122,30,105,109,112,111,114, + 116,108,105,98,32,114,101,113,117,105,114,101,115,32,112,111, + 115,105,120,32,111,114,32,110,116,114,3,0,0,0,114,25, + 0,0,0,114,21,0,0,0,114,30,0,0,0,90,7,95, + 116,104,114,101,97,100,78,90,8,95,119,101,97,107,114,101, + 102,90,6,119,105,110,114,101,103,114,163,0,0,0,114,6, + 0,0,0,122,4,46,112,121,119,122,6,95,100,46,112,121, + 100,84,41,4,122,3,95,105,111,122,9,95,119,97,114,110, + 105,110,103,115,122,8,98,117,105,108,116,105,110,115,122,7, + 109,97,114,115,104,97,108,41,19,114,114,0,0,0,114,7, + 0,0,0,114,139,0,0,0,114,233,0,0,0,114,105,0, + 0,0,90,18,95,98,117,105,108,116,105,110,95,102,114,111, + 109,95,110,97,109,101,114,109,0,0,0,218,3,97,108,108, + 218,14,65,115,115,101,114,116,105,111,110,69,114,114,111,114, + 114,99,0,0,0,114,26,0,0,0,114,11,0,0,0,114, + 223,0,0,0,114,143,0,0,0,114,21,1,0,0,114,84, + 0,0,0,114,157,0,0,0,114,162,0,0,0,114,167,0, + 0,0,41,12,218,17,95,98,111,111,116,115,116,114,97,112, + 95,109,111,100,117,108,101,90,11,115,101,108,102,95,109,111, + 100,117,108,101,90,12,98,117,105,108,116,105,110,95,110,97, + 109,101,90,14,98,117,105,108,116,105,110,95,109,111,100,117, + 108,101,90,10,111,115,95,100,101,116,97,105,108,115,90,10, + 98,117,105,108,116,105,110,95,111,115,114,21,0,0,0,114, + 25,0,0,0,90,9,111,115,95,109,111,100,117,108,101,90, + 13,116,104,114,101,97,100,95,109,111,100,117,108,101,90,14, + 119,101,97,107,114,101,102,95,109,111,100,117,108,101,90,13, + 119,105,110,114,101,103,95,109,111,100,117,108,101,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,6,95,115, + 101,116,117,112,68,5,0,0,115,82,0,0,0,0,8,6, + 1,9,1,9,3,13,1,13,1,15,1,18,2,13,1,20, + 3,33,1,19,2,31,1,10,1,15,1,13,1,4,2,3, + 1,15,1,5,1,13,1,12,2,12,1,16,1,16,1,25, + 3,3,1,19,1,13,2,11,1,16,3,15,1,16,3,12, + 1,15,1,16,3,19,1,19,1,12,1,13,1,12,1,114, + 29,1,0,0,99,1,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,67,0,0,0,115,116,0,0,0,116,0, + 0,124,0,0,131,1,0,1,116,1,0,131,0,0,125,1, + 0,116,2,0,106,3,0,106,4,0,116,5,0,106,6,0, + 124,1,0,140,0,0,103,1,0,131,1,0,1,116,7,0, + 106,8,0,100,1,0,107,2,0,114,78,0,116,2,0,106, + 9,0,106,10,0,116,11,0,131,1,0,1,116,2,0,106, + 9,0,106,10,0,116,12,0,131,1,0,1,116,5,0,124, + 0,0,95,5,0,116,13,0,124,0,0,95,13,0,100,2, + 0,83,41,3,122,41,73,110,115,116,97,108,108,32,116,104, + 101,32,112,97,116,104,45,98,97,115,101,100,32,105,109,112, + 111,114,116,32,99,111,109,112,111,110,101,110,116,115,46,114, + 24,1,0,0,78,41,14,114,29,1,0,0,114,155,0,0, + 0,114,7,0,0,0,114,250,0,0,0,114,143,0,0,0, + 114,2,1,0,0,114,15,1,0,0,114,3,0,0,0,114, + 105,0,0,0,218,9,109,101,116,97,95,112,97,116,104,114, + 157,0,0,0,114,162,0,0,0,114,245,0,0,0,114,212, + 0,0,0,41,2,114,28,1,0,0,90,17,115,117,112,112, + 111,114,116,101,100,95,108,111,97,100,101,114,115,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,8,95,105, + 110,115,116,97,108,108,136,5,0,0,115,16,0,0,0,0, + 2,10,1,9,1,28,1,15,1,16,1,16,4,9,1,114, + 31,1,0,0,41,3,122,3,119,105,110,114,1,0,0,0, + 114,2,0,0,0,41,56,114,107,0,0,0,114,10,0,0, + 0,114,11,0,0,0,114,17,0,0,0,114,19,0,0,0, + 114,28,0,0,0,114,38,0,0,0,114,39,0,0,0,114, + 43,0,0,0,114,44,0,0,0,114,46,0,0,0,114,55, + 0,0,0,218,4,116,121,112,101,218,8,95,95,99,111,100, + 101,95,95,114,138,0,0,0,114,15,0,0,0,114,128,0, + 0,0,114,14,0,0,0,114,18,0,0,0,90,17,95,82, + 65,87,95,77,65,71,73,67,95,78,85,77,66,69,82,114, + 73,0,0,0,114,72,0,0,0,114,84,0,0,0,114,74, + 0,0,0,90,23,68,69,66,85,71,95,66,89,84,69,67, + 79,68,69,95,83,85,70,70,73,88,69,83,90,27,79,80, + 84,73,77,73,90,69,68,95,66,89,84,69,67,79,68,69, + 95,83,85,70,70,73,88,69,83,114,79,0,0,0,114,85, + 0,0,0,114,91,0,0,0,114,95,0,0,0,114,97,0, + 0,0,114,116,0,0,0,114,123,0,0,0,114,135,0,0, + 0,114,141,0,0,0,114,144,0,0,0,114,149,0,0,0, + 218,6,111,98,106,101,99,116,114,156,0,0,0,114,161,0, + 0,0,114,162,0,0,0,114,178,0,0,0,114,188,0,0, + 0,114,204,0,0,0,114,212,0,0,0,114,217,0,0,0, + 114,223,0,0,0,114,218,0,0,0,114,224,0,0,0,114, + 243,0,0,0,114,245,0,0,0,114,2,1,0,0,114,20, + 1,0,0,114,155,0,0,0,114,29,1,0,0,114,31,1, + 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,8,60,109,111,100,117,108,101,62, + 8,0,0,0,115,102,0,0,0,6,17,6,3,12,12,12, + 5,12,5,12,6,12,12,12,10,12,9,12,5,12,7,15, + 22,15,112,22,1,18,2,6,1,6,2,9,2,9,2,10, + 2,21,44,12,33,12,19,12,12,12,12,12,28,12,17,21, + 55,21,12,18,10,12,14,9,3,12,1,15,65,19,64,19, + 29,22,110,19,41,25,45,25,16,6,3,25,53,19,60,19, + 42,19,127,0,7,19,127,0,20,15,23,12,11,12,68, }; -- cgit v0.12 From 32f2eb4941db115d4d5d0902ba086820406ef4b2 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 17 Mar 2016 07:50:22 +0000 Subject: Issue #21042: Revert Linux find_library() to return just filename This reverts most of revision 3092cf163eb4. The change worked on x86 architectures, but did not work on ARM, probably due to extra ABI flags in the ldconfig output. --- Doc/library/ctypes.rst | 11 ++++------- Lib/ctypes/test/test_find.py | 40 ++++++++++++++++------------------------ Lib/ctypes/util.py | 2 +- Misc/NEWS | 3 --- 4 files changed, 21 insertions(+), 35 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 4a7309e..828d7ca4 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1258,15 +1258,15 @@ The exact functionality is system dependent. On Linux, :func:`find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, and ``objdump``) to find the library file. It -returns the absolute path of the library file. Here are some examples:: +returns the filename of the library file. Here are some examples:: >>> from ctypes.util import find_library >>> find_library("m") - '/lib/x86_64-linux-gnu/libm.so.6' + 'libm.so.6' >>> find_library("c") - '/lib/x86_64-linux-gnu/libc.so.6' + 'libc.so.6' >>> find_library("bz2") - '/lib/x86_64-linux-gnu/libbz2.so.1.0' + 'libbz2.so.1.0' >>> On OS X, :func:`find_library` tries several predefined naming schemes and paths @@ -1835,9 +1835,6 @@ Utility functions The exact functionality is system dependent. - .. versionchanged:: 3.6 - On Linux it returns an absolute path. - .. function:: find_msvcrt() :module: ctypes.util diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py index 1845bb0..e6bc19d 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/ctypes/test/test_find.py @@ -9,39 +9,39 @@ from ctypes.util import find_library class Test_OpenGL_libs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.lib_gl = cls.lib_glu = cls.lib_gle = None + lib_gl = lib_glu = lib_gle = None if sys.platform == "win32": - cls.lib_gl = find_library("OpenGL32") - cls.lib_glu = find_library("Glu32") + lib_gl = find_library("OpenGL32") + lib_glu = find_library("Glu32") elif sys.platform == "darwin": - cls.lib_gl = cls.lib_glu = find_library("OpenGL") + lib_gl = lib_glu = find_library("OpenGL") else: - cls.lib_gl = find_library("GL") - cls.lib_glu = find_library("GLU") - cls.lib_gle = find_library("gle") + lib_gl = find_library("GL") + lib_glu = find_library("GLU") + lib_gle = find_library("gle") ## print, for debugging if test.support.verbose: print("OpenGL libraries:") - for item in (("GL", cls.lib_gl), - ("GLU", cls.lib_glu), - ("gle", cls.lib_gle)): + for item in (("GL", lib_gl), + ("GLU", lib_glu), + ("gle", lib_gle)): print("\t", item) cls.gl = cls.glu = cls.gle = None - if cls.lib_gl: + if lib_gl: try: - cls.gl = CDLL(cls.lib_gl, mode=RTLD_GLOBAL) + cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) except OSError: pass - if cls.lib_glu: + if lib_glu: try: - cls.glu = CDLL(cls.lib_glu, RTLD_GLOBAL) + cls.glu = CDLL(lib_glu, RTLD_GLOBAL) except OSError: pass - if cls.lib_gle: + if lib_gle: try: - cls.gle = CDLL(cls.lib_gle) + cls.gle = CDLL(lib_gle) except OSError: pass @@ -64,14 +64,6 @@ class Test_OpenGL_libs(unittest.TestCase): self.skipTest('lib_gle not available') self.gle.gleGetJoinStyle - def test_abspath(self): - if self.lib_gl: - self.assertTrue(os.path.isabs(self.lib_gl)) - if self.lib_glu: - self.assertTrue(os.path.isabs(self.lib_glu)) - if self.lib_gle: - self.assertTrue(os.path.isabs(self.lib_gle)) - # On platforms where the default shared library suffix is '.so', # at least some libraries can be loaded as attributes of the cdll # object, since ctypes now tries loading the lib again diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index d8e3bfa..38bd57f 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -221,7 +221,7 @@ elif os.name == "posix": abi_type = mach_map.get(machine, 'libc6') # XXX assuming GLIBC's ldconfig (with option -p) - regex = r'lib%s\.[^\s]+\s$%s(?:,\s.*)?$\s=>\s(.*)' + regex = r'\s+(lib%s\.[^\s]+)\s+\(%s' regex = os.fsencode(regex % (re.escape(name), abi_type)) try: with subprocess.Popen(['/sbin/ldconfig', '-p'], diff --git a/Misc/NEWS b/Misc/NEWS index 8338a84..7f93b2b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -245,9 +245,6 @@ Library - Issue #26177: Fixed the keys() method for Canvas and Scrollbar widgets. -- Issue #21042: Make ctypes.util.find_library() return the full path on - Linux, similar to other platforms. Patch by Tamás Bence Gedai. - - Issue #15068: Got rid of excessive buffering in fileinput. The bufsize parameter is now deprecated and ignored. -- cgit v0.12 From 013024ef67b7e5989e4be03f4ff2be22aa753ae0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 09:43:14 +0100 Subject: Fix compilation error of traceback.c on Windows Issue #26564. --- Python/traceback.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/traceback.c b/Python/traceback.c index a40dbd1..8383c16 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -509,13 +509,13 @@ _Py_DumpDecimal(int fd, unsigned long value) static void dump_hexadecimal(int fd, unsigned long value, Py_ssize_t width) { - Py_ssize_t size = sizeof(unsigned long) * 2; - char buffer[size + 1], *ptr, *end; + char buffer[sizeof(unsigned long) * 2 + 1], *ptr, *end; + const Py_ssize_t size = Py_ARRAY_LENGTH(buffer) - 1; if (width > size) width = size; - end = &buffer[Py_ARRAY_LENGTH(buffer) - 1]; + end = &buffer[size]; ptr = end; *ptr = '\0'; do { -- cgit v0.12 From ad524375af042a549d28ec252f3071a595b892b2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 12:12:53 +0100 Subject: Fail if PyMem_Malloc() is called without holding the GIL Issue #26563: Debug hooks on Python memory allocators now raise a fatal error if functions of the PyMem_Malloc() family are called without holding the GIL. --- Lib/test/test_capi.py | 17 +++++++++++++---- Misc/NEWS | 4 ++++ Modules/_testcapimodule.c | 19 +++++++++++++++++++ Objects/obmalloc.c | 14 +++++++------- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 8e6245b..8f4836a 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -602,15 +602,24 @@ class PyMemDebugTests(unittest.TestCase): regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) - def test_pyobject_malloc_without_gil(self): - # Calling PyObject_Malloc() without holding the GIL must raise an - # error in debug mode. - code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + def check_malloc_without_gil(self, code): out = self.check(code) expected = ('Fatal Python error: Python memory allocator called ' 'without holding the GIL') self.assertIn(expected, out) + def test_pymem_malloc_without_gil(self): + # Debug hooks must raise an error if PyMem_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def test_pyobject_malloc_without_gil(self): + # Debug hooks must raise an error if PyObject_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + self.check_malloc_without_gil(code) + class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' diff --git a/Misc/NEWS b/Misc/NEWS index 7f93b2b..adfa04b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26563: Debug hooks on Python memory allocators now raise a fatal + error if functions of the :c:func:`PyMem_Malloc` family are called without + holding the GIL. + - Issue #26564: On error, the debug hooks on Python memory allocators now use the :mod:`tracemalloc` module to get the traceback where a memory block was allocated. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b3d8818..0fc7cbc 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3644,10 +3644,28 @@ pymem_api_misuse(PyObject *self, PyObject *args) } static PyObject* +pymem_malloc_without_gil(PyObject *self, PyObject *args) +{ + char *buffer; + + /* Deliberate bug to test debug hooks on Python memory allocators: + call PyMem_Malloc() without holding the GIL */ + Py_BEGIN_ALLOW_THREADS + buffer = PyMem_Malloc(10); + Py_END_ALLOW_THREADS + + PyMem_Free(buffer); + + Py_RETURN_NONE; +} + +static PyObject* pyobject_malloc_without_gil(PyObject *self, PyObject *args) { char *buffer; + /* Deliberate bug to test debug hooks on Python memory allocators: + call PyObject_Malloc() without holding the GIL */ Py_BEGIN_ALLOW_THREADS buffer = PyObject_Malloc(10); Py_END_ALLOW_THREADS @@ -3841,6 +3859,7 @@ static PyMethodDef TestMethods[] = { {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, + {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS}, {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 8812f59..503fcdf 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -198,7 +198,7 @@ static PyMemAllocatorEx _PyMem_Raw = { static PyMemAllocatorEx _PyMem = { #ifdef Py_DEBUG - &_PyMem_Debug.mem, PYRAWDBG_FUNCS + &_PyMem_Debug.mem, PYDBG_FUNCS #else NULL, PYMEM_FUNCS #endif @@ -321,17 +321,17 @@ PyMem_SetupDebugHooks(void) PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); } - if (_PyMem.malloc != _PyMem_DebugRawMalloc) { - alloc.ctx = &_PyMem_Debug.mem; - PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); - } - alloc.malloc = _PyMem_DebugMalloc; alloc.calloc = _PyMem_DebugCalloc; alloc.realloc = _PyMem_DebugRealloc; alloc.free = _PyMem_DebugFree; + if (_PyMem.malloc != _PyMem_DebugMalloc) { + alloc.ctx = &_PyMem_Debug.mem; + PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + } + if (_PyObject.malloc != _PyMem_DebugMalloc) { alloc.ctx = &_PyMem_Debug.obj; PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); -- cgit v0.12 From c36674a2c52ecb30e180b3bcced2b8c529cf72fb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 14:30:16 +0100 Subject: Fix usage of PyMem_Malloc() in os.stat() Issue #26563: Replace PyMem_Malloc() with PyMem_RawMalloc() in the Windows implementation of os.stat(), since the code is called without holding the GIL. --- Modules/posixmodule.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7e89878..65b20be 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1463,7 +1463,7 @@ get_target_path(HANDLE hdl, wchar_t **target_path) if(!buf_size) return FALSE; - buf = PyMem_New(wchar_t, buf_size+1); + buf = (wchar_t *)PyMem_RawMalloc((buf_size + 1) * sizeof(wchar_t)); if (!buf) { SetLastError(ERROR_OUTOFMEMORY); return FALSE; @@ -1473,12 +1473,12 @@ get_target_path(HANDLE hdl, wchar_t **target_path) buf, buf_size, VOLUME_NAME_DOS); if(!result_length) { - PyMem_Free(buf); + PyMem_RawFree(buf); return FALSE; } if(!CloseHandle(hdl)) { - PyMem_Free(buf); + PyMem_RawFree(buf); return FALSE; } @@ -1563,7 +1563,7 @@ win32_xstat_impl(const char *path, struct _Py_stat_struct *result, return -1; code = win32_xstat_impl_w(target_path, result, FALSE); - PyMem_Free(target_path); + PyMem_RawFree(target_path); return code; } } else @@ -1653,7 +1653,7 @@ win32_xstat_impl_w(const wchar_t *path, struct _Py_stat_struct *result, return -1; code = win32_xstat_impl_w(target_path, result, FALSE); - PyMem_Free(target_path); + PyMem_RawFree(target_path); return code; } } else -- cgit v0.12 From 861d9abfcf6a4a34790e4edc5e440a68534137e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 22:45:24 +0100 Subject: faulthandler now works in non-Python threads Issue #26563: * Add _PyGILState_GetInterpreterStateUnsafe() function: the single PyInterpreterState used by this process' GILState implementation. * Enhance _Py_DumpTracebackThreads() to retrieve the interpreter state from autoInterpreterState in last resort. The function now accepts NULL for interp and current_tstate parameters. * test_faulthandler: fix a ResourceWarning when test is interrupted by CTRL+c --- Include/pystate.h | 16 +++++++--- Include/traceback.h | 21 ++++++++++--- Lib/test/test_faulthandler.py | 36 +++++++++++++++-------- Modules/faulthandler.c | 68 ++++++++++++++++++++++++++++++++++++------- Python/pylifecycle.c | 16 +--------- Python/pystate.c | 6 ++++ Python/traceback.c | 49 +++++++++++++++++++++++++++++-- 7 files changed, 164 insertions(+), 48 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 550c332..9423bc7 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -243,15 +243,23 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); */ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); -/* Helper/diagnostic function - return 1 if the current thread - * currently holds the GIL, 0 otherwise - */ #ifndef Py_LIMITED_API /* Issue #26558: Flag to disable PyGILState_Check(). - If set, PyGILState_Check() always return 1. */ + If set to non-zero, PyGILState_Check() always return 1. */ PyAPI_DATA(int) _PyGILState_check_enabled; +/* Helper/diagnostic function - return 1 if the current thread + currently holds the GIL, 0 otherwise. + + The function returns 1 if _PyGILState_check_enabled is non-zero. */ PyAPI_FUNC(int) PyGILState_Check(void); + +/* Unsafe function to get the single PyInterpreterState used by this process' + GILState implementation. + + Return NULL before _PyGILState_Init() is called and after _PyGILState_Fini() + is called. */ +PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); #endif #endif /* #ifdef WITH_THREAD */ diff --git a/Include/traceback.h b/Include/traceback.h index 7c630e3..76e169a 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -53,19 +53,32 @@ PyAPI_DATA(void) _Py_DumpTraceback( PyThreadState *tstate); /* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. Return NULL on success, or an error message on error. + NULL. + + Return NULL on success, or an error message on error. This function is written for debug purpose only. It calls _Py_DumpTraceback() for each thread, and so has the same limitations. It only write the traceback of the first 100 threads: write "..." if there are more threads. + If current_tstate is NULL, the function tries to get the Python thread state + of the current thread. It is not an error if the function is unable to get + the current Python thread state. + + If interp is NULL, the function tries to get the interpreter state from + the current Python thread state, or from + _PyGILState_GetInterpreterStateUnsafe() in last resort. + + It is better to pass NULL to interp and current_tstate, the function tries + different options to retrieve these informations. + This function is signal safe. */ PyAPI_DATA(const char*) _Py_DumpTracebackThreads( - int fd, PyInterpreterState *interp, - PyThreadState *current_thread); - + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate); #ifndef Py_LIMITED_API diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 12969d5..c3cd657 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -58,8 +58,9 @@ class FaultHandlerTests(unittest.TestCase): pass_fds.append(fd) with support.SuppressCrashReport(): process = script_helper.spawn_python('-c', code, pass_fds=pass_fds) - stdout, stderr = process.communicate() - exitcode = process.wait() + with process: + stdout, stderr = process.communicate() + exitcode = process.wait() output = support.strip_python_stderr(stdout) output = output.decode('ascii', 'backslashreplace') if filename: @@ -73,14 +74,11 @@ class FaultHandlerTests(unittest.TestCase): with open(fd, "rb", closefd=False) as fp: output = fp.read() output = output.decode('ascii', 'backslashreplace') - output = re.sub('Current thread 0x[0-9a-f]+', - 'Current thread XXX', - output) return output.splitlines(), exitcode def check_fatal_error(self, code, line_number, name_regex, filename=None, all_threads=True, other_regex=None, - fd=None): + fd=None, know_current_thread=True): """ Check that the fault handler for fatal errors is enabled and check the traceback from the child process output. @@ -88,19 +86,22 @@ class FaultHandlerTests(unittest.TestCase): Raise an error if the output doesn't match the expected format. """ if all_threads: - header = 'Current thread XXX (most recent call first)' + if know_current_thread: + header = 'Current thread 0x[0-9a-f]+' + else: + header = 'Thread 0x[0-9a-f]+' else: - header = 'Stack (most recent call first)' + header = 'Stack' regex = """ ^Fatal Python error: {name} - {header}: + {header} $most recent call first$: File "", line {lineno} in """ regex = dedent(regex.format( lineno=line_number, name=name_regex, - header=re.escape(header))).strip() + header=header)).strip() if other_regex: regex += '|' + other_regex output, exitcode = self.get_output(code, filename=filename, fd=fd) @@ -129,6 +130,17 @@ class FaultHandlerTests(unittest.TestCase): 3, 'Segmentation fault') + @unittest.skipIf(not HAVE_THREADS, 'need threads') + def test_fatal_error_c_thread(self): + self.check_fatal_error(""" + import faulthandler + faulthandler.enable() + faulthandler._fatal_error_c_thread() + """, + 3, + 'in new thread', + know_current_thread=False) + def test_sigabrt(self): self.check_fatal_error(""" import faulthandler @@ -465,7 +477,7 @@ class FaultHandlerTests(unittest.TestCase): File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap - Current thread XXX $most recent call first$: + Current thread 0x[0-9a-f]+ $most recent call first$: File "", line {lineno} in dump File "", line 28 in $ """ @@ -637,7 +649,7 @@ class FaultHandlerTests(unittest.TestCase): trace = '\n'.join(trace) if not unregister: if all_threads: - regex = 'Current thread XXX $most recent call first$:\n' + regex = 'Current thread 0x[0-9a-f]+ $most recent call first$:\n' else: regex = 'Stack $most recent call first$:\n' regex = expected_traceback(14, 32, regex) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 45c9fcb..1c247b7 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -202,8 +202,9 @@ faulthandler_get_fileno(PyObject **file_ptr) static PyThreadState* get_thread_state(void) { - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_UncheckedGet(); if (tstate == NULL) { + /* just in case but very unlikely... */ PyErr_SetString(PyExc_RuntimeError, "unable to get the current thread state"); return NULL; @@ -234,11 +235,12 @@ faulthandler_dump_traceback(int fd, int all_threads, PyGILState_GetThisThreadState(). */ tstate = PyGILState_GetThisThreadState(); #else - tstate = PyThreadState_Get(); + tstate = _PyThreadState_UncheckedGet(); #endif - if (all_threads) - _Py_DumpTracebackThreads(fd, interp, tstate); + if (all_threads) { + (void)_Py_DumpTracebackThreads(fd, NULL, tstate); + } else { if (tstate != NULL) _Py_DumpTraceback(fd, tstate); @@ -272,7 +274,7 @@ faulthandler_dump_traceback_py(PyObject *self, return NULL; if (all_threads) { - errmsg = _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate); if (errmsg != NULL) { PyErr_SetString(PyExc_RuntimeError, errmsg); return NULL; @@ -469,7 +471,6 @@ faulthandler_thread(void *unused) { PyLockStatus st; const char* errmsg; - PyThreadState *current; int ok; #if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) sigset_t set; @@ -489,12 +490,9 @@ faulthandler_thread(void *unused) /* Timeout => dump traceback */ assert(st == PY_LOCK_FAILURE); - /* get the thread holding the GIL, NULL if no thread hold the GIL */ - current = _PyThreadState_UncheckedGet(); - _Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); - errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, current); + errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL); ok = (errmsg == NULL); if (thread.exit) @@ -894,7 +892,7 @@ static PyObject * faulthandler_sigsegv(PyObject *self, PyObject *args) { int release_gil = 0; - if (!PyArg_ParseTuple(args, "|i:_read_null", &release_gil)) + if (!PyArg_ParseTuple(args, "|i:_sigsegv", &release_gil)) return NULL; if (release_gil) { @@ -907,6 +905,49 @@ faulthandler_sigsegv(PyObject *self, PyObject *args) Py_RETURN_NONE; } +#ifdef WITH_THREAD +static void +faulthandler_fatal_error_thread(void *plock) +{ + PyThread_type_lock *lock = (PyThread_type_lock *)plock; + + Py_FatalError("in new thread"); + + /* notify the caller that we are done */ + PyThread_release_lock(lock); +} + +static PyObject * +faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) +{ + long thread; + PyThread_type_lock lock; + + faulthandler_suppress_crash_report(); + + lock = PyThread_allocate_lock(); + if (lock == NULL) + return PyErr_NoMemory(); + + PyThread_acquire_lock(lock, WAIT_LOCK); + + thread = PyThread_start_new_thread(faulthandler_fatal_error_thread, lock); + if (thread == -1) { + PyThread_free_lock(lock); + PyErr_SetString(PyExc_RuntimeError, "unable to start the thread"); + return NULL; + } + + /* wait until the thread completes: it will never occur, since Py_FatalError() + exits the process immedialty. */ + PyThread_acquire_lock(lock, WAIT_LOCK); + PyThread_release_lock(lock); + PyThread_free_lock(lock); + + Py_RETURN_NONE; +} +#endif + static PyObject * faulthandler_sigfpe(PyObject *self, PyObject *args) { @@ -1065,6 +1106,11 @@ static PyMethodDef module_methods[] = { "a SIGSEGV or SIGBUS signal depending on the platform")}, {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, PyDoc_STR("_sigsegv(release_gil=False): raise a SIGSEGV signal")}, +#ifdef WITH_THREAD + {"_fatal_error_c_thread", faulthandler_fatal_error_c_thread, METH_NOARGS, + PyDoc_STR("fatal_error_c_thread(): " + "call Py_FatalError() in a new C thread.")}, +#endif {"_sigabrt", faulthandler_sigabrt, METH_NOARGS, PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4fc6a15..41528cd 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1275,25 +1275,11 @@ initstdio(void) static void _Py_FatalError_DumpTracebacks(int fd) { - PyThreadState *tstate; - -#ifdef WITH_THREAD - /* PyGILState_GetThisThreadState() works even if the GIL was released */ - tstate = PyGILState_GetThisThreadState(); -#else - tstate = PyThreadState_GET(); -#endif - if (tstate == NULL) { - /* _Py_DumpTracebackThreads() requires the thread state to display - * frames */ - return; - } - fputc('\n', stderr); fflush(stderr); /* display the current Python stack */ - _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + _Py_DumpTracebackThreads(fd, NULL, NULL); } /* Print the current exception (if an exception is set) with its traceback, diff --git a/Python/pystate.c b/Python/pystate.c index e8026c5..0503f32 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -714,6 +714,12 @@ _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) _PyGILState_NoteThreadState(t); } +PyInterpreterState * +_PyGILState_GetInterpreterStateUnsafe(void) +{ + return autoInterpreterState; +} + void _PyGILState_Fini(void) { diff --git a/Python/traceback.c b/Python/traceback.c index 8383c16..403dba5 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -707,11 +707,56 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) handlers if signals were received. */ const char* _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, - PyThreadState *current_thread) + PyThreadState *current_tstate) { PyThreadState *tstate; unsigned int nthreads; +#ifdef WITH_THREAD + if (current_tstate == NULL) { + /* _Py_DumpTracebackThreads() is called from signal handlers by + faulthandler. + + SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals + and are thus delivered to the thread that caused the fault. Get the + Python thread state of the current thread. + + PyThreadState_Get() doesn't give the state of the thread that caused + the fault if the thread released the GIL, and so this function + cannot be used. Read the thread local storage (TLS) instead: call + PyGILState_GetThisThreadState(). */ + current_tstate = PyGILState_GetThisThreadState(); + } + + if (interp == NULL) { + if (current_tstate == NULL) { + interp = _PyGILState_GetInterpreterStateUnsafe(); + if (interp == NULL) { + /* We need the interpreter state to get Python threads */ + return "unable to get the interpreter state"; + } + } + else { + interp = current_tstate->interp; + } + } +#else + if (current_tstate == NULL) { + /* Call _PyThreadState_UncheckedGet() instead of PyThreadState_Get() + to not fail with a fatal error if the thread state is NULL. */ + current_thread = _PyThreadState_UncheckedGet(); + } + + if (interp == NULL) { + if (current_tstate == NULL) { + /* We need the interpreter state to get Python threads */ + return "unable to get the interpreter state"; + } + interp = current_tstate->interp; + } +#endif + assert(interp != NULL); + /* Get the current interpreter from the current thread */ tstate = PyInterpreterState_ThreadHead(interp); if (tstate == NULL) @@ -729,7 +774,7 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, PUTS(fd, "...\n"); break; } - write_thread_id(fd, tstate, tstate == current_thread); + write_thread_id(fd, tstate, tstate == current_tstate); dump_traceback(fd, tstate, 0); tstate = PyThreadState_Next(tstate); nthreads++; -- cgit v0.12 From 2025d7839b1d0c5c3cc83601568f16177b1d85b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 23:19:15 +0100 Subject: Py_FatalError: disable faulthandler earlier Issue #26563: Py_FatalError: disable faulthandler before trying to flush sys.stdout and sys.stderr. --- Python/pylifecycle.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 41528cd..502a1e6 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1371,17 +1371,17 @@ Py_FatalError(const char *msg) if (!_Py_FatalError_PrintExc(fd)) _Py_FatalError_DumpTracebacks(fd); + /* The main purpose of faulthandler is to display the traceback. We already + * did our best to display it. So faulthandler can now be disabled. + * (Don't trigger it on abort().) */ + _PyFaulthandler_Fini(); + /* Check if the current Python thread hold the GIL */ if (PyThreadState_GET() != NULL) { /* Flush sys.stdout and sys.stderr */ flush_std_files(); } - /* The main purpose of faulthandler is to display the traceback. We already - * did our best to display it. So faulthandler can now be disabled. - * (Don't trigger it on abort().) */ - _PyFaulthandler_Fini(); - #ifdef MS_WINDOWS len = strlen(msg); -- cgit v0.12 From 13be7db34c698010d2a10e7ebbb1503f3495b20c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 23:25:02 +0100 Subject: Fix usage of PyMem_Malloc() in overlapped.c Issue #26563: Replace PyMem_Malloc() with PyMem_RawFree() since PostToQueueCallback() calls PyMem_RawFree() (previously PyMem_Free()) in a new C thread which doesn't hold the GIL. --- Modules/overlapped.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Modules/overlapped.c b/Modules/overlapped.c index ef77c88..8e6d397 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -238,7 +238,7 @@ PostToQueueCallback(PVOID lpParameter, BOOL TimerOrWaitFired) PostQueuedCompletionStatus(p->CompletionPort, TimerOrWaitFired, 0, p->Overlapped); /* ignore possible error! */ - PyMem_Free(p); + PyMem_RawFree(p); } PyDoc_STRVAR( @@ -262,7 +262,10 @@ overlapped_RegisterWaitWithQueue(PyObject *self, PyObject *args) &Milliseconds)) return NULL; - pdata = PyMem_Malloc(sizeof(struct PostCallbackData)); + /* Use PyMem_RawMalloc() rather than PyMem_Malloc(), since + PostToQueueCallback() will call PyMem_Free() from a new C thread + which doesn't hold the GIL. */ + pdata = PyMem_RawMalloc(sizeof(struct PostCallbackData)); if (pdata == NULL) return SetFromWindowsErr(0); @@ -273,7 +276,7 @@ overlapped_RegisterWaitWithQueue(PyObject *self, PyObject *args) pdata, Milliseconds, WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE)) { - PyMem_Free(pdata); + PyMem_RawFree(pdata); return SetFromWindowsErr(0); } -- cgit v0.12 From c2fc56836f6cb02ce6436d9eb0342dc595738e9d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Mar 2016 11:04:31 +0100 Subject: Enhance documentation on malloc debug hooks Issue #26564, #26516, #26563. --- Doc/c-api/memory.rst | 9 ++++++--- Doc/using/cmdline.rst | 16 ++++++++++------ Doc/whatsnew/3.6.rst | 6 ++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 843ccac..1787292 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -346,8 +346,9 @@ Customize Memory Allocators - Detect write before the start of the buffer (buffer underflow) - Detect write after the end of the buffer (buffer overflow) - Check that the :term:`GIL ` is held when - allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: - :c:func:`PyObject_Malloc`) are called + allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: + :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: + :c:func:`PyMem_Malloc`) domains are called On error, the debug hooks use the :mod:`tracemalloc` module to get the traceback where a memory block was allocated. The traceback is only @@ -361,7 +362,9 @@ Customize Memory Allocators .. versionchanged:: 3.6 This function now also works on Python compiled in release mode. On error, the debug hooks now use :mod:`tracemalloc` to get the traceback - where a memory block was allocated. + where a memory block was allocated. The debug hooks now also check + if the GIL is hold when functions of :c:data:`PYMEM_DOMAIN_OBJ` and + :c:data:`PYMEM_DOMAIN_MEM` domains are called. .. _pymalloc: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 684ccb6..4555982 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -638,16 +638,20 @@ conflict. Install debug hooks: * ``debug``: install debug hooks on top of the default memory allocator - * ``malloc_debug``: same than ``malloc`` but also install debug hooks - * ``pymalloc_debug``: same than ``malloc`` but also install debug hooks + * ``malloc_debug``: same as ``malloc`` but also install debug hooks + * ``pymalloc_debug``: same as ``pyalloc`` but also install debug hooks + + When is compiled in release mode, the default is ``pymalloc``. When Python + is compiled in debug mode, the default is ``pymalloc_debug``: debug hooks + are installed. + + If Python is configured without ``pymalloc`` support, ``pymalloc`` and + ``pymalloc_debug`` are not available, the default is ``malloc`` in release + mode and ``malloc_debug`` in debug mode. See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. - .. note:: - ``pymalloc`` and ``pymalloc_debug`` are not available if Python is - configured without ``pymalloc`` support. - .. versionadded:: 3.6 diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 443e46a..411332f 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -118,8 +118,10 @@ compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: * Detect write before the start of the buffer (buffer underflow) * Detect write after the end of the buffer (buffer overflow) * Check that the :term:`GIL ` is held when allocator - functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: - :c:func:`PyObject_Malloc`) are called + functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and + :c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. + +Checking if the GIL is hold is also a new feature of Python 3.6. See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. -- cgit v0.12 From 9b46a5730268f6f22e13cd0a99adada2932f062b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Mar 2016 15:10:43 +0100 Subject: Doc: fix typos, patch written by Stefan Behnel --- Doc/c-api/memory.rst | 2 +- Doc/using/cmdline.rst | 8 ++++---- Doc/whatsnew/3.6.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 1787292..c2e2442 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -363,7 +363,7 @@ Customize Memory Allocators This function now also works on Python compiled in release mode. On error, the debug hooks now use :mod:`tracemalloc` to get the traceback where a memory block was allocated. The debug hooks now also check - if the GIL is hold when functions of :c:data:`PYMEM_DOMAIN_OBJ` and + if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are called. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4555982..7ff9361 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -639,11 +639,11 @@ conflict. * ``debug``: install debug hooks on top of the default memory allocator * ``malloc_debug``: same as ``malloc`` but also install debug hooks - * ``pymalloc_debug``: same as ``pyalloc`` but also install debug hooks + * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks - When is compiled in release mode, the default is ``pymalloc``. When Python - is compiled in debug mode, the default is ``pymalloc_debug``: debug hooks - are installed. + When Python is compiled in release mode, the default is ``pymalloc``. When + compiled in debug mode, the default is ``pymalloc_debug`` and the debug hooks + are used automatically. If Python is configured without ``pymalloc`` support, ``pymalloc`` and ``pymalloc_debug`` are not available, the default is ``malloc`` in release diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 411332f..cc63589 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -121,7 +121,7 @@ compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. -Checking if the GIL is hold is also a new feature of Python 3.6. +Checking if the GIL is held is also a new feature of Python 3.6. See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. -- cgit v0.12 From 5936313651119b859420930b4126529cd38d277e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 18 Mar 2016 11:54:22 -0700 Subject: Issue #26252: Add an example on how to register a finder --- Doc/library/importlib.rst | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 2bb586c3..1a1348f 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1335,7 +1335,7 @@ import, then you should use :func:`importlib.util.find_spec`. if spec is None: print("can't find the itertools module") else: - # If you chose to perform the actual import. + # If you chose to perform the actual import ... module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # Adding the module to sys.modules is optional. @@ -1359,11 +1359,41 @@ To import a Python source file directly, use the following recipe # by name later. sys.modules[module_name] = module +For deep customizations of import, you typically want to implement an +:term:`importer`. This means managing both the :term:`finder` and :term:`loader` +side of things. For finders there are two flavours to choose from depending on +your needs: a :term:`meta path finder` or a :term:`path entry finder`. The +former is what you would put on :attr:`sys.meta_path` while the latter is what +you create using a :term:`path entry hook` on :attr:`sys.path_hooks` which works +with :attr:`sys.path` entries to potentially create a finder. This example will +show you how to register your own importers so that import will use them (for +creating an importer for yourself, read the documentation for the appropriate +classes defined within this package):: + + import importlib.machinery + import sys + + # For illustrative purposes only. + SpamMetaPathFinder = importlib.machinery.PathFinder + SpamPathEntryFinder = importlib.machinery.FileFinder + loader_details = (importlib.machinery.SourceFileLoader, + importlib.machinery.SOURCE_SUFFIXES) + + # Setting up a meta path finder. + # Make sure to put the finder in the proper location in the list in terms of + # priority. + sys.meta_path.append(SpamMetaPathFinder) + + # Setting up a path entry finder. + # Make sure to put the path hook in the proper location in the list in terms + # of priority. + sys.path_hooks.append(SpamPathEntryFinder.path_hook(loader_details)) + Import itself is implemented in Python code, making it possible to expose most of the import machinery through importlib. The following helps illustrate the various APIs that importlib exposes by providing an approximate implementation of -:func:`importlib.import_module` (Python 3.4 and newer for importlib usage, +:func:`importlib.import_module` (Python 3.4 and newer for the importlib usage, Python 3.6 and newer for other parts of the code). :: -- cgit v0.12 From 1231a4615fd447f0988a72a134a1fc5e7d4e8d69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 00:47:17 +0100 Subject: Add _showwarnmsg() and _formatwarnmsg() to warnings Issue #26568: add new _showwarnmsg() and _formatwarnmsg() functions to the warnings module. The C function warn_explicit() now calls warnings._showwarnmsg() with a warnings.WarningMessage as parameter, instead of calling warnings.showwarning() with multiple parameters. _showwarnmsg() calls warnings.showwarning() if warnings.showwarning() was replaced. Same for _formatwarnmsg(): call warnings.formatwarning() if it was replaced. --- Lib/test/test_warnings/__init__.py | 11 ++++++ Lib/warnings.py | 69 +++++++++++++++++++++++++-------- Python/_warnings.c | 78 ++++++++++++++++++++++++++------------ 3 files changed, 117 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index cea9c57..70eae4c 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -651,6 +651,17 @@ class _WarningsTests(BaseTest, unittest.TestCase): result = stream.getvalue() self.assertIn(text, result) + def test_showwarnmsg_missing(self): + # Test that _showwarnmsg() missing is okay. + text = 'del _showwarnmsg test' + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings("always", category=UserWarning) + del self.module._showwarnmsg + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + self.assertIn(text, result) + def test_showwarning_not_callable(self): with original_warnings.catch_warnings(module=self.module): self.module.filterwarnings("always", category=UserWarning) diff --git a/Lib/warnings.py b/Lib/warnings.py index 1d4fb20..f54726a 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -6,24 +6,63 @@ __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings"] - def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" + msg = WarningMessage(message, category, filename, lineno, file, line) + _showwarnmsg(msg) + +def formatwarning(message, category, filename, lineno, line=None): + """Function to format a warning the standard way.""" + msg = WarningMessage(message, category, filename, lineno, None, line) + return _formatwarnmsg(msg) + +# Keep references to check if the functions were replaced +_showwarning = showwarning +_formatwarning = formatwarning + +def _showwarnmsg(msg): + """Hook to write a warning to a file; replace if you like.""" + showwarning = globals().get('showwarning', _showwarning) + if showwarning is not _showwarning: + # warnings.showwarning() was replaced + if not callable(showwarning): + raise TypeError("warnings.showwarning() must be set to a " + "function or method") + + showwarning(msg.message, msg.category, msg.filename, msg.lineno, + msg.file, msg.line) + return + + file = msg.file if file is None: file = sys.stderr if file is None: - # sys.stderr is None when run with pythonw.exe - warnings get lost + # sys.stderr is None when run with pythonw.exe: + # warnings get lost return + text = _formatwarnmsg(msg) try: - file.write(formatwarning(message, category, filename, lineno, line)) + file.write(text) except OSError: - pass # the file (probably stderr) is invalid - this warning gets lost. + # the file (probably stderr) is invalid - this warning gets lost. + pass -def formatwarning(message, category, filename, lineno, line=None): +def _formatwarnmsg(msg): """Function to format a warning the standard way.""" + formatwarning = globals().get('formatwarning', _formatwarning) + if formatwarning is not _formatwarning: + # warnings.formatwarning() was replaced + return formatwarning(msg.message, msg.category, + msg.filename, msg.lineno, line=msg.line) + import linecache - s = "%s:%s: %s: %s\n" % (filename, lineno, category.__name__, message) - line = linecache.getline(filename, lineno) if line is None else line + s = ("%s:%s: %s: %s\n" + % (msg.filename, msg.lineno, msg.category.__name__, + msg.message)) + if msg.line is None: + line = linecache.getline(msg.filename, msg.lineno) + else: + line = msg.line if line: line = line.strip() s += " %s\n" % line @@ -293,17 +332,13 @@ def warn_explicit(message, category, filename, lineno, raise RuntimeError( "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) - if not callable(showwarning): - raise TypeError("warnings.showwarning() must be set to a " - "function or method") # Print message and context - showwarning(message, category, filename, lineno) + msg = WarningMessage(message, category, filename, lineno) + _showwarnmsg(msg) class WarningMessage(object): - """Holds the result of a single showwarning() call.""" - _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", "line") @@ -366,11 +401,12 @@ class catch_warnings(object): self._module.filters = self._filters[:] self._module._filters_mutated() self._showwarning = self._module.showwarning + self._showwarnmsg = self._module._showwarnmsg if self._record: log = [] - def showwarning(*args, **kwargs): - log.append(WarningMessage(*args, **kwargs)) - self._module.showwarning = showwarning + def showarnmsg(msg): + log.append(msg) + self._module._showwarnmsg = showarnmsg return log else: return None @@ -381,6 +417,7 @@ class catch_warnings(object): self._module.filters = self._filters self._module._filters_mutated() self._module.showwarning = self._showwarning + self._module._showwarnmsg = self._showwarnmsg # filters contains a sequence of filter 5-tuples diff --git a/Python/_warnings.c b/Python/_warnings.c index daa1355..a8c3703 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -359,6 +359,56 @@ error: PyErr_Clear(); } +static int +call_show_warning(PyObject *category, PyObject *text, PyObject *message, + PyObject *filename, int lineno, PyObject *lineno_obj, + PyObject *sourceline) +{ + PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; + + show_fn = get_warnings_attr("_showwarnmsg"); + if (show_fn == NULL) { + if (PyErr_Occurred()) + return -1; + show_warning(filename, lineno, text, category, sourceline); + return 0; + } + + if (!PyCallable_Check(show_fn)) { + PyErr_SetString(PyExc_TypeError, + "warnings._showwarnmsg() must be set to a callable"); + goto error; + } + + warnmsg_cls = get_warnings_attr("WarningMessage"); + if (warnmsg_cls == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "unable to get warnings.WarningMessage"); + goto error; + } + + msg = PyObject_CallFunctionObjArgs(warnmsg_cls, message, category, + filename, lineno_obj, + NULL); + Py_DECREF(warnmsg_cls); + if (msg == NULL) + goto error; + + res = PyObject_CallFunctionObjArgs(show_fn, msg, NULL); + Py_DECREF(show_fn); + Py_DECREF(msg); + + if (res == NULL) + return -1; + + Py_DECREF(res); + return 0; + +error: + Py_XDECREF(show_fn); + return -1; +} + static PyObject * warn_explicit(PyObject *category, PyObject *message, PyObject *filename, int lineno, @@ -470,31 +520,9 @@ warn_explicit(PyObject *category, PyObject *message, if (rc == 1) /* Already warned for this module. */ goto return_none; if (rc == 0) { - PyObject *show_fxn = get_warnings_attr("showwarning"); - if (show_fxn == NULL) { - if (PyErr_Occurred()) - goto cleanup; - show_warning(filename, lineno, text, category, sourceline); - } - else { - PyObject *res; - - if (!PyCallable_Check(show_fxn)) { - PyErr_SetString(PyExc_TypeError, - "warnings.showwarning() must be set to a " - "callable"); - Py_DECREF(show_fxn); - goto cleanup; - } - - res = PyObject_CallFunctionObjArgs(show_fxn, message, category, - filename, lineno_obj, - NULL); - Py_DECREF(show_fxn); - Py_XDECREF(res); - if (res == NULL) - goto cleanup; - } + if (call_show_warning(category, text, message, filename, lineno, + lineno_obj, sourceline) < 0) + goto cleanup; } else /* if (rc == -1) */ goto cleanup; -- cgit v0.12 From 914cde89d4c94b0b9206d0fa22322a1142833a56 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 01:03:51 +0100 Subject: On ResourceWarning, log traceback where the object was allocated Issue #26567: * Add a new function PyErr_ResourceWarning() function to pass the destroyed object * Add a source attribute to warnings.WarningMessage * Add warnings._showwarnmsg() which uses tracemalloc to get the traceback where source object was allocated. --- Doc/c-api/exceptions.rst | 8 ++++ Doc/library/warnings.rst | 8 +++- Doc/whatsnew/3.6.rst | 34 ++++++++++++++ Include/warnings.h | 7 +++ Lib/test/test_warnings/__init__.py | 30 +++++++++++++ Lib/warnings.py | 22 +++++++-- Misc/NEWS | 5 +++ Modules/_io/fileio.c | 3 +- Modules/posixmodule.c | 4 +- Modules/socketmodule.c | 3 +- Python/_warnings.c | 91 ++++++++++++++++++++++++++------------ 11 files changed, 175 insertions(+), 40 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 1e708a8..57f36ac 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -334,6 +334,14 @@ an error value). .. versionadded:: 3.2 +.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnFormat`, but *category* is + :exc:`ResourceWarning` and pass *source* to :func:`warnings.WarningMessage`. + + .. versionadded:: 3.6 + + Querying the error indicator ============================ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 8a538ad..4ce88ab 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -319,7 +319,7 @@ Available Functions of the warning message). -.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None) +.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) This is a low-level interface to the functionality of :func:`warn`, passing in explicitly the message, category, filename and line number, and optionally the @@ -335,6 +335,12 @@ Available Functions source for modules found in zipfiles or other non-filesystem import sources). + *source*, if supplied, is the destroyed object which emitted a + :exc:`ResourceWarning`. + + .. versionchanged:: 3.6 + Add the *source* parameter. + .. function:: showwarning(message, category, filename, lineno, file=None, line=None) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index cc63589..b709917 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -258,6 +258,40 @@ urllib.robotparser (Contributed by Nikolay Bogoychev in :issue:`16099`.) +warnings +-------- + +A new optional *source* parameter has been added to the +:func:`warnings.warn_explicit` function: the destroyed object which emitted a +:exc:`ResourceWarning`. A *source* attribute has also been added to +:class:`warnings.WarningMessage` (contributed by Victor Stinner in +:issue:`26568` and :issue:`26567`). + +When a :exc:`ResourceWarning` warning is logged, the :mod:`tracemalloc` is now +used to try to retrieve the traceback where the detroyed object was allocated. + +Example with the script ``example.py``:: + + def func(): + f = open(__file__) + f = None + + func() + +Output of the command ``python3.6 -Wd -X tracemalloc=5 example.py``:: + + example.py:3: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call first): + File "example.py", lineno 2 + f = open(__file__) + File "example.py", lineno 5 + func() + +The "Object allocated at" traceback is new and only displayed if +:mod:`tracemalloc` is tracing Python memory allocations. + + zipfile ------- diff --git a/Include/warnings.h b/Include/warnings.h index effb9fad..c1c6992 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -17,6 +17,13 @@ PyAPI_FUNC(int) PyErr_WarnFormat( Py_ssize_t stack_level, const char *format, /* ASCII-encoded string */ ...); + +/* Emit a ResourceWarning warning */ +PyAPI_FUNC(int) PyErr_ResourceWarning( + PyObject *source, + Py_ssize_t stack_level, + const char *format, /* ASCII-encoded string */ + ...); #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyErr_WarnExplicitObject( PyObject *category, diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 70eae4c..a1b3dba 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -2,7 +2,10 @@ from contextlib import contextmanager import linecache import os from io import StringIO +import re import sys +import tempfile +import textwrap import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure @@ -763,12 +766,39 @@ class WarningsDisplayTests(BaseTest): file_object, expected_file_line) self.assertEqual(expect, file_object.getvalue()) + class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = c_warnings class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings + def test_tracemalloc(self): + with tempfile.NamedTemporaryFile("w", suffix=".py") as tmpfile: + tmpfile.write(textwrap.dedent(""" + def func(): + f = open(__file__) + # Emit ResourceWarning + f = None + + func() + """)) + tmpfile.flush() + fname = tmpfile.name + res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', fname) + stderr = res.err.decode('ascii', 'replace') + stderr = re.sub('<.*>', '<...>', stderr) + expected = textwrap.dedent(f''' + {fname}:5: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call first): + File "{fname}", lineno 3 + f = open(__file__) + File "{fname}", lineno 7 + func() + ''').strip() + self.assertEqual(stderr, expected) + class CatchWarningTests(BaseTest): diff --git a/Lib/warnings.py b/Lib/warnings.py index f54726a..1566065 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -2,6 +2,7 @@ import sys + __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings"] @@ -66,6 +67,18 @@ def _formatwarnmsg(msg): if line: line = line.strip() s += " %s\n" % line + if msg.source is not None: + import tracemalloc + tb = tracemalloc.get_object_traceback(msg.source) + if tb is not None: + s += 'Object allocated at (most recent call first):\n' + for frame in tb: + s += (' File "%s", lineno %s\n' + % (frame.filename, frame.lineno)) + line = linecache.getline(frame.filename, frame.lineno) + if line: + line = line.strip() + s += ' %s\n' % line return s def filterwarnings(action, message="", category=Warning, module="", lineno=0, @@ -267,7 +280,8 @@ def warn(message, category=None, stacklevel=1): globals) def warn_explicit(message, category, filename, lineno, - module=None, registry=None, module_globals=None): + module=None, registry=None, module_globals=None, + source=None): lineno = int(lineno) if module is None: module = filename or "" @@ -333,17 +347,17 @@ def warn_explicit(message, category, filename, lineno, "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) # Print message and context - msg = WarningMessage(message, category, filename, lineno) + msg = WarningMessage(message, category, filename, lineno, source) _showwarnmsg(msg) class WarningMessage(object): _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line") + "line", "source") def __init__(self, message, category, filename, lineno, file=None, - line=None): + line=None, source=None): local_values = locals() for attr in self._WARNING_DETAILS: setattr(self, attr, local_values[attr]) diff --git a/Misc/NEWS b/Misc/NEWS index cef2b81..832d1ac 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -226,6 +226,11 @@ Core and Builtins Library ------- +- Issue #26567: Add a new function :c:func:`PyErr_ResourceWarning` function to + pass the destroyed object. Add a *source* attribute to + :class:`warnings.WarningMessage`. Add warnings._showwarnmsg() which uses + tracemalloc to get the traceback where source object was allocated. + - Issue #26313: ssl.py _load_windows_store_certs fails if windows cert store is empty. Patch by Baji. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8bf3922..a02a9c1 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -92,8 +92,7 @@ fileio_dealloc_warn(fileio *self, PyObject *source) if (self->fd >= 0 && self->closefd) { PyObject *exc, *val, *tb; PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed file %R", source)) { + if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) self); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 65b20be..3f22d14 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12111,8 +12111,8 @@ ScandirIterator_dealloc(ScandirIterator *iterator) */ ++Py_REFCNT(iterator); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed scandir iterator %R", iterator)) { + if (PyErr_ResourceWarning((PyObject *)iterator, 1, + "unclosed scandir iterator %R", iterator)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) iterator); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 77a6b31..657b04b 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4170,8 +4170,7 @@ sock_dealloc(PySocketSockObject *s) Py_ssize_t old_refcount = Py_REFCNT(s); ++Py_REFCNT(s); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed %R", s)) + if (PyErr_ResourceWarning(s, 1, "unclosed %R", s)) /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) s); diff --git a/Python/_warnings.c b/Python/_warnings.c index a8c3703..25299fb 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -287,8 +287,8 @@ update_registry(PyObject *registry, PyObject *text, PyObject *category, } static void -show_warning(PyObject *filename, int lineno, PyObject *text, PyObject - *category, PyObject *sourceline) +show_warning(PyObject *filename, int lineno, PyObject *text, + PyObject *category, PyObject *sourceline) { PyObject *f_stderr; PyObject *name; @@ -362,7 +362,7 @@ error: static int call_show_warning(PyObject *category, PyObject *text, PyObject *message, PyObject *filename, int lineno, PyObject *lineno_obj, - PyObject *sourceline) + PyObject *sourceline, PyObject *source) { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; @@ -388,7 +388,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, } msg = PyObject_CallFunctionObjArgs(warnmsg_cls, message, category, - filename, lineno_obj, + filename, lineno_obj, Py_None, Py_None, source, NULL); Py_DECREF(warnmsg_cls); if (msg == NULL) @@ -412,7 +412,8 @@ error: static PyObject * warn_explicit(PyObject *category, PyObject *message, PyObject *filename, int lineno, - PyObject *module, PyObject *registry, PyObject *sourceline) + PyObject *module, PyObject *registry, PyObject *sourceline, + PyObject *source) { PyObject *key = NULL, *text = NULL, *result = NULL, *lineno_obj = NULL; PyObject *item = NULL; @@ -521,7 +522,7 @@ warn_explicit(PyObject *category, PyObject *message, goto return_none; if (rc == 0) { if (call_show_warning(category, text, message, filename, lineno, - lineno_obj, sourceline) < 0) + lineno_obj, sourceline, source) < 0) goto cleanup; } else /* if (rc == -1) */ @@ -766,7 +767,8 @@ get_category(PyObject *message, PyObject *category) } static PyObject * -do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level) +do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, + PyObject *source) { PyObject *filename, *module, *registry, *res; int lineno; @@ -775,7 +777,7 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level) return NULL; res = warn_explicit(category, message, filename, lineno, module, registry, - NULL); + NULL, source); Py_DECREF(filename); Py_DECREF(registry); Py_DECREF(module); @@ -796,14 +798,15 @@ warnings_warn(PyObject *self, PyObject *args, PyObject *kwds) category = get_category(message, category); if (category == NULL) return NULL; - return do_warn(message, category, stack_level); + return do_warn(message, category, stack_level, NULL); } static PyObject * warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwd_list[] = {"message", "category", "filename", "lineno", - "module", "registry", "module_globals", 0}; + "module", "registry", "module_globals", + "source", 0}; PyObject *message; PyObject *category; PyObject *filename; @@ -811,10 +814,11 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) PyObject *module = NULL; PyObject *registry = NULL; PyObject *module_globals = NULL; + PyObject *sourceobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOO:warn_explicit", + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOOO:warn_explicit", kwd_list, &message, &category, &filename, &lineno, &module, - ®istry, &module_globals)) + ®istry, &module_globals, &sourceobj)) return NULL; if (module_globals) { @@ -870,14 +874,14 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) /* Handle the warning. */ returned = warn_explicit(category, message, filename, lineno, module, - registry, source_line); + registry, source_line, sourceobj); Py_DECREF(source_list); return returned; } standard_call: return warn_explicit(category, message, filename, lineno, module, - registry, NULL); + registry, NULL, sourceobj); } static PyObject * @@ -892,14 +896,14 @@ warnings_filters_mutated(PyObject *self, PyObject *args) static int warn_unicode(PyObject *category, PyObject *message, - Py_ssize_t stack_level) + Py_ssize_t stack_level, PyObject *source) { PyObject *res; if (category == NULL) category = PyExc_RuntimeWarning; - res = do_warn(message, category, stack_level); + res = do_warn(message, category, stack_level, source); if (res == NULL) return -1; Py_DECREF(res); @@ -907,12 +911,28 @@ warn_unicode(PyObject *category, PyObject *message, return 0; } +static int +_PyErr_WarnFormatV(PyObject *source, + PyObject *category, Py_ssize_t stack_level, + const char *format, va_list vargs) +{ + PyObject *message; + int res; + + message = PyUnicode_FromFormatV(format, vargs); + if (message == NULL) + return -1; + + res = warn_unicode(category, message, stack_level, source); + Py_DECREF(message); + return res; +} + int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) { - int ret; - PyObject *message; + int res; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES @@ -920,25 +940,38 @@ PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, #else va_start(vargs); #endif - message = PyUnicode_FromFormatV(format, vargs); - if (message != NULL) { - ret = warn_unicode(category, message, stack_level); - Py_DECREF(message); - } - else - ret = -1; + res = _PyErr_WarnFormatV(NULL, category, stack_level, format, vargs); va_end(vargs); - return ret; + return res; } int +PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, + const char *format, ...) +{ + int res; + va_list vargs; + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + res = _PyErr_WarnFormatV(source, PyExc_ResourceWarning, + stack_level, format, vargs); + va_end(vargs); + return res; +} + + +int PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) { int ret; PyObject *message = PyUnicode_FromString(text); if (message == NULL) return -1; - ret = warn_unicode(category, message, stack_level); + ret = warn_unicode(category, message, stack_level, NULL); Py_DECREF(message); return ret; } @@ -964,7 +997,7 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, if (category == NULL) category = PyExc_RuntimeWarning; res = warn_explicit(category, message, filename, lineno, - module, registry, NULL); + module, registry, NULL, NULL); if (res == NULL) return -1; Py_DECREF(res); @@ -1028,7 +1061,7 @@ PyErr_WarnExplicitFormat(PyObject *category, if (message != NULL) { PyObject *res; res = warn_explicit(category, message, filename, lineno, - module, registry, NULL); + module, registry, NULL, NULL); Py_DECREF(message); if (res != NULL) { Py_DECREF(res); -- cgit v0.12 From eedf13fe231b49b10c9a836ae8c9120d215deb84 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 02:11:56 +0100 Subject: Fix test_logging Issue #26568: Fix implementation of showwarning() and formatwarning() for test_logging. --- Lib/warnings.py | 60 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/Lib/warnings.py b/Lib/warnings.py index 1566065..9fb21a8 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -10,30 +10,14 @@ __all__ = ["warn", "warn_explicit", "showwarning", def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" msg = WarningMessage(message, category, filename, lineno, file, line) - _showwarnmsg(msg) + _showwarnmsg_impl(msg) def formatwarning(message, category, filename, lineno, line=None): """Function to format a warning the standard way.""" msg = WarningMessage(message, category, filename, lineno, None, line) - return _formatwarnmsg(msg) - -# Keep references to check if the functions were replaced -_showwarning = showwarning -_formatwarning = formatwarning - -def _showwarnmsg(msg): - """Hook to write a warning to a file; replace if you like.""" - showwarning = globals().get('showwarning', _showwarning) - if showwarning is not _showwarning: - # warnings.showwarning() was replaced - if not callable(showwarning): - raise TypeError("warnings.showwarning() must be set to a " - "function or method") - - showwarning(msg.message, msg.category, msg.filename, msg.lineno, - msg.file, msg.line) - return + return _formatwarnmsg_impl(msg) +def _showwarnmsg_impl(msg): file = msg.file if file is None: file = sys.stderr @@ -48,14 +32,7 @@ def _showwarnmsg(msg): # the file (probably stderr) is invalid - this warning gets lost. pass -def _formatwarnmsg(msg): - """Function to format a warning the standard way.""" - formatwarning = globals().get('formatwarning', _formatwarning) - if formatwarning is not _formatwarning: - # warnings.formatwarning() was replaced - return formatwarning(msg.message, msg.category, - msg.filename, msg.lineno, line=msg.line) - +def _formatwarnmsg_impl(msg): import linecache s = ("%s:%s: %s: %s\n" % (msg.filename, msg.lineno, msg.category.__name__, @@ -81,6 +58,35 @@ def _formatwarnmsg(msg): s += ' %s\n' % line return s +# Keep a reference to check if the function was replaced +_showwarning = showwarning + +def _showwarnmsg(msg): + """Hook to write a warning to a file; replace if you like.""" + showwarning = globals().get('showwarning', _showwarning) + if showwarning is not _showwarning: + # warnings.showwarning() was replaced + if not callable(showwarning): + raise TypeError("warnings.showwarning() must be set to a " + "function or method") + + showwarning(msg.message, msg.category, msg.filename, msg.lineno, + msg.file, msg.line) + return + _showwarnmsg_impl(msg) + +# Keep a reference to check if the function was replaced +_formatwarning = formatwarning + +def _formatwarnmsg(msg): + """Function to format a warning the standard way.""" + formatwarning = globals().get('formatwarning', _formatwarning) + if formatwarning is not _formatwarning: + # warnings.formatwarning() was replaced + return formatwarning(msg.message, msg.category, + msg.filename, msg.lineno, line=msg.line) + return _formatwarnmsg_impl(msg) + def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=False): """Insert an entry into the list of warnings filters (at the front). -- cgit v0.12 From f664dc58344fc615f8f5649752bf5cb54efe5b3b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 02:01:48 +0100 Subject: ResourceWarning: Revert change on socket and scandir io.FileIO has a safe implementation of destructor, but not socket nor scandir. --- Modules/posixmodule.c | 4 ++-- Modules/socketmodule.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 3f22d14..65b20be 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12111,8 +12111,8 @@ ScandirIterator_dealloc(ScandirIterator *iterator) */ ++Py_REFCNT(iterator); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_ResourceWarning((PyObject *)iterator, 1, - "unclosed scandir iterator %R", iterator)) { + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed scandir iterator %R", iterator)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) iterator); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 657b04b..77a6b31 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4170,7 +4170,8 @@ sock_dealloc(PySocketSockObject *s) Py_ssize_t old_refcount = Py_REFCNT(s); ++Py_REFCNT(s); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_ResourceWarning(s, 1, "unclosed %R", s)) + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed %R", s)) /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) s); -- cgit v0.12 From bfab932971593b6468024e5774653bb2f5047b56 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 02:51:45 +0100 Subject: Try to fix test_warnings on Windows Issue #26567. --- Lib/test/test_warnings/__init__.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index a1b3dba..be80b05 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -4,7 +4,6 @@ import os from io import StringIO import re import sys -import tempfile import textwrap import unittest from test import support @@ -774,8 +773,10 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings def test_tracemalloc(self): - with tempfile.NamedTemporaryFile("w", suffix=".py") as tmpfile: - tmpfile.write(textwrap.dedent(""" + self.addCleanup(support.unlink, support.TESTFN) + + with open(support.TESTFN, 'w') as fp: + fp.write(textwrap.dedent(""" def func(): f = open(__file__) # Emit ResourceWarning @@ -783,12 +784,12 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): func() """)) - tmpfile.flush() - fname = tmpfile.name - res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', fname) + + res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', support.TESTFN) + stderr = res.err.decode('ascii', 'replace') stderr = re.sub('<.*>', '<...>', stderr) - expected = textwrap.dedent(f''' + expected = textwrap.dedent(''' {fname}:5: ResourceWarning: unclosed file <...> f = None Object allocated at (most recent call first): @@ -796,7 +797,8 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): f = open(__file__) File "{fname}", lineno 7 func() - ''').strip() + ''') + expected = expected.format(fname=support.TESTFN).strip() self.assertEqual(stderr, expected) -- cgit v0.12 From 74879e41791c9c033597250476d556815921a8e3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:00:08 +0100 Subject: Try again to fix test_warnings on Windows Issue #26567: normalize newlines in test_tracemalloc. --- Lib/test/test_warnings/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index be80b05..e6f47cd 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -788,6 +788,8 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', support.TESTFN) stderr = res.err.decode('ascii', 'replace') + # normalize newlines + stderr = '\n'.join(stderr.splitlines()) stderr = re.sub('<.*>', '<...>', stderr) expected = textwrap.dedent(''' {fname}:5: ResourceWarning: unclosed file <...> -- cgit v0.12 From ee803a8d2ccf50e60d18ca1bb46f7a1cf3db66f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:33:25 +0100 Subject: Issue #26567: enhance ResourceWarning example --- Doc/whatsnew/3.6.rst | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b709917..1129cb3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -272,24 +272,27 @@ used to try to retrieve the traceback where the detroyed object was allocated. Example with the script ``example.py``:: + import warnings + def func(): - f = open(__file__) - f = None + return open(__file__) - func() + f = func() + f = None Output of the command ``python3.6 -Wd -X tracemalloc=5 example.py``:: - example.py:3: ResourceWarning: unclosed file <...> + example.py:7: ResourceWarning: unclosed file <_io.TextIOWrapper name='example.py' mode='r' encoding='UTF-8'> f = None Object allocated at (most recent call first): - File "example.py", lineno 2 - f = open(__file__) - File "example.py", lineno 5 - func() + File "example.py", lineno 4 + return open(__file__) + File "example.py", lineno 6 + f = func() The "Object allocated at" traceback is new and only displayed if -:mod:`tracemalloc` is tracing Python memory allocations. +:mod:`tracemalloc` is tracing Python memory allocations and if the +:mod:`warnings` was already imported. zipfile -- cgit v0.12 From af4a1f20bafe5bb38215d8b2cb1470bdf8fe4625 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:33:50 +0100 Subject: fix indentation in Py_DECREF() --- Include/object.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/object.h b/Include/object.h index 0e67a4e..a032005 100644 --- a/Include/object.h +++ b/Include/object.h @@ -785,7 +785,7 @@ PyAPI_FUNC(void) _Py_Dealloc(PyObject *); --(_py_decref_tmp)->ob_refcnt != 0) \ _Py_CHECK_REFCNT(_py_decref_tmp) \ else \ - _Py_Dealloc(_py_decref_tmp); \ + _Py_Dealloc(_py_decref_tmp); \ } while (0) /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear -- cgit v0.12 From 3c3d7f4b99b1f62567592ef57216e07677f17d7f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 19 Mar 2016 11:44:17 +0200 Subject: Issue #18787: spwd.getspnam() now raises a PermissionError if the user doesn't have privileges. --- Doc/library/spwd.rst | 3 +++ Doc/whatsnew/3.6.rst | 2 ++ Lib/test/test_spwd.py | 10 ++++++++++ Misc/NEWS | 3 +++ Modules/spwdmodule.c | 5 ++++- 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst index 58be78f..53f8c09 100644 --- a/Doc/library/spwd.rst +++ b/Doc/library/spwd.rst @@ -54,6 +54,9 @@ The following functions are defined: Return the shadow password database entry for the given user name. + .. versionchanged:: 3.6 + Raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user + doesn't have privileges. .. function:: getspall() diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 1129cb3..9046058 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -471,6 +471,8 @@ Changes in the Python API the exception will stop a single-threaded server. (Contributed by Martin Panter in :issue:`23430`.) +* :func:`spwd.getspnam` now raises a :exc:`PermissionError` instead of + :exc:`KeyError` if the user doesn't have privileges. Changes in the C API -------------------- diff --git a/Lib/test/test_spwd.py b/Lib/test/test_spwd.py index bea7ab1..fca809e 100644 --- a/Lib/test/test_spwd.py +++ b/Lib/test/test_spwd.py @@ -56,5 +56,15 @@ class TestSpwdRoot(unittest.TestCase): self.assertRaises(TypeError, spwd.getspnam, bytes_name) +@unittest.skipUnless(hasattr(os, 'geteuid') and os.geteuid() != 0, + 'non-root user required') +class TestSpwdNonRoot(unittest.TestCase): + + def test_getspnam_exception(self): + with self.assertRaises(PermissionError) as cm: + spwd.getspnam('bin') + self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index fe3071e..ae0a100 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -226,6 +226,9 @@ Core and Builtins Library ------- +- Issue #18787: spwd.getspnam() now raises a PermissionError if the user + doesn't have privileges. + - Issue #26560: Avoid potential ValueError in BaseHandler.start_response. Initial patch by Peter Inglesby. diff --git a/Modules/spwdmodule.c b/Modules/spwdmodule.c index 49324d5..e715d01 100644 --- a/Modules/spwdmodule.c +++ b/Modules/spwdmodule.c @@ -137,7 +137,10 @@ spwd_getspnam_impl(PyModuleDef *module, PyObject *arg) if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1) goto out; if ((p = getspnam(name)) == NULL) { - PyErr_SetString(PyExc_KeyError, "getspnam(): name not found"); + if (errno != 0) + PyErr_SetFromErrno(PyExc_OSError); + else + PyErr_SetString(PyExc_KeyError, "getspnam(): name not found"); goto out; } retval = mkspent(p); -- cgit v0.12 From 51b846c47a9b1db927939ccfb037a5a0ff6ff99c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Mar 2016 21:52:22 +0100 Subject: _tracemalloc: add domain to trace keys * hashtable.h: key has now a variable size * _tracemalloc uses (pointer: void*, domain: unsigned int) as key for traces --- Modules/_tracemalloc.c | 186 ++++++++++++++++++++++++++++++++++++------------- Modules/hashtable.c | 105 ++++++++++++++++------------ Modules/hashtable.h | 89 ++++++++++++++++------- Python/marshal.c | 11 +-- 4 files changed, 270 insertions(+), 121 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5752904..73aa53b 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -3,6 +3,7 @@ #include "frameobject.h" #include "pythread.h" #include "osdefs.h" +#include /* For offsetof */ /* Trace memory blocks allocated by PyMem_RawMalloc() */ #define TRACE_RAW_MALLOC @@ -54,6 +55,26 @@ static PyThread_type_lock tables_lock; # define TABLES_UNLOCK() #endif +typedef unsigned int domain_t; + +/* FIXME: pack also? */ +typedef struct { + void *ptr; + domain_t domain; +} pointer_t; + +/* Size of pointer_t content, it can be smaller than sizeof(pointer_t) */ +#define POINTER_T_SIZE \ + (offsetof(pointer_t, domain) + sizeof(domain_t)) + +#define POINTER_T_FILL_PADDING(key) \ + do { \ + if (POINTER_T_SIZE != sizeof(pointer_t)) { \ + memset((char *)&(key) + POINTER_T_SIZE, 0, \ + sizeof(pointer_t) - POINTER_T_SIZE); \ + } \ + } while (0) + /* Pack the frame_t structure to reduce the memory footprint on 64-bit architectures: 12 bytes instead of 16. This optimization might produce SIGBUS on architectures not supporting unaligned memory accesses (64-bit @@ -196,23 +217,56 @@ set_reentrant(int reentrant) } #endif +static Py_uhash_t +hashtable_hash_pointer(size_t key_size, const void *pkey) +{ + pointer_t ptr; + Py_uhash_t hash; + + assert(sizeof(ptr) == key_size); + ptr = *(pointer_t *)pkey; + + hash = (Py_uhash_t)_Py_HashPointer(ptr.ptr); + hash ^= ptr.domain; + return hash; +} + +static Py_uhash_t +hashtable_hash_pyobject(size_t key_size, const void *pkey) +{ + PyObject *obj; + + assert(key_size == sizeof(PyObject *)); + obj = *(PyObject **)pkey; + + return PyObject_Hash(obj); +} + static int -hashtable_compare_unicode(const void *key, const _Py_hashtable_entry_t *entry) +hashtable_compare_unicode(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - if (key != NULL && entry->key != NULL) - return (PyUnicode_Compare((PyObject *)key, (PyObject *)entry->key) == 0); + PyObject *key, *entry_key; + + assert(sizeof(key) == key_size); + key = *(PyObject **)pkey; + assert(sizeof(entry_key) == key_size); + entry_key = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); + + if (key != NULL && entry_key != NULL) + return (PyUnicode_Compare(key, entry_key) == 0); else - return key == entry->key; + return key == entry_key; } static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; static _Py_hashtable_t * -hashtable_new(size_t data_size, +hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, 0, + return _Py_hashtable_new_full(key_size, data_size, 0, hash_func, compare_func, NULL, NULL, NULL, &hashtable_alloc); } @@ -230,20 +284,26 @@ raw_free(void *ptr) } static Py_uhash_t -hashtable_hash_traceback(const void *key) +hashtable_hash_traceback(size_t key_size, const void *pkey) { - const traceback_t *traceback = key; + const traceback_t *traceback = *(const traceback_t **)pkey; + assert(key_size == sizeof(const traceback_t *)); return traceback->hash; } static int -hashtable_compare_traceback(const traceback_t *traceback1, +hashtable_compare_traceback(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he) { - const traceback_t *traceback2 = he->key; + traceback_t *traceback1, *traceback2; const frame_t *frame1, *frame2; int i; + assert(sizeof(traceback1) == key_size); + assert(sizeof(traceback2) == key_size); + traceback1 = *(traceback_t **)pkey; + traceback2 = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(he); + if (traceback1->nframe != traceback2->nframe) return 0; @@ -312,15 +372,16 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) } /* intern the filename */ - entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); if (entry != NULL) { - filename = (PyObject *)entry->key; + assert(sizeof(filename) == tracemalloc_filenames->key_size); + filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); } else { /* tracemalloc_filenames is responsible to keep a reference to the filename */ Py_INCREF(filename); - if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_filenames, filename) < 0) { Py_DECREF(filename); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the filename"); @@ -403,9 +464,10 @@ traceback_new(void) traceback->hash = traceback_hash(traceback); /* intern the traceback */ - entry = _Py_hashtable_get_entry(tracemalloc_tracebacks, traceback); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); if (entry != NULL) { - traceback = (traceback_t *)entry->key; + assert(sizeof(traceback) == tracemalloc_tracebacks->key_size); + traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); } else { traceback_t *copy; @@ -422,7 +484,7 @@ traceback_new(void) } memcpy(copy, traceback, traceback_size); - if (_Py_hashtable_set(tracemalloc_tracebacks, copy, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_tracebacks, copy) < 0) { raw_free(copy); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the traceback: putdata failed"); @@ -435,8 +497,9 @@ traceback_new(void) } static int -tracemalloc_add_trace(void *ptr, size_t size) +tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) { + pointer_t key; traceback_t *traceback; trace_t trace; int res; @@ -445,10 +508,14 @@ tracemalloc_add_trace(void *ptr, size_t size) if (traceback == NULL) return -1; + key.ptr = ptr; + key.domain = 0; + POINTER_T_FILL_PADDING(key); + trace.size = size; trace.traceback = traceback; - res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); if (res == 0) { assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); tracemalloc_traced_memory += size; @@ -460,11 +527,16 @@ tracemalloc_add_trace(void *ptr, size_t size) } static void -tracemalloc_remove_trace(void *ptr) +tracemalloc_remove_trace(void *ptr, domain_t domain) { + pointer_t key; trace_t trace; - if (_Py_hashtable_pop(tracemalloc_traces, ptr, &trace, sizeof(trace))) { + key.ptr = ptr; + key.domain = domain; + POINTER_T_FILL_PADDING(key); + + if (_Py_HASHTABLE_POP(tracemalloc_traces, key, trace)) { assert(tracemalloc_traced_memory >= trace.size); tracemalloc_traced_memory -= trace.size; } @@ -486,7 +558,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) { + if (tracemalloc_add_trace(ptr, 0, nelem * elsize) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr); @@ -510,9 +582,9 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to the caller, because realloc() may already have shrinked the memory block and so removed bytes. @@ -530,7 +602,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* new allocation */ TABLES_LOCK(); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); @@ -555,7 +627,7 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); TABLES_UNLOCK(); } @@ -610,7 +682,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); TABLES_UNLOCK(); } return ptr2; @@ -689,7 +761,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); TABLES_UNLOCK(); } return ptr2; @@ -714,17 +786,27 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) #endif /* TRACE_RAW_MALLOC */ static int -tracemalloc_clear_filename(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *filename = (PyObject *)entry->key; + PyObject *filename; + + assert(sizeof(filename) == ht->key_size); + filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); + Py_DECREF(filename); return 0; } static int -traceback_free_traceback(_Py_hashtable_entry_t *entry, void *user_data) +traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - traceback_t *traceback = (traceback_t *)entry->key; + traceback_t *traceback; + + assert(sizeof(traceback) == ht->key_size); + traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); + raw_free(traceback); return 0; } @@ -791,16 +873,16 @@ tracemalloc_init(void) } #endif - tracemalloc_filenames = hashtable_new(0, - (_Py_hashtable_hash_func)PyObject_Hash, + tracemalloc_filenames = hashtable_new(sizeof(PyObject *), 0, + hashtable_hash_pyobject, hashtable_compare_unicode); - tracemalloc_tracebacks = hashtable_new(0, - (_Py_hashtable_hash_func)hashtable_hash_traceback, - (_Py_hashtable_compare_func)hashtable_compare_traceback); + tracemalloc_tracebacks = hashtable_new(sizeof(traceback_t *), 0, + hashtable_hash_traceback, + hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(trace_t), - _Py_hashtable_hash_ptr, + tracemalloc_traces = hashtable_new(sizeof(pointer_t), sizeof(trace_t), + hashtable_hash_pointer, _Py_hashtable_compare_direct); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL @@ -1065,14 +1147,15 @@ typedef struct { } get_traces_t; static int -tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entry, + void *user_data) { get_traces_t *get_traces = user_data; trace_t *trace; PyObject *tracemalloc_obj; int res; - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(entry); + trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) @@ -1087,9 +1170,11 @@ tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) } static int -tracemalloc_pyobject_decref_cb(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, + _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); Py_DECREF(obj); return 0; } @@ -1120,7 +1205,7 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1152,7 +1237,7 @@ error: finally: if (get_traces.tracebacks != NULL) { _Py_hashtable_foreach(get_traces.tracebacks, - tracemalloc_pyobject_decref_cb, NULL); + tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } if (get_traces.traces != NULL) @@ -1162,16 +1247,21 @@ finally: } static traceback_t* -tracemalloc_get_traceback(const void *ptr) +tracemalloc_get_traceback(const void *ptr, domain_t domain) { + pointer_t key; trace_t trace; int found; if (!tracemalloc_config.tracing) return NULL; + key.ptr = (void *)ptr; + key.domain = domain; + POINTER_T_FILL_PADDING(key); + TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); TABLES_UNLOCK(); if (!found) @@ -1202,7 +1292,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(ptr, 0); if (traceback == NULL) Py_RETURN_NONE; @@ -1229,7 +1319,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(ptr, 0); if (traceback == NULL) return; diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 7de154b..002c0a9 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -59,7 +59,7 @@ #define ENTRY_NEXT(ENTRY) \ ((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY)) #define HASHTABLE_ITEM_SIZE(HT) \ - (sizeof(_Py_hashtable_entry_t) + (HT)->data_size) + (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -88,21 +88,21 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, } Py_uhash_t -_Py_hashtable_hash_int(const void *key) +_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) { - return (Py_uhash_t)key; -} + void *key; + + assert(key_size == sizeof(void *)); + key = *(void**)pkey; -Py_uhash_t -_Py_hashtable_hash_ptr(const void *key) -{ return (Py_uhash_t)_Py_HashPointer((void *)key); } int -_Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry) +_Py_hashtable_compare_direct(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - return entry->key == key; + return (memcmp(pkey, _Py_HASHTABLE_ENTRY_KEY(entry), key_size) == 0); } /* makes sure the real size of the buckets array is a power of 2 */ @@ -119,7 +119,8 @@ round_size(size_t s) } _Py_hashtable_t * -_Py_hashtable_new_full(size_t data_size, size_t init_size, +_Py_hashtable_new_full(size_t key_size, size_t data_size, + size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, _Py_hashtable_copy_data_func copy_data_func, @@ -144,6 +145,7 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, ht->num_buckets = round_size(init_size); ht->entries = 0; + ht->key_size = key_size; ht->data_size = data_size; buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); @@ -164,11 +166,12 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, } _Py_hashtable_t * -_Py_hashtable_new(size_t data_size, +_Py_hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE, + return _Py_hashtable_new_full(key_size, data_size, + HASHTABLE_MIN_SIZE, hash_func, compare_func, NULL, NULL, NULL, NULL); } @@ -195,7 +198,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { void *data; - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); size += ht->get_data_size_func(data); } } @@ -245,17 +248,21 @@ _Py_hashtable_print_stats(_Py_hashtable_t *ht) /* Get an entry. Return NULL if the key does not exist. */ _Py_hashtable_entry_t * -_Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_get_entry(_Py_hashtable_t *ht, + size_t key_size, const void *pkey) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; - key_hash = ht->hash_func(key); + assert(key_size == ht->key_size); + + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; } @@ -263,18 +270,20 @@ _Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) } static int -_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry, *previous; - key_hash = ht->hash_func(key); + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; previous = entry; } @@ -298,8 +307,8 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t da /* Add a new entry to the hash. The key must not be present in the hash table. Return 0 on success, -1 on memory error. */ int -_Py_hashtable_set(_Py_hashtable_t *ht, const void *key, - void *data, size_t data_size) +_Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { Py_uhash_t key_hash; size_t index; @@ -310,11 +319,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, /* Don't write the assertion on a single line because it is interesting to know the duplicated entry if the assertion failed. The entry can be read using a debugger. */ - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); assert(entry == NULL); #endif - key_hash = ht->hash_func(key); + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -323,11 +332,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, return -1; } - entry->key = (void *)key; entry->key_hash = key_hash; + memcpy(_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(entry), data, data_size); + memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -340,13 +349,14 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, /* Get data from an entry. Copy entry data into data and return 1 if the entry exists, return 0 if the entry does not exist. */ int -_Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, + size_t data_size, void *data) { _Py_hashtable_entry_t *entry; assert(data != NULL); - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); @@ -354,22 +364,23 @@ _Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_ } int -_Py_hashtable_pop(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { assert(data != NULL); assert(ht->free_data_func == NULL); - return _hashtable_pop_entry(ht, key, data, data_size); + return _hashtable_pop_entry(ht, key_size, pkey, data, data_size); } /* Delete an entry. The entry must exist. */ void -_Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) { #ifndef NDEBUG - int found = _hashtable_pop_entry(ht, key, NULL, 0); + int found = _hashtable_pop_entry(ht, key_size, pkey, NULL, 0); assert(found); #else - (void)_hashtable_pop_entry(ht, key, NULL, 0); + (void)_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); #endif } @@ -378,7 +389,7 @@ _Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) stops if a non-zero value is returned. */ int _Py_hashtable_foreach(_Py_hashtable_t *ht, - int (*func) (_Py_hashtable_entry_t *entry, void *arg), + _Py_hashtable_foreach_func func, void *arg) { _Py_hashtable_entry_t *entry; @@ -386,7 +397,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, for (hv = 0; hv < ht->num_buckets; hv++) { for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - int res = func(entry, arg); + int res = func(ht, entry, arg); if (res) return res; } @@ -397,6 +408,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, static void hashtable_rehash(_Py_hashtable_t *ht) { + const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -425,7 +437,8 @@ hashtable_rehash(_Py_hashtable_t *ht) for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { size_t entry_index; - assert(ht->hash_func(entry->key) == entry->key_hash); + + assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -446,7 +459,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -465,7 +478,7 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) while (entry) { _Py_slist_item_t *entry_next = entry->next; if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); entry = entry_next; } @@ -479,13 +492,16 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) _Py_hashtable_t * _Py_hashtable_copy(_Py_hashtable_t *src) { + const size_t key_size = src->key_size; + const size_t data_size = src->data_size; _Py_hashtable_t *dst; _Py_hashtable_entry_t *entry; size_t bucket; int err; void *data, *new_data; - dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, + dst = _Py_hashtable_new_full(key_size, data_size, + src->num_buckets, src->hash_func, src->compare_func, src->copy_data_func, src->free_data_func, src->get_data_size_func, &src->alloc); @@ -496,17 +512,20 @@ _Py_hashtable_copy(_Py_hashtable_t *src) entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); new_data = src->copy_data_func(data); if (new_data != NULL) - err = _Py_hashtable_set(dst, entry->key, - &new_data, src->data_size); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, &new_data); else err = 1; } else { - data = _Py_HASHTABLE_ENTRY_DATA(entry); - err = _Py_hashtable_set(dst, entry->key, data, src->data_size); + data = _Py_HASHTABLE_ENTRY_DATA(src, entry); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, data); } if (err) { _Py_hashtable_destroy(dst); diff --git a/Modules/hashtable.h b/Modules/hashtable.h index a9f9993..aed3ed0 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -20,26 +20,31 @@ typedef struct { /* used by _Py_hashtable_t.buckets to link entries */ _Py_slist_item_t _Py_slist_item; - const void *key; Py_uhash_t key_hash; /* data follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_DATA(ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t)) +#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ + ((void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(ENTRY)) +#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ + ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) + +#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ + (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) #define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \ + memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ } while (0) -typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key); -typedef int (*_Py_hashtable_compare_func) (const void *key, const _Py_hashtable_entry_t *he); +typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, + const void *pkey); +typedef int (*_Py_hashtable_compare_func) (size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *he); typedef void* (*_Py_hashtable_copy_data_func)(void *data); typedef void (*_Py_hashtable_free_data_func)(void *data); typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); @@ -56,6 +61,7 @@ typedef struct { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; + size_t key_size; size_t data_size; _Py_hashtable_hash_func hash_func; @@ -67,15 +73,21 @@ typedef struct { } _Py_hashtable_t; /* hash and compare functions for integers and pointers */ -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key); -PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry); +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( + size_t key_size, + const void *pkey); +PyAPI_FUNC(int) _Py_hashtable_compare_direct( + size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *entry); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( + size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( + size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, @@ -88,40 +100,65 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src); PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht); PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); -typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_entry_t *entry, void *arg); +typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_t *ht, + _Py_hashtable_entry_t *entry, + void *arg); PyAPI_FUNC(int) _Py_hashtable_foreach( _Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, void *arg); + _Py_hashtable_foreach_func func, + void *arg); PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht); PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( _Py_hashtable_t *ht, - const void *key); + size_t key_size, + const void *pkey); + +/* Don't call directly this function, + but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */ PyAPI_FUNC(int) _Py_hashtable_set( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +/* Don't call directly this function, but use _Py_HASHTABLE_GET() macro */ PyAPI_FUNC(int) _Py_hashtable_get( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +/* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */ PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + PyAPI_FUNC(void) _Py_hashtable_delete( _Py_hashtable_t *ht, - const void *key); + size_t key_size, + const void *pkey); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA)) + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) + +#define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \ + _Py_hashtable_get_entry(TABLE, sizeof(KEY), &(KEY)) #define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ - _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA)) + _Py_hashtable_get(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \ + _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) #endif /* Py_LIMITED_API */ diff --git a/Python/marshal.c b/Python/marshal.c index 7a4b9d2..64084f4 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -263,7 +263,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (Py_REFCNT(v) == 1) return 0; - entry = _Py_hashtable_get_entry(p->hashtable, v); + entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry); @@ -571,7 +571,8 @@ static int w_init_refs(WFILE *wf, int version) { if (version >= 3) { - wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr, + wf->hashtable = _Py_hashtable_new(sizeof(void *), sizeof(int), + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (wf->hashtable == NULL) { PyErr_NoMemory(); @@ -582,9 +583,11 @@ w_init_refs(WFILE *wf, int version) } static int -w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) +w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) { - Py_XDECREF(entry->key); + void *entry_key = *(void **)_Py_HASHTABLE_ENTRY_KEY(entry); + assert(ht->key_size == sizeof(entry_key)); + Py_XDECREF(entry_key); return 0; } -- cgit v0.12 From fac395681fb758401d17974f258b17d285336c57 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 10:38:58 +0100 Subject: Optimize bytes.replace(b'', b'.') Issue #26574: Optimize bytes.replace(b'', b'.') and bytearray.replace(b'', b'.'): up to 80% faster. Patch written by Josh Snider. --- Doc/whatsnew/3.6.rst | 3 +++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ Objects/bytearrayobject.c | 28 +++++++++++++++++++--------- Objects/bytesobject.c | 28 +++++++++++++++++++--------- 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 9046058..986c145 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -339,6 +339,9 @@ Optimizations * Optimize :meth:`bytes.fromhex` and :meth:`bytearray.fromhex`: they are now between 2x and 3.5x faster. (Contributed by Victor Stinner in :issue:`25401`). +* Optimize ``bytes.replace(b'', b'.')`` and ``bytearray.replace(b'', b'.')``: + up to 80% faster. (Contributed by Josh Snider in :issue:`26574`). + Build and C API Changes ======================= diff --git a/Misc/ACKS b/Misc/ACKS index e67f6d1..52eae69 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1376,6 +1376,7 @@ Mark Smith Roy Smith Ryan Smith-Roberts Rafal Smotrzyk +Josh Snider Eric Snow Dirk Soede Nir Soffer diff --git a/Misc/NEWS b/Misc/NEWS index 6f5c7ab..2fa82f3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #26574: Optimize ``bytes.replace(b'', b'.')`` and + ``bytearray.replace(b'', b'.')``. Patch written by Josh Snider. + - Issue #26581: If coding cookie is specified multiple times on a line in Python source code file, only the first one is taken to account. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 9e8ba39..209a641 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1705,17 +1705,27 @@ replace_interleave(PyByteArrayObject *self, self_s = PyByteArray_AS_STRING(self); result_s = PyByteArray_AS_STRING(result); - /* TODO: special case single character, which doesn't need memcpy */ - - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; - - for (i=0; i 1) { + /* Lay the first one down (guaranteed this will occur) */ Py_MEMCPY(result_s, to_s, to_len); result_s += to_len; + count -= 1; + + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } + } + else { + result_s[0] = to_s[0]; + result_s += to_len; + count -= 1; + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + result_s[0] = to_s[0]; + result_s += to_len; + } } /* Copy the rest of the original string */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 602dea6..5b9006e 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2464,17 +2464,27 @@ replace_interleave(PyBytesObject *self, self_s = PyBytes_AS_STRING(self); result_s = PyBytes_AS_STRING(result); - /* TODO: special case single character, which doesn't need memcpy */ - - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; - - for (i=0; i 1) { + /* Lay the first one down (guaranteed this will occur) */ Py_MEMCPY(result_s, to_s, to_len); result_s += to_len; + count -= 1; + + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } + } + else { + result_s[0] = to_s[0]; + result_s += to_len; + count -= 1; + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + result_s[0] = to_s[0]; + result_s += to_len; + } } /* Copy the rest of the original string */ -- cgit v0.12 From 322bc12c3142e7816dd34c6c3085929ab29d3ed8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 14:36:39 +0100 Subject: Ooops, revert changeset ea9efa06c137 Change pushed by mistake, the patch is still under review :-/ """ _tracemalloc: add domain to trace keys * hashtable.h: key has now a variable size * _tracemalloc uses (pointer: void*, domain: unsigned int) as key for traces """ --- Modules/_tracemalloc.c | 186 +++++++++++++------------------------------------ Modules/hashtable.c | 105 ++++++++++++---------------- Modules/hashtable.h | 89 +++++++---------------- Python/marshal.c | 11 ++- 4 files changed, 121 insertions(+), 270 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 73aa53b..5752904 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -3,7 +3,6 @@ #include "frameobject.h" #include "pythread.h" #include "osdefs.h" -#include /* For offsetof */ /* Trace memory blocks allocated by PyMem_RawMalloc() */ #define TRACE_RAW_MALLOC @@ -55,26 +54,6 @@ static PyThread_type_lock tables_lock; # define TABLES_UNLOCK() #endif -typedef unsigned int domain_t; - -/* FIXME: pack also? */ -typedef struct { - void *ptr; - domain_t domain; -} pointer_t; - -/* Size of pointer_t content, it can be smaller than sizeof(pointer_t) */ -#define POINTER_T_SIZE \ - (offsetof(pointer_t, domain) + sizeof(domain_t)) - -#define POINTER_T_FILL_PADDING(key) \ - do { \ - if (POINTER_T_SIZE != sizeof(pointer_t)) { \ - memset((char *)&(key) + POINTER_T_SIZE, 0, \ - sizeof(pointer_t) - POINTER_T_SIZE); \ - } \ - } while (0) - /* Pack the frame_t structure to reduce the memory footprint on 64-bit architectures: 12 bytes instead of 16. This optimization might produce SIGBUS on architectures not supporting unaligned memory accesses (64-bit @@ -217,56 +196,23 @@ set_reentrant(int reentrant) } #endif -static Py_uhash_t -hashtable_hash_pointer(size_t key_size, const void *pkey) -{ - pointer_t ptr; - Py_uhash_t hash; - - assert(sizeof(ptr) == key_size); - ptr = *(pointer_t *)pkey; - - hash = (Py_uhash_t)_Py_HashPointer(ptr.ptr); - hash ^= ptr.domain; - return hash; -} - -static Py_uhash_t -hashtable_hash_pyobject(size_t key_size, const void *pkey) -{ - PyObject *obj; - - assert(key_size == sizeof(PyObject *)); - obj = *(PyObject **)pkey; - - return PyObject_Hash(obj); -} - static int -hashtable_compare_unicode(size_t key_size, const void *pkey, - const _Py_hashtable_entry_t *entry) +hashtable_compare_unicode(const void *key, const _Py_hashtable_entry_t *entry) { - PyObject *key, *entry_key; - - assert(sizeof(key) == key_size); - key = *(PyObject **)pkey; - assert(sizeof(entry_key) == key_size); - entry_key = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); - - if (key != NULL && entry_key != NULL) - return (PyUnicode_Compare(key, entry_key) == 0); + if (key != NULL && entry->key != NULL) + return (PyUnicode_Compare((PyObject *)key, (PyObject *)entry->key) == 0); else - return key == entry_key; + return key == entry->key; } static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; static _Py_hashtable_t * -hashtable_new(size_t key_size, size_t data_size, +hashtable_new(size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(key_size, data_size, 0, + return _Py_hashtable_new_full(data_size, 0, hash_func, compare_func, NULL, NULL, NULL, &hashtable_alloc); } @@ -284,26 +230,20 @@ raw_free(void *ptr) } static Py_uhash_t -hashtable_hash_traceback(size_t key_size, const void *pkey) +hashtable_hash_traceback(const void *key) { - const traceback_t *traceback = *(const traceback_t **)pkey; - assert(key_size == sizeof(const traceback_t *)); + const traceback_t *traceback = key; return traceback->hash; } static int -hashtable_compare_traceback(size_t key_size, const void *pkey, +hashtable_compare_traceback(const traceback_t *traceback1, const _Py_hashtable_entry_t *he) { - traceback_t *traceback1, *traceback2; + const traceback_t *traceback2 = he->key; const frame_t *frame1, *frame2; int i; - assert(sizeof(traceback1) == key_size); - assert(sizeof(traceback2) == key_size); - traceback1 = *(traceback_t **)pkey; - traceback2 = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(he); - if (traceback1->nframe != traceback2->nframe) return 0; @@ -372,16 +312,15 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) } /* intern the filename */ - entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); + entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); if (entry != NULL) { - assert(sizeof(filename) == tracemalloc_filenames->key_size); - filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); + filename = (PyObject *)entry->key; } else { /* tracemalloc_filenames is responsible to keep a reference to the filename */ Py_INCREF(filename); - if (_Py_HASHTABLE_SET_NODATA(tracemalloc_filenames, filename) < 0) { + if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL, 0) < 0) { Py_DECREF(filename); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the filename"); @@ -464,10 +403,9 @@ traceback_new(void) traceback->hash = traceback_hash(traceback); /* intern the traceback */ - entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); + entry = _Py_hashtable_get_entry(tracemalloc_tracebacks, traceback); if (entry != NULL) { - assert(sizeof(traceback) == tracemalloc_tracebacks->key_size); - traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); + traceback = (traceback_t *)entry->key; } else { traceback_t *copy; @@ -484,7 +422,7 @@ traceback_new(void) } memcpy(copy, traceback, traceback_size); - if (_Py_HASHTABLE_SET_NODATA(tracemalloc_tracebacks, copy) < 0) { + if (_Py_hashtable_set(tracemalloc_tracebacks, copy, NULL, 0) < 0) { raw_free(copy); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the traceback: putdata failed"); @@ -497,9 +435,8 @@ traceback_new(void) } static int -tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) +tracemalloc_add_trace(void *ptr, size_t size) { - pointer_t key; traceback_t *traceback; trace_t trace; int res; @@ -508,14 +445,10 @@ tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) if (traceback == NULL) return -1; - key.ptr = ptr; - key.domain = 0; - POINTER_T_FILL_PADDING(key); - trace.size = size; trace.traceback = traceback; - res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); if (res == 0) { assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); tracemalloc_traced_memory += size; @@ -527,16 +460,11 @@ tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) } static void -tracemalloc_remove_trace(void *ptr, domain_t domain) +tracemalloc_remove_trace(void *ptr) { - pointer_t key; trace_t trace; - key.ptr = ptr; - key.domain = domain; - POINTER_T_FILL_PADDING(key); - - if (_Py_HASHTABLE_POP(tracemalloc_traces, key, trace)) { + if (_Py_hashtable_pop(tracemalloc_traces, ptr, &trace, sizeof(trace))) { assert(tracemalloc_traced_memory >= trace.size); tracemalloc_traced_memory -= trace.size; } @@ -558,7 +486,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (tracemalloc_add_trace(ptr, 0, nelem * elsize) < 0) { + if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr); @@ -582,9 +510,9 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); - if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to the caller, because realloc() may already have shrinked the memory block and so removed bytes. @@ -602,7 +530,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* new allocation */ TABLES_LOCK(); - if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); @@ -627,7 +555,7 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); TABLES_UNLOCK(); } @@ -682,7 +610,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); TABLES_UNLOCK(); } return ptr2; @@ -761,7 +689,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); TABLES_UNLOCK(); } return ptr2; @@ -786,27 +714,17 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) #endif /* TRACE_RAW_MALLOC */ static int -tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, - void *user_data) +tracemalloc_clear_filename(_Py_hashtable_entry_t *entry, void *user_data) { - PyObject *filename; - - assert(sizeof(filename) == ht->key_size); - filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); - + PyObject *filename = (PyObject *)entry->key; Py_DECREF(filename); return 0; } static int -traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, - void *user_data) +traceback_free_traceback(_Py_hashtable_entry_t *entry, void *user_data) { - traceback_t *traceback; - - assert(sizeof(traceback) == ht->key_size); - traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); - + traceback_t *traceback = (traceback_t *)entry->key; raw_free(traceback); return 0; } @@ -873,16 +791,16 @@ tracemalloc_init(void) } #endif - tracemalloc_filenames = hashtable_new(sizeof(PyObject *), 0, - hashtable_hash_pyobject, + tracemalloc_filenames = hashtable_new(0, + (_Py_hashtable_hash_func)PyObject_Hash, hashtable_compare_unicode); - tracemalloc_tracebacks = hashtable_new(sizeof(traceback_t *), 0, - hashtable_hash_traceback, - hashtable_compare_traceback); + tracemalloc_tracebacks = hashtable_new(0, + (_Py_hashtable_hash_func)hashtable_hash_traceback, + (_Py_hashtable_compare_func)hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(pointer_t), sizeof(trace_t), - hashtable_hash_pointer, + tracemalloc_traces = hashtable_new(sizeof(trace_t), + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL @@ -1147,15 +1065,14 @@ typedef struct { } get_traces_t; static int -tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entry, - void *user_data) +tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) { get_traces_t *get_traces = user_data; trace_t *trace; PyObject *tracemalloc_obj; int res; - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); + trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(entry); tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) @@ -1170,11 +1087,9 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr } static int -tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, - _Py_hashtable_entry_t *entry, - void *user_data) +tracemalloc_pyobject_decref_cb(_Py_hashtable_entry_t *entry, void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); + PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); Py_DECREF(obj); return 0; } @@ -1205,7 +1120,7 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1237,7 +1152,7 @@ error: finally: if (get_traces.tracebacks != NULL) { _Py_hashtable_foreach(get_traces.tracebacks, - tracemalloc_pyobject_decref_cb, NULL); + tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } if (get_traces.traces != NULL) @@ -1247,21 +1162,16 @@ finally: } static traceback_t* -tracemalloc_get_traceback(const void *ptr, domain_t domain) +tracemalloc_get_traceback(const void *ptr) { - pointer_t key; trace_t trace; int found; if (!tracemalloc_config.tracing) return NULL; - key.ptr = (void *)ptr; - key.domain = domain; - POINTER_T_FILL_PADDING(key); - TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); TABLES_UNLOCK(); if (!found) @@ -1292,7 +1202,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(ptr, 0); + traceback = tracemalloc_get_traceback(ptr); if (traceback == NULL) Py_RETURN_NONE; @@ -1319,7 +1229,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(ptr, 0); + traceback = tracemalloc_get_traceback(ptr); if (traceback == NULL) return; diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 002c0a9..7de154b 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -59,7 +59,7 @@ #define ENTRY_NEXT(ENTRY) \ ((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY)) #define HASHTABLE_ITEM_SIZE(HT) \ - (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) + (sizeof(_Py_hashtable_entry_t) + (HT)->data_size) /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -88,21 +88,21 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, } Py_uhash_t -_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) +_Py_hashtable_hash_int(const void *key) { - void *key; - - assert(key_size == sizeof(void *)); - key = *(void**)pkey; + return (Py_uhash_t)key; +} +Py_uhash_t +_Py_hashtable_hash_ptr(const void *key) +{ return (Py_uhash_t)_Py_HashPointer((void *)key); } int -_Py_hashtable_compare_direct(size_t key_size, const void *pkey, - const _Py_hashtable_entry_t *entry) +_Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry) { - return (memcmp(pkey, _Py_HASHTABLE_ENTRY_KEY(entry), key_size) == 0); + return entry->key == key; } /* makes sure the real size of the buckets array is a power of 2 */ @@ -119,8 +119,7 @@ round_size(size_t s) } _Py_hashtable_t * -_Py_hashtable_new_full(size_t key_size, size_t data_size, - size_t init_size, +_Py_hashtable_new_full(size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, _Py_hashtable_copy_data_func copy_data_func, @@ -145,7 +144,6 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, ht->num_buckets = round_size(init_size); ht->entries = 0; - ht->key_size = key_size; ht->data_size = data_size; buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); @@ -166,12 +164,11 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, } _Py_hashtable_t * -_Py_hashtable_new(size_t key_size, size_t data_size, +_Py_hashtable_new(size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(key_size, data_size, - HASHTABLE_MIN_SIZE, + return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE, hash_func, compare_func, NULL, NULL, NULL, NULL); } @@ -198,7 +195,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { void *data; - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); size += ht->get_data_size_func(data); } } @@ -248,21 +245,17 @@ _Py_hashtable_print_stats(_Py_hashtable_t *ht) /* Get an entry. Return NULL if the key does not exist. */ _Py_hashtable_entry_t * -_Py_hashtable_get_entry(_Py_hashtable_t *ht, - size_t key_size, const void *pkey) +_Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; - assert(key_size == ht->key_size); - - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(key); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) break; } @@ -270,20 +263,18 @@ _Py_hashtable_get_entry(_Py_hashtable_t *ht, } static int -_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - void *data, size_t data_size) +_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry, *previous; - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(key); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) break; previous = entry; } @@ -307,8 +298,8 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, /* Add a new entry to the hash. The key must not be present in the hash table. Return 0 on success, -1 on memory error. */ int -_Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - size_t data_size, void *data) +_Py_hashtable_set(_Py_hashtable_t *ht, const void *key, + void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; @@ -319,11 +310,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, /* Don't write the assertion on a single line because it is interesting to know the duplicated entry if the assertion failed. The entry can be read using a debugger. */ - entry = _Py_hashtable_get_entry(ht, key_size, pkey); + entry = _Py_hashtable_get_entry(ht, key); assert(entry == NULL); #endif - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(key); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -332,11 +323,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, return -1; } + entry->key = (void *)key; entry->key_hash = key_hash; - memcpy(_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); + memcpy(_Py_HASHTABLE_ENTRY_DATA(entry), data, data_size); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -349,14 +340,13 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, /* Get data from an entry. Copy entry data into data and return 1 if the entry exists, return 0 if the entry does not exist. */ int -_Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, - size_t data_size, void *data) +_Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) { _Py_hashtable_entry_t *entry; assert(data != NULL); - entry = _Py_hashtable_get_entry(ht, key_size, pkey); + entry = _Py_hashtable_get_entry(ht, key); if (entry == NULL) return 0; _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); @@ -364,23 +354,22 @@ _Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, } int -_Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - size_t data_size, void *data) +_Py_hashtable_pop(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) { assert(data != NULL); assert(ht->free_data_func == NULL); - return _hashtable_pop_entry(ht, key_size, pkey, data, data_size); + return _hashtable_pop_entry(ht, key, data, data_size); } /* Delete an entry. The entry must exist. */ void -_Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) +_Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) { #ifndef NDEBUG - int found = _hashtable_pop_entry(ht, key_size, pkey, NULL, 0); + int found = _hashtable_pop_entry(ht, key, NULL, 0); assert(found); #else - (void)_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); + (void)_hashtable_pop_entry(ht, key, NULL, 0); #endif } @@ -389,7 +378,7 @@ _Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) stops if a non-zero value is returned. */ int _Py_hashtable_foreach(_Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, + int (*func) (_Py_hashtable_entry_t *entry, void *arg), void *arg) { _Py_hashtable_entry_t *entry; @@ -397,7 +386,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, for (hv = 0; hv < ht->num_buckets; hv++) { for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - int res = func(ht, entry, arg); + int res = func(entry, arg); if (res) return res; } @@ -408,7 +397,6 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, static void hashtable_rehash(_Py_hashtable_t *ht) { - const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -437,8 +425,7 @@ hashtable_rehash(_Py_hashtable_t *ht) for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { size_t entry_index; - - assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); + assert(ht->hash_func(entry->key) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -459,7 +446,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -478,7 +465,7 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) while (entry) { _Py_slist_item_t *entry_next = entry->next; if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); ht->alloc.free(entry); entry = entry_next; } @@ -492,16 +479,13 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) _Py_hashtable_t * _Py_hashtable_copy(_Py_hashtable_t *src) { - const size_t key_size = src->key_size; - const size_t data_size = src->data_size; _Py_hashtable_t *dst; _Py_hashtable_entry_t *entry; size_t bucket; int err; void *data, *new_data; - dst = _Py_hashtable_new_full(key_size, data_size, - src->num_buckets, + dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, src->hash_func, src->compare_func, src->copy_data_func, src->free_data_func, src->get_data_size_func, &src->alloc); @@ -512,20 +496,17 @@ _Py_hashtable_copy(_Py_hashtable_t *src) entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); new_data = src->copy_data_func(data); if (new_data != NULL) - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, &new_data); + err = _Py_hashtable_set(dst, entry->key, + &new_data, src->data_size); else err = 1; } else { - data = _Py_HASHTABLE_ENTRY_DATA(src, entry); - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, data); + data = _Py_HASHTABLE_ENTRY_DATA(entry); + err = _Py_hashtable_set(dst, entry->key, data, src->data_size); } if (err) { _Py_hashtable_destroy(dst); diff --git a/Modules/hashtable.h b/Modules/hashtable.h index aed3ed0..a9f9993 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -20,31 +20,26 @@ typedef struct { /* used by _Py_hashtable_t.buckets to link entries */ _Py_slist_item_t _Py_slist_item; + const void *key; Py_uhash_t key_hash; /* data follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ - ((void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) +#define _Py_HASHTABLE_ENTRY_DATA(ENTRY) \ + ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t)) -#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) - -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) +#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \ + (*(void **)_Py_HASHTABLE_ENTRY_DATA(ENTRY)) #define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ + memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \ } while (0) -typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, - const void *pkey); -typedef int (*_Py_hashtable_compare_func) (size_t key_size, - const void *pkey, - const _Py_hashtable_entry_t *he); +typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key); +typedef int (*_Py_hashtable_compare_func) (const void *key, const _Py_hashtable_entry_t *he); typedef void* (*_Py_hashtable_copy_data_func)(void *data); typedef void (*_Py_hashtable_free_data_func)(void *data); typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); @@ -61,7 +56,6 @@ typedef struct { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; - size_t key_size; size_t data_size; _Py_hashtable_hash_func hash_func; @@ -73,21 +67,15 @@ typedef struct { } _Py_hashtable_t; /* hash and compare functions for integers and pointers */ -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( - size_t key_size, - const void *pkey); -PyAPI_FUNC(int) _Py_hashtable_compare_direct( - size_t key_size, - const void *pkey, - const _Py_hashtable_entry_t *entry); +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key); +PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( - size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( - size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, @@ -100,65 +88,40 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src); PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht); PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); -typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_t *ht, - _Py_hashtable_entry_t *entry, - void *arg); +typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_entry_t *entry, void *arg); PyAPI_FUNC(int) _Py_hashtable_foreach( _Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, - void *arg); + _Py_hashtable_foreach_func func, void *arg); PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht); PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey); - -/* Don't call directly this function, - but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */ + const void *key); PyAPI_FUNC(int) _Py_hashtable_set( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey, - size_t data_size, - void *data); - -/* Don't call directly this function, but use _Py_HASHTABLE_GET() macro */ + const void *key, + void *data, + size_t data_size); PyAPI_FUNC(int) _Py_hashtable_get( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey, - size_t data_size, - void *data); - -/* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */ + const void *key, + void *data, + size_t data_size); PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey, - size_t data_size, - void *data); - + const void *key, + void *data, + size_t data_size); PyAPI_FUNC(void) _Py_hashtable_delete( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey); + const void *key); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) - -#define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) - -#define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \ - _Py_hashtable_get_entry(TABLE, sizeof(KEY), &(KEY)) + _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA)) #define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ - _Py_hashtable_get(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) - -#define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \ - _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) + _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA)) #endif /* Py_LIMITED_API */ diff --git a/Python/marshal.c b/Python/marshal.c index 64084f4..7a4b9d2 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -263,7 +263,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (Py_REFCNT(v) == 1) return 0; - entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); + entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry); @@ -571,8 +571,7 @@ static int w_init_refs(WFILE *wf, int version) { if (version >= 3) { - wf->hashtable = _Py_hashtable_new(sizeof(void *), sizeof(int), - _Py_hashtable_hash_ptr, + wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (wf->hashtable == NULL) { PyErr_NoMemory(); @@ -583,11 +582,9 @@ w_init_refs(WFILE *wf, int version) } static int -w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) +w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) { - void *entry_key = *(void **)_Py_HASHTABLE_ENTRY_KEY(entry); - assert(ht->key_size == sizeof(entry_key)); - Py_XDECREF(entry_key); + Py_XDECREF(entry->key); return 0; } -- cgit v0.12 From 19a8e844e455a26419f35bd4b57d4a7d19b61b69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 16:36:48 +0100 Subject: Add socket finalizer Issue #26590: Implement a safe finalizer for the _socket.socket type. It now releases the GIL to close the socket. Use PyErr_ResourceWarning() to raise the ResourceWarning to pass the socket object to the warning logger, to get the traceback where the socket was created (allocated). --- Misc/NEWS | 3 +++ Modules/socketmodule.c | 71 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 2fa82f3..6dfac97 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,9 @@ Core and Builtins Library ------- +- Issue #26590: Implement a safe finalizer for the _socket.socket type. It now + releases the GIL to close the socket. + - Issue #18787: spwd.getspnam() now raises a PermissionError if the user doesn't have privileges. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 77a6b31..ba3cefd 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2564,12 +2564,14 @@ sock_close(PySocketSockObject *s) { SOCKET_T fd; - /* We do not want to retry upon EINTR: see http://lwn.net/Articles/576478/ - * and http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html - * for more details. - */ - if ((fd = s->sock_fd) != -1) { + fd = s->sock_fd; + if (fd != -1) { s->sock_fd = -1; + + /* We do not want to retry upon EINTR: see + http://lwn.net/Articles/576478/ and + http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html + for more details. */ Py_BEGIN_ALLOW_THREADS (void) SOCKETCLOSE(fd); Py_END_ALLOW_THREADS @@ -4163,22 +4165,45 @@ static PyGetSetDef sock_getsetlist[] = { First close the file description. */ static void -sock_dealloc(PySocketSockObject *s) +sock_finalize(PySocketSockObject *s) { + SOCKET_T fd; + PyObject *error_type, *error_value, *error_traceback; + + /* Save the current exception, if any. */ + PyErr_Fetch(&error_type, &error_value, &error_traceback); + if (s->sock_fd != -1) { - PyObject *exc, *val, *tb; - Py_ssize_t old_refcount = Py_REFCNT(s); - ++Py_REFCNT(s); - PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed %R", s)) + if (PyErr_ResourceWarning((PyObject *)s, 1, "unclosed %R", s)) { /* Spurious errors can appear at shutdown */ - if (PyErr_ExceptionMatches(PyExc_Warning)) - PyErr_WriteUnraisable((PyObject *) s); - PyErr_Restore(exc, val, tb); - (void) SOCKETCLOSE(s->sock_fd); - Py_REFCNT(s) = old_refcount; + if (PyErr_ExceptionMatches(PyExc_Warning)) { + PyErr_WriteUnraisable((PyObject *)s); + } + } + + /* Only close the socket *after* logging the ResourceWarning warning + to allow the logger to call socket methods like + socket.getsockname(). If the socket is closed before, socket + methods fails with the EBADF error. */ + fd = s->sock_fd; + s->sock_fd = -1; + + /* We do not want to retry upon EINTR: see sock_close() */ + Py_BEGIN_ALLOW_THREADS + (void) SOCKETCLOSE(fd); + Py_END_ALLOW_THREADS } + + /* Restore the saved exception. */ + PyErr_Restore(error_type, error_value, error_traceback); +} + +static void +sock_dealloc(PySocketSockObject *s) +{ + if (PyObject_CallFinalizerFromDealloc((PyObject *)s) < 0) + return; + Py_TYPE(s)->tp_free((PyObject *)s); } @@ -4395,7 +4420,8 @@ static PyTypeObject sock_type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ sock_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -4415,6 +4441,15 @@ static PyTypeObject sock_type = { PyType_GenericAlloc, /* tp_alloc */ sock_new, /* tp_new */ PyObject_Del, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + (destructor)sock_finalize, /* tp_finalize */ }; -- cgit v0.12 From e0b75b7e87a385ca8b125e35e7c558626811ca99 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 17:26:04 +0100 Subject: Fix test_ssl.test_refcycle() Issue #26590: support.check_warnings() stores warnins, but ResourceWarning now comes with a reference to the socket object which indirectly keeps the socket alive. --- Lib/test/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 9a48483..e0c31a8 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -328,7 +328,7 @@ class BasicSocketTests(unittest.TestCase): wr = weakref.ref(ss) with support.check_warnings(("", ResourceWarning)): del ss - self.assertEqual(wr(), None) + self.assertEqual(wr(), None) def test_wrapped_unconnected(self): # Methods on an unconnected SSLSocket propagate the original -- cgit v0.12 From 928bff0b26adb643a7078575c9075b4b709c1b16 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:36:36 +0100 Subject: cleanup iobase.c casting iobase_finalize to destructor is not needed --- Modules/_io/iobase.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 025007e..e289a10 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -827,7 +827,7 @@ PyTypeObject PyIOBase_Type = { 0, /* tp_weaklist */ 0, /* tp_del */ 0, /* tp_version_tag */ - (destructor)iobase_finalize, /* tp_finalize */ + iobase_finalize, /* tp_finalize */ }; -- cgit v0.12 From 285cf0a6014af147b82a3446d9e088ad0332720d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 22:00:58 +0100 Subject: hashtable.h now supports keys of any size Issue #26588: hashtable.h now supports keys of any size, not only sizeof(void*). It allows to support key larger than sizeof(void*), but also to use less memory for key smaller than sizeof(void*). --- Modules/_tracemalloc.c | 105 ++++++++++++++++++++----------- Modules/hashtable.c | 147 ++++++++++++++++++++++++++----------------- Modules/hashtable.h | 165 +++++++++++++++++++++++++++++++++++++------------ Python/marshal.c | 15 +++-- 4 files changed, 294 insertions(+), 138 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5752904..6799eb6 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -196,23 +196,38 @@ set_reentrant(int reentrant) } #endif +static Py_uhash_t +hashtable_hash_pyobject(size_t key_size, const void *pkey) +{ + PyObject *obj; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, obj); + return PyObject_Hash(obj); +} + static int -hashtable_compare_unicode(const void *key, const _Py_hashtable_entry_t *entry) +hashtable_compare_unicode(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - if (key != NULL && entry->key != NULL) - return (PyUnicode_Compare((PyObject *)key, (PyObject *)entry->key) == 0); + PyObject *key, *entry_key; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, key); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, entry_key); + + if (key != NULL && entry_key != NULL) + return (PyUnicode_Compare(key, entry_key) == 0); else - return key == entry->key; + return key == entry_key; } static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; static _Py_hashtable_t * -hashtable_new(size_t data_size, +hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, 0, + return _Py_hashtable_new_full(key_size, data_size, 0, hash_func, compare_func, NULL, NULL, NULL, &hashtable_alloc); } @@ -230,20 +245,25 @@ raw_free(void *ptr) } static Py_uhash_t -hashtable_hash_traceback(const void *key) +hashtable_hash_traceback(size_t key_size, const void *pkey) { - const traceback_t *traceback = key; + const traceback_t *traceback; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback); return traceback->hash; } static int -hashtable_compare_traceback(const traceback_t *traceback1, +hashtable_compare_traceback(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he) { - const traceback_t *traceback2 = he->key; + traceback_t *traceback1, *traceback2; const frame_t *frame1, *frame2; int i; + _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback1); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, he, traceback2); + if (traceback1->nframe != traceback2->nframe) return 0; @@ -312,15 +332,16 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) } /* intern the filename */ - entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); if (entry != NULL) { - filename = (PyObject *)entry->key; + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_filenames->key_size, entry, + filename); } else { /* tracemalloc_filenames is responsible to keep a reference to the filename */ Py_INCREF(filename); - if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_filenames, filename) < 0) { Py_DECREF(filename); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the filename"); @@ -403,9 +424,10 @@ traceback_new(void) traceback->hash = traceback_hash(traceback); /* intern the traceback */ - entry = _Py_hashtable_get_entry(tracemalloc_tracebacks, traceback); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); if (entry != NULL) { - traceback = (traceback_t *)entry->key; + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_tracebacks->key_size, entry, + traceback); } else { traceback_t *copy; @@ -422,7 +444,7 @@ traceback_new(void) } memcpy(copy, traceback, traceback_size); - if (_Py_hashtable_set(tracemalloc_tracebacks, copy, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_tracebacks, copy) < 0) { raw_free(copy); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the traceback: putdata failed"); @@ -464,7 +486,7 @@ tracemalloc_remove_trace(void *ptr) { trace_t trace; - if (_Py_hashtable_pop(tracemalloc_traces, ptr, &trace, sizeof(trace))) { + if (_Py_HASHTABLE_POP(tracemalloc_traces, ptr, trace)) { assert(tracemalloc_traced_memory >= trace.size); tracemalloc_traced_memory -= trace.size; } @@ -714,17 +736,23 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) #endif /* TRACE_RAW_MALLOC */ static int -tracemalloc_clear_filename(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *filename = (PyObject *)entry->key; + PyObject *filename; + + _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, filename); Py_DECREF(filename); return 0; } static int -traceback_free_traceback(_Py_hashtable_entry_t *entry, void *user_data) +traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - traceback_t *traceback = (traceback_t *)entry->key; + traceback_t *traceback; + + _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, traceback); raw_free(traceback); return 0; } @@ -791,21 +819,20 @@ tracemalloc_init(void) } #endif - tracemalloc_filenames = hashtable_new(0, - (_Py_hashtable_hash_func)PyObject_Hash, + tracemalloc_filenames = hashtable_new(sizeof(PyObject *), 0, + hashtable_hash_pyobject, hashtable_compare_unicode); - tracemalloc_tracebacks = hashtable_new(0, - (_Py_hashtable_hash_func)hashtable_hash_traceback, - (_Py_hashtable_compare_func)hashtable_compare_traceback); + tracemalloc_tracebacks = hashtable_new(sizeof(traceback_t *), 0, + hashtable_hash_traceback, + hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(trace_t), + tracemalloc_traces = hashtable_new(sizeof(void*), sizeof(trace_t), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL - || tracemalloc_traces == NULL) - { + || tracemalloc_traces == NULL) { PyErr_NoMemory(); return -1; } @@ -840,9 +867,9 @@ tracemalloc_deinit(void) tracemalloc_stop(); /* destroy hash tables */ - _Py_hashtable_destroy(tracemalloc_traces); _Py_hashtable_destroy(tracemalloc_tracebacks); _Py_hashtable_destroy(tracemalloc_filenames); + _Py_hashtable_destroy(tracemalloc_traces); #if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) if (tables_lock != NULL) { @@ -935,8 +962,9 @@ tracemalloc_stop(void) PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); - /* release memory */ tracemalloc_clear_traces(); + + /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; } @@ -1065,14 +1093,15 @@ typedef struct { } get_traces_t; static int -tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entry, + void *user_data) { get_traces_t *get_traces = user_data; trace_t *trace; PyObject *tracemalloc_obj; int res; - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(entry); + trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) @@ -1087,9 +1116,11 @@ tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) } static int -tracemalloc_pyobject_decref_cb(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, + _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); Py_DECREF(obj); return 0; } @@ -1120,7 +1151,7 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1152,7 +1183,7 @@ error: finally: if (get_traces.tracebacks != NULL) { _Py_hashtable_foreach(get_traces.tracebacks, - tracemalloc_pyobject_decref_cb, NULL); + tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } if (get_traces.traces != NULL) diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 7de154b..d33f0d7 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -1,5 +1,5 @@ -/* The implementation of the hash table (_Py_hashtable_t) is based on the cfuhash - project: +/* The implementation of the hash table (_Py_hashtable_t) is based on the + cfuhash project: http://sourceforge.net/projects/libcfu/ Copyright of cfuhash: @@ -59,7 +59,7 @@ #define ENTRY_NEXT(ENTRY) \ ((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY)) #define HASHTABLE_ITEM_SIZE(HT) \ - (sizeof(_Py_hashtable_entry_t) + (HT)->data_size) + (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -70,6 +70,7 @@ _Py_slist_init(_Py_slist_t *list) list->head = NULL; } + static void _Py_slist_prepend(_Py_slist_t *list, _Py_slist_item_t *item) { @@ -77,6 +78,7 @@ _Py_slist_prepend(_Py_slist_t *list, _Py_slist_item_t *item) list->head = item; } + static void _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, _Py_slist_item_t *item) @@ -87,24 +89,26 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, list->head = item->next; } -Py_uhash_t -_Py_hashtable_hash_int(const void *key) -{ - return (Py_uhash_t)key; -} Py_uhash_t -_Py_hashtable_hash_ptr(const void *key) +_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) { + void *key; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, key); return (Py_uhash_t)_Py_HashPointer((void *)key); } + int -_Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry) +_Py_hashtable_compare_direct(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - return entry->key == key; + const void *pkey2 = _Py_HASHTABLE_ENTRY_KEY(entry); + return (memcmp(pkey, pkey2, key_size) == 0); } + /* makes sure the real size of the buckets array is a power of 2 */ static size_t round_size(size_t s) @@ -118,8 +122,10 @@ round_size(size_t s) return i; } + _Py_hashtable_t * -_Py_hashtable_new_full(size_t data_size, size_t init_size, +_Py_hashtable_new_full(size_t key_size, size_t data_size, + size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, _Py_hashtable_copy_data_func copy_data_func, @@ -144,6 +150,7 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, ht->num_buckets = round_size(init_size); ht->entries = 0; + ht->key_size = key_size; ht->data_size = data_size; buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); @@ -163,16 +170,19 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, return ht; } + _Py_hashtable_t * -_Py_hashtable_new(size_t data_size, +_Py_hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE, + return _Py_hashtable_new_full(key_size, data_size, + HASHTABLE_MIN_SIZE, hash_func, compare_func, NULL, NULL, NULL, NULL); } + size_t _Py_hashtable_size(_Py_hashtable_t *ht) { @@ -195,7 +205,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { void *data; - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); size += ht->get_data_size_func(data); } } @@ -203,6 +213,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) return size; } + #ifdef Py_DEBUG void _Py_hashtable_print_stats(_Py_hashtable_t *ht) @@ -243,38 +254,47 @@ _Py_hashtable_print_stats(_Py_hashtable_t *ht) } #endif -/* Get an entry. Return NULL if the key does not exist. */ + _Py_hashtable_entry_t * -_Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_get_entry(_Py_hashtable_t *ht, + size_t key_size, const void *pkey) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; - key_hash = ht->hash_func(key); + assert(key_size == ht->key_size); + + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; } return entry; } + static int -_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry, *previous; - key_hash = ht->hash_func(key); + assert(key_size == ht->key_size); + + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; previous = entry; } @@ -287,7 +307,7 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t da ht->entries--; if (data != NULL) - _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); + _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); ht->alloc.free(entry); if ((float)ht->entries / (float)ht->num_buckets < HASHTABLE_LOW) @@ -295,26 +315,27 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t da return 1; } -/* Add a new entry to the hash. The key must not be present in the hash table. - Return 0 on success, -1 on memory error. */ + int -_Py_hashtable_set(_Py_hashtable_t *ht, const void *key, - void *data, size_t data_size) +_Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; + assert(key_size == ht->key_size); + assert(data != NULL || data_size == 0); #ifndef NDEBUG /* Don't write the assertion on a single line because it is interesting to know the duplicated entry if the assertion failed. The entry can be read using a debugger. */ - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); assert(entry == NULL); #endif - key_hash = ht->hash_func(key); + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -323,11 +344,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, return -1; } - entry->key = (void *)key; entry->key_hash = key_hash; + memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(entry), data, data_size); + memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -337,48 +358,48 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, return 0; } -/* Get data from an entry. Copy entry data into data and return 1 if the entry - exists, return 0 if the entry does not exist. */ + int -_Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, + size_t data_size, void *data) { _Py_hashtable_entry_t *entry; assert(data != NULL); - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; - _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); + _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); return 1; } + int -_Py_hashtable_pop(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { assert(data != NULL); assert(ht->free_data_func == NULL); - return _hashtable_pop_entry(ht, key, data, data_size); + return _Py_hashtable_pop_entry(ht, key_size, pkey, data, data_size); } -/* Delete an entry. The entry must exist. */ + void -_Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) { #ifndef NDEBUG - int found = _hashtable_pop_entry(ht, key, NULL, 0); + int found = _Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); assert(found); #else - (void)_hashtable_pop_entry(ht, key, NULL, 0); + (void)_Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); #endif } -/* Prototype for a pointer to a function to be called foreach - key/value pair in the hash by hashtable_foreach(). Iteration - stops if a non-zero value is returned. */ + int _Py_hashtable_foreach(_Py_hashtable_t *ht, - int (*func) (_Py_hashtable_entry_t *entry, void *arg), + _Py_hashtable_foreach_func func, void *arg) { _Py_hashtable_entry_t *entry; @@ -386,7 +407,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, for (hv = 0; hv < ht->num_buckets; hv++) { for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - int res = func(entry, arg); + int res = func(ht, entry, arg); if (res) return res; } @@ -394,9 +415,11 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, return 0; } + static void hashtable_rehash(_Py_hashtable_t *ht) { + const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -425,7 +448,8 @@ hashtable_rehash(_Py_hashtable_t *ht) for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { size_t entry_index; - assert(ht->hash_func(entry->key) == entry->key_hash); + + assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -436,6 +460,7 @@ hashtable_rehash(_Py_hashtable_t *ht) ht->alloc.free(old_buckets); } + void _Py_hashtable_clear(_Py_hashtable_t *ht) { @@ -446,7 +471,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -455,6 +480,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) hashtable_rehash(ht); } + void _Py_hashtable_destroy(_Py_hashtable_t *ht) { @@ -465,7 +491,7 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) while (entry) { _Py_slist_item_t *entry_next = entry->next; if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); entry = entry_next; } @@ -475,17 +501,20 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) ht->alloc.free(ht); } -/* Return a copy of the hash table */ + _Py_hashtable_t * _Py_hashtable_copy(_Py_hashtable_t *src) { + const size_t key_size = src->key_size; + const size_t data_size = src->data_size; _Py_hashtable_t *dst; _Py_hashtable_entry_t *entry; size_t bucket; int err; void *data, *new_data; - dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, + dst = _Py_hashtable_new_full(key_size, data_size, + src->num_buckets, src->hash_func, src->compare_func, src->copy_data_func, src->free_data_func, src->get_data_size_func, &src->alloc); @@ -496,17 +525,20 @@ _Py_hashtable_copy(_Py_hashtable_t *src) entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); new_data = src->copy_data_func(data); if (new_data != NULL) - err = _Py_hashtable_set(dst, entry->key, - &new_data, src->data_size); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, &new_data); else err = 1; } else { - data = _Py_HASHTABLE_ENTRY_DATA(entry); - err = _Py_hashtable_set(dst, entry->key, data, src->data_size); + data = _Py_HASHTABLE_ENTRY_DATA(src, entry); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, data); } if (err) { _Py_hashtable_destroy(dst); @@ -516,4 +548,3 @@ _Py_hashtable_copy(_Py_hashtable_t *src) } return dst; } - diff --git a/Modules/hashtable.h b/Modules/hashtable.h index a9f9993..6eb5737 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -1,9 +1,10 @@ #ifndef Py_HASHTABLE_H #define Py_HASHTABLE_H - /* The whole API is private */ #ifndef Py_LIMITED_API +/* Single linked list */ + typedef struct _Py_slist_item_s { struct _Py_slist_item_s *next; } _Py_slist_item_t; @@ -16,30 +17,55 @@ typedef struct { #define _Py_SLIST_HEAD(SLIST) (((_Py_slist_t *)SLIST)->head) + +/* _Py_hashtable: table entry */ + typedef struct { /* used by _Py_hashtable_t.buckets to link entries */ _Py_slist_item_t _Py_slist_item; - const void *key; Py_uhash_t key_hash; - /* data follows */ + /* key (key_size bytes) and then data (data_size bytes) follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_DATA(ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t)) +#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ + ((const void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) + +#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ + ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) + +#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ + (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) + +/* Get a key value from pkey: use memcpy() rather than a pointer dereference + to avoid memory alignment issues. */ +#define _Py_HASHTABLE_READ_KEY(KEY_SIZE, PKEY, DST_KEY) \ + do { \ + assert(sizeof(DST_KEY) == (KEY_SIZE)); \ + memcpy(&(DST_KEY), (PKEY), sizeof(DST_KEY)); \ + } while (0) -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(ENTRY)) +#define _Py_HASHTABLE_ENTRY_READ_KEY(KEY_SIZE, ENTRY, KEY) \ + do { \ + assert(sizeof(KEY) == (KEY_SIZE)); \ + memcpy(&(KEY), _Py_HASHTABLE_ENTRY_KEY(ENTRY), sizeof(KEY)); \ + } while (0) -#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ +#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA_SIZE, DATA) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \ + memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ } while (0) -typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key); -typedef int (*_Py_hashtable_compare_func) (const void *key, const _Py_hashtable_entry_t *he); + +/* _Py_hashtable: prototypes */ + +typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, + const void *pkey); +typedef int (*_Py_hashtable_compare_func) (size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *he); typedef void* (*_Py_hashtable_copy_data_func)(void *data); typedef void (*_Py_hashtable_free_data_func)(void *data); typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); @@ -52,10 +78,14 @@ typedef struct { void (*free) (void *ptr); } _Py_hashtable_allocator_t; + +/* _Py_hashtable: table */ + typedef struct { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; + size_t key_size; size_t data_size; _Py_hashtable_hash_func hash_func; @@ -66,16 +96,25 @@ typedef struct { _Py_hashtable_allocator_t alloc; } _Py_hashtable_t; -/* hash and compare functions for integers and pointers */ -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key); -PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry); +/* hash a pointer (void*) */ +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( + size_t key_size, + const void *pkey); + +/* comparison using memcmp() */ +PyAPI_FUNC(int) _Py_hashtable_compare_direct( + size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *entry); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( + size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func); + PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( + size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, @@ -84,45 +123,95 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( _Py_hashtable_free_data_func free_data_func, _Py_hashtable_get_data_size_func get_data_size_func, _Py_hashtable_allocator_t *allocator); + +PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); + +/* Return a copy of the hash table */ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src); + PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht); -PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); -typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_entry_t *entry, void *arg); +typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_t *ht, + _Py_hashtable_entry_t *entry, + void *arg); +/* Call func() on each entry of the hashtable. + Iteration stops if func() result is non-zero, in this case it's the result + of the call. Otherwise, the function returns 0. */ PyAPI_FUNC(int) _Py_hashtable_foreach( _Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, void *arg); + _Py_hashtable_foreach_func func, + void *arg); + PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht); -PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( - _Py_hashtable_t *ht, - const void *key); +/* Add a new entry to the hash. The key must not be present in the hash table. + Return 0 on success, -1 on memory error. + + Don't call directly this function, + but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */ PyAPI_FUNC(int) _Py_hashtable_set( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +#define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) + + +/* Get an entry. + Return NULL if the key does not exist. + + Don't call directly this function, but use _Py_HASHTABLE_GET_ENTRY() + macro */ +PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( + _Py_hashtable_t *ht, + size_t key_size, + const void *pkey); + +#define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \ + _Py_hashtable_get_entry(TABLE, sizeof(KEY), &(KEY)) + + +/* Get data from an entry. Copy entry data into data and return 1 if the entry + exists, return 0 if the entry does not exist. + + Don't call directly this function, but use _Py_HASHTABLE_GET() macro */ PyAPI_FUNC(int) _Py_hashtable_get( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +#define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ + _Py_hashtable_get(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) + + +/* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */ PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); -PyAPI_FUNC(void) _Py_hashtable_delete( - _Py_hashtable_t *ht, - const void *key); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); -#define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA)) +#define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \ + _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) -#define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ - _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA)) -#endif /* Py_LIMITED_API */ +/* Delete an entry. + + WARNING: The entry must exist. */ +PyAPI_FUNC(void) _Py_hashtable_delete( + _Py_hashtable_t *ht, + size_t key_size, + const void *pkey); +#endif /* Py_LIMITED_API */ #endif diff --git a/Python/marshal.c b/Python/marshal.c index 7a4b9d2..83a1885 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -263,10 +263,10 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (Py_REFCNT(v) == 1) return 0; - entry = _Py_hashtable_get_entry(p->hashtable, v); + entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry); + _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, sizeof(w), &w); /* we don't store "long" indices in the dict */ assert(0 <= w && w <= 0x7fffffff); w_byte(TYPE_REF, p); @@ -571,7 +571,8 @@ static int w_init_refs(WFILE *wf, int version) { if (version >= 3) { - wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr, + wf->hashtable = _Py_hashtable_new(sizeof(PyObject *), sizeof(int), + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (wf->hashtable == NULL) { PyErr_NoMemory(); @@ -582,9 +583,13 @@ w_init_refs(WFILE *wf, int version) } static int -w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) +w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *Py_UNUSED(data)) { - Py_XDECREF(entry->key); + PyObject *entry_key; + + _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, entry_key); + Py_XDECREF(entry_key); return 0; } -- cgit v0.12 From b32a7edb22805daa28f486271ca5052e1ad87633 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 23:05:08 +0100 Subject: Issue #26588: Fix compilation warning on Windows --- Modules/_tracemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 6799eb6..3c5319b 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -247,7 +247,7 @@ raw_free(void *ptr) static Py_uhash_t hashtable_hash_traceback(size_t key_size, const void *pkey) { - const traceback_t *traceback; + traceback_t *traceback; _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback); return traceback->hash; -- cgit v0.12 From c9553876ae88b7f1494cff826c8f7a08c2ac5614 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 12:13:01 +0100 Subject: Simplify implementation of hashtable.c Issue #26588: Remove copy_data, free_data and get_data_size callbacks from hashtable.h. These callbacks are not used in Python and makes the code more complex. Remove also the _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P() macro which uses an unsafe pointer dereference (can cause memory alignment issue). Replace the macro usage with _Py_HASHTABLE_ENTRY_READ_DATA() which is implemented with the safe memcpy() function. --- Modules/_tracemalloc.c | 14 +++++++------ Modules/hashtable.c | 55 ++++++++------------------------------------------ Modules/hashtable.h | 14 +------------ 3 files changed, 17 insertions(+), 66 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 3c5319b..c48dd08 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -220,16 +220,15 @@ hashtable_compare_unicode(size_t key_size, const void *pkey, return key == entry_key; } -static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; - static _Py_hashtable_t * hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { + _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; return _Py_hashtable_new_full(key_size, data_size, 0, hash_func, compare_func, - NULL, NULL, NULL, &hashtable_alloc); + &hashtable_alloc); } static void* @@ -1120,7 +1119,8 @@ tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, _Py_hashtable_entry_t *entry, void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); + PyObject *obj; + _Py_HASHTABLE_ENTRY_READ_DATA(tracebacks, entry, sizeof(obj), &obj); Py_DECREF(obj); return 0; } @@ -1151,7 +1151,8 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), + sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1186,8 +1187,9 @@ finally: tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } - if (get_traces.traces != NULL) + if (get_traces.traces != NULL) { _Py_hashtable_destroy(get_traces.traces); + } return get_traces.list; } diff --git a/Modules/hashtable.c b/Modules/hashtable.c index d33f0d7..7094b95 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -128,9 +128,6 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, - _Py_hashtable_copy_data_func copy_data_func, - _Py_hashtable_free_data_func free_data_func, - _Py_hashtable_get_data_size_func get_data_size_func, _Py_hashtable_allocator_t *allocator) { _Py_hashtable_t *ht; @@ -163,9 +160,6 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, ht->hash_func = hash_func; ht->compare_func = compare_func; - ht->copy_data_func = copy_data_func; - ht->free_data_func = free_data_func; - ht->get_data_size_func = get_data_size_func; ht->alloc = alloc; return ht; } @@ -179,7 +173,7 @@ _Py_hashtable_new(size_t key_size, size_t data_size, return _Py_hashtable_new_full(key_size, data_size, HASHTABLE_MIN_SIZE, hash_func, compare_func, - NULL, NULL, NULL, NULL); + NULL); } @@ -187,7 +181,6 @@ size_t _Py_hashtable_size(_Py_hashtable_t *ht) { size_t size; - size_t hv; size = sizeof(_Py_hashtable_t); @@ -197,19 +190,6 @@ _Py_hashtable_size(_Py_hashtable_t *ht) /* entries */ size += ht->entries * HASHTABLE_ITEM_SIZE(ht); - /* data linked from entries */ - if (ht->get_data_size_func) { - for (hv = 0; hv < ht->num_buckets; hv++) { - _Py_hashtable_entry_t *entry; - - for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - void *data; - - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); - size += ht->get_data_size_func(data); - } - } - } return size; } @@ -318,7 +298,7 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, int _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - size_t data_size, void *data) + size_t data_size, const void *data) { Py_uhash_t key_hash; size_t index; @@ -380,7 +360,6 @@ _Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, size_t data_size, void *data) { assert(data != NULL); - assert(ht->free_data_func == NULL); return _Py_hashtable_pop_entry(ht, key_size, pkey, data, data_size); } @@ -470,8 +449,6 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (i=0; i < ht->num_buckets; i++) { for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); - if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -490,8 +467,6 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) _Py_slist_item_t *entry = ht->buckets[i].head; while (entry) { _Py_slist_item_t *entry_next = entry->next; - if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); entry = entry_next; } @@ -511,35 +486,21 @@ _Py_hashtable_copy(_Py_hashtable_t *src) _Py_hashtable_entry_t *entry; size_t bucket; int err; - void *data, *new_data; dst = _Py_hashtable_new_full(key_size, data_size, src->num_buckets, - src->hash_func, src->compare_func, - src->copy_data_func, src->free_data_func, - src->get_data_size_func, &src->alloc); + src->hash_func, + src->compare_func, + &src->alloc); if (dst == NULL) return NULL; for (bucket=0; bucket < src->num_buckets; bucket++) { entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { - if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); - new_data = src->copy_data_func(data); - if (new_data != NULL) - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, &new_data); - else - err = 1; - } - else { - data = _Py_HASHTABLE_ENTRY_DATA(src, entry); - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, data); - } + const void *pkey = _Py_HASHTABLE_ENTRY_KEY(entry); + const void *data = _Py_HASHTABLE_ENTRY_DATA(src, entry); + err = _Py_hashtable_set(dst, key_size, pkey, data_size, data); if (err) { _Py_hashtable_destroy(dst); return NULL; diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 6eb5737..eede038 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -35,9 +35,6 @@ typedef struct { #define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) - /* Get a key value from pkey: use memcpy() rather than a pointer dereference to avoid memory alignment issues. */ #define _Py_HASHTABLE_READ_KEY(KEY_SIZE, PKEY, DST_KEY) \ @@ -66,9 +63,6 @@ typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, typedef int (*_Py_hashtable_compare_func) (size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he); -typedef void* (*_Py_hashtable_copy_data_func)(void *data); -typedef void (*_Py_hashtable_free_data_func)(void *data); -typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); typedef struct { /* allocate a memory block */ @@ -90,9 +84,6 @@ typedef struct { _Py_hashtable_hash_func hash_func; _Py_hashtable_compare_func compare_func; - _Py_hashtable_copy_data_func copy_data_func; - _Py_hashtable_free_data_func free_data_func; - _Py_hashtable_get_data_size_func get_data_size_func; _Py_hashtable_allocator_t alloc; } _Py_hashtable_t; @@ -119,9 +110,6 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, - _Py_hashtable_copy_data_func copy_data_func, - _Py_hashtable_free_data_func free_data_func, - _Py_hashtable_get_data_size_func get_data_size_func, _Py_hashtable_allocator_t *allocator); PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); @@ -155,7 +143,7 @@ PyAPI_FUNC(int) _Py_hashtable_set( size_t key_size, const void *pkey, size_t data_size, - void *data); + const void *data); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) -- cgit v0.12 From 58100059acb89179530036b4649f91cc679ea12b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 12:25:04 +0100 Subject: Remove _Py_hashtable_delete() Issue #26588: Remove _Py_hashtable_delete() from hashtable.h since the function is not used. Keep the C code in hashtable.c as commented code if someone needs it later. --- Modules/hashtable.c | 3 +++ Modules/hashtable.h | 8 -------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 7094b95..bb20cce 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -364,6 +364,8 @@ _Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, } +/* Code commented since the function is not needed in Python */ +#if 0 void _Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) { @@ -374,6 +376,7 @@ _Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) (void)_Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); #endif } +#endif int diff --git a/Modules/hashtable.h b/Modules/hashtable.h index eede038..41542d2 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -193,13 +193,5 @@ PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) -/* Delete an entry. - - WARNING: The entry must exist. */ -PyAPI_FUNC(void) _Py_hashtable_delete( - _Py_hashtable_t *ht, - size_t key_size, - const void *pkey); - #endif /* Py_LIMITED_API */ #endif -- cgit v0.12 From e492ae50e251c4fcd48bc37b1eaa4821894f1fdb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 12:58:23 +0100 Subject: tracemalloc now supports domains Issue #26588: * The _tracemalloc now supports tracing memory allocations of multiple address spaces (domains). * Add domain parameter to tracemalloc_add_trace() and tracemalloc_remove_trace(). * tracemalloc_add_trace() now starts by removing the previous trace, if any. * _tracemalloc._get_traces() now returns a list of (domain, size, traceback_frames): the domain is new. * Add tracemalloc.DomainFilter * tracemalloc.Filter: add an optional domain parameter to the constructor and a domain attribute * Sublte change: use Py_uintptr_t rather than void* in the traces key. * Add tracemalloc_config.use_domain, currently hardcoded to 1 --- Doc/library/tracemalloc.rst | 45 +++++++- Lib/test/test_tracemalloc.py | 104 ++++++++++++----- Lib/tracemalloc.py | 73 +++++++++--- Misc/NEWS | 3 + Modules/_tracemalloc.c | 267 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 388 insertions(+), 104 deletions(-) diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 5feb2d9..9d1cb17 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -355,10 +355,32 @@ Functions See also the :func:`get_object_traceback` function. +DomainFilter +^^^^^^^^^^^^ + +.. class:: DomainFilter(inclusive: bool, domain: int) + + Filter traces of memory blocks by their address space (domain). + + .. versionadded:: 3.6 + + .. attribute:: inclusive + + If *inclusive* is ``True`` (include), match memory blocks allocated + in the address space :attr:`domain`. + + If *inclusive* is ``False`` (exclude), match memory blocks not allocated + in the address space :attr:`domain`. + + .. attribute:: domain + + Address space of a memory block (``int``). Read-only property. + + Filter ^^^^^^ -.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False) +.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False, domain: int=None) Filter on traces of memory blocks. @@ -378,9 +400,17 @@ Filter .. versionchanged:: 3.5 The ``'.pyo'`` file extension is no longer replaced with ``'.py'``. + .. versionchanged:: 3.6 + Added the :attr:`domain` attribute. + + + .. attribute:: domain + + Address space of a memory block (``int`` or ``None``). + .. attribute:: inclusive - If *inclusive* is ``True`` (include), only trace memory blocks allocated + If *inclusive* is ``True`` (include), only match memory blocks allocated in a file with a name matching :attr:`filename_pattern` at line number :attr:`lineno`. @@ -395,7 +425,7 @@ Filter .. attribute:: filename_pattern - Filename pattern of the filter (``str``). + Filename pattern of the filter (``str``). Read-only property. .. attribute:: all_frames @@ -458,14 +488,17 @@ Snapshot .. method:: filter_traces(filters) Create a new :class:`Snapshot` instance with a filtered :attr:`traces` - sequence, *filters* is a list of :class:`Filter` instances. If *filters* - is an empty list, return a new :class:`Snapshot` instance with a copy of - the traces. + sequence, *filters* is a list of :class:`DomainFilter` and + :class:`Filter` instances. If *filters* is an empty list, return a new + :class:`Snapshot` instance with a copy of the traces. All inclusive filters are applied at once, a trace is ignored if no inclusive filters match it. A trace is ignored if at least one exclusive filter matches it. + .. versionchanged:: 3.6 + :class:`DomainFilter` instances are now also accepted in *filters*. + .. classmethod:: load(filename) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index f65e361..7b92b87 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -37,28 +37,31 @@ def allocate_bytes(size): def create_snapshots(): traceback_limit = 2 + # _tracemalloc._get_traces() returns a list of (domain, size, + # traceback_frames) tuples. traceback_frames is a tuple of (filename, + # line_number) tuples. raw_traces = [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), - (66, (('b.py', 1),)), + (2, 66, (('b.py', 1),)), - (7, (('', 0),)), + (3, 7, (('', 0),)), ] snapshot = tracemalloc.Snapshot(raw_traces, traceback_limit) raw_traces2 = [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), - (5000, (('a.py', 5), ('b.py', 4))), + (2, 2, (('a.py', 5), ('b.py', 4))), + (2, 5000, (('a.py', 5), ('b.py', 4))), - (400, (('c.py', 578),)), + (4, 400, (('c.py', 578),)), ] snapshot2 = tracemalloc.Snapshot(raw_traces2, traceback_limit) @@ -126,7 +129,7 @@ class TestTracemallocEnabled(unittest.TestCase): def find_trace(self, traces, traceback): for trace in traces: - if trace[1] == traceback._frames: + if trace[2] == traceback._frames: return trace self.fail("trace not found") @@ -140,7 +143,7 @@ class TestTracemallocEnabled(unittest.TestCase): trace = self.find_trace(traces, obj_traceback) self.assertIsInstance(trace, tuple) - size, traceback = trace + domain, size, traceback = trace self.assertEqual(size, obj_size) self.assertEqual(traceback, obj_traceback._frames) @@ -167,9 +170,8 @@ class TestTracemallocEnabled(unittest.TestCase): trace1 = self.find_trace(traces, obj1_traceback) trace2 = self.find_trace(traces, obj2_traceback) - size1, traceback1 = trace1 - size2, traceback2 = trace2 - self.assertEqual(traceback2, traceback1) + domain1, size1, traceback1 = trace1 + domain2, size2, traceback2 = trace2 self.assertIs(traceback2, traceback1) def test_get_traced_memory(self): @@ -292,7 +294,7 @@ class TestSnapshot(unittest.TestCase): maxDiff = 4000 def test_create_snapshot(self): - raw_traces = [(5, (('a.py', 2),))] + raw_traces = [(0, 5, (('a.py', 2),))] with contextlib.ExitStack() as stack: stack.enter_context(patch.object(tracemalloc, 'is_tracing', @@ -322,11 +324,11 @@ class TestSnapshot(unittest.TestCase): # exclude b.py snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), - (7, (('', 0),)), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), + (3, 7, (('', 0),)), ]) # filter_traces() must not touch the original snapshot @@ -335,10 +337,10 @@ class TestSnapshot(unittest.TestCase): # only include two lines of a.py snapshot4 = snapshot3.filter_traces((filter2, filter3)) self.assertEqual(snapshot4.traces._traces, [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), ]) # No filter: just duplicate the snapshot @@ -349,6 +351,54 @@ class TestSnapshot(unittest.TestCase): self.assertRaises(TypeError, snapshot.filter_traces, filter1) + def test_filter_traces_domain(self): + snapshot, snapshot2 = create_snapshots() + filter1 = tracemalloc.Filter(False, "a.py", domain=1) + filter2 = tracemalloc.Filter(True, "a.py", domain=1) + + original_traces = list(snapshot.traces._traces) + + # exclude a.py of domain 1 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + (3, 7, (('', 0),)), + ]) + + # include domain 1 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + (3, 7, (('', 0),)), + ]) + + def test_filter_traces_domain_filter(self): + snapshot, snapshot2 = create_snapshots() + filter1 = tracemalloc.DomainFilter(False, domain=3) + filter2 = tracemalloc.DomainFilter(True, domain=3) + + # exclude domain 2 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + ]) + + # include domain 2 + snapshot3 = snapshot.filter_traces((filter2,)) + self.assertEqual(snapshot3.traces._traces, [ + (3, 7, (('', 0),)), + ]) + def test_snapshot_group_by_line(self): snapshot, snapshot2 = create_snapshots() tb_0 = traceback_lineno('', 0) diff --git a/Lib/tracemalloc.py b/Lib/tracemalloc.py index 6288da8..75b3918 100644 --- a/Lib/tracemalloc.py +++ b/Lib/tracemalloc.py @@ -244,17 +244,21 @@ class Trace: __slots__ = ("_trace",) def __init__(self, trace): - # trace is a tuple: (size, traceback), see Traceback constructor - # for the format of the traceback tuple + # trace is a tuple: (domain: int, size: int, traceback: tuple). + # See Traceback constructor for the format of the traceback tuple. self._trace = trace @property - def size(self): + def domain(self): return self._trace[0] @property + def size(self): + return self._trace[1] + + @property def traceback(self): - return Traceback(self._trace[1]) + return Traceback(self._trace[2]) def __eq__(self, other): return (self._trace == other._trace) @@ -266,8 +270,8 @@ class Trace: return "%s: %s" % (self.traceback, _format_size(self.size, False)) def __repr__(self): - return ("" - % (_format_size(self.size, False), self.traceback)) + return ("" + % (self.domain, _format_size(self.size, False), self.traceback)) class _Traces(Sequence): @@ -302,19 +306,29 @@ def _normalize_filename(filename): return filename -class Filter: +class BaseFilter: + def __init__(self, inclusive): + self.inclusive = inclusive + + def _match(self, trace): + raise NotImplementedError + + +class Filter(BaseFilter): def __init__(self, inclusive, filename_pattern, - lineno=None, all_frames=False): + lineno=None, all_frames=False, domain=None): + super().__init__(inclusive) self.inclusive = inclusive self._filename_pattern = _normalize_filename(filename_pattern) self.lineno = lineno self.all_frames = all_frames + self.domain = domain @property def filename_pattern(self): return self._filename_pattern - def __match_frame(self, filename, lineno): + def _match_frame_impl(self, filename, lineno): filename = _normalize_filename(filename) if not fnmatch.fnmatch(filename, self._filename_pattern): return False @@ -324,11 +338,11 @@ class Filter: return (lineno == self.lineno) def _match_frame(self, filename, lineno): - return self.__match_frame(filename, lineno) ^ (not self.inclusive) + return self._match_frame_impl(filename, lineno) ^ (not self.inclusive) def _match_traceback(self, traceback): if self.all_frames: - if any(self.__match_frame(filename, lineno) + if any(self._match_frame_impl(filename, lineno) for filename, lineno in traceback): return self.inclusive else: @@ -337,6 +351,30 @@ class Filter: filename, lineno = traceback[0] return self._match_frame(filename, lineno) + def _match(self, trace): + domain, size, traceback = trace + res = self._match_traceback(traceback) + if self.domain is not None: + if self.inclusive: + return res and (domain == self.domain) + else: + return res or (domain != self.domain) + return res + + +class DomainFilter(BaseFilter): + def __init__(self, inclusive, domain): + super().__init__(inclusive) + self._domain = domain + + @property + def domain(self): + return self._domain + + def _match(self, trace): + domain, size, traceback = trace + return (domain == self.domain) ^ (not self.inclusive) + class Snapshot: """ @@ -365,13 +403,12 @@ class Snapshot: return pickle.load(fp) def _filter_trace(self, include_filters, exclude_filters, trace): - traceback = trace[1] if include_filters: - if not any(trace_filter._match_traceback(traceback) + if not any(trace_filter._match(trace) for trace_filter in include_filters): return False if exclude_filters: - if any(not trace_filter._match_traceback(traceback) + if any(not trace_filter._match(trace) for trace_filter in exclude_filters): return False return True @@ -379,8 +416,8 @@ class Snapshot: def filter_traces(self, filters): """ Create a new Snapshot instance with a filtered traces sequence, filters - is a list of Filter instances. If filters is an empty list, return a - new Snapshot instance with a copy of the traces. + is a list of Filter or DomainFilter instances. If filters is an empty + list, return a new Snapshot instance with a copy of the traces. """ if not isinstance(filters, Iterable): raise TypeError("filters must be a list of filters, not %s" @@ -412,7 +449,7 @@ class Snapshot: tracebacks = {} if not cumulative: for trace in self.traces._traces: - size, trace_traceback = trace + domain, size, trace_traceback = trace try: traceback = tracebacks[trace_traceback] except KeyError: @@ -433,7 +470,7 @@ class Snapshot: else: # cumulative statistics for trace in self.traces._traces: - size, trace_traceback = trace + domain, size, trace_traceback = trace for frame in trace_traceback: try: traceback = tracebacks[frame] diff --git a/Misc/NEWS b/Misc/NEWS index e6d69a2..841f8a0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,9 @@ Core and Builtins Library ------- +- Issue #26588: The _tracemalloc now supports tracing memory allocations of + multiple address spaces (domains). + - Issue #24266: Ctrl+C during Readline history search now cancels the search mode when compiled with Readline 7. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index c48dd08..784157f 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -39,7 +39,11 @@ static struct { /* limit of the number of frames in a traceback, 1 by default. Variable protected by the GIL. */ int max_nframe; -} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1}; + + /* use domain in trace key? + Variable protected by the GIL. */ + int use_domain; +} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without @@ -54,10 +58,23 @@ static PyThread_type_lock tables_lock; # define TABLES_UNLOCK() #endif + +#define DEFAULT_DOMAIN 0 + +typedef unsigned int domain_t; + +/* Pack the frame_t structure to reduce the memory footprint. */ +typedef struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +{ + Py_uintptr_t ptr; + domain_t domain; +} pointer_t; + /* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. This optimization might produce - SIGBUS on architectures not supporting unaligned memory accesses (64-bit - MIPS CPU?): on such architecture, the structure must not be packed. */ + architectures: 12 bytes instead of 16. */ typedef struct #ifdef __GNUC__ __attribute__((packed)) @@ -71,6 +88,7 @@ _declspec(align(4)) unsigned int lineno; } frame_t; + typedef struct { Py_uhash_t hash; int nframe; @@ -83,6 +101,7 @@ typedef struct { #define MAX_NFRAME \ ((INT_MAX - (int)sizeof(traceback_t)) / (int)sizeof(frame_t) + 1) + static PyObject *unknown_filename = NULL; static traceback_t tracemalloc_empty_traceback; @@ -95,6 +114,7 @@ typedef struct { traceback_t *traceback; } trace_t; + /* Size in bytes of currently traced memory. Protected by TABLES_LOCK(). */ static size_t tracemalloc_traced_memory = 0; @@ -121,6 +141,7 @@ static _Py_hashtable_t *tracemalloc_tracebacks = NULL; Protected by TABLES_LOCK(). */ static _Py_hashtable_t *tracemalloc_traces = NULL; + #ifdef TRACE_DEBUG static void tracemalloc_error(const char *format, ...) @@ -135,6 +156,7 @@ tracemalloc_error(const char *format, ...) } #endif + #if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) #define REENTRANT_THREADLOCAL @@ -196,6 +218,7 @@ set_reentrant(int reentrant) } #endif + static Py_uhash_t hashtable_hash_pyobject(size_t key_size, const void *pkey) { @@ -205,21 +228,53 @@ hashtable_hash_pyobject(size_t key_size, const void *pkey) return PyObject_Hash(obj); } + static int hashtable_compare_unicode(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *entry) { - PyObject *key, *entry_key; + PyObject *key1, *key2; - _Py_HASHTABLE_READ_KEY(key_size, pkey, key); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, entry_key); + _Py_HASHTABLE_READ_KEY(key_size, pkey, key1); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, key2); - if (key != NULL && entry_key != NULL) - return (PyUnicode_Compare(key, entry_key) == 0); + if (key1 != NULL && key2 != NULL) + return (PyUnicode_Compare(key1, key2) == 0); else - return key == entry_key; + return key1 == key2; +} + + +static Py_uhash_t +hashtable_hash_pointer_t(size_t key_size, const void *pkey) +{ + pointer_t ptr; + Py_uhash_t hash; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr); + + hash = (Py_uhash_t)_Py_HashPointer((void*)ptr.ptr); + hash ^= ptr.domain; + return hash; +} + + +int +hashtable_compare_pointer_t(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) +{ + pointer_t ptr1, ptr2; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr1); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, ptr2); + + /* compare pointer before domain, because pointer is more likely to be + different */ + return (ptr1.ptr == ptr2.ptr && ptr1.domain == ptr2.domain); + } + static _Py_hashtable_t * hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, @@ -231,6 +286,7 @@ hashtable_new(size_t key_size, size_t data_size, &hashtable_alloc); } + static void* raw_malloc(size_t size) { @@ -243,6 +299,7 @@ raw_free(void *ptr) allocators.raw.free(allocators.raw.ctx, ptr); } + static Py_uhash_t hashtable_hash_traceback(size_t key_size, const void *pkey) { @@ -252,6 +309,7 @@ hashtable_hash_traceback(size_t key_size, const void *pkey) return traceback->hash; } + static int hashtable_compare_traceback(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he) @@ -281,6 +339,7 @@ hashtable_compare_traceback(size_t key_size, const void *pkey, return 1; } + static void tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) { @@ -353,6 +412,7 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) frame->filename = filename; } + static Py_uhash_t traceback_hash(traceback_t *traceback) { @@ -377,6 +437,7 @@ traceback_hash(traceback_t *traceback) return x; } + static void traceback_get_frames(traceback_t *traceback) { @@ -404,6 +465,7 @@ traceback_get_frames(traceback_t *traceback) } } + static traceback_t * traceback_new(void) { @@ -455,41 +517,72 @@ traceback_new(void) return traceback; } + +static void +tracemalloc_remove_trace(domain_t domain, Py_uintptr_t ptr) +{ + trace_t trace; + int removed; + + if (tracemalloc_config.use_domain) { + pointer_t key = {ptr, domain}; + removed = _Py_HASHTABLE_POP(tracemalloc_traces, key, trace); + } + else { + removed = _Py_HASHTABLE_POP(tracemalloc_traces, ptr, trace); + } + if (!removed) { + return; + } + + assert(tracemalloc_traced_memory >= trace.size); + tracemalloc_traced_memory -= trace.size; +} + +#define REMOVE_TRACE(ptr) \ + tracemalloc_remove_trace(DEFAULT_DOMAIN, (Py_uintptr_t)(ptr)) + + static int -tracemalloc_add_trace(void *ptr, size_t size) +tracemalloc_add_trace(domain_t domain, Py_uintptr_t ptr, size_t size) { traceback_t *traceback; trace_t trace; int res; + /* first, remove the previous trace (if any) */ + tracemalloc_remove_trace(domain, ptr); + traceback = traceback_new(); - if (traceback == NULL) + if (traceback == NULL) { return -1; + } trace.size = size; trace.traceback = traceback; - res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); - if (res == 0) { - assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); - tracemalloc_traced_memory += size; - if (tracemalloc_traced_memory > tracemalloc_peak_traced_memory) - tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + if (tracemalloc_config.use_domain) { + pointer_t key = {ptr, domain}; + res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + } + else { + res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + } + + if (res != 0) { + return res; } - return res; + assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); + tracemalloc_traced_memory += size; + if (tracemalloc_traced_memory > tracemalloc_peak_traced_memory) + tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + return 0; } -static void -tracemalloc_remove_trace(void *ptr) -{ - trace_t trace; +#define ADD_TRACE(ptr, size) \ + tracemalloc_add_trace(DEFAULT_DOMAIN, (Py_uintptr_t)(ptr), size) - if (_Py_HASHTABLE_POP(tracemalloc_traces, ptr, trace)) { - assert(tracemalloc_traced_memory >= trace.size); - tracemalloc_traced_memory -= trace.size; - } -} static void* tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) @@ -507,7 +600,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) { + if (ADD_TRACE(ptr, nelem * elsize) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr); @@ -517,6 +610,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return ptr; } + static void* tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) { @@ -531,9 +625,9 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (ADD_TRACE(ptr2, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to the caller, because realloc() may already have shrinked the memory block and so removed bytes. @@ -551,7 +645,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* new allocation */ TABLES_LOCK(); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (ADD_TRACE(ptr2, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); @@ -562,6 +656,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) return ptr2; } + static void tracemalloc_free(void *ctx, void *ptr) { @@ -576,10 +671,11 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); TABLES_UNLOCK(); } + static void* tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize) { @@ -604,18 +700,21 @@ tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize) return ptr; } + static void* tracemalloc_malloc_gil(void *ctx, size_t size) { return tracemalloc_alloc_gil(0, ctx, 1, size); } + static void* tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize) { return tracemalloc_alloc_gil(1, ctx, nelem, elsize); } + static void* tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) { @@ -631,7 +730,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); TABLES_UNLOCK(); } return ptr2; @@ -648,6 +747,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) return ptr2; } + #ifdef TRACE_RAW_MALLOC static void* tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) @@ -682,18 +782,21 @@ tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return ptr; } + static void* tracemalloc_raw_malloc(void *ctx, size_t size) { return tracemalloc_raw_alloc(0, ctx, 1, size); } + static void* tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize) { return tracemalloc_raw_alloc(1, ctx, nelem, elsize); } + static void* tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) { @@ -710,7 +813,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); TABLES_UNLOCK(); } return ptr2; @@ -734,6 +837,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) } #endif /* TRACE_RAW_MALLOC */ + static int tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *user_data) @@ -745,6 +849,7 @@ tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, return 0; } + static int traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *user_data) @@ -756,6 +861,7 @@ traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, return 0; } + /* reentrant flag must be set to call this function and GIL must be held */ static void tracemalloc_clear_traces(void) @@ -782,6 +888,7 @@ tracemalloc_clear_traces(void) _Py_hashtable_clear(tracemalloc_filenames); } + static int tracemalloc_init(void) { @@ -826,9 +933,18 @@ tracemalloc_init(void) hashtable_hash_traceback, hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(void*), sizeof(trace_t), - _Py_hashtable_hash_ptr, - _Py_hashtable_compare_direct); + if (tracemalloc_config.use_domain) { + tracemalloc_traces = hashtable_new(sizeof(pointer_t), + sizeof(trace_t), + hashtable_hash_pointer_t, + hashtable_compare_pointer_t); + } + else { + tracemalloc_traces = hashtable_new(sizeof(Py_uintptr_t), + sizeof(trace_t), + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + } if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL || tracemalloc_traces == NULL) { @@ -856,6 +972,7 @@ tracemalloc_init(void) return 0; } + static void tracemalloc_deinit(void) { @@ -884,6 +1001,7 @@ tracemalloc_deinit(void) Py_XDECREF(unknown_filename); } + static int tracemalloc_start(int max_nframe) { @@ -941,6 +1059,7 @@ tracemalloc_start(int max_nframe) return 0; } + static void tracemalloc_stop(void) { @@ -974,6 +1093,7 @@ PyDoc_STRVAR(tracemalloc_is_tracing_doc, "True if the tracemalloc module is tracing Python memory allocations,\n" "False otherwise."); + static PyObject* py_tracemalloc_is_tracing(PyObject *self) { @@ -985,6 +1105,7 @@ PyDoc_STRVAR(tracemalloc_clear_traces_doc, "\n" "Clear traces of memory blocks allocated by Python."); + static PyObject* py_tracemalloc_clear_traces(PyObject *self) { @@ -998,6 +1119,7 @@ py_tracemalloc_clear_traces(PyObject *self) Py_RETURN_NONE; } + static PyObject* frame_to_pyobject(frame_t *frame) { @@ -1020,6 +1142,7 @@ frame_to_pyobject(frame_t *frame) return frame_obj; } + static PyObject* traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) { @@ -1058,33 +1181,43 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) return frames; } + static PyObject* -trace_to_pyobject(trace_t *trace, _Py_hashtable_t *intern_tracebacks) +trace_to_pyobject(domain_t domain, trace_t *trace, + _Py_hashtable_t *intern_tracebacks) { PyObject *trace_obj = NULL; - PyObject *size, *traceback; + PyObject *obj; - trace_obj = PyTuple_New(2); + trace_obj = PyTuple_New(3); if (trace_obj == NULL) return NULL; - size = PyLong_FromSize_t(trace->size); - if (size == NULL) { + obj = PyLong_FromSize_t(domain); + if (obj == NULL) { Py_DECREF(trace_obj); return NULL; } - PyTuple_SET_ITEM(trace_obj, 0, size); + PyTuple_SET_ITEM(trace_obj, 0, obj); - traceback = traceback_to_pyobject(trace->traceback, intern_tracebacks); - if (traceback == NULL) { + obj = PyLong_FromSize_t(trace->size); + if (obj == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 1, obj); + + obj = traceback_to_pyobject(trace->traceback, intern_tracebacks); + if (obj == NULL) { Py_DECREF(trace_obj); return NULL; } - PyTuple_SET_ITEM(trace_obj, 1, traceback); + PyTuple_SET_ITEM(trace_obj, 2, obj); return trace_obj; } + typedef struct { _Py_hashtable_t *traces; _Py_hashtable_t *tracebacks; @@ -1096,13 +1229,22 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr void *user_data) { get_traces_t *get_traces = user_data; + domain_t domain; trace_t *trace; PyObject *tracemalloc_obj; int res; + if (tracemalloc_config.use_domain) { + pointer_t key; + _Py_HASHTABLE_ENTRY_READ_KEY(traces->key_size, entry, key); + domain = key.domain; + } + else { + domain = DEFAULT_DOMAIN; + } trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); - tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); + tracemalloc_obj = trace_to_pyobject(domain, trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) return 1; @@ -1114,6 +1256,7 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr return 0; } + static int tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, _Py_hashtable_entry_t *entry, @@ -1125,6 +1268,7 @@ tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, return 0; } + PyDoc_STRVAR(tracemalloc_get_traces_doc, "_get_traces() -> list\n" "\n" @@ -1194,8 +1338,9 @@ finally: return get_traces.list; } + static traceback_t* -tracemalloc_get_traceback(const void *ptr) +tracemalloc_get_traceback(domain_t domain, const void *ptr) { trace_t trace; int found; @@ -1204,7 +1349,13 @@ tracemalloc_get_traceback(const void *ptr) return NULL; TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + if (tracemalloc_config.use_domain) { + pointer_t key = {(Py_uintptr_t)ptr, domain}; + found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); + } + else { + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + } TABLES_UNLOCK(); if (!found) @@ -1213,6 +1364,7 @@ tracemalloc_get_traceback(const void *ptr) return trace.traceback; } + PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, "_get_object_traceback(obj)\n" "\n" @@ -1235,13 +1387,14 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); if (traceback == NULL) Py_RETURN_NONE; return traceback_to_pyobject(traceback, NULL); } + #define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) static void @@ -1262,7 +1415,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); if (traceback == NULL) return; @@ -1275,6 +1428,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) #undef PUTS + PyDoc_STRVAR(tracemalloc_start_doc, "start(nframe: int=1)\n" "\n" @@ -1310,6 +1464,7 @@ PyDoc_STRVAR(tracemalloc_stop_doc, "Stop tracing Python memory allocations and clear traces\n" "of memory blocks allocated by Python."); + static PyObject* py_tracemalloc_stop(PyObject *self) { @@ -1317,6 +1472,7 @@ py_tracemalloc_stop(PyObject *self) Py_RETURN_NONE; } + PyDoc_STRVAR(tracemalloc_get_traceback_limit_doc, "get_traceback_limit() -> int\n" "\n" @@ -1332,6 +1488,7 @@ py_tracemalloc_get_traceback_limit(PyObject *self) return PyLong_FromLong(tracemalloc_config.max_nframe); } + PyDoc_STRVAR(tracemalloc_get_tracemalloc_memory_doc, "get_tracemalloc_memory() -> int\n" "\n" @@ -1355,6 +1512,7 @@ tracemalloc_get_tracemalloc_memory(PyObject *self) return Py_BuildValue("N", size_obj); } + PyDoc_STRVAR(tracemalloc_get_traced_memory_doc, "get_traced_memory() -> (int, int)\n" "\n" @@ -1380,6 +1538,7 @@ tracemalloc_get_traced_memory(PyObject *self) return Py_BuildValue("NN", size_obj, peak_size_obj); } + static PyMethodDef module_methods[] = { {"is_tracing", (PyCFunction)py_tracemalloc_is_tracing, METH_NOARGS, tracemalloc_is_tracing_doc}, @@ -1430,6 +1589,7 @@ PyInit__tracemalloc(void) return m; } + static int parse_sys_xoptions(PyObject *value) { @@ -1458,6 +1618,7 @@ parse_sys_xoptions(PyObject *value) return Py_SAFE_DOWNCAST(nframe, long, int); } + int _PyTraceMalloc_Init(void) { @@ -1516,6 +1677,7 @@ _PyTraceMalloc_Init(void) return tracemalloc_start(nframe); } + void _PyTraceMalloc_Fini(void) { @@ -1524,4 +1686,3 @@ _PyTraceMalloc_Fini(void) #endif tracemalloc_deinit(); } - -- cgit v0.12 From 10b73e17489048419b512f6710aecba62ff5b91a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 13:39:05 +0100 Subject: Add C functions _PyTraceMalloc_Track() Issue #26530: * Add C functions _PyTraceMalloc_Track() and _PyTraceMalloc_Untrack() to track memory blocks using the tracemalloc module. * Add _PyTraceMalloc_GetTraceback() to get the traceback of an object. --- Include/pymem.h | 34 +++++++++++++ Lib/test/test_tracemalloc.py | 115 +++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 5 ++ Modules/_testcapimodule.c | 75 ++++++++++++++++++++++++++++ Modules/_tracemalloc.c | 82 +++++++++++++++++++++++++----- 5 files changed, 300 insertions(+), 11 deletions(-) diff --git a/Include/pymem.h b/Include/pymem.h index b1f06ef..941e00f 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -25,6 +25,40 @@ PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt); PyAPI_FUNC(int) _PyMem_PymallocEnabled(void); #endif +/* Identifier of an address space (domain) in tracemalloc */ +typedef unsigned int _PyTraceMalloc_domain_t; + +/* Track an allocated memory block in the tracemalloc module. + Return 0 on success, return -1 on error (failed to allocate memory to store + the trace). + + Return -2 if tracemalloc is disabled. + + If memory block was already tracked, begin by removing the old trace. */ +PyAPI_FUNC(int) _PyTraceMalloc_Track( + _PyTraceMalloc_domain_t domain, + Py_uintptr_t ptr, + size_t size); + +/* Untrack an allocated memory block in the tracemalloc module. + Do nothing if the block was not tracked. + + Return -2 if tracemalloc is disabled, otherwise return 0. */ +PyAPI_FUNC(int) _PyTraceMalloc_Untrack( + _PyTraceMalloc_domain_t domain, + Py_uintptr_t ptr); + +/* Get the traceback where a memory block was allocated. + + Return a tuple of (filename: str, lineno: int) tuples. + + Return None if the tracemalloc module is disabled or if the memory block + is not tracked by tracemalloc. + + Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( + _PyTraceMalloc_domain_t domain, + Py_uintptr_t ptr); #endif /* !Py_LIMITED_API */ diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 7b92b87..359d9c0 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -11,9 +11,15 @@ try: import threading except ImportError: threading = None +try: + import _testcapi +except ImportError: + _testcapi = None + EMPTY_STRING_SIZE = sys.getsizeof(b'') + def get_frames(nframe, lineno_delta): frames = [] frame = sys._getframe(1) @@ -866,12 +872,121 @@ class TestCommandLine(unittest.TestCase): assert_python_ok('-X', 'tracemalloc', '-c', code) +@unittest.skipIf(_testcapi is None, 'need _testcapi') +class TestCAPI(unittest.TestCase): + maxDiff = 80 * 20 + + def setUp(self): + if tracemalloc.is_tracing(): + self.skipTest("tracemalloc must be stopped before the test") + + self.domain = 5 + self.size = 123 + self.obj = allocate_bytes(self.size)[0] + + # for the type "object", id(obj) is the address of its memory block. + # This type is not tracked by the garbage collector + self.ptr = id(self.obj) + + def tearDown(self): + tracemalloc.stop() + + def get_traceback(self): + frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) + if frames is not None: + return tracemalloc.Traceback(frames) + else: + return None + + def track(self, release_gil=False, nframe=1): + frames = get_frames(nframe, 2) + _testcapi.tracemalloc_track(self.domain, self.ptr, self.size, + release_gil) + return frames + + def untrack(self): + _testcapi.tracemalloc_untrack(self.domain, self.ptr) + + def get_traced_memory(self): + # Get the traced size in the domain + snapshot = tracemalloc.take_snapshot() + domain_filter = tracemalloc.DomainFilter(True, self.domain) + snapshot = snapshot.filter_traces([domain_filter]) + return sum(trace.size for trace in snapshot.traces) + + def check_track(self, release_gil): + nframe = 5 + tracemalloc.start(nframe) + + size = tracemalloc.get_traced_memory()[0] + + frames = self.track(release_gil, nframe) + self.assertEqual(self.get_traceback(), + tracemalloc.Traceback(frames)) + + self.assertEqual(self.get_traced_memory(), self.size) + + def test_track(self): + self.check_track(False) + + def test_track_without_gil(self): + # check that calling _PyTraceMalloc_Track() without holding the GIL + # works too + self.check_track(True) + + def test_track_already_tracked(self): + nframe = 5 + tracemalloc.start(nframe) + + # track a first time + self.track() + + # calling _PyTraceMalloc_Track() must remove the old trace and add + # a new trace with the new traceback + frames = self.track(nframe=nframe) + self.assertEqual(self.get_traceback(), + tracemalloc.Traceback(frames)) + + def test_untrack(self): + tracemalloc.start() + + self.track() + self.assertIsNotNone(self.get_traceback()) + self.assertEqual(self.get_traced_memory(), self.size) + + # untrack must remove the trace + self.untrack() + self.assertIsNone(self.get_traceback()) + self.assertEqual(self.get_traced_memory(), 0) + + # calling _PyTraceMalloc_Untrack() multiple times must not crash + self.untrack() + self.untrack() + + def test_stop_track(self): + tracemalloc.start() + tracemalloc.stop() + + with self.assertRaises(RuntimeError): + self.track() + self.assertIsNone(self.get_traceback()) + + def test_stop_untrack(self): + tracemalloc.start() + self.track() + + tracemalloc.stop() + with self.assertRaises(RuntimeError): + self.untrack() + + def test_main(): support.run_unittest( TestTracemallocEnabled, TestSnapshot, TestFilters, TestCommandLine, + TestCAPI, ) if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index 841f8a0..29fc65d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,11 @@ Core and Builtins Library ------- +- Issue #26530: Add C functions :c:func:`_PyTraceMalloc_Track` and + :c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the + :mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get + the traceback of an object. + - Issue #26588: The _tracemalloc now supports tracing memory allocations of multiple address spaces (domains). diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0fc7cbc..8c79485 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3675,6 +3675,78 @@ pyobject_malloc_without_gil(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +tracemalloc_track(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + void *ptr; + Py_ssize_t size; + int release_gil = 0; + int res; + + if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil)) + return NULL; + ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) + return NULL; + + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = _PyTraceMalloc_Track(domain, (Py_uintptr_t)ptr, size); + Py_END_ALLOW_THREADS + } + else { + res = _PyTraceMalloc_Track(domain, (Py_uintptr_t)ptr, size); + } + + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "_PyTraceMalloc_Track error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_untrack(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + void *ptr; + int res; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) + return NULL; + ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) + return NULL; + + res = _PyTraceMalloc_Untrack(domain, (Py_uintptr_t)ptr); + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "_PyTraceMalloc_Track error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_get_traceback(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + void *ptr; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) + return NULL; + ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) + return NULL; + + return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr); +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3861,6 +3933,9 @@ static PyMethodDef TestMethods[] = { {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS}, {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS}, + {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, + {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, + {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 784157f..5ff1f84 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -61,8 +61,6 @@ static PyThread_type_lock tables_lock; #define DEFAULT_DOMAIN 0 -typedef unsigned int domain_t; - /* Pack the frame_t structure to reduce the memory footprint. */ typedef struct #ifdef __GNUC__ @@ -70,7 +68,7 @@ __attribute__((packed)) #endif { Py_uintptr_t ptr; - domain_t domain; + _PyTraceMalloc_domain_t domain; } pointer_t; /* Pack the frame_t structure to reduce the memory footprint on 64-bit @@ -519,11 +517,13 @@ traceback_new(void) static void -tracemalloc_remove_trace(domain_t domain, Py_uintptr_t ptr) +tracemalloc_remove_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { trace_t trace; int removed; + assert(tracemalloc_config.tracing); + if (tracemalloc_config.use_domain) { pointer_t key = {ptr, domain}; removed = _Py_HASHTABLE_POP(tracemalloc_traces, key, trace); @@ -544,12 +544,15 @@ tracemalloc_remove_trace(domain_t domain, Py_uintptr_t ptr) static int -tracemalloc_add_trace(domain_t domain, Py_uintptr_t ptr, size_t size) +tracemalloc_add_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, + size_t size) { traceback_t *traceback; trace_t trace; int res; + assert(tracemalloc_config.tracing); + /* first, remove the previous trace (if any) */ tracemalloc_remove_trace(domain, ptr); @@ -1183,7 +1186,7 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) static PyObject* -trace_to_pyobject(domain_t domain, trace_t *trace, +trace_to_pyobject(_PyTraceMalloc_domain_t domain, trace_t *trace, _Py_hashtable_t *intern_tracebacks) { PyObject *trace_obj = NULL; @@ -1229,7 +1232,7 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr void *user_data) { get_traces_t *get_traces = user_data; - domain_t domain; + _PyTraceMalloc_domain_t domain; trace_t *trace; PyObject *tracemalloc_obj; int res; @@ -1340,7 +1343,7 @@ finally: static traceback_t* -tracemalloc_get_traceback(domain_t domain, const void *ptr) +tracemalloc_get_traceback(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { trace_t trace; int found; @@ -1350,7 +1353,7 @@ tracemalloc_get_traceback(domain_t domain, const void *ptr) TABLES_LOCK(); if (tracemalloc_config.use_domain) { - pointer_t key = {(Py_uintptr_t)ptr, domain}; + pointer_t key = {ptr, domain}; found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); } else { @@ -1387,7 +1390,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (Py_uintptr_t)ptr); if (traceback == NULL) Py_RETURN_NONE; @@ -1415,7 +1418,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (Py_uintptr_t)ptr); if (traceback == NULL) return; @@ -1686,3 +1689,60 @@ _PyTraceMalloc_Fini(void) #endif tracemalloc_deinit(); } + +int +_PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, + size_t size) +{ + int res; +#ifdef WITH_THREAD + PyGILState_STATE gil_state; +#endif + + if (!tracemalloc_config.tracing) { + /* tracemalloc is not tracing: do nothing */ + return -2; + } + +#ifdef WITH_THREAD + gil_state = PyGILState_Ensure(); +#endif + + TABLES_LOCK(); + res = tracemalloc_add_trace(domain, ptr, size); + TABLES_UNLOCK(); + +#ifdef WITH_THREAD + PyGILState_Release(gil_state); +#endif + return res; +} + + +int +_PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) +{ + if (!tracemalloc_config.tracing) { + /* tracemalloc is not tracing: do nothing */ + return -2; + } + + TABLES_LOCK(); + tracemalloc_remove_trace(domain, ptr); + TABLES_UNLOCK(); + + return 0; +} + + +PyObject* +_PyTraceMalloc_GetTraceback(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) +{ + traceback_t *traceback; + + traceback = tracemalloc_get_traceback(domain, ptr); + if (traceback == NULL) + Py_RETURN_NONE; + + return traceback_to_pyobject(traceback, NULL); +} -- cgit v0.12 From 24f949e10c4031c059118dd4b49cace16d4e3e5d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 15:14:09 +0100 Subject: regrtest: add time to output Timestamps should help to debug slow buildbots, and timeout and hang on buildbots. --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++++----- Lib/test/test_regrtest.py | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 82788ad..1c99f2b 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import datetime import faulthandler import os import platform @@ -7,6 +8,7 @@ import sys import sysconfig import tempfile import textwrap +import time from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, @@ -79,6 +81,7 @@ class Regrtest: self.found_garbage = [] # used to display the progress bar "[ 3/100]" + self.start_time = time.monotonic() self.test_count = '' self.test_count_width = 1 @@ -102,16 +105,24 @@ class Regrtest: self.skipped.append(test) self.resource_denieds.append(test) + def time_delta(self): + seconds = time.monotonic() - self.start_time + return datetime.timedelta(seconds=int(seconds)) + def display_progress(self, test_index, test): if self.ns.quiet: return if self.bad and not self.ns.pgo: - fmt = "[{1:{0}}{2}/{3}] {4}" + fmt = "{time} [{test_index:{count_width}}{test_count}/{nbad}] {test_name}" else: - fmt = "[{1:{0}}{2}] {4}" - print(fmt.format(self.test_count_width, test_index, - self.test_count, len(self.bad), test), - flush=True) + fmt = "{time} [{test_index:{count_width}}{test_count}] {test_name}" + line = fmt.format(count_width=self.test_count_width, + test_index=test_index, + test_count=self.test_count, + nbad=len(self.bad), + test_name=test, + time=self.time_delta()) + print(line, flush=True) def parse_args(self, kwargs): ns = _parse_args(sys.argv[1:], **kwargs) @@ -368,6 +379,8 @@ class Regrtest: r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) + print("Total duration: %s" % self.time_delta()) + if self.ns.runleaks: os.system("leaks %d" % os.getpid()) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 59f8c9d..18d0f46 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -351,7 +351,7 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = r'^\[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + regex = r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) -- cgit v0.12 From 84aab09421330711607671ed8bc637ea2a8e273c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 16:13:31 +0100 Subject: Issue #26588: add debug traces Try to debug random failure on buildbots. --- Lib/test/test_tracemalloc.py | 10 ++++++++++ Modules/_testcapimodule.c | 14 ++++++++++++++ Modules/_tracemalloc.c | 40 +++++++++++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 359d9c0..44cd539 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -88,6 +88,9 @@ def traceback_filename(filename): class TestTracemallocEnabled(unittest.TestCase): def setUp(self): + if _testcapi: + _testcapi.tracemalloc_set_debug(True) + if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -95,6 +98,8 @@ class TestTracemallocEnabled(unittest.TestCase): def tearDown(self): tracemalloc.stop() + if _testcapi: + _testcapi.tracemalloc_set_debug(False) def test_get_tracemalloc_memory(self): data = [allocate_bytes(123) for count in range(1000)] @@ -877,6 +882,9 @@ class TestCAPI(unittest.TestCase): maxDiff = 80 * 20 def setUp(self): + if _testcapi: + _testcapi.tracemalloc_set_debug(True) + if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -890,6 +898,8 @@ class TestCAPI(unittest.TestCase): def tearDown(self): tracemalloc.stop() + if _testcapi: + _testcapi.tracemalloc_set_debug(False) def get_traceback(self): frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 8c79485..f952aec 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3747,6 +3747,19 @@ tracemalloc_get_traceback(PyObject *self, PyObject *args) return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr); } +PyObject* +tracemalloc_set_debug(PyObject *self, PyObject *args) +{ + int debug; + extern int tracemalloc_debug; + + if (!PyArg_ParseTuple(args, "i", &debug)) + return NULL; + + tracemalloc_debug = debug; + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3936,6 +3949,7 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, + {"tracemalloc_set_debug", tracemalloc_set_debug, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5ff1f84..551bade 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -45,6 +45,8 @@ static struct { int use_domain; } tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; +int tracemalloc_debug = 0; + #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it @@ -891,18 +893,24 @@ tracemalloc_clear_traces(void) _Py_hashtable_clear(tracemalloc_filenames); } +#define DEBUG(MSG) \ + if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li] " MSG "\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); } + static int tracemalloc_init(void) { +DEBUG("tracemalloc_init()"); if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { PyErr_SetString(PyExc_RuntimeError, "the tracemalloc module has been unloaded"); return -1; } - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) + if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) { +DEBUG("tracemalloc_init(): exit (already initialized)"); return 0; + } PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); @@ -969,9 +977,11 @@ tracemalloc_init(void) /* Disable tracing allocations until hooks are installed. Set also the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ +DEBUG("tracemalloc_init(): set_reentrant(1)"); set_reentrant(1); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; +DEBUG("tracemalloc_init(): done"); return 0; } @@ -979,8 +989,11 @@ tracemalloc_init(void) static void tracemalloc_deinit(void) { - if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) +DEBUG("tracemalloc_deinit()"); + if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) { +DEBUG("tracemalloc_deinit(): exit (not initialized)"); return; + } tracemalloc_config.initialized = TRACEMALLOC_FINALIZED; tracemalloc_stop(); @@ -997,11 +1010,13 @@ tracemalloc_deinit(void) } #endif +DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL PyThread_delete_key(tracemalloc_reentrant_key); #endif Py_XDECREF(unknown_filename); +DEBUG("tracemalloc_deinit(): done"); } @@ -1011,11 +1026,15 @@ tracemalloc_start(int max_nframe) PyMemAllocatorEx alloc; size_t size; - if (tracemalloc_init() < 0) +DEBUG("tracemalloc_start()"); + if (tracemalloc_init() < 0) { +DEBUG("tracemalloc_start(): ERROR! init failed!"); return -1; + } if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ +DEBUG("tracemalloc_start(): exit (already tracing)"); return 0; } @@ -1057,8 +1076,11 @@ tracemalloc_start(int max_nframe) /* everything is ready: start tracing Python memory allocations */ tracemalloc_config.tracing = 1; + +DEBUG("tracemalloc_start(): set_reentrant(0)"); set_reentrant(0); +DEBUG("tracemalloc_start(): done"); return 0; } @@ -1066,14 +1088,18 @@ tracemalloc_start(int max_nframe) static void tracemalloc_stop(void) { - if (!tracemalloc_config.tracing) +DEBUG("tracemalloc_stop()"); + if (!tracemalloc_config.tracing) { +DEBUG("tracemalloc_stop(): exit (not tracing)"); return; + } /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; /* set the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ +DEBUG("tracemalloc_stop(): set_reentrant(1)"); set_reentrant(1); /* unregister the hook on memory allocators */ @@ -1088,6 +1114,7 @@ tracemalloc_stop(void) /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; +DEBUG("tracemalloc_stop(): done"); } PyDoc_STRVAR(tracemalloc_is_tracing_doc, @@ -1455,8 +1482,11 @@ py_tracemalloc_start(PyObject *self, PyObject *args) } nframe_int = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); - if (tracemalloc_start(nframe_int) < 0) + if (tracemalloc_start(nframe_int) < 0) { +DEBUG("start(): ERROR!"); return NULL; + } +DEBUG("start(): done"); Py_RETURN_NONE; } -- cgit v0.12 From ff9c5346ea4388bb0f77c9f92415fbfbc92e429f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 16:29:02 +0100 Subject: Issue #26588: fix compilation on Windows --- Include/pymem.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Include/pymem.h b/Include/pymem.h index 941e00f..3fac523 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -59,6 +59,8 @@ PyAPI_FUNC(int) _PyTraceMalloc_Untrack( PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( _PyTraceMalloc_domain_t domain, Py_uintptr_t ptr); + +PyAPI_DATA(int) tracemalloc_debug; #endif /* !Py_LIMITED_API */ -- cgit v0.12 From 0cfc058d619cea398baf34bfdd94b52620275f88 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 17:40:07 +0100 Subject: Issue #26588: more assertions --- Modules/_tracemalloc.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 551bade..baeb58c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -189,11 +189,11 @@ set_reentrant(int reentrant) { assert(reentrant == 0 || reentrant == 1); if (reentrant) { - assert(PyThread_get_key_value(tracemalloc_reentrant_key) == NULL); + assert(!get_reentrant()); PyThread_set_key_value(tracemalloc_reentrant_key, REENTRANT); } else { - assert(PyThread_get_key_value(tracemalloc_reentrant_key) == REENTRANT); + assert(get_reentrant()); PyThread_set_key_value(tracemalloc_reentrant_key, NULL); } } @@ -901,6 +901,11 @@ static int tracemalloc_init(void) { DEBUG("tracemalloc_init()"); + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { PyErr_SetString(PyExc_RuntimeError, "the tracemalloc module has been unloaded"); @@ -1027,6 +1032,11 @@ tracemalloc_start(int max_nframe) size_t size; DEBUG("tracemalloc_start()"); + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (tracemalloc_init() < 0) { DEBUG("tracemalloc_start(): ERROR! init failed!"); return -1; @@ -1035,8 +1045,10 @@ DEBUG("tracemalloc_start(): ERROR! init failed!"); if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ DEBUG("tracemalloc_start(): exit (already tracing)"); +assert(!get_reentrant()); return 0; } +assert(get_reentrant()); assert(1 <= max_nframe && max_nframe <= MAX_NFRAME); tracemalloc_config.max_nframe = max_nframe; @@ -1081,6 +1093,7 @@ DEBUG("tracemalloc_start(): set_reentrant(0)"); set_reentrant(0); DEBUG("tracemalloc_start(): done"); +assert(!get_reentrant()); return 0; } @@ -1089,10 +1102,17 @@ static void tracemalloc_stop(void) { DEBUG("tracemalloc_stop()"); + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (!tracemalloc_config.tracing) { DEBUG("tracemalloc_stop(): exit (not tracing)"); +assert(get_reentrant()); return; } +assert(!get_reentrant()); /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; @@ -1115,6 +1135,7 @@ DEBUG("tracemalloc_stop(): set_reentrant(1)"); raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; DEBUG("tracemalloc_stop(): done"); +assert(get_reentrant()); } PyDoc_STRVAR(tracemalloc_is_tracing_doc, -- cgit v0.12 From 4a06647534162c7a1bed0ec01d544758b56a446b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 17:45:09 +0100 Subject: Add assertions on tracemalloc_reentrant_key Issue #26588. --- Modules/_tracemalloc.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index baeb58c..5c9f69e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -167,7 +167,7 @@ tracemalloc_error(const char *format, ...) # error "need native thread local storage (TLS)" #endif -static int tracemalloc_reentrant_key; +static int tracemalloc_reentrant_key = -1; /* Any non-NULL pointer can be used */ #define REENTRANT Py_True @@ -175,7 +175,10 @@ static int tracemalloc_reentrant_key; static int get_reentrant(void) { - void *ptr = PyThread_get_key_value(tracemalloc_reentrant_key); + void *ptr; + + assert(tracemalloc_reentrant_key != -1); + ptr = PyThread_get_key_value(tracemalloc_reentrant_key); if (ptr != NULL) { assert(ptr == REENTRANT); return 1; @@ -188,6 +191,8 @@ static void set_reentrant(int reentrant) { assert(reentrant == 0 || reentrant == 1); + assert(tracemalloc_reentrant_key != -1); + if (reentrant) { assert(!get_reentrant()); PyThread_set_key_value(tracemalloc_reentrant_key, REENTRANT); @@ -1018,6 +1023,7 @@ DEBUG("tracemalloc_deinit(): exit (not initialized)"); DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL PyThread_delete_key(tracemalloc_reentrant_key); + tracemalloc_reentrant_key = -1; #endif Py_XDECREF(unknown_filename); -- cgit v0.12 From e31e35d34e9db28d36481c9921b3ffdcc65d750d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 18:48:50 +0100 Subject: Issue #26588: one more assertion --- Modules/_tracemalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5c9f69e..288942a 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -992,6 +992,7 @@ DEBUG("tracemalloc_init(): set_reentrant(1)"); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; DEBUG("tracemalloc_init(): done"); +assert(get_reentrant()); return 0; } -- cgit v0.12 From b8d927266d6c1f098b3490b897e160222d035e5c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 20:56:49 +0100 Subject: Issue #26588: Don't call tracemalloc_init() at module initilization So it's possible to get debug messages in test_tracemalloc. --- Modules/_tracemalloc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 288942a..79a7164 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1644,8 +1644,10 @@ PyInit__tracemalloc(void) if (m == NULL) return NULL; +#if 0 if (tracemalloc_init() < 0) return NULL; +#endif return m; } -- cgit v0.12 From eddb074c534e32caad468c952c7d00adcbd76a41 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 21:06:07 +0100 Subject: Issue #26588: more debug traces --- Modules/_tracemalloc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 79a7164..a674d0c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -899,7 +899,7 @@ tracemalloc_clear_traces(void) } #define DEBUG(MSG) \ - if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li] " MSG "\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); } + if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li, reentrant key %i] " MSG "\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); } static int @@ -926,6 +926,7 @@ DEBUG("tracemalloc_init(): exit (already initialized)"); #ifdef REENTRANT_THREADLOCAL tracemalloc_reentrant_key = PyThread_create_key(); +fprintf(stderr, "[pid %li, tid %li] PyThread_create_key() -> %i\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); if (tracemalloc_reentrant_key == -1) { #ifdef MS_WINDOWS PyErr_SetFromWindowsErr(0); @@ -1023,6 +1024,7 @@ DEBUG("tracemalloc_deinit(): exit (not initialized)"); DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL +fprintf(stderr, "[pid %li, tid %li] PyThread_delete_key(%i)\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); PyThread_delete_key(tracemalloc_reentrant_key); tracemalloc_reentrant_key = -1; #endif @@ -1640,6 +1642,9 @@ PyMODINIT_FUNC PyInit__tracemalloc(void) { PyObject *m; + +fprintf(stderr, "[pid %li, tid %li] PyInit__tracemalloc\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); + m = PyModule_Create(&module_def); if (m == NULL) return NULL; -- cgit v0.12 From 92c21d7a7c3a925358e606ffa7ab25cc7e73f0b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 21:26:31 +0100 Subject: Issue #26588: skip test_warnings.test_tracemalloc() --- Lib/test/test_warnings/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index e6f47cd..f2401c8 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -772,6 +772,7 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings + @unittest.skipIf(True, "FIXME: Issue #26588") def test_tracemalloc(self): self.addCleanup(support.unlink, support.TESTFN) -- cgit v0.12 From f9a71153e9122a3d2bb86fc479bdda2f2adc859e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 23:54:42 +0100 Subject: Issue #26588: remove debug traces from _tracemalloc. --- Include/pymem.h | 2 -- Lib/test/test_tracemalloc.py | 10 ------ Lib/test/test_warnings/__init__.py | 1 - Modules/_testcapimodule.c | 14 -------- Modules/_tracemalloc.c | 69 +++----------------------------------- 5 files changed, 5 insertions(+), 91 deletions(-) diff --git a/Include/pymem.h b/Include/pymem.h index 3fac523..941e00f 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -59,8 +59,6 @@ PyAPI_FUNC(int) _PyTraceMalloc_Untrack( PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( _PyTraceMalloc_domain_t domain, Py_uintptr_t ptr); - -PyAPI_DATA(int) tracemalloc_debug; #endif /* !Py_LIMITED_API */ diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 44cd539..359d9c0 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -88,9 +88,6 @@ def traceback_filename(filename): class TestTracemallocEnabled(unittest.TestCase): def setUp(self): - if _testcapi: - _testcapi.tracemalloc_set_debug(True) - if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -98,8 +95,6 @@ class TestTracemallocEnabled(unittest.TestCase): def tearDown(self): tracemalloc.stop() - if _testcapi: - _testcapi.tracemalloc_set_debug(False) def test_get_tracemalloc_memory(self): data = [allocate_bytes(123) for count in range(1000)] @@ -882,9 +877,6 @@ class TestCAPI(unittest.TestCase): maxDiff = 80 * 20 def setUp(self): - if _testcapi: - _testcapi.tracemalloc_set_debug(True) - if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -898,8 +890,6 @@ class TestCAPI(unittest.TestCase): def tearDown(self): tracemalloc.stop() - if _testcapi: - _testcapi.tracemalloc_set_debug(False) def get_traceback(self): frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index f2401c8..e6f47cd 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -772,7 +772,6 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings - @unittest.skipIf(True, "FIXME: Issue #26588") def test_tracemalloc(self): self.addCleanup(support.unlink, support.TESTFN) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f952aec..8c79485 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3747,19 +3747,6 @@ tracemalloc_get_traceback(PyObject *self, PyObject *args) return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr); } -PyObject* -tracemalloc_set_debug(PyObject *self, PyObject *args) -{ - int debug; - extern int tracemalloc_debug; - - if (!PyArg_ParseTuple(args, "i", &debug)) - return NULL; - - tracemalloc_debug = debug; - Py_RETURN_NONE; -} - static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3949,7 +3936,6 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, - {"tracemalloc_set_debug", tracemalloc_set_debug, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index a674d0c..77742de 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -45,8 +45,6 @@ static struct { int use_domain; } tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; -int tracemalloc_debug = 0; - #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it @@ -898,35 +896,23 @@ tracemalloc_clear_traces(void) _Py_hashtable_clear(tracemalloc_filenames); } -#define DEBUG(MSG) \ - if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li, reentrant key %i] " MSG "\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); } - static int tracemalloc_init(void) { -DEBUG("tracemalloc_init()"); - -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { PyErr_SetString(PyExc_RuntimeError, "the tracemalloc module has been unloaded"); return -1; } - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) { -DEBUG("tracemalloc_init(): exit (already initialized)"); + if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) return 0; - } PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); #ifdef REENTRANT_THREADLOCAL tracemalloc_reentrant_key = PyThread_create_key(); -fprintf(stderr, "[pid %li, tid %li] PyThread_create_key() -> %i\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); if (tracemalloc_reentrant_key == -1) { #ifdef MS_WINDOWS PyErr_SetFromWindowsErr(0); @@ -988,12 +974,9 @@ fprintf(stderr, "[pid %li, tid %li] PyThread_create_key() -> %i\n", (long)getpid /* Disable tracing allocations until hooks are installed. Set also the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ -DEBUG("tracemalloc_init(): set_reentrant(1)"); set_reentrant(1); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; -DEBUG("tracemalloc_init(): done"); -assert(get_reentrant()); return 0; } @@ -1001,11 +984,8 @@ assert(get_reentrant()); static void tracemalloc_deinit(void) { -DEBUG("tracemalloc_deinit()"); - if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) { -DEBUG("tracemalloc_deinit(): exit (not initialized)"); + if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) return; - } tracemalloc_config.initialized = TRACEMALLOC_FINALIZED; tracemalloc_stop(); @@ -1022,15 +1002,12 @@ DEBUG("tracemalloc_deinit(): exit (not initialized)"); } #endif -DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL -fprintf(stderr, "[pid %li, tid %li] PyThread_delete_key(%i)\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); PyThread_delete_key(tracemalloc_reentrant_key); tracemalloc_reentrant_key = -1; #endif Py_XDECREF(unknown_filename); -DEBUG("tracemalloc_deinit(): done"); } @@ -1040,24 +1017,13 @@ tracemalloc_start(int max_nframe) PyMemAllocatorEx alloc; size_t size; -DEBUG("tracemalloc_start()"); - -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - - if (tracemalloc_init() < 0) { -DEBUG("tracemalloc_start(): ERROR! init failed!"); + if (tracemalloc_init() < 0) return -1; - } if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ -DEBUG("tracemalloc_start(): exit (already tracing)"); -assert(!get_reentrant()); return 0; } -assert(get_reentrant()); assert(1 <= max_nframe && max_nframe <= MAX_NFRAME); tracemalloc_config.max_nframe = max_nframe; @@ -1097,12 +1063,8 @@ assert(get_reentrant()); /* everything is ready: start tracing Python memory allocations */ tracemalloc_config.tracing = 1; - -DEBUG("tracemalloc_start(): set_reentrant(0)"); set_reentrant(0); -DEBUG("tracemalloc_start(): done"); -assert(!get_reentrant()); return 0; } @@ -1110,25 +1072,14 @@ assert(!get_reentrant()); static void tracemalloc_stop(void) { -DEBUG("tracemalloc_stop()"); - -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - - if (!tracemalloc_config.tracing) { -DEBUG("tracemalloc_stop(): exit (not tracing)"); -assert(get_reentrant()); + if (!tracemalloc_config.tracing) return; - } -assert(!get_reentrant()); /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; /* set the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ -DEBUG("tracemalloc_stop(): set_reentrant(1)"); set_reentrant(1); /* unregister the hook on memory allocators */ @@ -1143,8 +1094,6 @@ DEBUG("tracemalloc_stop(): set_reentrant(1)"); /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; -DEBUG("tracemalloc_stop(): done"); -assert(get_reentrant()); } PyDoc_STRVAR(tracemalloc_is_tracing_doc, @@ -1512,11 +1461,8 @@ py_tracemalloc_start(PyObject *self, PyObject *args) } nframe_int = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); - if (tracemalloc_start(nframe_int) < 0) { -DEBUG("start(): ERROR!"); + if (tracemalloc_start(nframe_int) < 0) return NULL; - } -DEBUG("start(): done"); Py_RETURN_NONE; } @@ -1642,17 +1588,12 @@ PyMODINIT_FUNC PyInit__tracemalloc(void) { PyObject *m; - -fprintf(stderr, "[pid %li, tid %li] PyInit__tracemalloc\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); - m = PyModule_Create(&module_def); if (m == NULL) return NULL; -#if 0 if (tracemalloc_init() < 0) return NULL; -#endif return m; } -- cgit v0.12 From 060f9bb6024580272440f57c612ca34d9beaa169 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:18:36 +0100 Subject: Fix macros in hashtable.h Add parenthesis. --- Modules/hashtable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 41542d2..4199aab 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -146,10 +146,10 @@ PyAPI_FUNC(int) _Py_hashtable_set( const void *data); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) + _Py_hashtable_set(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) #define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) + _Py_hashtable_set(TABLE, sizeof(KEY), &(KEY), 0, NULL) /* Get an entry. -- cgit v0.12 From e19558af1b6979edf27f7a05a6e47a8b5d113390 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:28:08 +0100 Subject: Add a source parameter to warnings.warn() Issue #26604: * Add a new optional source parameter to _warnings.warn() and warnings.warn() * Modify asyncore, asyncio and _pyio modules to set the source parameter when logging a ResourceWarning warning --- Doc/library/warnings.rst | 8 +++++++- Lib/_pyio.py | 2 +- Lib/asyncio/base_events.py | 3 ++- Lib/asyncio/base_subprocess.py | 3 ++- Lib/asyncio/proactor_events.py | 3 ++- Lib/asyncio/selector_events.py | 3 ++- Lib/asyncio/sslproto.py | 3 ++- Lib/asyncio/unix_events.py | 6 ++++-- Lib/asyncio/windows_utils.py | 3 ++- Lib/asyncore.py | 3 ++- Lib/tempfile.py | 1 - Lib/warnings.py | 4 ++-- Python/_warnings.c | 11 ++++++----- 13 files changed, 34 insertions(+), 19 deletions(-) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 4ce88ab..4fec365 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -300,7 +300,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1) +.. function:: warn(message, category=None, stacklevel=1, source=None) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a warning category class (see above); it defaults to @@ -318,6 +318,12 @@ Available Functions source of :func:`deprecation` itself (since the latter would defeat the purpose of the warning message). + *source*, if supplied, is the destroyed object which emitted a + :exc:`ResourceWarning`. + + .. versionchanged:: 3.6 + Added *source* parameter. + .. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index c3ad81e..972c082 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1514,7 +1514,7 @@ class FileIO(RawIOBase): if self._fd >= 0 and self._closefd and not self.closed: import warnings warnings.warn('unclosed file %r' % (self,), ResourceWarning, - stacklevel=2) + stacklevel=2, source=self) self.close() def __getstate__(self): diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 9d07673..3a42b10 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -412,7 +412,8 @@ class BaseEventLoop(events.AbstractEventLoop): if compat.PY34: def __del__(self): if not self.is_closed(): - warnings.warn("unclosed event loop %r" % self, ResourceWarning) + warnings.warn("unclosed event loop %r" % self, ResourceWarning, + source=self) if not self.is_running(): self.close() diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 73425d9..efe0831 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -122,7 +122,8 @@ class BaseSubprocessTransport(transports.SubprocessTransport): if compat.PY34: def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def get_pid(self): diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 14c0659..2671a94 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -86,7 +86,8 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 812fac1..cbb3625 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -573,7 +573,8 @@ class _SelectorTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._sock.close() def _fatal_error(self, exc, message='Fatal error on transport'): diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index dde980b..1cea850 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -324,7 +324,8 @@ class _SSLProtocolTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def pause_reading(self): diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 7747ff4..2beba3e 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -378,7 +378,8 @@ class _UnixReadPipeTransport(transports.ReadTransport): if compat.PY34: def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._pipe.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): @@ -567,7 +568,8 @@ class _UnixWritePipeTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._pipe.close() def abort(self): diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index 870cd13..7c63fb9 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -159,7 +159,8 @@ class PipeHandle: def __del__(self): if self._handle is not None: - warnings.warn("unclosed %r" % self, ResourceWarning) + warnings.warn("unclosed %r" % self, ResourceWarning, + source=self) self.close() def __enter__(self): diff --git a/Lib/asyncore.py b/Lib/asyncore.py index 3b51f0f..4b046d6 100644 --- a/Lib/asyncore.py +++ b/Lib/asyncore.py @@ -595,7 +595,8 @@ if os.name == 'posix': def __del__(self): if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning) + warnings.warn("unclosed file %r" % self, ResourceWarning, + source=self) self.close() def recv(self, *args): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index ad687b9..6146235 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -797,7 +797,6 @@ class TemporaryDirectory(object): _shutil.rmtree(name) _warnings.warn(warn_message, ResourceWarning) - def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.name) diff --git a/Lib/warnings.py b/Lib/warnings.py index 9fb21a8..d4f591e 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -233,7 +233,7 @@ def _next_external_frame(frame): # Code typically replaced by _warnings -def warn(message, category=None, stacklevel=1): +def warn(message, category=None, stacklevel=1, source=None): """Issue a warning, or maybe ignore it or raise an exception.""" # Check if message is already a Warning object if isinstance(message, Warning): @@ -283,7 +283,7 @@ def warn(message, category=None, stacklevel=1): filename = module registry = globals.setdefault("__warningregistry__", {}) warn_explicit(message, category, filename, lineno, module, registry, - globals) + globals, source) def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, diff --git a/Python/_warnings.c b/Python/_warnings.c index 25299fb..dcac57b 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -787,18 +787,19 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, static PyObject * warnings_warn(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kw_list[] = { "message", "category", "stacklevel", 0 }; - PyObject *message, *category = NULL; + static char *kw_list[] = {"message", "category", "stacklevel", + "source", NULL}; + PyObject *message, *category = NULL, *source = NULL; Py_ssize_t stack_level = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|On:warn", kw_list, - &message, &category, &stack_level)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OnO:warn", kw_list, + &message, &category, &stack_level, &source)) return NULL; category = get_category(message, category); if (category == NULL) return NULL; - return do_warn(message, category, stack_level, NULL); + return do_warn(message, category, stack_level, source); } static PyObject * -- cgit v0.12 From 7bfa409ff841fd84dfa194dd9052650d0a28585d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:43:54 +0100 Subject: Implement finalizer for os.scandir() iterator Issue #26603: * Implement finalizer for os.scandir() iterator * Set the source parameter when emitting the ResourceWarning warning * Close the iterator before emitting the warning --- Modules/posixmodule.c | 64 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 65b20be..1cd0f24 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12101,29 +12101,38 @@ ScandirIterator_exit(ScandirIterator *self, PyObject *args) } static void -ScandirIterator_dealloc(ScandirIterator *iterator) +ScandirIterator_finalize(ScandirIterator *iterator) { + PyObject *error_type, *error_value, *error_traceback; + + /* Save the current exception, if any. */ + PyErr_Fetch(&error_type, &error_value, &error_traceback); + if (!ScandirIterator_is_closed(iterator)) { - PyObject *exc, *val, *tb; - Py_ssize_t old_refcount = Py_REFCNT(iterator); - /* Py_INCREF/Py_DECREF cannot be used, because the refcount is - * likely zero, Py_DECREF would call again the destructor. - */ - ++Py_REFCNT(iterator); - PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed scandir iterator %R", iterator)) { + ScandirIterator_closedir(iterator); + + if (PyErr_ResourceWarning((PyObject *)iterator, 1, + "unclosed scandir iterator %R", iterator)) { /* Spurious errors can appear at shutdown */ - if (PyErr_ExceptionMatches(PyExc_Warning)) + if (PyErr_ExceptionMatches(PyExc_Warning)) { PyErr_WriteUnraisable((PyObject *) iterator); + } } - PyErr_Restore(exc, val, tb); - Py_REFCNT(iterator) = old_refcount; - - ScandirIterator_closedir(iterator); } - Py_XDECREF(iterator->path.object); + + Py_CLEAR(iterator->path.object); path_cleanup(&iterator->path); + + /* Restore the saved exception. */ + PyErr_Restore(error_type, error_value, error_traceback); +} + +static void +ScandirIterator_dealloc(ScandirIterator *iterator) +{ + if (PyObject_CallFinalizerFromDealloc((PyObject *)iterator) < 0) + return; + Py_TYPE(iterator)->tp_free((PyObject *)iterator); } @@ -12155,7 +12164,8 @@ static PyTypeObject ScandirIteratorType = { 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ 0, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -12164,6 +12174,26 @@ static PyTypeObject ScandirIteratorType = { PyObject_SelfIter, /* tp_iter */ (iternextfunc)ScandirIterator_iternext, /* tp_iternext */ ScandirIterator_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + (destructor)ScandirIterator_finalize, /* tp_finalize */ }; static PyObject * -- cgit v0.12 From e98445a4deb2b2eb97de26e03fc8c4c2a5f256d4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:54:48 +0100 Subject: _warnings.warn_explicit(): try to import warnings Issue #26592: _warnings.warn_explicit() now tries to import the warnings module (Python implementation) if the source parameter is set to be able to log the traceback where the source was allocated. --- Python/_warnings.c | 50 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index dcac57b..41eaf53 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -40,11 +40,11 @@ check_matched(PyObject *obj, PyObject *arg) A NULL return value can mean false or an error. */ static PyObject * -get_warnings_attr(const char *attr) +get_warnings_attr(const char *attr, int try_import) { static PyObject *warnings_str = NULL; PyObject *all_modules; - PyObject *warnings_module; + PyObject *warnings_module, *obj; int result; if (warnings_str == NULL) { @@ -53,15 +53,34 @@ get_warnings_attr(const char *attr) return NULL; } - all_modules = PyImport_GetModuleDict(); - result = PyDict_Contains(all_modules, warnings_str); - if (result == -1 || result == 0) + /* don't try to import after the start of the Python finallization */ + if (try_import && _Py_Finalizing == NULL) { + warnings_module = PyImport_Import(warnings_str); + if (warnings_module == NULL) { + /* Fallback to the C implementation if we cannot get + the Python implementation */ + PyErr_Clear(); + return NULL; + } + } + else { + all_modules = PyImport_GetModuleDict(); + result = PyDict_Contains(all_modules, warnings_str); + if (result == -1 || result == 0) + return NULL; + + warnings_module = PyDict_GetItem(all_modules, warnings_str); + Py_INCREF(warnings_module); + } + + if (!PyObject_HasAttrString(warnings_module, attr)) { + Py_DECREF(warnings_module); return NULL; + } - warnings_module = PyDict_GetItem(all_modules, warnings_str); - if (!PyObject_HasAttrString(warnings_module, attr)) - return NULL; - return PyObject_GetAttrString(warnings_module, attr); + obj = PyObject_GetAttrString(warnings_module, attr); + Py_DECREF(warnings_module); + return obj; } @@ -70,7 +89,7 @@ get_once_registry(void) { PyObject *registry; - registry = get_warnings_attr("onceregistry"); + registry = get_warnings_attr("onceregistry", 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; @@ -87,7 +106,7 @@ get_default_action(void) { PyObject *default_action; - default_action = get_warnings_attr("defaultaction"); + default_action = get_warnings_attr("defaultaction", 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; @@ -110,7 +129,7 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, Py_ssize_t i; PyObject *warnings_filters; - warnings_filters = get_warnings_attr("filters"); + warnings_filters = get_warnings_attr("filters", 0); if (warnings_filters == NULL) { if (PyErr_Occurred()) return NULL; @@ -366,7 +385,10 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; - show_fn = get_warnings_attr("_showwarnmsg"); + /* If the source parameter is set, try to get the Python implementation. + The Python implementation is able to log the traceback where the source + was allocated, whereas the C implementation doesnt. */ + show_fn = get_warnings_attr("_showwarnmsg", source != NULL); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; @@ -380,7 +402,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, goto error; } - warnmsg_cls = get_warnings_attr("WarningMessage"); + warnmsg_cls = get_warnings_attr("WarningMessage", 0); if (warnmsg_cls == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get warnings.WarningMessage"); -- cgit v0.12 From 6d7f4f6675b683475a8eaa8f0f5dff35a0b165f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 02:04:32 +0100 Subject: regrtest: add timeout to main process when using -jN libregrtest: add a watchdog to run_tests_multiprocess() using faulthandler.dump_traceback_later(). --- Lib/test/libregrtest/runtest_mp.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 0ca7dd7..5e847a0 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,3 +1,4 @@ +import faulthandler import json import os import queue @@ -151,6 +152,8 @@ class MultiprocessThread(threading.Thread): def run_tests_multiprocess(regrtest): output = queue.Queue() pending = MultiprocessIterator(regrtest.tests) + test_timeout = regrtest.ns.timeout + use_timeout = (test_timeout is not None) workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] @@ -170,11 +173,14 @@ def run_tests_multiprocess(regrtest): finished = 0 test_index = 1 - timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) + get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: while finished < regrtest.ns.use_mp: + if use_timeout: + faulthandler.dump_traceback_later(test_timeout, exit=True) + try: - item = output.get(timeout=timeout) + item = output.get(timeout=get_timeout) except queue.Empty: running = get_running(workers) if running and not regrtest.ns.pgo: @@ -215,6 +221,9 @@ def run_tests_multiprocess(regrtest): regrtest.interrupted = True pending.interrupted = True print() + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() running = [worker.current_test for worker in workers] running = list(filter(bool, running)) -- cgit v0.12 From d65e0c7560f8a5ad31d31892cee6f1416293ca39 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 02:05:39 +0100 Subject: Makefile: change default value of TESTTIMEOUT from 1 hour to 15 min The whole test suite takes 6 minutes on my laptop. It takes less than 30 minutes on most buildbots. The TESTTIMEOUT is the timeout for a single test file. --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6514bf8..5c90daf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -974,7 +974,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) TESTOPTS= $(EXTRATESTOPTS) TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py -TESTTIMEOUT= 3600 +TESTTIMEOUT= 900 # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -- cgit v0.12 From 42bcf37fcffa65592e401c43aa8c0190452b28b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:08:08 +0100 Subject: Issue #26588: Optimize tracemalloc_realloc() No need to remove the old trace if the memory block didn't move. --- Modules/_tracemalloc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 3d96860..139b169 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -633,7 +633,12 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - REMOVE_TRACE(ptr); + + /* tracemalloc_add_trace() updates the trace if there is already + a trace at address (domain, ptr2) */ + if (ptr2 != ptr) { + REMOVE_TRACE(ptr); + } if (ADD_TRACE(ptr2, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to -- cgit v0.12 From e8c6b2fd1bb75c6a3865745de30f26e235d3f12f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:25:01 +0100 Subject: Issue #26588: * _Py_HASHTABLE_ENTRY_DATA: change type from "char *" to "const void *" * Add _Py_HASHTABLE_ENTRY_WRITE_PKEY() macro * Rename _Py_HASHTABLE_ENTRY_WRITE_DATA() macro to _Py_HASHTABLE_ENTRY_WRITE_PDATA() * Add _Py_HASHTABLE_ENTRY_WRITE_DATA() macro --- Modules/_tracemalloc.c | 2 +- Modules/hashtable.c | 10 ++++------ Modules/hashtable.h | 27 +++++++++++++++++++++++---- Python/marshal.c | 2 +- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 139b169..0bab540 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1263,7 +1263,7 @@ tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, void *user_data) { PyObject *obj; - _Py_HASHTABLE_ENTRY_READ_DATA(tracebacks, entry, sizeof(obj), &obj); + _Py_HASHTABLE_ENTRY_READ_DATA(tracebacks, entry, obj); Py_DECREF(obj); return 0; } diff --git a/Modules/hashtable.c b/Modules/hashtable.c index bb20cce..d80acc6 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -287,7 +287,7 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, ht->entries--; if (data != NULL) - _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); + _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); ht->alloc.free(entry); if ((float)ht->entries / (float)ht->num_buckets < HASHTABLE_LOW) @@ -325,10 +325,8 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, } entry->key_hash = key_hash; - memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); - - assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); + _Py_HASHTABLE_ENTRY_WRITE_PKEY(key_size, entry, pkey); + _Py_HASHTABLE_ENTRY_WRITE_PDATA(ht, entry, data_size, data); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -350,7 +348,7 @@ _Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; - _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); + _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); return 1; } diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 4199aab..9c3fbdd 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -30,10 +30,13 @@ typedef struct { } _Py_hashtable_entry_t; #define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ - ((const void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) + ((const void *)((char *)(ENTRY) \ + + sizeof(_Py_hashtable_entry_t))) #define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) + ((const void *)((char *)(ENTRY) \ + + sizeof(_Py_hashtable_entry_t) \ + + (TABLE)->key_size)) /* Get a key value from pkey: use memcpy() rather than a pointer dereference to avoid memory alignment issues. */ @@ -49,10 +52,26 @@ typedef struct { memcpy(&(KEY), _Py_HASHTABLE_ENTRY_KEY(ENTRY), sizeof(KEY)); \ } while (0) -#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA_SIZE, DATA) \ +#define _Py_HASHTABLE_ENTRY_WRITE_PKEY(KEY_SIZE, ENTRY, PKEY) \ + do { \ + memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(ENTRY), (PKEY), (KEY_SIZE)); \ + } while (0) + +#define _Py_HASHTABLE_ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + memcpy((PDATA), _Py_HASHTABLE_ENTRY_DATA(TABLE, (ENTRY)), \ + (DATA_SIZE)); \ + } while (0) + +#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA) \ + _Py_HASHTABLE_ENTRY_READ_PDATA((TABLE), (ENTRY), sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ + memcpy((void *)_Py_HASHTABLE_ENTRY_DATA((TABLE), (ENTRY)), \ + (PDATA), (DATA_SIZE)); \ } while (0) diff --git a/Python/marshal.c b/Python/marshal.c index 83a1885..3be77a8 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -266,7 +266,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, sizeof(w), &w); + _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, w); /* we don't store "long" indices in the dict */ assert(0 <= w && w <= 0x7fffffff); w_byte(TYPE_REF, p); -- cgit v0.12 From ca79ccd9e69641330d4002acac1bfeeb2dccda32 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:38:54 +0100 Subject: Issue #26588: * Optimize tracemalloc_add_trace(): modify hashtable entry data (trace) if the memory block is already tracked, rather than trying to remove the old trace and then add a new trace. * Add _Py_HASHTABLE_ENTRY_WRITE_DATA() macro --- Include/pymem.h | 2 +- Modules/_tracemalloc.c | 38 +++++++++++++++++++++++++++----------- Modules/hashtable.h | 3 +++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/Include/pymem.h b/Include/pymem.h index 941e00f..431e5b6 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -34,7 +34,7 @@ typedef unsigned int _PyTraceMalloc_domain_t; Return -2 if tracemalloc is disabled. - If memory block was already tracked, begin by removing the old trace. */ + If memory block is already tracked, update the existing trace. */ PyAPI_FUNC(int) _PyTraceMalloc_Track( _PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 0bab540..e6465a3 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -552,33 +552,49 @@ static int tracemalloc_add_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, size_t size) { + pointer_t key = {ptr, domain}; traceback_t *traceback; trace_t trace; + _Py_hashtable_entry_t* entry; int res; assert(tracemalloc_config.tracing); - /* first, remove the previous trace (if any) */ - tracemalloc_remove_trace(domain, ptr); - traceback = traceback_new(); if (traceback == NULL) { return -1; } - trace.size = size; - trace.traceback = traceback; - if (tracemalloc_config.use_domain) { - pointer_t key = {ptr, domain}; - res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_traces, key); } else { - res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_traces, ptr); } - if (res != 0) { - return res; + if (entry != NULL) { + /* the memory block is already tracked */ + _Py_HASHTABLE_ENTRY_READ_DATA(tracemalloc_traces, entry, trace); + assert(tracemalloc_traced_memory >= trace.size); + tracemalloc_traced_memory -= trace.size; + + trace.size = size; + trace.traceback = traceback; + _Py_HASHTABLE_ENTRY_WRITE_DATA(tracemalloc_traces, entry, trace); + } + else { + trace.size = size; + trace.traceback = traceback; + + if (tracemalloc_config.use_domain) { + res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + } + else { + res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + } + if (res != 0) { + return res; + } } assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 9c3fbdd..e3e8148 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -74,6 +74,9 @@ typedef struct { (PDATA), (DATA_SIZE)); \ } while (0) +#define _Py_HASHTABLE_ENTRY_WRITE_DATA(TABLE, ENTRY, DATA) \ + _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, sizeof(DATA), &(DATA)) + /* _Py_hashtable: prototypes */ -- cgit v0.12 From 5dacbd4c42171e447e2f07144faf502774dc921a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:52:13 +0100 Subject: Cleanup hashtable.h Issue #26588: * Pass the hash table rather than the key size to hash and compare functions * _Py_HASHTABLE_READ_KEY() and _Py_HASHTABLE_ENTRY_READ_KEY() macros now expect the hash table as the first parameter, rather than the key size * tracemalloc_get_traces_fill(): use _Py_HASHTABLE_ENTRY_READ_DATA() rather than pointer dereference * Remove the _Py_HASHTABLE_ENTRY_WRITE_PKEY() macro * Move "PKEY" and "PDATA" macros inside hashtable.c --- Modules/_tracemalloc.c | 50 ++++++++++++++++++++++---------------------- Modules/hashtable.c | 55 +++++++++++++++++++++++++++++-------------------- Modules/hashtable.h | 56 ++++++++++++++++++++++---------------------------- Python/marshal.c | 2 +- 4 files changed, 82 insertions(+), 81 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index e6465a3..169fd2c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -223,23 +223,23 @@ set_reentrant(int reentrant) static Py_uhash_t -hashtable_hash_pyobject(size_t key_size, const void *pkey) +hashtable_hash_pyobject(_Py_hashtable_t *ht, const void *pkey) { PyObject *obj; - _Py_HASHTABLE_READ_KEY(key_size, pkey, obj); + _Py_HASHTABLE_READ_KEY(ht, pkey, obj); return PyObject_Hash(obj); } static int -hashtable_compare_unicode(size_t key_size, const void *pkey, +hashtable_compare_unicode(_Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry) { PyObject *key1, *key2; - _Py_HASHTABLE_READ_KEY(key_size, pkey, key1); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, key2); + _Py_HASHTABLE_READ_KEY(ht, pkey, key1); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, key2); if (key1 != NULL && key2 != NULL) return (PyUnicode_Compare(key1, key2) == 0); @@ -249,12 +249,12 @@ hashtable_compare_unicode(size_t key_size, const void *pkey, static Py_uhash_t -hashtable_hash_pointer_t(size_t key_size, const void *pkey) +hashtable_hash_pointer_t(_Py_hashtable_t *ht, const void *pkey) { pointer_t ptr; Py_uhash_t hash; - _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr); + _Py_HASHTABLE_READ_KEY(ht, pkey, ptr); hash = (Py_uhash_t)_Py_HashPointer((void*)ptr.ptr); hash ^= ptr.domain; @@ -263,13 +263,13 @@ hashtable_hash_pointer_t(size_t key_size, const void *pkey) int -hashtable_compare_pointer_t(size_t key_size, const void *pkey, +hashtable_compare_pointer_t(_Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry) { pointer_t ptr1, ptr2; - _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr1); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, ptr2); + _Py_HASHTABLE_READ_KEY(ht, pkey, ptr1); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, ptr2); /* compare pointer before domain, because pointer is more likely to be different */ @@ -304,25 +304,25 @@ raw_free(void *ptr) static Py_uhash_t -hashtable_hash_traceback(size_t key_size, const void *pkey) +hashtable_hash_traceback(_Py_hashtable_t *ht, const void *pkey) { traceback_t *traceback; - _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback); + _Py_HASHTABLE_READ_KEY(ht, pkey, traceback); return traceback->hash; } static int -hashtable_compare_traceback(size_t key_size, const void *pkey, - const _Py_hashtable_entry_t *he) +hashtable_compare_traceback(_Py_hashtable_t *ht, const void *pkey, + const _Py_hashtable_entry_t *entry) { traceback_t *traceback1, *traceback2; const frame_t *frame1, *frame2; int i; - _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback1); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, he, traceback2); + _Py_HASHTABLE_READ_KEY(ht, pkey, traceback1); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, traceback2); if (traceback1->nframe != traceback2->nframe) return 0; @@ -395,8 +395,7 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) /* intern the filename */ entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); if (entry != NULL) { - _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_filenames->key_size, entry, - filename); + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_filenames, entry, filename); } else { /* tracemalloc_filenames is responsible to keep a reference @@ -490,8 +489,7 @@ traceback_new(void) /* intern the traceback */ entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); if (entry != NULL) { - _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_tracebacks->key_size, entry, - traceback); + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_tracebacks, entry, traceback); } else { traceback_t *copy; @@ -873,7 +871,7 @@ tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, { PyObject *filename; - _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, filename); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, filename); Py_DECREF(filename); return 0; } @@ -885,7 +883,7 @@ traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, { traceback_t *traceback; - _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, traceback); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, traceback); raw_free(traceback); return 0; } @@ -1246,21 +1244,21 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr { get_traces_t *get_traces = user_data; _PyTraceMalloc_domain_t domain; - trace_t *trace; + trace_t trace; PyObject *tracemalloc_obj; int res; if (tracemalloc_config.use_domain) { pointer_t key; - _Py_HASHTABLE_ENTRY_READ_KEY(traces->key_size, entry, key); + _Py_HASHTABLE_ENTRY_READ_KEY(traces, entry, key); domain = key.domain; } else { domain = DEFAULT_DOMAIN; } - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); + _Py_HASHTABLE_ENTRY_READ_DATA(traces, entry, trace); - tracemalloc_obj = trace_to_pyobject(domain, trace, get_traces->tracebacks); + tracemalloc_obj = trace_to_pyobject(domain, &trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) return 1; diff --git a/Modules/hashtable.c b/Modules/hashtable.c index d80acc6..b53cc24 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -61,6 +61,20 @@ #define HASHTABLE_ITEM_SIZE(HT) \ (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) +#define ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + Py_MEMCPY((PDATA), _Py_HASHTABLE_ENTRY_PDATA(TABLE, (ENTRY)), \ + (DATA_SIZE)); \ + } while (0) + +#define ENTRY_WRITE_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + Py_MEMCPY((void *)_Py_HASHTABLE_ENTRY_PDATA((TABLE), (ENTRY)), \ + (PDATA), (DATA_SIZE)); \ + } while (0) + /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -91,21 +105,21 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, Py_uhash_t -_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) +_Py_hashtable_hash_ptr(struct _Py_hashtable_t *ht, const void *pkey) { void *key; - _Py_HASHTABLE_READ_KEY(key_size, pkey, key); - return (Py_uhash_t)_Py_HashPointer((void *)key); + _Py_HASHTABLE_READ_KEY(ht, pkey, key); + return (Py_uhash_t)_Py_HashPointer(key); } int -_Py_hashtable_compare_direct(size_t key_size, const void *pkey, +_Py_hashtable_compare_direct(_Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry) { - const void *pkey2 = _Py_HASHTABLE_ENTRY_KEY(entry); - return (memcmp(pkey, pkey2, key_size) == 0); + const void *pkey2 = _Py_HASHTABLE_ENTRY_PKEY(entry); + return (memcmp(pkey, pkey2, ht->key_size) == 0); } @@ -245,12 +259,11 @@ _Py_hashtable_get_entry(_Py_hashtable_t *ht, assert(key_size == ht->key_size); - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(ht, pkey); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(ht, pkey, entry)) break; } @@ -268,13 +281,12 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, assert(key_size == ht->key_size); - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(ht, pkey); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(ht, pkey, entry)) break; previous = entry; } @@ -287,7 +299,7 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, ht->entries--; if (data != NULL) - _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); + ENTRY_READ_PDATA(ht, entry, data_size, data); ht->alloc.free(entry); if ((float)ht->entries / (float)ht->num_buckets < HASHTABLE_LOW) @@ -315,7 +327,7 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, assert(entry == NULL); #endif - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(ht, pkey); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -325,8 +337,8 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, } entry->key_hash = key_hash; - _Py_HASHTABLE_ENTRY_WRITE_PKEY(key_size, entry, pkey); - _Py_HASHTABLE_ENTRY_WRITE_PDATA(ht, entry, data_size, data); + Py_MEMCPY((void *)_Py_HASHTABLE_ENTRY_PKEY(entry), pkey, ht->key_size); + ENTRY_WRITE_PDATA(ht, entry, data_size, data); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -348,7 +360,7 @@ _Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; - _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); + ENTRY_READ_PDATA(ht, entry, data_size, data); return 1; } @@ -399,7 +411,6 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, static void hashtable_rehash(_Py_hashtable_t *ht) { - const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -429,7 +440,7 @@ hashtable_rehash(_Py_hashtable_t *ht) size_t entry_index; - assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); + assert(ht->hash_func(ht, _Py_HASHTABLE_ENTRY_PKEY(entry)) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -499,9 +510,9 @@ _Py_hashtable_copy(_Py_hashtable_t *src) for (bucket=0; bucket < src->num_buckets; bucket++) { entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { - const void *pkey = _Py_HASHTABLE_ENTRY_KEY(entry); - const void *data = _Py_HASHTABLE_ENTRY_DATA(src, entry); - err = _Py_hashtable_set(dst, key_size, pkey, data_size, data); + const void *pkey = _Py_HASHTABLE_ENTRY_PKEY(entry); + const void *pdata = _Py_HASHTABLE_ENTRY_PDATA(src, entry); + err = _Py_hashtable_set(dst, key_size, pkey, data_size, pdata); if (err) { _Py_hashtable_destroy(dst); return NULL; diff --git a/Modules/hashtable.h b/Modules/hashtable.h index e3e8148..18fed09 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -29,60 +29,52 @@ typedef struct { /* key (key_size bytes) and then data (data_size bytes) follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ +#define _Py_HASHTABLE_ENTRY_PKEY(ENTRY) \ ((const void *)((char *)(ENTRY) \ + sizeof(_Py_hashtable_entry_t))) -#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ +#define _Py_HASHTABLE_ENTRY_PDATA(TABLE, ENTRY) \ ((const void *)((char *)(ENTRY) \ + sizeof(_Py_hashtable_entry_t) \ + (TABLE)->key_size)) /* Get a key value from pkey: use memcpy() rather than a pointer dereference to avoid memory alignment issues. */ -#define _Py_HASHTABLE_READ_KEY(KEY_SIZE, PKEY, DST_KEY) \ +#define _Py_HASHTABLE_READ_KEY(TABLE, PKEY, DST_KEY) \ do { \ - assert(sizeof(DST_KEY) == (KEY_SIZE)); \ - memcpy(&(DST_KEY), (PKEY), sizeof(DST_KEY)); \ + assert(sizeof(DST_KEY) == (TABLE)->key_size); \ + Py_MEMCPY(&(DST_KEY), (PKEY), sizeof(DST_KEY)); \ } while (0) -#define _Py_HASHTABLE_ENTRY_READ_KEY(KEY_SIZE, ENTRY, KEY) \ +#define _Py_HASHTABLE_ENTRY_READ_KEY(TABLE, ENTRY, KEY) \ do { \ - assert(sizeof(KEY) == (KEY_SIZE)); \ - memcpy(&(KEY), _Py_HASHTABLE_ENTRY_KEY(ENTRY), sizeof(KEY)); \ - } while (0) - -#define _Py_HASHTABLE_ENTRY_WRITE_PKEY(KEY_SIZE, ENTRY, PKEY) \ - do { \ - memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(ENTRY), (PKEY), (KEY_SIZE)); \ - } while (0) - -#define _Py_HASHTABLE_ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ - do { \ - assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy((PDATA), _Py_HASHTABLE_ENTRY_DATA(TABLE, (ENTRY)), \ - (DATA_SIZE)); \ + assert(sizeof(KEY) == (TABLE)->key_size); \ + Py_MEMCPY(&(KEY), _Py_HASHTABLE_ENTRY_PKEY(ENTRY), sizeof(KEY)); \ } while (0) #define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA) \ - _Py_HASHTABLE_ENTRY_READ_PDATA((TABLE), (ENTRY), sizeof(DATA), &(DATA)) - -#define _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ do { \ - assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy((void *)_Py_HASHTABLE_ENTRY_DATA((TABLE), (ENTRY)), \ - (PDATA), (DATA_SIZE)); \ + assert(sizeof(DATA) == (TABLE)->data_size); \ + Py_MEMCPY(&(DATA), _Py_HASHTABLE_ENTRY_PDATA(TABLE, (ENTRY)), \ + sizeof(DATA)); \ } while (0) #define _Py_HASHTABLE_ENTRY_WRITE_DATA(TABLE, ENTRY, DATA) \ - _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, sizeof(DATA), &(DATA)) + do { \ + assert(sizeof(DATA) == (TABLE)->data_size); \ + Py_MEMCPY((void *)_Py_HASHTABLE_ENTRY_PDATA((TABLE), (ENTRY)), \ + &(DATA), sizeof(DATA)); \ + } while (0) /* _Py_hashtable: prototypes */ -typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, +/* Forward declaration */ +struct _Py_hashtable_t; + +typedef Py_uhash_t (*_Py_hashtable_hash_func) (struct _Py_hashtable_t *ht, const void *pkey); -typedef int (*_Py_hashtable_compare_func) (size_t key_size, +typedef int (*_Py_hashtable_compare_func) (struct _Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *he); @@ -97,7 +89,7 @@ typedef struct { /* _Py_hashtable: table */ -typedef struct { +typedef struct _Py_hashtable_t { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; @@ -111,12 +103,12 @@ typedef struct { /* hash a pointer (void*) */ PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( - size_t key_size, + struct _Py_hashtable_t *ht, const void *pkey); /* comparison using memcmp() */ PyAPI_FUNC(int) _Py_hashtable_compare_direct( - size_t key_size, + _Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry); diff --git a/Python/marshal.c b/Python/marshal.c index 3be77a8..627a842 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -588,7 +588,7 @@ w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, { PyObject *entry_key; - _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, entry_key); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, entry_key); Py_XDECREF(entry_key); return 0; } -- cgit v0.12 From bd31b7c48317eecf981215afbb1f30c81769acbf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 10:32:26 +0100 Subject: Issue #23848: Expose _Py_DumpHexadecimal() This function will be reused by faulthandler. --- Include/traceback.h | 15 ++++++++++++++- Python/traceback.c | 17 +++++++++-------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Include/traceback.h b/Include/traceback.h index 76e169a..f767ea8 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -94,7 +94,20 @@ PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text); /* Format an integer as decimal into the file descriptor fd. This function is signal safe. */ -PyAPI_FUNC(void) _Py_DumpDecimal(int fd, unsigned long value); +PyAPI_FUNC(void) _Py_DumpDecimal( + int fd, + unsigned long value); + +/* Format an integer as hexadecimal into the file descriptor fd with at least + width digits. + + The maximum width is sizeof(unsigned long)*2 digits. + + This function is signal safe. */ +PyAPI_FUNC(void) _Py_DumpHexadecimal( + int fd, + unsigned long value, + Py_ssize_t width); #endif /* !Py_LIMITED_API */ diff --git a/Python/traceback.c b/Python/traceback.c index 403dba5..3259482 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -506,14 +506,15 @@ _Py_DumpDecimal(int fd, unsigned long value) This function is signal safe. */ -static void -dump_hexadecimal(int fd, unsigned long value, Py_ssize_t width) +void +_Py_DumpHexadecimal(int fd, unsigned long value, Py_ssize_t width) { char buffer[sizeof(unsigned long) * 2 + 1], *ptr, *end; const Py_ssize_t size = Py_ARRAY_LENGTH(buffer) - 1; if (width > size) width = size; + /* it's ok if width is negative */ end = &buffer[size]; ptr = end; @@ -582,15 +583,15 @@ _Py_DumpASCII(int fd, PyObject *text) } else if (ch <= 0xff) { PUTS(fd, "\\x"); - dump_hexadecimal(fd, ch, 2); + _Py_DumpHexadecimal(fd, ch, 2); } else if (ch <= 0xffff) { PUTS(fd, "\\u"); - dump_hexadecimal(fd, ch, 4); + _Py_DumpHexadecimal(fd, ch, 4); } else { PUTS(fd, "\\U"); - dump_hexadecimal(fd, ch, 8); + _Py_DumpHexadecimal(fd, ch, 8); } } if (truncated) { @@ -693,9 +694,9 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) PUTS(fd, "Current thread 0x"); else PUTS(fd, "Thread 0x"); - dump_hexadecimal(fd, - (unsigned long)tstate->thread_id, - sizeof(unsigned long) * 2); + _Py_DumpHexadecimal(fd, + (unsigned long)tstate->thread_id, + sizeof(unsigned long) * 2); PUTS(fd, " (most recent call first):\n"); } -- cgit v0.12 From 404cdc5a924a294a17c04b745df5549c32e08130 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 10:39:17 +0100 Subject: faulthandler: add Windows exception handler Issue #23848: On Windows, faulthandler.enable() now also installs an exception handler to dump the traceback of all Python threads on any Windows exception, not only on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). --- Doc/library/faulthandler.rst | 3 + Doc/whatsnew/3.6.rst | 8 ++ Lib/test/test_faulthandler.py | 62 ++++++++--- Misc/NEWS | 4 + Modules/faulthandler.c | 253 +++++++++++++++++++++++++++++++----------- 5 files changed, 253 insertions(+), 77 deletions(-) diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 3a5badd..3c49649 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -68,6 +68,9 @@ Fault handler state .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.6 + On Windows, a handler for Windows exception is also installed. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 986c145..928d748 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -199,6 +199,14 @@ directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) +faulthandler +------------ + +On Windows, the :mod:`faulthandler` module now installs an handler for Windows +exceptions: see :func:`faulthandler.enable`. (Contributed by Victor Stinner in +:issue:`23848`.) + + os -- diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index c3cd657..fbd535b 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -23,6 +23,7 @@ except ImportError: _testcapi = None TIMEOUT = 0.5 +MS_WINDOWS = (os.name == 'nt') def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header @@ -76,9 +77,9 @@ class FaultHandlerTests(unittest.TestCase): output = output.decode('ascii', 'backslashreplace') return output.splitlines(), exitcode - def check_fatal_error(self, code, line_number, name_regex, - filename=None, all_threads=True, other_regex=None, - fd=None, know_current_thread=True): + def check_error(self, code, line_number, fatal_error, *, + filename=None, all_threads=True, other_regex=None, + fd=None, know_current_thread=True): """ Check that the fault handler for fatal errors is enabled and check the traceback from the child process output. @@ -93,14 +94,14 @@ class FaultHandlerTests(unittest.TestCase): else: header = 'Stack' regex = """ - ^Fatal Python error: {name} + ^{fatal_error} {header} $most recent call first$: File "", line {lineno} in """ regex = dedent(regex.format( lineno=line_number, - name=name_regex, + fatal_error=fatal_error, header=header)).strip() if other_regex: regex += '|' + other_regex @@ -109,17 +110,36 @@ class FaultHandlerTests(unittest.TestCase): self.assertRegex(output, regex) self.assertNotEqual(exitcode, 0) + def check_fatal_error(self, code, line_number, name_regex, **kw): + fatal_error = 'Fatal Python error: %s' % name_regex + self.check_error(code, line_number, fatal_error, **kw) + + def check_windows_exception(self, code, line_number, name_regex, **kw): + fatal_error = 'Windows exception: %s' % name_regex + self.check_error(code, line_number, fatal_error, **kw) + @unittest.skipIf(sys.platform.startswith('aix'), "the first page of memory is a mapped read-only on AIX") def test_read_null(self): - self.check_fatal_error(""" - import faulthandler - faulthandler.enable() - faulthandler._read_null() - """, - 3, - # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion - '(?:Segmentation fault|Bus error|Illegal instruction)') + if not MS_WINDOWS: + self.check_fatal_error(""" + import faulthandler + faulthandler.enable() + faulthandler._read_null() + """, + 3, + # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion + '(?:Segmentation fault' + '|Bus error' + '|Illegal instruction)') + else: + self.check_windows_exception(""" + import faulthandler + faulthandler.enable() + faulthandler._read_null() + """, + 3, + 'access violation') def test_sigsegv(self): self.check_fatal_error(""" @@ -708,6 +728,22 @@ class FaultHandlerTests(unittest.TestCase): with self.check_stderr_none(): faulthandler.register(signal.SIGUSR1) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_raise_exception(self): + for exc, name in ( + ('EXCEPTION_ACCESS_VIOLATION', 'access violation'), + ('EXCEPTION_INT_DIVIDE_BY_ZERO', 'int divide by zero'), + ('EXCEPTION_STACK_OVERFLOW', 'stack overflow'), + ): + self.check_windows_exception(f""" + import faulthandler + faulthandler.enable() + faulthandler._raise_exception(faulthandler._{exc}) + """, + 3, + name) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 29fc65d..70ff3de 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,10 @@ Core and Builtins Library ------- +- Issue #23848: On Windows, faulthandler.enable() now also installs an + exception handler to dump the traceback of all Python threads on any Windows + exception, not only on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). + - Issue #26530: Add C functions :c:func:`_PyTraceMalloc_Track` and :c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the :mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c247b7..2214466 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -119,7 +119,7 @@ static fault_handler_t faulthandler_handlers[] = { handler fails in faulthandler_fatal_error() */ {SIGSEGV, 0, "Segmentation fault", } }; -static const unsigned char faulthandler_nsignals = \ +static const size_t faulthandler_nsignals = \ Py_ARRAY_LENGTH(faulthandler_handlers); #ifdef HAVE_SIGALTSTACK @@ -290,6 +290,19 @@ faulthandler_dump_traceback_py(PyObject *self, Py_RETURN_NONE; } +static void +faulthandler_disable_fatal_handler(fault_handler_t *handler) +{ + if (!handler->enabled) + return; + handler->enabled = 0; +#ifdef HAVE_SIGACTION + (void)sigaction(handler->signum, &handler->previous, NULL); +#else + (void)signal(handler->signum, handler->previous); +#endif +} + /* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. @@ -308,7 +321,7 @@ static void faulthandler_fatal_error(int signum) { const int fd = fatal_error.fd; - unsigned int i; + size_t i; fault_handler_t *handler = NULL; int save_errno = errno; @@ -326,12 +339,7 @@ faulthandler_fatal_error(int signum) } /* restore the previous handler */ -#ifdef HAVE_SIGACTION - (void)sigaction(signum, &handler->previous, NULL); -#else - (void)signal(signum, handler->previous); -#endif - handler->enabled = 0; + faulthandler_disable_fatal_handler(handler); PUTS(fd, "Fatal Python error: "); PUTS(fd, handler->name); @@ -353,20 +361,110 @@ faulthandler_fatal_error(int signum) raise(signum); } +#ifdef MS_WINDOWS +static LONG WINAPI +faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) +{ + const int fd = fatal_error.fd; + DWORD code = exc_info->ExceptionRecord->ExceptionCode; + + PUTS(fd, "Windows exception: "); + switch (code) + { + /* only format most common errors */ + case EXCEPTION_ACCESS_VIOLATION: PUTS(fd, "access violation"); break; + case EXCEPTION_FLT_DIVIDE_BY_ZERO: PUTS(fd, "float divide by zero"); break; + case EXCEPTION_FLT_OVERFLOW: PUTS(fd, "float overflow"); break; + case EXCEPTION_INT_DIVIDE_BY_ZERO: PUTS(fd, "int divide by zero"); break; + case EXCEPTION_INT_OVERFLOW: PUTS(fd, "integer overflow"); break; + case EXCEPTION_IN_PAGE_ERROR: PUTS(fd, "page error"); break; + case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; + default: + PUTS(fd, "code 0x"); + _Py_DumpHexadecimal(fd, code, sizeof(DWORD)); + } + PUTS(fd, "\n\n"); + + if (code == EXCEPTION_ACCESS_VIOLATION) { + /* disable signal handler for SIGSEGV */ + size_t i; + for (i=0; i < faulthandler_nsignals; i++) { + fault_handler_t *handler = &faulthandler_handlers[i]; + if (handler->signum == SIGSEGV) { + faulthandler_disable_fatal_handler(handler); + break; + } + } + } + + faulthandler_dump_traceback(fd, fatal_error.all_threads, + fatal_error.interp); + + /* call the next exception handler */ + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + /* Install the handler for fatal signals, faulthandler_fatal_error(). */ -static PyObject* -faulthandler_enable(PyObject *self, PyObject *args, PyObject *kwargs) +int +faulthandler_enable(void) { - static char *kwlist[] = {"file", "all_threads", NULL}; - PyObject *file = NULL; - int all_threads = 1; - unsigned int i; + size_t i; fault_handler_t *handler; #ifdef HAVE_SIGACTION struct sigaction action; #endif int err; + + if (fatal_error.enabled) { + return 0; + } + + fatal_error.enabled = 1; + + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + +#ifdef HAVE_SIGACTION + action.sa_handler = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER; +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } +#endif + err = sigaction(handler->signum, &action, &handler->previous); +#else + handler->previous = signal(handler->signum, + faulthandler_fatal_error); + err = (handler->previous == SIG_ERR); +#endif + if (err) { + PyErr_SetFromErrno(PyExc_RuntimeError); + return -1; + } + + handler->enabled = 1; + } + +#ifdef MS_WINDOWS + AddVectoredExceptionHandler(1, faulthandler_exc_handler); +#endif + return 0; +} + +static PyObject* +faulthandler_py_enable(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"file", "all_threads", NULL}; + PyObject *file = NULL; + int all_threads = 1; int fd; PyThreadState *tstate; @@ -388,37 +486,10 @@ faulthandler_enable(PyObject *self, PyObject *args, PyObject *kwargs) fatal_error.all_threads = all_threads; fatal_error.interp = tstate->interp; - if (!fatal_error.enabled) { - fatal_error.enabled = 1; - - for (i=0; i < faulthandler_nsignals; i++) { - handler = &faulthandler_handlers[i]; -#ifdef HAVE_SIGACTION - action.sa_handler = faulthandler_fatal_error; - sigemptyset(&action.sa_mask); - /* Do not prevent the signal from being received from within - its own signal handler */ - action.sa_flags = SA_NODEFER; -#ifdef HAVE_SIGALTSTACK - if (stack.ss_sp != NULL) { - /* Call the signal handler on an alternate signal stack - provided by sigaltstack() */ - action.sa_flags |= SA_ONSTACK; - } -#endif - err = sigaction(handler->signum, &action, &handler->previous); -#else - handler->previous = signal(handler->signum, - faulthandler_fatal_error); - err = (handler->previous == SIG_ERR); -#endif - if (err) { - PyErr_SetFromErrno(PyExc_RuntimeError); - return NULL; - } - handler->enabled = 1; - } + if (faulthandler_enable() < 0) { + return NULL; } + Py_RETURN_NONE; } @@ -432,14 +503,7 @@ faulthandler_disable(void) fatal_error.enabled = 0; for (i=0; i < faulthandler_nsignals; i++) { handler = &faulthandler_handlers[i]; - if (!handler->enabled) - continue; -#ifdef HAVE_SIGACTION - (void)sigaction(handler->signum, &handler->previous, NULL); -#else - (void)signal(handler->signum, handler->previous); -#endif - handler->enabled = 0; + faulthandler_disable_fatal_handler(handler); } } @@ -991,7 +1055,10 @@ faulthandler_fatal_error_py(PyObject *self, PyObject *args) Py_RETURN_NONE; } + #if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) +#define FAULTHANDLER_STACK_OVERFLOW + #ifdef __INTEL_COMPILER /* Issue #23654: Turn off ICC's tail call optimization for the * stack_overflow generator. ICC turns the recursive tail call into @@ -1005,12 +1072,21 @@ stack_overflow(Py_uintptr_t min_sp, Py_uintptr_t max_sp, size_t *depth) /* allocate 4096 bytes on the stack at each call */ unsigned char buffer[4096]; Py_uintptr_t sp = (Py_uintptr_t)&buffer; + Py_uintptr_t stop; + *depth += 1; - if (sp < min_sp || max_sp < sp) + if (sp < min_sp || max_sp < sp) { + printf("call #%lu\n", (unsigned long)*depth); return sp; - buffer[0] = 1; - buffer[4095] = 0; - return stack_overflow(min_sp, max_sp, depth); + } + + memset(buffer, (unsigned char)*depth, sizeof(buffer)); + stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; + + memset(buffer, (unsigned char)stop, sizeof(buffer)); + stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; + + return stop; } static PyObject * @@ -1018,13 +1094,19 @@ faulthandler_stack_overflow(PyObject *self) { size_t depth, size; Py_uintptr_t sp = (Py_uintptr_t)&depth; - Py_uintptr_t stop; + Py_uintptr_t min_sp, max_sp, stop; faulthandler_suppress_crash_report(); + depth = 0; - stop = stack_overflow(sp - STACK_OVERFLOW_MAX_SIZE, - sp + STACK_OVERFLOW_MAX_SIZE, - &depth); + if (sp > STACK_OVERFLOW_MAX_SIZE) + min_sp = sp - STACK_OVERFLOW_MAX_SIZE; + else + min_sp = 0; + max_sp = sp + STACK_OVERFLOW_MAX_SIZE; + + stop = stack_overflow(min_sp, max_sp, &depth); + if (sp < stop) size = stop - sp; else @@ -1035,7 +1117,7 @@ faulthandler_stack_overflow(PyObject *self) size, depth); return NULL; } -#endif +#endif /* (defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION)) ... */ static int @@ -1058,12 +1140,25 @@ faulthandler_traverse(PyObject *module, visitproc visit, void *arg) return 0; } +#ifdef MS_WINDOWS +static PyObject * +faulthandler_raise_exception(PyObject *self, PyObject *args) +{ + unsigned int code, flags = 0; + if (!PyArg_ParseTuple(args, "I|I:_raise_exception", &code, &flags)) + return NULL; + faulthandler_suppress_crash_report(); + RaiseException(code, flags, 0, NULL); + Py_RETURN_NONE; +} +#endif + PyDoc_STRVAR(module_doc, "faulthandler module."); static PyMethodDef module_methods[] = { {"enable", - (PyCFunction)faulthandler_enable, METH_VARARGS|METH_KEYWORDS, + (PyCFunction)faulthandler_py_enable, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("enable(file=sys.stderr, all_threads=True): " "enable the fault handler")}, {"disable", (PyCFunction)faulthandler_disable_py, METH_NOARGS, @@ -1117,10 +1212,14 @@ static PyMethodDef module_methods[] = { PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, {"_fatal_error", faulthandler_fatal_error_py, METH_VARARGS, PyDoc_STR("_fatal_error(message): call Py_FatalError(message)")}, -#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) +#ifdef FAULTHANDLER_STACK_OVERFLOW {"_stack_overflow", (PyCFunction)faulthandler_stack_overflow, METH_NOARGS, PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, #endif +#ifdef MS_WINDOWS + {"_raise_exception", faulthandler_raise_exception, METH_VARARGS, + PyDoc_STR("raise_exception(code, flags=0): Call RaiseException(code, flags).")}, +#endif {NULL, NULL} /* sentinel */ }; @@ -1139,7 +1238,33 @@ static struct PyModuleDef module_def = { PyMODINIT_FUNC PyInit_faulthandler(void) { - return PyModule_Create(&module_def); + PyObject *m = PyModule_Create(&module_def); + if (m == NULL) + return NULL; + + /* Add constants for unit tests */ +#ifdef MS_WINDOWS + /* RaiseException() codes (prefixed by an underscore) */ + if (PyModule_AddIntConstant(m, "_EXCEPTION_ACCESS_VIOLATION", + EXCEPTION_ACCESS_VIOLATION)) + return NULL; + if (PyModule_AddIntConstant(m, "_EXCEPTION_INT_DIVIDE_BY_ZERO", + EXCEPTION_INT_DIVIDE_BY_ZERO)) + return NULL; + if (PyModule_AddIntConstant(m, "_EXCEPTION_STACK_OVERFLOW", + EXCEPTION_STACK_OVERFLOW)) + return NULL; + + /* RaiseException() flags (prefixed by an underscore) */ + if (PyModule_AddIntConstant(m, "_EXCEPTION_NONCONTINUABLE", + EXCEPTION_NONCONTINUABLE)) + return NULL; + if (PyModule_AddIntConstant(m, "_EXCEPTION_NONCONTINUABLE_EXCEPTION", + EXCEPTION_NONCONTINUABLE_EXCEPTION)) + return NULL; +#endif + + return m; } /* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable -- cgit v0.12 From 0aed3a4ebc42afcf41ea926b5fca721a0923b43e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:30:43 +0100 Subject: _PyMem_DebugFree(): fix compiler warning on Windows Don't return a void value. --- Objects/obmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 503fcdf..40c9fcd 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2039,7 +2039,7 @@ static void _PyMem_DebugFree(void *ctx, void *ptr) { _PyMem_DebugCheckGIL(); - return _PyMem_DebugRawFree(ctx, ptr); + _PyMem_DebugRawFree(ctx, ptr); } static void * -- cgit v0.12 From ccb1f8cb1a9faabe3a3ef90b64257f632052c368 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:31:58 +0100 Subject: getpathp.c: fix compiler warning wcsnlen_s() result type is size_t. --- PC/getpathp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PC/getpathp.c b/PC/getpathp.c index c7ddf1e..cb9f1db 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -135,7 +135,7 @@ exists(wchar_t *filename) static int ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ { - int n; + size_t n; if (exists(filename)) return 1; @@ -553,7 +553,7 @@ calculate_path(void) envpath = NULL; pythonhome = argv0_path; } - + /* Look for a 'home' variable and set argv0_path to it, if found */ if (find_env_config_value(env_file, L"home", tmpbuffer)) { wcscpy_s(argv0_path, MAXPATHLEN+1, tmpbuffer); -- cgit v0.12 From 976bb4099c41af2eda00213eeac3083829f12c98 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:36:19 +0100 Subject: compiler.c: fix compiler warnings on Windows --- Python/compile.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index e86b293..92fea3d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -195,7 +195,7 @@ static int expr_constant(struct compiler *, expr_ty); static int compiler_with(struct compiler *, stmt_ty, int); static int compiler_async_with(struct compiler *, stmt_ty, int); static int compiler_async_for(struct compiler *, stmt_ty); -static int compiler_call_helper(struct compiler *c, Py_ssize_t n, +static int compiler_call_helper(struct compiler *c, int n, asdl_seq *args, asdl_seq *keywords); static int compiler_try_except(struct compiler *, stmt_ty); @@ -476,9 +476,9 @@ compiler_unit_check(struct compiler_unit *u) { basicblock *block; for (block = u->u_blocks; block != NULL; block = block->b_list) { - assert((void *)block != (void *)0xcbcbcbcb); - assert((void *)block != (void *)0xfbfbfbfb); - assert((void *)block != (void *)0xdbdbdbdb); + assert((Py_uintptr_t)block != 0xcbcbcbcbU); + assert((Py_uintptr_t)block != 0xfbfbfbfbU); + assert((Py_uintptr_t)block != 0xdbdbdbdbU); if (block->b_instr != NULL) { assert(block->b_ialloc > 0); assert(block->b_iused > 0); @@ -3097,7 +3097,8 @@ compiler_set(struct compiler *c, expr_ty e) static int compiler_dict(struct compiler *c, expr_ty e) { - Py_ssize_t i, n, containers, elements; + Py_ssize_t i, n, elements; + int containers; int is_unpacking = 0; n = asdl_seq_LEN(e->v.Dict.values); containers = 0; @@ -3267,12 +3268,13 @@ compiler_formatted_value(struct compiler *c, expr_ty e) /* shared code between compiler_call and compiler_class */ static int compiler_call_helper(struct compiler *c, - Py_ssize_t n, /* Args already pushed */ + int n, /* Args already pushed */ asdl_seq *args, asdl_seq *keywords) { int code = 0; - Py_ssize_t nelts, i, nseen, nkw; + Py_ssize_t nelts, i, nseen; + int nkw; /* the number of tuples and dictionaries on the stack */ Py_ssize_t nsubargs = 0, nsubkwargs = 0; -- cgit v0.12 From e985726553b4e5571f110f00e14da5a6ae5d7994 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:37:41 +0100 Subject: _msi.c: try to fix compiler warnings --- PC/_msi.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/PC/_msi.c b/PC/_msi.c index 86a5943..b66b18b 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -1032,12 +1032,12 @@ PyInit__msi(void) if (m == NULL) return NULL; - PyModule_AddIntConstant(m, "MSIDBOPEN_CREATEDIRECT", (int)MSIDBOPEN_CREATEDIRECT); - PyModule_AddIntConstant(m, "MSIDBOPEN_CREATE", (int)MSIDBOPEN_CREATE); - PyModule_AddIntConstant(m, "MSIDBOPEN_DIRECT", (int)MSIDBOPEN_DIRECT); - PyModule_AddIntConstant(m, "MSIDBOPEN_READONLY", (int)MSIDBOPEN_READONLY); - PyModule_AddIntConstant(m, "MSIDBOPEN_TRANSACT", (int)MSIDBOPEN_TRANSACT); - PyModule_AddIntConstant(m, "MSIDBOPEN_PATCHFILE", (int)MSIDBOPEN_PATCHFILE); + PyModule_AddIntConstant(m, "MSIDBOPEN_CREATEDIRECT", (long)MSIDBOPEN_CREATEDIRECT); + PyModule_AddIntConstant(m, "MSIDBOPEN_CREATE", (long)MSIDBOPEN_CREATE); + PyModule_AddIntConstant(m, "MSIDBOPEN_DIRECT", (long)MSIDBOPEN_DIRECT); + PyModule_AddIntConstant(m, "MSIDBOPEN_READONLY", (long)MSIDBOPEN_READONLY); + PyModule_AddIntConstant(m, "MSIDBOPEN_TRANSACT", (long)MSIDBOPEN_TRANSACT); + PyModule_AddIntConstant(m, "MSIDBOPEN_PATCHFILE", (long)MSIDBOPEN_PATCHFILE); PyModule_AddIntMacro(m, MSICOLINFO_NAMES); PyModule_AddIntMacro(m, MSICOLINFO_TYPES); -- cgit v0.12 From 69649f21f02907da59264df8faf73849d2557f5e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 12:14:10 +0100 Subject: regrtest: display test duration in sequential mode Only display duration if a test takes more than 30 seconds. --- Lib/test/libregrtest/main.py | 21 +++++++++++++++++++-- Lib/test/libregrtest/runtest.py | 5 +++++ Lib/test/libregrtest/runtest_mp.py | 7 ++----- Lib/test/test_regrtest.py | 5 +++-- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1c99f2b..b954db5 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -13,7 +13,8 @@ from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, - INTERRUPTED, CHILD_ERROR) + INTERRUPTED, CHILD_ERROR, + PROGRESS_MIN_TIME) from test.libregrtest.setup import setup_tests from test import support try: @@ -293,8 +294,15 @@ class Regrtest: save_modules = sys.modules.keys() + previous_test = None for test_index, test in enumerate(self.tests, 1): - self.display_progress(test_index, test) + start_time = time.monotonic() + + text = test + if previous_test: + text = '%s -- %s' % (text, previous_test) + self.display_progress(test_index, text) + if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. @@ -311,6 +319,12 @@ class Regrtest: else: self.accumulate_result(test, result) + test_time = time.monotonic() - start_time + if test_time >= PROGRESS_MIN_TIME: + previous_test = '%s took %.0f sec' % (test, test_time) + else: + previous_test = None + if self.ns.findleaks: gc.collect() if gc.garbage: @@ -326,6 +340,9 @@ class Regrtest: if module not in save_modules and module.startswith("test."): support.unload(module) + if previous_test: + print(previous_test) + def _test_forever(self, tests): while True: for test in tests: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 043f23c..daff476 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -20,6 +20,11 @@ RESOURCE_DENIED = -3 INTERRUPTED = -4 CHILD_ERROR = -5 # error in a child process +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 5e847a0..e51b100 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,14 +13,11 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import ( + runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME) from test.libregrtest.setup import setup_tests -# Minimum duration of a test to display its duration or to mention that -# the test is running in background -PROGRESS_MIN_TIME = 30.0 # seconds - # Display the running tests if nothing happened last N seconds PROGRESS_UPDATE = 30.0 # seconds diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 18d0f46..03e7e1d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -304,7 +304,7 @@ class ParseArgsTestCase(unittest.TestCase): class BaseTestCase(unittest.TestCase): TEST_UNIQUE_ID = 1 TESTNAME_PREFIX = 'test_regrtest_' - TESTNAME_REGEX = r'test_[a-z0-9_]+' + TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+' def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) @@ -351,7 +351,8 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' + % self.TESTNAME_REGEX) parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) -- cgit v0.12 From 96f6e7a1edfdbfbb85538d897ec6001f218b351e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 12:38:01 +0100 Subject: Buildbots: change also Windows timeout from 1 hour to 15 min --- Tools/buildbot/test.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index ff7d167..c01400c 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -16,4 +16,4 @@ if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=3600 %regrtest_args% +call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=900 %regrtest_args% -- cgit v0.12 From 412a5e7e2349689ab408c9475401c83886108e75 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 14:44:14 +0100 Subject: faulthandler: only log fatal exceptions Issue #23848, #26622: * faulthandler now only logs fatal Windows exceptions. * write error code as decimal, not as hexadecimal * replace "Windows exception" with "Windows fatal exception" --- Lib/test/test_faulthandler.py | 2 +- Modules/faulthandler.c | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index fbd535b..a3a2116 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -115,7 +115,7 @@ class FaultHandlerTests(unittest.TestCase): self.check_error(code, line_number, fatal_error, **kw) def check_windows_exception(self, code, line_number, name_regex, **kw): - fatal_error = 'Windows exception: %s' % name_regex + fatal_error = 'Windows fatal exception: %s' % name_regex self.check_error(code, line_number, fatal_error, **kw) @unittest.skipIf(sys.platform.startswith('aix'), diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 2214466..e86f2bb 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -367,8 +367,15 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) { const int fd = fatal_error.fd; DWORD code = exc_info->ExceptionRecord->ExceptionCode; + DWORD flags = exc_info->ExceptionRecord->ExceptionFlags; - PUTS(fd, "Windows exception: "); + /* only log fatal exceptions */ + if (flags & EXCEPTION_NONCONTINUABLE) { + /* call the next exception handler */ + return EXCEPTION_CONTINUE_SEARCH; + } + + PUTS(fd, "Windows fatal exception: "); switch (code) { /* only format most common errors */ @@ -380,8 +387,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) case EXCEPTION_IN_PAGE_ERROR: PUTS(fd, "page error"); break; case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; default: - PUTS(fd, "code 0x"); - _Py_DumpHexadecimal(fd, code, sizeof(DWORD)); + PUTS(fd, "code "); + _Py_DumpDecimal(fd, code, sizeof(DWORD)); } PUTS(fd, "\n\n"); -- cgit v0.12 From 928ad285b5eb0f25d1dabd48ff3761c205efb6f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 15:19:12 +0100 Subject: Issue #23848: Try to fix test_faulthandler on ARM Restore the previous code for stack_overflow(). --- Modules/faulthandler.c | 48 ++++++++++++++++-------------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index e86f2bb..03afe0e 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -418,21 +418,21 @@ int faulthandler_enable(void) { size_t i; - fault_handler_t *handler; -#ifdef HAVE_SIGACTION - struct sigaction action; -#endif - int err; if (fatal_error.enabled) { return 0; } - fatal_error.enabled = 1; for (i=0; i < faulthandler_nsignals; i++) { - handler = &faulthandler_handlers[i]; + fault_handler_t *handler; +#ifdef HAVE_SIGACTION + struct sigaction action; +#endif + int err; + handler = &faulthandler_handlers[i]; + assert(!handler->enabled); #ifdef HAVE_SIGACTION action.sa_handler = faulthandler_fatal_error; sigemptyset(&action.sa_mask); @@ -1062,7 +1062,6 @@ faulthandler_fatal_error_py(PyObject *self, PyObject *args) Py_RETURN_NONE; } - #if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) #define FAULTHANDLER_STACK_OVERFLOW @@ -1079,21 +1078,12 @@ stack_overflow(Py_uintptr_t min_sp, Py_uintptr_t max_sp, size_t *depth) /* allocate 4096 bytes on the stack at each call */ unsigned char buffer[4096]; Py_uintptr_t sp = (Py_uintptr_t)&buffer; - Py_uintptr_t stop; - *depth += 1; - if (sp < min_sp || max_sp < sp) { - printf("call #%lu\n", (unsigned long)*depth); + if (sp < min_sp || max_sp < sp) return sp; - } - - memset(buffer, (unsigned char)*depth, sizeof(buffer)); - stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; - - memset(buffer, (unsigned char)stop, sizeof(buffer)); - stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; - - return stop; + buffer[0] = 1; + buffer[4095] = 0; + return stack_overflow(min_sp, max_sp, depth); } static PyObject * @@ -1101,19 +1091,13 @@ faulthandler_stack_overflow(PyObject *self) { size_t depth, size; Py_uintptr_t sp = (Py_uintptr_t)&depth; - Py_uintptr_t min_sp, max_sp, stop; + Py_uintptr_t stop; faulthandler_suppress_crash_report(); - depth = 0; - if (sp > STACK_OVERFLOW_MAX_SIZE) - min_sp = sp - STACK_OVERFLOW_MAX_SIZE; - else - min_sp = 0; - max_sp = sp + STACK_OVERFLOW_MAX_SIZE; - - stop = stack_overflow(min_sp, max_sp, &depth); - + stop = stack_overflow(sp - STACK_OVERFLOW_MAX_SIZE, + sp + STACK_OVERFLOW_MAX_SIZE, + &depth); if (sp < stop) size = stop - sp; else @@ -1124,7 +1108,7 @@ faulthandler_stack_overflow(PyObject *self) size, depth); return NULL; } -#endif /* (defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION)) ... */ +#endif /* defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) */ static int -- cgit v0.12 From 1c3069aed6098804f3ce4c810e07dbd02fc797c8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 16:10:07 +0100 Subject: Rework _Py_DumpASCII() to make Coverity happy --- Python/traceback.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Python/traceback.c b/Python/traceback.c index 3259482..62a6b1e 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -545,23 +545,23 @@ _Py_DumpASCII(int fd, PyObject *text) size = ascii->length; kind = ascii->state.kind; - if (ascii->state.compact) { + if (kind == PyUnicode_WCHAR_KIND) { + wstr = ((PyASCIIObject *)text)->wstr; + if (wstr == NULL) + return; + size = ((PyCompactUnicodeObject *)text)->wstr_length; + } + else if (ascii->state.compact) { if (ascii->state.ascii) data = ((PyASCIIObject*)text) + 1; else data = ((PyCompactUnicodeObject*)text) + 1; } - else if (kind != PyUnicode_WCHAR_KIND) { + else { data = ((PyUnicodeObject *)text)->data.any; if (data == NULL) return; } - else { - wstr = ((PyASCIIObject *)text)->wstr; - if (wstr == NULL) - return; - size = ((PyCompactUnicodeObject *)text)->wstr_length; - } if (MAX_STRING_LENGTH < size) { size = MAX_STRING_LENGTH; -- cgit v0.12 From d1700a9360e55f05b2dec5654353bbd947ba19e8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 16:57:51 +0100 Subject: Fix typo in doc: avoid the french "& cie" :-) --- Doc/c-api/memory.rst | 2 +- Doc/using/cmdline.rst | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index c2e2442..bd0bc12 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -378,7 +378,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_OBJ` domain -(:c:func:`PyObject_Malloc` & cie). +(ex: :c:func:`PyObject_Malloc`). The arena allocator uses the following functions: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 7ff9361..c1c700c 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -628,12 +628,12 @@ conflict. Set the family of memory allocators used by Python: * ``malloc``: use the :c:func:`malloc` function of the C library - for all Python memory allocators (:c:func:`PyMem_RawMalloc`, - :c:func:`PyMem_Malloc`, :c:func:`PyObject_Malloc` & cie). + for all Python memory allocators (ex: :c:func:`PyMem_RawMalloc`, + :c:func:`PyMem_Malloc` and :c:func:`PyObject_Malloc`). * ``pymalloc``: :c:func:`PyObject_Malloc`, :c:func:`PyObject_Calloc` and :c:func:`PyObject_Realloc` use the :ref:`pymalloc allocator `. - Other Python memory allocators (:c:func:`PyMem_RawMalloc`, - :c:func:`PyMem_Malloc` & cie) use :c:func:`malloc`. + Other Python memory allocators (ex: :c:func:`PyMem_RawMalloc` and + :c:func:`PyMem_Malloc`) use :c:func:`malloc`. Install debug hooks: -- cgit v0.12 From 023654fa68c73e34da01610cad330f98d2db79cf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 17:48:22 +0100 Subject: get_warnings_attr(): Fix coverity warning Don't check if the dict key exists before getting the key. Instead get the key and handle error. --- Python/_warnings.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 41eaf53..40f5c8e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -45,7 +45,6 @@ get_warnings_attr(const char *attr, int try_import) static PyObject *warnings_str = NULL; PyObject *all_modules; PyObject *warnings_module, *obj; - int result; if (warnings_str == NULL) { warnings_str = PyUnicode_InternFromString("warnings"); @@ -65,11 +64,11 @@ get_warnings_attr(const char *attr, int try_import) } else { all_modules = PyImport_GetModuleDict(); - result = PyDict_Contains(all_modules, warnings_str); - if (result == -1 || result == 0) - return NULL; warnings_module = PyDict_GetItem(all_modules, warnings_str); + if (warnings_module == NULL) + return NULL; + Py_INCREF(warnings_module); } -- cgit v0.12 From 904f5def5cc6da106a1e2e9feb0830cdb1433519 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:32:54 +0100 Subject: Try to fix test_gdb on s390x buildbots --- Lib/test/test_gdb.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 3fe15e4..8e3ccb0 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -199,25 +199,13 @@ class DebuggerTests(unittest.TestCase): # Ignore some benign messages on stderr. ignore_patterns = ( 'Function "%s" not defined.' % breakpoint, - "warning: no loadable sections found in added symbol-file" - " system-supplied DSO", - "warning: Unable to find libthread_db matching" - " inferior's thread library, thread debugging will" - " not be available.", - "warning: Cannot initialize thread debugging" - " library: Debugger service failed", - 'warning: Could not load shared library symbols for ' - 'linux-vdso.so', - 'warning: Could not load shared library symbols for ' - 'linux-gate.so', - 'warning: Could not load shared library symbols for ' - 'linux-vdso64.so', 'Do you need "set solib-search-path" or ' '"set sysroot"?', - 'warning: Source file is more recent than executable.', - # Issue #19753: missing symbols on System Z - 'Missing separate debuginfo for ', - 'Try: zypper install -C ', + # BFD: /usr/lib/debug/(...): unable to initialize decompress + # status for section .debug_aranges + 'BFD: ', + # ignore all warnings + 'warning: ', ) for line in errlines: if not line.startswith(ignore_patterns): -- cgit v0.12 From f963c135979838893772619e83f94db97c536048 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:36:54 +0100 Subject: longobject.c: fix compilation warning on Windows 64-bit We know that Py_SIZE(b) is -1 or 1 an so fits into the sdigit type. --- Objects/longobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 3f9837f..70d8cfc 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3522,7 +3522,7 @@ fast_mod(PyLongObject *a, PyLongObject *b) mod = right - 1 - (left - 1) % right; } - return PyLong_FromLong(mod * Py_SIZE(b)); + return PyLong_FromLong(mod * (sdigit)Py_SIZE(b)); } /* Fast floor division for single-digit longs. */ -- cgit v0.12 From 82d44f0598a38034174dae4416c315d2c3926456 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:37:54 +0100 Subject: Issue #23848: Fix usage of _Py_DumpDecimal() --- Modules/faulthandler.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 03afe0e..a990d25 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -388,7 +388,7 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; default: PUTS(fd, "code "); - _Py_DumpDecimal(fd, code, sizeof(DWORD)); + _Py_DumpDecimal(fd, code); } PUTS(fd, "\n\n"); -- cgit v0.12 From 4a1c7d2c3699a0f201ec2bb277ebcadf3f863759 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:45:55 +0100 Subject: Try to fix test_spwd on OpenIndiana Issue #18787: try to get the "root" entry which should exist on all UNIX instead of "bin" which doesn't exist on OpenIndiana. --- Lib/test/test_spwd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_spwd.py b/Lib/test/test_spwd.py index fca809e..3a11a2d 100644 --- a/Lib/test/test_spwd.py +++ b/Lib/test/test_spwd.py @@ -62,7 +62,7 @@ class TestSpwdNonRoot(unittest.TestCase): def test_getspnam_exception(self): with self.assertRaises(PermissionError) as cm: - spwd.getspnam('bin') + spwd.getspnam('root') self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') -- cgit v0.12 From 66e9d03bf445b595861a3749df75b269ba3634e4 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Wed, 23 Mar 2016 20:50:10 +0100 Subject: Issue #26621: Update libmpdec version and remove unnecessary test case. --- Lib/_pydecimal.py | 2 +- Lib/test/test_decimal.py | 1 - Modules/_decimal/libmpdec/mpdecimal.h | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 02365ca..60723f3 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -148,7 +148,7 @@ __xname__ = __name__ # sys.modules lookup (--without-threads) __name__ = 'decimal' # For pickling __version__ = '1.70' # Highest version of the spec this complies with # See http://speleotrove.com/decimal/ -__libmpdec_version__ = "2.4.1" # compatible libmpdec version +__libmpdec_version__ = "2.4.2" # compatible libmpdec version import math as _math import numbers as _numbers diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index f6d58ff..f56f9ad 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4206,7 +4206,6 @@ class CheckAttributes(unittest.TestCase): self.assertTrue(P.HAVE_THREADS is True or P.HAVE_THREADS is False) self.assertEqual(C.__version__, P.__version__) - self.assertEqual(C.__libmpdec_version__, P.__libmpdec_version__) self.assertEqual(dir(C), dir(P)) diff --git a/Modules/_decimal/libmpdec/mpdecimal.h b/Modules/_decimal/libmpdec/mpdecimal.h index 5ca7413..56e4887 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.h +++ b/Modules/_decimal/libmpdec/mpdecimal.h @@ -108,9 +108,9 @@ MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) #define MPD_MAJOR_VERSION 2 #define MPD_MINOR_VERSION 4 -#define MPD_MICRO_VERSION 1 +#define MPD_MICRO_VERSION 2 -#define MPD_VERSION "2.4.1" +#define MPD_VERSION "2.4.2" #define MPD_VERSION_HEX ((MPD_MAJOR_VERSION << 24) | \ (MPD_MINOR_VERSION << 16) | \ -- cgit v0.12 From c53195bbf0b9f5b3bf4e8ddd1c6b12b59731a822 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 21:08:25 +0100 Subject: Try to fix test_gdb on s390x SLES 3.x Ignore empty lines in stderr. --- Lib/test/test_gdb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 8e3ccb0..ccef422 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -208,6 +208,8 @@ class DebuggerTests(unittest.TestCase): 'warning: ', ) for line in errlines: + if not line: + continue if not line.startswith(ignore_patterns): unexpected_errlines.append(line) -- cgit v0.12 From 0069aef51a176ec90fb93f4601636e8763e07c42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 21:15:55 +0100 Subject: Fix test_spwd on OpenIndiana Issue #18787: restore "bin" name in test_spwd but catch KeyError. --- Lib/test/test_spwd.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_spwd.py b/Lib/test/test_spwd.py index 3a11a2d..e893f3a 100644 --- a/Lib/test/test_spwd.py +++ b/Lib/test/test_spwd.py @@ -61,9 +61,14 @@ class TestSpwdRoot(unittest.TestCase): class TestSpwdNonRoot(unittest.TestCase): def test_getspnam_exception(self): - with self.assertRaises(PermissionError) as cm: - spwd.getspnam('root') - self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') + name = 'bin' + try: + with self.assertRaises(PermissionError) as cm: + spwd.getspnam(name) + except KeyError as exc: + self.skipTest("spwd entry %r doesn't exist: %s" % (name, exc)) + else: + self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') if __name__ == "__main__": -- cgit v0.12 From cc73932125c9e1d6cada036d90873821ddda091a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 21:35:29 +0100 Subject: socketmodule.c: error if option larger than INT_MAX On Windows, socket.setsockopt() raises an OverflowError if the socket option is larger than INT_MAX bytes. --- Modules/socketmodule.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index ba3cefd..8df735d 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2458,13 +2458,26 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) if (!PyArg_ParseTuple(args, "iiy*:setsockopt", &level, &optname, &optval)) return NULL; +#ifdef MS_WINDOWS + if (optval.len > INT_MAX) { + PyBuffer_Release(&optval); + PyErr_Format(PyExc_OverflowError, + "socket option is larger than %i bytes", + INT_MAX); + return NULL; + } + res = setsockopt(s->sock_fd, level, optname, + optval.buf, (int)optval.len); +#else res = setsockopt(s->sock_fd, level, optname, optval.buf, optval.len); +#endif PyBuffer_Release(&optval); } - if (res < 0) + if (res < 0) { return s->errorhandler(); - Py_INCREF(Py_None); - return Py_None; + } + + Py_RETURN_NONE; } PyDoc_STRVAR(setsockopt_doc, -- cgit v0.12 From 5e14a38e8e84aa8a80651324c72c00f1d407e07c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 22:03:55 +0100 Subject: _tracemalloc: use compact key for traces Issue #26588: Optimize memory footprint of _tracemalloc before non-zero domain is used. Start with compact key (Py_uintptr_t) and also switch to pointer_t key when the first memory block with a non-zero domain is tracked. --- Modules/_tracemalloc.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 169fd2c..ca0ed3b 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -43,7 +43,7 @@ static struct { /* use domain in trace key? Variable protected by the GIL. */ int use_domain; -} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; +} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 0}; #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without @@ -519,6 +519,58 @@ traceback_new(void) } +static int +tracemalloc_use_domain_cb(_Py_hashtable_t *old_traces, + _Py_hashtable_entry_t *entry, void *user_data) +{ + Py_uintptr_t ptr; + pointer_t key; + _Py_hashtable_t *new_traces = (_Py_hashtable_t *)user_data; + const void *pdata = _Py_HASHTABLE_ENTRY_PDATA(old_traces, entry); + + _Py_HASHTABLE_ENTRY_READ_KEY(old_traces, entry, ptr); + key.ptr = ptr; + key.domain = DEFAULT_DOMAIN; + + return _Py_hashtable_set(new_traces, + sizeof(key), &key, + old_traces->data_size, pdata); +} + + +/* Convert tracemalloc_traces from compact key (Py_uintptr_t) to pointer_t key. + * Return 0 on success, -1 on error. */ +static int +tracemalloc_use_domain(void) +{ + _Py_hashtable_t *new_traces = NULL; + + assert(!tracemalloc_config.use_domain); + + new_traces = hashtable_new(sizeof(pointer_t), + sizeof(trace_t), + hashtable_hash_pointer_t, + hashtable_compare_pointer_t); + if (new_traces == NULL) { + return -1; + } + + if (_Py_hashtable_foreach(tracemalloc_traces, tracemalloc_use_domain_cb, + new_traces) < 0) + { + _Py_hashtable_destroy(new_traces); + return -1; + } + + _Py_hashtable_destroy(tracemalloc_traces); + tracemalloc_traces = new_traces; + + tracemalloc_config.use_domain = 1; + + return 0; +} + + static void tracemalloc_remove_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { @@ -563,6 +615,14 @@ tracemalloc_add_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, return -1; } + if (!tracemalloc_config.use_domain && domain != DEFAULT_DOMAIN) { + /* first trace using a non-zero domain whereas traces use compact + (Py_uintptr_t) keys: switch to pointer_t keys. */ + if (tracemalloc_use_domain() < 0) { + return -1; + } + } + if (tracemalloc_config.use_domain) { entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_traces, key); } -- cgit v0.12 From 923590e397e255ae0f34a04eb662264de9b67f09 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 09:11:48 +0100 Subject: Fix DeprecationWarning on Windows Issue #25911: Use support.check_warnings() to expect or ignore DeprecationWarning in test_os. --- Lib/test/test_os.py | 116 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 1c0dc5b..bac839d 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -66,6 +66,7 @@ except ImportError: from test.support.script_helper import assert_python_ok + root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) @@ -82,6 +83,23 @@ else: # Issue #14110: Some tests fail on FreeBSD if the user is in the wheel group. HAVE_WHEEL_GROUP = sys.platform.startswith('freebsd') and os.getgid() == 0 + +@contextlib.contextmanager +def ignore_deprecation_warnings(msg_regex, quiet=False): + with support.check_warnings((msg_regex, DeprecationWarning), quiet=quiet): + yield + + +@contextlib.contextmanager +def bytes_filename_warn(expected): + msg = 'The Windows bytes API has been deprecated' + if os.name == 'nt': + with ignore_deprecation_warnings(msg, quiet=not expected): + yield + else: + yield + + # Tests creating TESTFN class FileTests(unittest.TestCase): def setUp(self): @@ -305,8 +323,7 @@ class StatAttributeTests(unittest.TestCase): fname = self.fname.encode(sys.getfilesystemencoding()) except UnicodeEncodeError: self.skipTest("cannot encode %a for the filesystem" % self.fname) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with bytes_filename_warn(True): self.check_stat_attributes(fname) def test_stat_result_pickle(self): @@ -443,15 +460,11 @@ class UtimeTests(unittest.TestCase): fp.write(b"ABC") def restore_float_times(state): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - + with ignore_deprecation_warnings('stat_float_times'): os.stat_float_times(state) # ensure that st_atime and st_mtime are float - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - + with ignore_deprecation_warnings('stat_float_times'): old_float_times = os.stat_float_times(-1) self.addCleanup(restore_float_times, old_float_times) @@ -1024,8 +1037,7 @@ class BytesWalkTests(WalkTests): super().setUp() self.stack = contextlib.ExitStack() if os.name == 'nt': - self.stack.enter_context(warnings.catch_warnings()) - warnings.simplefilter("ignore", DeprecationWarning) + self.stack.enter_context(bytes_filename_warn(False)) def tearDown(self): self.stack.close() @@ -1580,8 +1592,7 @@ class LinkTests(unittest.TestCase): with open(file1, "w") as f1: f1.write("test") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with bytes_filename_warn(False): os.link(file1, file2) with open(file1, "r") as f1, open(file2, "r") as f2: self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno())) @@ -1873,10 +1884,12 @@ class Win32ListdirTests(unittest.TestCase): self.assertEqual( sorted(os.listdir(support.TESTFN)), self.created_paths) + # bytes - self.assertEqual( - sorted(os.listdir(os.fsencode(support.TESTFN))), - [os.fsencode(path) for path in self.created_paths]) + with bytes_filename_warn(False): + self.assertEqual( + sorted(os.listdir(os.fsencode(support.TESTFN))), + [os.fsencode(path) for path in self.created_paths]) def test_listdir_extended_path(self): """Test when the path starts with '\\\\?\\'.""" @@ -1886,11 +1899,13 @@ class Win32ListdirTests(unittest.TestCase): self.assertEqual( sorted(os.listdir(path)), self.created_paths) + # bytes - path = b'\\\\?\\' + os.fsencode(os.path.abspath(support.TESTFN)) - self.assertEqual( - sorted(os.listdir(path)), - [os.fsencode(path) for path in self.created_paths]) + with bytes_filename_warn(False): + path = b'\\\\?\\' + os.fsencode(os.path.abspath(support.TESTFN)) + self.assertEqual( + sorted(os.listdir(path)), + [os.fsencode(path) for path in self.created_paths]) @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") @@ -1965,9 +1980,9 @@ class Win32SymlinkTests(unittest.TestCase): self.assertNotEqual(os.lstat(link), os.stat(link)) bytes_link = os.fsencode(link) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with bytes_filename_warn(True): self.assertEqual(os.stat(bytes_link), os.stat(target)) + with bytes_filename_warn(True): self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) def test_12084(self): @@ -2529,36 +2544,37 @@ class Win32DeprecatedBytesAPI(unittest.TestCase): def test_deprecated(self): import nt filename = os.fsencode(support.TESTFN) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - for func, *args in ( - (nt._getfullpathname, filename), - (nt._isdir, filename), - (os.access, filename, os.R_OK), - (os.chdir, filename), - (os.chmod, filename, 0o777), - (os.getcwdb,), - (os.link, filename, filename), - (os.listdir, filename), - (os.lstat, filename), - (os.mkdir, filename), - (os.open, filename, os.O_RDONLY), - (os.rename, filename, filename), - (os.rmdir, filename), - (os.startfile, filename), - (os.stat, filename), - (os.unlink, filename), - (os.utime, filename), - ): - self.assertRaises(DeprecationWarning, func, *args) + for func, *args in ( + (nt._getfullpathname, filename), + (nt._isdir, filename), + (os.access, filename, os.R_OK), + (os.chdir, filename), + (os.chmod, filename, 0o777), + (os.getcwdb,), + (os.link, filename, filename), + (os.listdir, filename), + (os.lstat, filename), + (os.mkdir, filename), + (os.open, filename, os.O_RDONLY), + (os.rename, filename, filename), + (os.rmdir, filename), + (os.startfile, filename), + (os.stat, filename), + (os.unlink, filename), + (os.utime, filename), + ): + with bytes_filename_warn(True): + try: + func(*args) + except OSError: + # ignore OSError, we only care about DeprecationWarning + pass @support.skip_unless_symlink def test_symlink(self): filename = os.fsencode(support.TESTFN) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, - os.symlink, filename, filename) + with bytes_filename_warn(True): + os.symlink(filename, filename) @unittest.skipUnless(hasattr(os, 'get_terminal_size'), "requires os.get_terminal_size") @@ -2696,7 +2712,8 @@ class OSErrorTests(unittest.TestCase): for filenames, func, *func_args in funcs: for name in filenames: try: - func(name, *func_args) + with bytes_filename_warn(False): + func(name, *func_args) except OSError as err: self.assertIs(err.filename, name) else: @@ -3011,7 +3028,8 @@ class TestScandir(unittest.TestCase): def test_bytes(self): if os.name == "nt": # On Windows, os.scandir(bytes) must raise an exception - self.assertRaises(TypeError, os.scandir, b'.') + with bytes_filename_warn(True): + self.assertRaises(TypeError, os.scandir, b'.') return self.create_file("file.txt") -- cgit v0.12 From 5de16e80c14c7698992da8b08ae169e4be1d2ca3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 09:43:00 +0100 Subject: regrtest: fix --fromfile feature * Update code for the name regrtest output format. * Enhance also test_regrtest test on --fromfile --- Lib/test/libregrtest/main.py | 23 +++++++++++++++++------ Lib/test/test_regrtest.py | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b954db5..de08f32 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -168,13 +168,21 @@ class Regrtest: if self.ns.fromfile: self.tests = [] + # regex to match 'test_builtin' in line: + # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' + regex = (r'^(?:[0-9]+:[0-9]+:[0-9]+ *)?' + r'(?:\[[0-9/ ]+\] *)?' + r'(test_[a-zA-Z0-9_]+)') + regex = re.compile(regex) with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - self.tests.extend(guts) + line = line.strip() + if line.startswith('#'): + continue + match = regex.match(line) + if match is None: + continue + self.tests.append(match.group(1)) removepy(self.tests) @@ -194,7 +202,10 @@ class Regrtest: else: alltests = findtests(self.ns.testdir, stdtests, nottests) - self.selected = self.tests or self.ns.args or alltests + if not self.ns.fromfile: + self.selected = self.tests or self.ns.args or alltests + else: + self.selected = self.tests if self.ns.single: self.selected = self.selected[:1] try: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 03e7e1d..df8447c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -628,6 +628,22 @@ class ArgsTestCase(BaseTestCase): # [2/2] test_2 filename = support.TESTFN self.addCleanup(support.unlink, filename) + + # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec' + with open(filename, "w") as fp: + previous = None + for index, name in enumerate(tests, 1): + line = ("00:00:%02i [%s/%s] %s" + % (index, index, len(tests), name)) + if previous: + line += " -- %s took 0 sec" % previous + print(line, file=fp) + previous = name + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + # test format '[2/7] test_opcodes' with open(filename, "w") as fp: for index, name in enumerate(tests, 1): print("[%s/%s] %s" % (index, len(tests), name), file=fp) @@ -635,6 +651,14 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + # test format 'test_opcodes' + with open(filename, "w") as fp: + for name in tests: + print(name, file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test("sigint", code=code) -- cgit v0.12 From 2b60b7237e81895b71e8763dd94946cd76bf2d16 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 11:55:29 +0100 Subject: regrtest: mention in tests run sequentially or in parallel --- Lib/test/libregrtest/main.py | 2 ++ Lib/test/libregrtest/runtest_mp.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index de08f32..c6d9ad0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -305,6 +305,8 @@ class Regrtest: save_modules = sys.modules.keys() + print("Run tests sequentially") + previous_test = None for test_index, test in enumerate(self.tests, 1): start_time = time.monotonic() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index e51b100..96db196 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -154,6 +154,8 @@ def run_tests_multiprocess(regrtest): workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] + print("Run tests in parallel using %s child processes" + % len(workers)) for worker in workers: worker.start() -- cgit v0.12 From 56db16cd445603f688084d9ace4c271b13e4ec01 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 12:04:15 +0100 Subject: regrtest: when parallel tests are interrupted, display progress --- Lib/test/libregrtest/runtest_mp.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 96db196..aa97f05 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,6 +21,9 @@ from test.libregrtest.setup import setup_tests # Display the running tests if nothing happened last N seconds PROGRESS_UPDATE = 30.0 # seconds +# If interrupted, display the wait process every N seconds +WAIT_PROGRESS = 2.0 # seconds + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -224,9 +227,18 @@ def run_tests_multiprocess(regrtest): if use_timeout: faulthandler.cancel_dump_traceback_later() - running = [worker.current_test for worker in workers] - running = list(filter(bool, running)) - if running: - print("Waiting for %s" % ', '.join(running)) - for worker in workers: - worker.join() + # If tests are interrupted, wait until tests complete + wait_start = time.monotonic() + while True: + running = [worker.current_test for worker in workers] + running = list(filter(bool, running)) + if not running: + break + + dt = time.monotonic() - wait_start + line = "Waiting for %s (%s tests)" % (', '.join(running), len(running)) + if dt >= WAIT_PROGRESS: + line = "%s since %.0f sec" % (line, dt) + print(line) + for worker in workers: + worker.join(WAIT_PROGRESS) -- cgit v0.12 From ba8b0a7db447fad7a7064e53424725b02ab5ccec Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 12:23:18 +0100 Subject: Enhance os._DummyDirEntry Issue #25911: * Try to fix test_os.BytesWalkTests on Windows * Try to mimick better the reference os.DirEntry on Windows * _DummyDirEntry now caches os.stat() result * _DummyDirEntry constructor now tries to get os.stat() --- Lib/os.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 0ea7b62..c3ce05d 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -439,15 +439,36 @@ def walk(top, topdown=True, onerror=None, followlinks=False): yield top, dirs, nondirs class _DummyDirEntry: + """Dummy implementation of DirEntry + + Only used internally by os.walk(bytes). Since os.walk() doesn't need the + follow_symlinks parameter: don't implement it, always follow symbolic + links. + """ + def __init__(self, dir, name): self.name = name self.path = path.join(dir, name) + # Mimick FindFirstFile/FindNextFile: we should get file attributes + # while iterating on a directory + self._stat = None + try: + self.stat() + except OSError: + pass + + def stat(self): + if self._stat is None: + self._stat = stat(self.path) + return self._stat def is_dir(self): - return path.isdir(self.path) + stat = self.stat() + return st.S_ISDIR(stat.st_mode) def is_symlink(self): - return path.islink(self.path) + stat = self.stat() + return st.S_ISLNK(stat.st_mode) class _dummy_scandir: # listdir-based implementation for bytes patches on Windows -- cgit v0.12 From e321274b2eeaf9cfaccbc8d40c4056ab72040e1b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 13:44:19 +0100 Subject: Enhance and modernize test_genericpath * Replace "try/finally: os.remove()" with self.addCleanup(support.unlink) or self.addCleanup(support.rmdir): the support function handles the case when the file doesn't exist * Replace "try/finally: f.close()" with "with open(...) as f:" * test_getsize: add a second test with a different size * Create file using "x" mode to ensure that the file didn't exist before, to detect bugs in tests * Open files in unbuffered mode (buferring=0) to write immediatly data on disk * Replace map() with simpler code * Split isdir() unit test into two units tests to make them less dependant, same change for isfile() test * test_samefile(): test also two different files --- Lib/test/test_genericpath.py | 270 +++++++++++++++++++++---------------------- 1 file changed, 134 insertions(+), 136 deletions(-) diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 86fc2de..9c28a68 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -10,11 +10,9 @@ import warnings from test import support -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass +def create_file(filename, data=b'foo'): + with open(filename, 'xb', 0) as fp: + fp.write(data) class GenericTest: @@ -97,52 +95,47 @@ class GenericTest: self.assertNotEqual(s1[n:n+1], s2[n:n+1]) def test_getsize(self): - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertEqual(self.pathmodule.getsize(support.TESTFN), 3) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) - def test_time(self): - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - f = open(support.TESTFN, "ab") + create_file(filename, b'Hello') + self.assertEqual(self.pathmodule.getsize(filename), 5) + os.remove(filename) + + create_file(filename, b'Hello World!') + self.assertEqual(self.pathmodule.getsize(filename), 12) + + def test_filetime(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + create_file(filename, b'foo') + + with open(filename, "ab", 0) as f: f.write(b"bar") - f.close() - f = open(support.TESTFN, "rb") - d = f.read() - f.close() - self.assertEqual(d, b"foobar") - - self.assertLessEqual( - self.pathmodule.getctime(support.TESTFN), - self.pathmodule.getmtime(support.TESTFN) - ) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) + + with open(filename, "rb", 0) as f: + data = f.read() + self.assertEqual(data, b"foobar") + + self.assertLessEqual( + self.pathmodule.getctime(filename), + self.pathmodule.getmtime(filename) + ) def test_exists(self): - self.assertIs(self.pathmodule.exists(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + self.assertIs(self.pathmodule.exists(filename), False) + + with open(filename, "xb") as f: f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.exists(support.TESTFN), True) - if not self.pathmodule == genericpath: - self.assertIs(self.pathmodule.lexists(support.TESTFN), - True) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) + + self.assertIs(self.pathmodule.exists(filename), True) + + if not self.pathmodule == genericpath: + self.assertIs(self.pathmodule.lexists(filename), True) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_exists_fd(self): @@ -154,53 +147,66 @@ class GenericTest: os.close(w) self.assertFalse(self.pathmodule.exists(r)) - def test_isdir(self): - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isdir(support.TESTFN), True) - os.rmdir(support.TESTFN) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) - - def test_isfile(self): - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isfile(support.TESTFN), True) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - os.rmdir(support.TESTFN) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) + def test_isdir_file(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + create_file(filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + def test_isdir_dir(self): + filename = support.TESTFN + self.addCleanup(support.rmdir, filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + os.mkdir(filename) + self.assertIs(self.pathmodule.isdir(filename), True) + + def test_isfile_file(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + self.assertIs(self.pathmodule.isfile(filename), False) - @staticmethod - def _create_file(filename): - with open(filename, 'wb') as f: - f.write(b'foo') + create_file(filename) + self.assertIs(self.pathmodule.isfile(filename), True) + + def test_isfile_dir(self): + filename = support.TESTFN + self.addCleanup(support.rmdir, filename) + self.assertIs(self.pathmodule.isfile(filename), False) + + os.mkdir(filename) + self.assertIs(self.pathmodule.isfile(filename), False) def test_samefile(self): - try: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - self.assertTrue(self.pathmodule.samefile(test_fn, test_fn)) - self.assertRaises(TypeError, self.pathmodule.samefile) - finally: - os.remove(test_fn) + file1 = support.TESTFN + file2 = support.TESTFN + "2" + self.addCleanup(support.unlink, file1) + self.addCleanup(support.unlink, file2) + + create_file(file1) + self.assertTrue(self.pathmodule.samefile(file1, file1)) + + create_file(file2) + self.assertFalse(self.pathmodule.samefile(file1, file2)) + + self.assertRaises(TypeError, self.pathmodule.samefile) + + def _test_samefile_on_link_func(self, func): + test_fn1 = support.TESTFN + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) + + create_file(test_fn1) + + func(test_fn1, test_fn2) + self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2)) + os.remove(test_fn2) + + create_file(test_fn2) + self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2)) @support.skip_unless_symlink def test_samefile_on_symlink(self): @@ -209,31 +215,37 @@ class GenericTest: def test_samefile_on_link(self): self._test_samefile_on_link_func(os.link) - def _test_samefile_on_link_func(self, func): - try: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) + def test_samestat(self): + test_fn1 = support.TESTFN + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) - func(test_fn1, test_fn2) - self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2)) - os.remove(test_fn2) + create_file(test_fn1) + stat1 = os.stat(test_fn1) + self.assertTrue(self.pathmodule.samestat(stat1, os.stat(test_fn1))) - self._create_file(test_fn2) - self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2)) - finally: - os.remove(test_fn1) - os.remove(test_fn2) + create_file(test_fn2) + stat2 = os.stat(test_fn2) + self.assertFalse(self.pathmodule.samestat(stat1, stat2)) - def test_samestat(self): - try: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - test_fns = [test_fn]*2 - stats = map(os.stat, test_fns) - self.assertTrue(self.pathmodule.samestat(*stats)) - finally: - os.remove(test_fn) + self.assertRaises(TypeError, self.pathmodule.samestat) + + def _test_samestat_on_link_func(self, func): + test_fn1 = support.TESTFN + "1" + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) + + create_file(test_fn1) + func(test_fn1, test_fn2) + self.assertTrue(self.pathmodule.samestat(os.stat(test_fn1), + os.stat(test_fn2))) + os.remove(test_fn2) + + create_file(test_fn2) + self.assertFalse(self.pathmodule.samestat(os.stat(test_fn1), + os.stat(test_fn2))) @support.skip_unless_symlink def test_samestat_on_symlink(self): @@ -242,31 +254,17 @@ class GenericTest: def test_samestat_on_link(self): self._test_samestat_on_link_func(os.link) - def _test_samestat_on_link_func(self, func): - try: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) - test_fns = (test_fn1, test_fn2) - func(*test_fns) - stats = map(os.stat, test_fns) - self.assertTrue(self.pathmodule.samestat(*stats)) - os.remove(test_fn2) - - self._create_file(test_fn2) - stats = map(os.stat, test_fns) - self.assertFalse(self.pathmodule.samestat(*stats)) - - self.assertRaises(TypeError, self.pathmodule.samestat) - finally: - os.remove(test_fn1) - os.remove(test_fn2) - def test_sameopenfile(self): - fname = support.TESTFN + "1" - with open(fname, "wb") as a, open(fname, "wb") as b: - self.assertTrue(self.pathmodule.sameopenfile( - a.fileno(), b.fileno())) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + create_file(filename) + + with open(filename, "rb", 0) as fp1: + fd1 = fp1.fileno() + with open(filename, "rb", 0) as fp2: + fd2 = fp2.fileno() + self.assertTrue(self.pathmodule.sameopenfile(fd1, fd2)) + class TestGenericTest(GenericTest, unittest.TestCase): # Issue 16852: GenericTest can't inherit from unittest.TestCase -- cgit v0.12 From bc6b72ed06318a80a68b4bbf513a0e055b102e11 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 13:55:58 +0100 Subject: Closes #26620: Fix ResourceWarning in test_urllib2_localnet * Use context manager on urllib objects to ensure that they are closed on error * Use self.addCleanup() to cleanup resources even if a test is interrupted with CTRL+c --- Lib/test/test_urllib2_localnet.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index c8b37ee..e9564fd 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -289,12 +289,12 @@ class BasicAuthTests(unittest.TestCase): def http_server_with_basic_auth_handler(*args, **kwargs): return BasicAuthHandler(*args, **kwargs) self.server = LoopbackHttpServerThread(http_server_with_basic_auth_handler) + self.addCleanup(self.server.stop) self.server_url = 'http://127.0.0.1:%s' % self.server.port self.server.start() self.server.ready.wait() def tearDown(self): - self.server.stop() super(BasicAuthTests, self).tearDown() def test_basic_auth_success(self): @@ -438,17 +438,13 @@ class TestUrlopen(unittest.TestCase): def setUp(self): super(TestUrlopen, self).setUp() + # Ignore proxies for localhost tests. - self.old_environ = os.environ.copy() + def restore_environ(old_environ): + os.environ.clear() + os.environ.update(old_environ) + self.addCleanup(restore_environ, os.environ.copy()) os.environ['NO_PROXY'] = '*' - self.server = None - - def tearDown(self): - if self.server is not None: - self.server.stop() - os.environ.clear() - os.environ.update(self.old_environ) - super(TestUrlopen, self).tearDown() def urlopen(self, url, data=None, **kwargs): l = [] @@ -469,6 +465,7 @@ class TestUrlopen(unittest.TestCase): handler = GetRequestHandler(responses) self.server = LoopbackHttpServerThread(handler) + self.addCleanup(self.server.stop) self.server.start() self.server.ready.wait() port = self.server.port @@ -592,7 +589,8 @@ class TestUrlopen(unittest.TestCase): handler = self.start_server() req = urllib.request.Request("http://localhost:%s/" % handler.port, headers={"Range": "bytes=20-39"}) - urllib.request.urlopen(req) + with urllib.request.urlopen(req): + pass self.assertEqual(handler.headers_received["Range"], "bytes=20-39") def test_basic(self): @@ -608,22 +606,21 @@ class TestUrlopen(unittest.TestCase): def test_info(self): handler = self.start_server() - try: - open_url = urllib.request.urlopen( - "http://localhost:%s" % handler.port) + open_url = urllib.request.urlopen( + "http://localhost:%s" % handler.port) + with open_url: info_obj = open_url.info() - self.assertIsInstance(info_obj, email.message.Message, - "object returned by 'info' is not an " - "instance of email.message.Message") - self.assertEqual(info_obj.get_content_subtype(), "plain") - finally: - self.server.stop() + self.assertIsInstance(info_obj, email.message.Message, + "object returned by 'info' is not an " + "instance of email.message.Message") + self.assertEqual(info_obj.get_content_subtype(), "plain") def test_geturl(self): # Make sure same URL as opened is returned by geturl. handler = self.start_server() open_url = urllib.request.urlopen("http://localhost:%s" % handler.port) - url = open_url.geturl() + with open_url: + url = open_url.geturl() self.assertEqual(url, "http://localhost:%s" % handler.port) def test_iteration(self): -- cgit v0.12 From f95a19b900262c32d1038a6ab9a6808a58656c34 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 16:50:41 +0100 Subject: test_os: use @support.requires_linux_version --- Lib/test/test_os.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bac839d..b317e33 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -15,7 +15,6 @@ import locale import mmap import os import pickle -import platform import re import shutil import signal @@ -2456,14 +2455,14 @@ def supports_extended_attributes(): return False finally: support.unlink(support.TESTFN) - # Kernels < 2.6.39 don't respect setxattr flags. - kernel_version = platform.release() - m = re.match("2.6.(\d{1,2})", kernel_version) - return m is None or int(m.group(1)) >= 39 + + return True @unittest.skipUnless(supports_extended_attributes(), "no non-broken extended attribute support") +# Kernels < 2.6.39 don't respect setxattr flags. +@support.requires_linux_version(2, 6, 39) class ExtendedAttributeTests(unittest.TestCase): def tearDown(self): -- cgit v0.12 From ae39d236b41ad369215dc523b63000f4860517cc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:12:55 +0100 Subject: Enhance and modernize test_os * add create_file() helper function * create files using "x" mode instead of "w" to detect when a previous test forget to remove a file * open file for writing in unbuferred mode (buffering=0) * replace "try/finally: unlink" with self.addCleanup(support.unlink) * register unlink cleanup function *before* creating new files --- Lib/test/test_os.py | 153 ++++++++++++++++++++++++++-------------------------- 1 file changed, 76 insertions(+), 77 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b317e33..17484fe 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -99,6 +99,11 @@ def bytes_filename_warn(expected): yield +def create_file(filename, content=b'content'): + with open(filename, "xb", 0) as fp: + fp.write(content) + + # Tests creating TESTFN class FileTests(unittest.TestCase): def setUp(self): @@ -157,9 +162,8 @@ class FileTests(unittest.TestCase): "needs INT_MAX < PY_SSIZE_T_MAX") @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) def test_large_read(self, size): - with open(support.TESTFN, "wb") as fp: - fp.write(b'test') self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b'test') # Issue #21932: Make sure that os.read() does not raise an # OverflowError for size larger than INT_MAX @@ -216,11 +220,12 @@ class FileTests(unittest.TestCase): def test_replace(self): TESTFN2 = support.TESTFN + ".2" - with open(support.TESTFN, 'w') as f: - f.write("1") - with open(TESTFN2, 'w') as f: - f.write("2") - self.addCleanup(os.unlink, TESTFN2) + self.addCleanup(support.unlink, support.TESTFN) + self.addCleanup(support.unlink, TESTFN2) + + create_file(support.TESTFN, b"1") + create_file(TESTFN2, b"2") + os.replace(support.TESTFN, TESTFN2) self.assertRaises(FileNotFoundError, os.stat, support.TESTFN) with open(TESTFN2, 'r') as f: @@ -245,8 +250,7 @@ class StatAttributeTests(unittest.TestCase): def setUp(self): self.fname = support.TESTFN self.addCleanup(support.unlink, self.fname) - with open(self.fname, 'wb') as fp: - fp.write(b"ABC") + create_file(self.fname, b"ABC") @unittest.skipUnless(hasattr(os, 'stat'), 'test needs os.stat()') def check_stat_attributes(self, fname): @@ -455,8 +459,7 @@ class UtimeTests(unittest.TestCase): self.addCleanup(support.rmtree, self.dirname) os.mkdir(self.dirname) - with open(self.fname, 'wb') as fp: - fp.write(b"ABC") + create_file(self.fname) def restore_float_times(state): with ignore_deprecation_warnings('stat_float_times'): @@ -555,7 +558,7 @@ class UtimeTests(unittest.TestCase): "fd support for utime required for this test.") def test_utime_fd(self): def set_time(filename, ns): - with open(filename, 'wb') as fp: + with open(filename, 'wb', 0) as fp: # use a file descriptor to test futimens(timespec) # or futimes(timeval) os.utime(fp.fileno(), ns=ns) @@ -1213,8 +1216,7 @@ class RemoveDirsTests(unittest.TestCase): os.mkdir(dira) dirb = os.path.join(dira, 'dirb') os.mkdir(dirb) - with open(os.path.join(dira, 'file.txt'), 'w') as f: - f.write('text') + create_file(os.path.join(dira, 'file.txt')) os.removedirs(dirb) self.assertFalse(os.path.exists(dirb)) self.assertTrue(os.path.exists(dira)) @@ -1225,8 +1227,7 @@ class RemoveDirsTests(unittest.TestCase): os.mkdir(dira) dirb = os.path.join(dira, 'dirb') os.mkdir(dirb) - with open(os.path.join(dirb, 'file.txt'), 'w') as f: - f.write('text') + create_file(os.path.join(dirb, 'file.txt')) with self.assertRaises(OSError): os.removedirs(dirb) self.assertTrue(os.path.exists(dirb)) @@ -1236,7 +1237,7 @@ class RemoveDirsTests(unittest.TestCase): class DevNullTests(unittest.TestCase): def test_devnull(self): - with open(os.devnull, 'wb') as f: + with open(os.devnull, 'wb', 0) as f: f.write(b'hello') f.close() with open(os.devnull, 'rb') as f: @@ -1323,9 +1324,9 @@ class URandomFDTests(unittest.TestCase): def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. - with open(support.TESTFN, 'wb') as f: - f.write(b"x" * 256) - self.addCleanup(os.unlink, support.TESTFN) + self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b"x" * 256) + code = """if 1: import os import sys @@ -1464,12 +1465,10 @@ class Win32ErrorTests(unittest.TestCase): self.assertRaises(OSError, os.chdir, support.TESTFN) def test_mkdir(self): - f = open(support.TESTFN, "w") - try: + self.addCleanup(support.unlink, support.TESTFN) + + with open(support.TESTFN, "w") as f: self.assertRaises(OSError, os.mkdir, support.TESTFN) - finally: - f.close() - os.unlink(support.TESTFN) def test_utime(self): self.assertRaises(OSError, os.utime, support.TESTFN, None) @@ -1988,42 +1987,36 @@ class Win32SymlinkTests(unittest.TestCase): level1 = os.path.abspath(support.TESTFN) level2 = os.path.join(level1, "level2") level3 = os.path.join(level2, "level3") - try: - os.mkdir(level1) - os.mkdir(level2) - os.mkdir(level3) + self.addCleanup(support.rmtree, level1) - file1 = os.path.abspath(os.path.join(level1, "file1")) + os.mkdir(level1) + os.mkdir(level2) + os.mkdir(level3) - with open(file1, "w") as f: - f.write("file1") + file1 = os.path.abspath(os.path.join(level1, "file1")) + create_file(file1) - orig_dir = os.getcwd() - try: - os.chdir(level2) - link = os.path.join(level2, "link") - os.symlink(os.path.relpath(file1), "link") - self.assertIn("link", os.listdir(os.getcwd())) - - # Check os.stat calls from the same dir as the link - self.assertEqual(os.stat(file1), os.stat("link")) - - # Check os.stat calls from a dir below the link - os.chdir(level1) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - - # Check os.stat calls from a dir above the link - os.chdir(level3) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - finally: - os.chdir(orig_dir) - except OSError as err: - self.fail(err) + orig_dir = os.getcwd() + try: + os.chdir(level2) + link = os.path.join(level2, "link") + os.symlink(os.path.relpath(file1), "link") + self.assertIn("link", os.listdir(os.getcwd())) + + # Check os.stat calls from the same dir as the link + self.assertEqual(os.stat(file1), os.stat("link")) + + # Check os.stat calls from a dir below the link + os.chdir(level1) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) + + # Check os.stat calls from a dir above the link + os.chdir(level3) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) finally: - os.remove(file1) - shutil.rmtree(level1) + os.chdir(orig_dir) @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") @@ -2159,8 +2152,8 @@ class ProgramPriorityTests(unittest.TestCase): try: new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) if base >= 19 and new_prio <= 19: - raise unittest.SkipTest( - "unable to reliably test setpriority at current nice level of %s" % base) + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) else: self.assertEqual(new_prio, base + 1) finally: @@ -2270,8 +2263,7 @@ class TestSendfile(unittest.TestCase): @classmethod def setUpClass(cls): cls.key = support.threading_setup() - with open(support.TESTFN, "wb") as f: - f.write(cls.DATA) + create_file(support.TESTFN, cls.DATA) @classmethod def tearDownClass(cls): @@ -2421,10 +2413,11 @@ class TestSendfile(unittest.TestCase): def test_trailers(self): TESTFN2 = support.TESTFN + "2" file_data = b"abcdef" - with open(TESTFN2, 'wb') as f: - f.write(file_data) - with open(TESTFN2, 'rb')as f: - self.addCleanup(os.remove, TESTFN2) + + self.addCleanup(support.unlink, TESTFN2) + create_file(TESTFN2, file_data) + + with open(TESTFN2, 'rb') as f: os.sendfile(self.sockno, f.fileno(), 0, len(file_data), trailers=[b"1234"]) self.client.close() @@ -2447,8 +2440,9 @@ class TestSendfile(unittest.TestCase): def supports_extended_attributes(): if not hasattr(os, "setxattr"): return False + try: - with open(support.TESTFN, "wb") as fp: + with open(support.TESTFN, "xb", 0) as fp: try: os.setxattr(fp.fileno(), b"user.test", b"") except OSError: @@ -2465,17 +2459,18 @@ def supports_extended_attributes(): @support.requires_linux_version(2, 6, 39) class ExtendedAttributeTests(unittest.TestCase): - def tearDown(self): - support.unlink(support.TESTFN) - def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwargs): fn = support.TESTFN - open(fn, "wb").close() + self.addCleanup(support.unlink, fn) + create_file(fn) + with self.assertRaises(OSError) as cm: getxattr(fn, s("user.test"), **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + init_xattr = listxattr(fn) self.assertIsInstance(init_xattr, list) + setxattr(fn, s("user.test"), b"", **kwargs) xattr = set(init_xattr) xattr.add("user.test") @@ -2483,19 +2478,24 @@ class ExtendedAttributeTests(unittest.TestCase): self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"") setxattr(fn, s("user.test"), b"hello", os.XATTR_REPLACE, **kwargs) self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"hello") + with self.assertRaises(OSError) as cm: setxattr(fn, s("user.test"), b"bye", os.XATTR_CREATE, **kwargs) self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(OSError) as cm: setxattr(fn, s("user.test2"), b"bye", os.XATTR_REPLACE, **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + setxattr(fn, s("user.test2"), b"foo", os.XATTR_CREATE, **kwargs) xattr.add("user.test2") self.assertEqual(set(listxattr(fn)), xattr) removexattr(fn, s("user.test"), **kwargs) + with self.assertRaises(OSError) as cm: getxattr(fn, s("user.test"), **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + xattr.remove("user.test") self.assertEqual(set(listxattr(fn)), xattr) self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") @@ -2508,11 +2508,11 @@ class ExtendedAttributeTests(unittest.TestCase): self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) def _check_xattrs(self, *args, **kwargs): - def make_bytes(s): - return bytes(s, "ascii") self._check_xattrs_str(str, *args, **kwargs) support.unlink(support.TESTFN) - self._check_xattrs_str(make_bytes, *args, **kwargs) + + self._check_xattrs_str(os.fsencode, *args, **kwargs) + support.unlink(support.TESTFN) def test_simple(self): self._check_xattrs(os.getxattr, os.setxattr, os.removexattr, @@ -2527,10 +2527,10 @@ class ExtendedAttributeTests(unittest.TestCase): with open(path, "rb") as fp: return os.getxattr(fp.fileno(), *args) def setxattr(path, *args): - with open(path, "wb") as fp: + with open(path, "wb", 0) as fp: os.setxattr(fp.fileno(), *args) def removexattr(path, *args): - with open(path, "wb") as fp: + with open(path, "wb", 0) as fp: os.removexattr(fp.fileno(), *args) def listxattr(path, *args): with open(path, "rb") as fp: @@ -2844,8 +2844,7 @@ class TestScandir(unittest.TestCase): def create_file(self, name="file.txt"): filename = os.path.join(self.path, name) - with open(filename, "wb") as fp: - fp.write(b'python') + create_file(filename, b'python') return filename def get_entries(self, names): -- cgit v0.12 From 3899b549e3d1acb1f0f7bc5d226cc3632675d36d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:21:17 +0100 Subject: test_os: use support.rmtree() to cleanup WalkTests --- Lib/test/test_os.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 17484fe..bc24e47 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -812,6 +812,7 @@ class WalkTests(unittest.TestCase): def setUp(self): join = os.path.join + self.addCleanup(support.rmtree, support.TESTFN) # Build: # TESTFN/ @@ -922,22 +923,6 @@ class WalkTests(unittest.TestCase): else: self.fail("Didn't follow symlink with followlinks=True") - def tearDown(self): - # Tear everything down. This is a decent use for bottom-up on - # Windows, which doesn't have a recursive delete command. The - # (not so) subtlety is that rmdir will fail unless the dir's - # kids are removed first, so bottom up is essential. - for root, dirs, files in os.walk(support.TESTFN, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - dirname = os.path.join(root, name) - if not os.path.islink(dirname): - os.rmdir(dirname) - else: - os.remove(dirname) - os.rmdir(support.TESTFN) - def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1020,19 +1005,6 @@ class FwalkTests(WalkTests): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) - def tearDown(self): - # cleanup - for root, dirs, files, rootfd in os.fwalk(support.TESTFN, topdown=False): - for name in files: - os.unlink(name, dir_fd=rootfd) - for name in dirs: - st = os.stat(name, dir_fd=rootfd, follow_symlinks=False) - if stat.S_ISDIR(st.st_mode): - os.rmdir(name, dir_fd=rootfd) - else: - os.unlink(name, dir_fd=rootfd) - os.rmdir(support.TESTFN) - class BytesWalkTests(WalkTests): """Tests for os.walk() with bytes.""" def setUp(self): -- cgit v0.12 From e40390473dc1303c550664457f5b7b2582592086 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:42:10 +0100 Subject: support.temp_dir(): call support.rmtree() instead of shutil.rmtree() --- Lib/test/support/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a2ef93d..7914943 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -902,7 +902,7 @@ def temp_dir(path=None, quiet=False): yield path finally: if dir_created: - shutil.rmtree(path) + rmtree(path) @contextlib.contextmanager def change_cwd(path, quiet=False): -- cgit v0.12 From 4ffcc3ee1e982b1c739a3c1d6fb948960b31c3dd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:43:53 +0100 Subject: Cleanup regrtest.py * Move code into a new _main() function * Fix loop to cleanup sys.path * Remove unused import --- Lib/test/regrtest.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index fcc3937..9cbb926 100644 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -11,21 +11,28 @@ import importlib import os import sys -from test.libregrtest import main, main_in_temp_cwd +from test.libregrtest import main_in_temp_cwd -if __name__ == '__main__': +# alias needed by other scripts +main = main_in_temp_cwd + + +def _main(): + global __file__ + # Remove regrtest.py's own directory from the module search path. Despite # the elimination of implicit relative imports, this is still needed to # ensure that submodules of the test package do not inappropriately appear # as top-level modules even when people (or buildbots!) invoke regrtest.py # directly instead of using the -m switch mydir = os.path.abspath(os.path.normpath(os.path.dirname(sys.argv[0]))) - i = len(sys.path) + i = len(sys.path) - 1 while i >= 0: - i -= 1 if os.path.abspath(os.path.normpath(sys.path[i])) == mydir: del sys.path[i] + else: + i -= 1 # findtestdir() gets the dirname out of __file__, so we have to make it # absolute before changing the working directory. @@ -36,4 +43,8 @@ if __name__ == '__main__': # sanity check assert __file__ == os.path.abspath(sys.argv[0]) - main_in_temp_cwd() + main() + + +if __name__ == '__main__': + _main() -- cgit v0.12 From 8c08e0db8fa47c0c8780607a0d803d4964fdffa6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:46:24 +0100 Subject: rt.bat: use -m test instead of Lib\test\regrtest.py --- PCbuild/rt.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 2d93b80..7d4d071 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -42,7 +42,7 @@ if "%1"=="-x64" (set prefix=%pcbuild%amd64\) & shift & goto CheckOpts if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts set exe=%prefix%python%suffix%.exe -set cmd="%exe%" %dashO% -Wd -E -bb "%pcbuild%..\lib\test\regrtest.py" %regrtestargs% +set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% if defined qmode goto Qmode echo Deleting .pyc/.pyo files ... -- cgit v0.12 From 3aac0adfe01b30fc58c638c5ab61844e80b3fd66 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:53:20 +0100 Subject: Cleanup regrtest "main()" function * Rename libregrtest.main_in_temp_cwd() to libregrtest.main() * Add regrtest.main_in_temp_cwd() alias to libregrtest.main() * Move old main_in_temp_cwd() code into libregrtest.Regrtest.main() * Update multiple scripts to call libregrtest.main() --- Lib/test/__main__.py | 5 ++--- Lib/test/autotest.py | 5 ++--- Lib/test/libregrtest/__init__.py | 5 ++++- Lib/test/libregrtest/main.py | 45 +++++++++++++++++++------------------ Lib/test/regrtest.py | 6 ++--- Lib/test/test_importlib/regrtest.py | 4 ++-- PC/testpy.py | 4 ++-- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index d5fbe15..19a6b2b 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,3 +1,2 @@ -from test import regrtest - -regrtest.main_in_temp_cwd() +from test.libregrtest import main +main() diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py index 41c2088..fa85cc1 100644 --- a/Lib/test/autotest.py +++ b/Lib/test/autotest.py @@ -1,6 +1,5 @@ # This should be equivalent to running regrtest.py from the cmdline. # It can be especially handy if you're in an interactive shell, e.g., # from test import autotest. - -from test import regrtest -regrtest.main() +from test.libregrtest import main +main() diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 9f7b1c1..7ba0e6e 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,2 +1,5 @@ +# We import importlib *ASAP* in order to test #15386 +import importlib + from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES -from test.libregrtest.main import main, main_in_temp_cwd +from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c6d9ad0..e1367da 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -415,6 +415,28 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): + global TEMPDIR + + if sysconfig.is_python_build(): + try: + os.mkdir(TEMPDIR) + except FileExistsError: + pass + + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + test_cwd = 'test_python_{}'.format(os.getpid()) + test_cwd = os.path.join(TEMPDIR, test_cwd) + + # Run the tests in a context manager that temporarily changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + self._main(tests, kwargs) + + def _main(self, tests, kwargs): self.ns = self.parse_args(kwargs) if self.ns.slaveargs is not None: @@ -473,26 +495,5 @@ def printlist(x, width=70, indent=4): def main(tests=None, **kwargs): + """Run the Python suite.""" Regrtest().main(tests=tests, **kwargs) - - -def main_in_temp_cwd(): - """Run main() in a temporary working directory.""" - if sysconfig.is_python_build(): - try: - os.mkdir(TEMPDIR) - except FileExistsError: - pass - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from support.SAVEDCWD. - with support.temp_cwd(test_cwd, quiet=True): - main() diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 9cbb926..21b0edf 100644 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -11,11 +11,11 @@ import importlib import os import sys -from test.libregrtest import main_in_temp_cwd +from test.libregrtest import main -# alias needed by other scripts -main = main_in_temp_cwd +# Alias for backward compatibility (just in case) +main_in_temp_cwd = main def _main(): diff --git a/Lib/test/test_importlib/regrtest.py b/Lib/test/test_importlib/regrtest.py index a5be11f..98c815c 100644 --- a/Lib/test/test_importlib/regrtest.py +++ b/Lib/test/test_importlib/regrtest.py @@ -8,10 +8,10 @@ this script. """ import importlib import sys -from test import regrtest +from test import libregrtest if __name__ == '__main__': __builtins__.__import__ = importlib.__import__ sys.path_importer_cache.clear() - regrtest.main(quiet=True, verbose2=True) + libregrtest.main(quiet=True, verbose2=True) diff --git a/PC/testpy.py b/PC/testpy.py index 4ef3d4f..709f35c 100644 --- a/PC/testpy.py +++ b/PC/testpy.py @@ -26,5 +26,5 @@ for dir in sys.path: # Add the "test" directory to PYTHONPATH. sys.path = sys.path + [test] -import regrtest # Standard Python tester. -regrtest.main() +import libregrtest # Standard Python tester. +libregrtest.main() -- cgit v0.12 From 4f17426437a5bdf458f8628936c3e37909fca448 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 00:40:59 +0100 Subject: Fix bug in __import__ during Python shutdown Issue #26637: The importlib module now emits an ImportError rather than a TypeError if __import__() is tried during the Python shutdown process but sys.path is already cleared (set to None). --- Lib/importlib/_bootstrap.py | 11 +- Misc/NEWS | 5 + Python/importlib.h | 884 ++++++++++++++++++++++---------------------- 3 files changed, 459 insertions(+), 441 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 880c493..fa99f56 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -878,13 +878,20 @@ def _find_spec_legacy(finder, name, path): def _find_spec(name, path, target=None): """Find a module's loader.""" - if sys.meta_path is not None and not sys.meta_path: + meta_path = sys.meta_path + if meta_path is None: + # PyImport_Cleanup() is running or has been called. + raise ImportError("sys.meta_path is None, Python is likely " + "shutting down") + + if not meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) + # We check sys.modules here for the reload case. While a passed-in # target will usually indicate a reload there is no guarantee, whereas # sys.modules provides one. is_reload = name in sys.modules - for finder in sys.meta_path: + for finder in meta_path: with _ImportLockContext(): try: find_spec = finder.find_spec diff --git a/Misc/NEWS b/Misc/NEWS index f483735..d684c82 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,11 @@ Core and Builtins Library ------- +- Issue #26637: The :mod:`importlib` module now emits an :exc:`ImportError` + rather than a :exc:`TypeError` if :func:`__import__` is tried during the + Python shutdown process but :data:`sys.path` is already cleared (set to + ``None``). + - Issue #21925: :func:`warnings.formatwarning` now catches exceptions when calling :func;`linecache.getline` and :func:`tracemalloc.get_object_traceback` to be able to log diff --git a/Python/importlib.h b/Python/importlib.h index d983114..685c51a 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1573,443 +1573,449 @@ const unsigned char _Py_M__importlib[] = { 11,0,0,0,218,17,95,102,105,110,100,95,115,112,101,99, 95,108,101,103,97,99,121,102,3,0,0,115,8,0,0,0, 0,3,18,1,12,1,4,1,114,174,0,0,0,99,3,0, - 0,0,0,0,0,0,9,0,0,0,27,0,0,0,67,0, - 0,0,115,42,1,0,0,116,0,0,106,1,0,100,1,0, - 107,9,0,114,41,0,116,0,0,106,1,0,12,114,41,0, - 116,2,0,106,3,0,100,2,0,116,4,0,131,2,0,1, - 124,0,0,116,0,0,106,5,0,107,6,0,125,3,0,120, - 235,0,116,0,0,106,1,0,68,93,220,0,125,4,0,116, - 6,0,131,0,0,143,90,0,1,121,13,0,124,4,0,106, - 7,0,125,5,0,87,110,51,0,4,116,8,0,107,10,0, - 114,148,0,1,1,1,116,9,0,124,4,0,124,0,0,124, - 1,0,131,3,0,125,6,0,124,6,0,100,1,0,107,8, - 0,114,144,0,119,66,0,89,110,19,0,88,124,5,0,124, - 0,0,124,1,0,124,2,0,131,3,0,125,6,0,87,100, - 1,0,81,82,88,124,6,0,100,1,0,107,9,0,114,66, - 0,124,3,0,12,114,26,1,124,0,0,116,0,0,106,5, - 0,107,6,0,114,26,1,116,0,0,106,5,0,124,0,0, - 25,125,7,0,121,13,0,124,7,0,106,10,0,125,8,0, - 87,110,22,0,4,116,8,0,107,10,0,114,2,1,1,1, - 1,124,6,0,83,89,113,30,1,88,124,8,0,100,1,0, - 107,8,0,114,19,1,124,6,0,83,124,8,0,83,113,66, - 0,124,6,0,83,113,66,0,87,100,1,0,83,100,1,0, - 83,41,3,122,23,70,105,110,100,32,97,32,109,111,100,117, - 108,101,39,115,32,108,111,97,100,101,114,46,78,122,22,115, - 121,115,46,109,101,116,97,95,112,97,116,104,32,105,115,32, - 101,109,112,116,121,41,11,114,14,0,0,0,218,9,109,101, - 116,97,95,112,97,116,104,114,142,0,0,0,114,143,0,0, - 0,218,13,73,109,112,111,114,116,87,97,114,110,105,110,103, - 114,21,0,0,0,114,166,0,0,0,114,155,0,0,0,114, - 96,0,0,0,114,174,0,0,0,114,95,0,0,0,41,9, - 114,15,0,0,0,114,153,0,0,0,114,154,0,0,0,90, - 9,105,115,95,114,101,108,111,97,100,114,173,0,0,0,114, - 155,0,0,0,114,88,0,0,0,114,89,0,0,0,114,95, - 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,218,10,95,102,105,110,100,95,115,112,101,99,111,3, - 0,0,115,48,0,0,0,0,2,25,1,16,4,15,1,16, - 1,10,1,3,1,13,1,13,1,18,1,12,1,8,2,25, - 1,12,2,22,1,13,1,3,1,13,1,13,4,9,2,12, - 1,4,2,7,2,8,2,114,177,0,0,0,99,3,0,0, - 0,0,0,0,0,4,0,0,0,4,0,0,0,67,0,0, - 0,115,206,0,0,0,116,0,0,124,0,0,116,1,0,131, - 2,0,115,42,0,116,2,0,100,1,0,106,3,0,116,4, - 0,124,0,0,131,1,0,131,1,0,131,1,0,130,1,0, - 124,2,0,100,2,0,107,0,0,114,66,0,116,5,0,100, - 3,0,131,1,0,130,1,0,124,2,0,100,2,0,107,4, - 0,114,171,0,116,0,0,124,1,0,116,1,0,131,2,0, - 115,108,0,116,2,0,100,4,0,131,1,0,130,1,0,110, - 63,0,124,1,0,115,129,0,116,6,0,100,5,0,131,1, - 0,130,1,0,110,42,0,124,1,0,116,7,0,106,8,0, - 107,7,0,114,171,0,100,6,0,125,3,0,116,9,0,124, - 3,0,106,3,0,124,1,0,131,1,0,131,1,0,130,1, - 0,124,0,0,12,114,202,0,124,2,0,100,2,0,107,2, - 0,114,202,0,116,5,0,100,7,0,131,1,0,130,1,0, - 100,8,0,83,41,9,122,28,86,101,114,105,102,121,32,97, - 114,103,117,109,101,110,116,115,32,97,114,101,32,34,115,97, - 110,101,34,46,122,31,109,111,100,117,108,101,32,110,97,109, - 101,32,109,117,115,116,32,98,101,32,115,116,114,44,32,110, - 111,116,32,123,125,114,33,0,0,0,122,18,108,101,118,101, - 108,32,109,117,115,116,32,98,101,32,62,61,32,48,122,31, - 95,95,112,97,99,107,97,103,101,95,95,32,110,111,116,32, - 115,101,116,32,116,111,32,97,32,115,116,114,105,110,103,122, - 54,97,116,116,101,109,112,116,101,100,32,114,101,108,97,116, - 105,118,101,32,105,109,112,111,114,116,32,119,105,116,104,32, - 110,111,32,107,110,111,119,110,32,112,97,114,101,110,116,32, - 112,97,99,107,97,103,101,122,61,80,97,114,101,110,116,32, - 109,111,100,117,108,101,32,123,33,114,125,32,110,111,116,32, - 108,111,97,100,101,100,44,32,99,97,110,110,111,116,32,112, - 101,114,102,111,114,109,32,114,101,108,97,116,105,118,101,32, - 105,109,112,111,114,116,122,17,69,109,112,116,121,32,109,111, - 100,117,108,101,32,110,97,109,101,78,41,10,218,10,105,115, - 105,110,115,116,97,110,99,101,218,3,115,116,114,218,9,84, - 121,112,101,69,114,114,111,114,114,50,0,0,0,114,13,0, - 0,0,114,169,0,0,0,114,77,0,0,0,114,14,0,0, - 0,114,21,0,0,0,218,11,83,121,115,116,101,109,69,114, - 114,111,114,41,4,114,15,0,0,0,114,170,0,0,0,114, - 171,0,0,0,114,148,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,218,13,95,115,97,110,105,116, - 121,95,99,104,101,99,107,151,3,0,0,115,28,0,0,0, - 0,2,15,1,27,1,12,1,12,1,12,1,15,1,15,1, - 6,1,15,2,15,1,6,2,21,1,19,1,114,182,0,0, - 0,122,16,78,111,32,109,111,100,117,108,101,32,110,97,109, - 101,100,32,122,4,123,33,114,125,99,2,0,0,0,0,0, - 0,0,8,0,0,0,12,0,0,0,67,0,0,0,115,40, - 1,0,0,100,0,0,125,2,0,124,0,0,106,0,0,100, - 1,0,131,1,0,100,2,0,25,125,3,0,124,3,0,114, - 175,0,124,3,0,116,1,0,106,2,0,107,7,0,114,59, - 0,116,3,0,124,1,0,124,3,0,131,2,0,1,124,0, - 0,116,1,0,106,2,0,107,6,0,114,85,0,116,1,0, - 106,2,0,124,0,0,25,83,116,1,0,106,2,0,124,3, - 0,25,125,4,0,121,13,0,124,4,0,106,4,0,125,2, - 0,87,110,61,0,4,116,5,0,107,10,0,114,174,0,1, - 1,1,116,6,0,100,3,0,23,106,7,0,124,0,0,124, - 3,0,131,2,0,125,5,0,116,8,0,124,5,0,100,4, - 0,124,0,0,131,1,1,100,0,0,130,2,0,89,110,1, - 0,88,116,9,0,124,0,0,124,2,0,131,2,0,125,6, - 0,124,6,0,100,0,0,107,8,0,114,232,0,116,8,0, - 116,6,0,106,7,0,124,0,0,131,1,0,100,4,0,124, - 0,0,131,1,1,130,1,0,110,12,0,116,10,0,124,6, - 0,131,1,0,125,7,0,124,3,0,114,36,1,116,1,0, - 106,2,0,124,3,0,25,125,4,0,116,11,0,124,4,0, - 124,0,0,106,0,0,100,1,0,131,1,0,100,5,0,25, - 124,7,0,131,3,0,1,124,7,0,83,41,6,78,114,121, - 0,0,0,114,33,0,0,0,122,23,59,32,123,33,114,125, - 32,105,115,32,110,111,116,32,97,32,112,97,99,107,97,103, - 101,114,15,0,0,0,114,141,0,0,0,41,12,114,122,0, - 0,0,114,14,0,0,0,114,21,0,0,0,114,65,0,0, - 0,114,131,0,0,0,114,96,0,0,0,218,8,95,69,82, - 82,95,77,83,71,114,50,0,0,0,114,77,0,0,0,114, - 177,0,0,0,114,150,0,0,0,114,5,0,0,0,41,8, - 114,15,0,0,0,218,7,105,109,112,111,114,116,95,114,153, - 0,0,0,114,123,0,0,0,90,13,112,97,114,101,110,116, - 95,109,111,100,117,108,101,114,148,0,0,0,114,88,0,0, - 0,114,89,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,218,23,95,102,105,110,100,95,97,110,100, - 95,108,111,97,100,95,117,110,108,111,99,107,101,100,174,3, - 0,0,115,42,0,0,0,0,1,6,1,19,1,6,1,15, - 1,13,2,15,1,11,1,13,1,3,1,13,1,13,1,22, - 1,26,1,15,1,12,1,30,2,12,1,6,2,13,1,29, - 1,114,185,0,0,0,99,2,0,0,0,0,0,0,0,2, - 0,0,0,10,0,0,0,67,0,0,0,115,37,0,0,0, - 116,0,0,124,0,0,131,1,0,143,18,0,1,116,1,0, - 124,0,0,124,1,0,131,2,0,83,87,100,1,0,81,82, - 88,100,1,0,83,41,2,122,54,70,105,110,100,32,97,110, - 100,32,108,111,97,100,32,116,104,101,32,109,111,100,117,108, - 101,44,32,97,110,100,32,114,101,108,101,97,115,101,32,116, - 104,101,32,105,109,112,111,114,116,32,108,111,99,107,46,78, - 41,2,114,54,0,0,0,114,185,0,0,0,41,2,114,15, - 0,0,0,114,184,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,218,14,95,102,105,110,100,95,97, - 110,100,95,108,111,97,100,201,3,0,0,115,4,0,0,0, - 0,2,13,1,114,186,0,0,0,114,33,0,0,0,99,3, - 0,0,0,0,0,0,0,5,0,0,0,4,0,0,0,67, - 0,0,0,115,166,0,0,0,116,0,0,124,0,0,124,1, - 0,124,2,0,131,3,0,1,124,2,0,100,1,0,107,4, - 0,114,46,0,116,1,0,124,0,0,124,1,0,124,2,0, - 131,3,0,125,0,0,116,2,0,106,3,0,131,0,0,1, - 124,0,0,116,4,0,106,5,0,107,7,0,114,84,0,116, - 6,0,124,0,0,116,7,0,131,2,0,83,116,4,0,106, - 5,0,124,0,0,25,125,3,0,124,3,0,100,2,0,107, - 8,0,114,152,0,116,2,0,106,8,0,131,0,0,1,100, - 3,0,106,9,0,124,0,0,131,1,0,125,4,0,116,10, - 0,124,4,0,100,4,0,124,0,0,131,1,1,130,1,0, - 116,11,0,124,0,0,131,1,0,1,124,3,0,83,41,5, - 97,50,1,0,0,73,109,112,111,114,116,32,97,110,100,32, - 114,101,116,117,114,110,32,116,104,101,32,109,111,100,117,108, - 101,32,98,97,115,101,100,32,111,110,32,105,116,115,32,110, - 97,109,101,44,32,116,104,101,32,112,97,99,107,97,103,101, - 32,116,104,101,32,99,97,108,108,32,105,115,10,32,32,32, - 32,98,101,105,110,103,32,109,97,100,101,32,102,114,111,109, - 44,32,97,110,100,32,116,104,101,32,108,101,118,101,108,32, - 97,100,106,117,115,116,109,101,110,116,46,10,10,32,32,32, - 32,84,104,105,115,32,102,117,110,99,116,105,111,110,32,114, - 101,112,114,101,115,101,110,116,115,32,116,104,101,32,103,114, - 101,97,116,101,115,116,32,99,111,109,109,111,110,32,100,101, - 110,111,109,105,110,97,116,111,114,32,111,102,32,102,117,110, - 99,116,105,111,110,97,108,105,116,121,10,32,32,32,32,98, - 101,116,119,101,101,110,32,105,109,112,111,114,116,95,109,111, - 100,117,108,101,32,97,110,100,32,95,95,105,109,112,111,114, - 116,95,95,46,32,84,104,105,115,32,105,110,99,108,117,100, - 101,115,32,115,101,116,116,105,110,103,32,95,95,112,97,99, - 107,97,103,101,95,95,32,105,102,10,32,32,32,32,116,104, - 101,32,108,111,97,100,101,114,32,100,105,100,32,110,111,116, - 46,10,10,32,32,32,32,114,33,0,0,0,78,122,40,105, - 109,112,111,114,116,32,111,102,32,123,125,32,104,97,108,116, - 101,100,59,32,78,111,110,101,32,105,110,32,115,121,115,46, - 109,111,100,117,108,101,115,114,15,0,0,0,41,12,114,182, - 0,0,0,114,172,0,0,0,114,57,0,0,0,114,146,0, - 0,0,114,14,0,0,0,114,21,0,0,0,114,186,0,0, - 0,218,11,95,103,99,100,95,105,109,112,111,114,116,114,58, - 0,0,0,114,50,0,0,0,114,77,0,0,0,114,63,0, - 0,0,41,5,114,15,0,0,0,114,170,0,0,0,114,171, - 0,0,0,114,89,0,0,0,114,74,0,0,0,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,114,187,0,0, - 0,207,3,0,0,115,28,0,0,0,0,9,16,1,12,1, - 18,1,10,1,15,1,13,1,13,1,12,1,10,1,6,1, - 9,1,18,1,10,1,114,187,0,0,0,99,3,0,0,0, - 0,0,0,0,6,0,0,0,17,0,0,0,67,0,0,0, - 115,239,0,0,0,116,0,0,124,0,0,100,1,0,131,2, - 0,114,235,0,100,2,0,124,1,0,107,6,0,114,83,0, - 116,1,0,124,1,0,131,1,0,125,1,0,124,1,0,106, - 2,0,100,2,0,131,1,0,1,116,0,0,124,0,0,100, - 3,0,131,2,0,114,83,0,124,1,0,106,3,0,124,0, - 0,106,4,0,131,1,0,1,120,149,0,124,1,0,68,93, - 141,0,125,3,0,116,0,0,124,0,0,124,3,0,131,2, - 0,115,90,0,100,4,0,106,5,0,124,0,0,106,6,0, - 124,3,0,131,2,0,125,4,0,121,17,0,116,7,0,124, - 2,0,124,4,0,131,2,0,1,87,113,90,0,4,116,8, - 0,107,10,0,114,230,0,1,125,5,0,1,122,47,0,116, - 9,0,124,5,0,131,1,0,106,10,0,116,11,0,131,1, - 0,114,209,0,124,5,0,106,12,0,124,4,0,107,2,0, - 114,209,0,119,90,0,130,0,0,87,89,100,5,0,100,5, - 0,125,5,0,126,5,0,88,113,90,0,88,113,90,0,87, - 124,0,0,83,41,6,122,238,70,105,103,117,114,101,32,111, - 117,116,32,119,104,97,116,32,95,95,105,109,112,111,114,116, - 95,95,32,115,104,111,117,108,100,32,114,101,116,117,114,110, - 46,10,10,32,32,32,32,84,104,101,32,105,109,112,111,114, - 116,95,32,112,97,114,97,109,101,116,101,114,32,105,115,32, - 97,32,99,97,108,108,97,98,108,101,32,119,104,105,99,104, - 32,116,97,107,101,115,32,116,104,101,32,110,97,109,101,32, - 111,102,32,109,111,100,117,108,101,32,116,111,10,32,32,32, - 32,105,109,112,111,114,116,46,32,73,116,32,105,115,32,114, - 101,113,117,105,114,101,100,32,116,111,32,100,101,99,111,117, - 112,108,101,32,116,104,101,32,102,117,110,99,116,105,111,110, - 32,102,114,111,109,32,97,115,115,117,109,105,110,103,32,105, - 109,112,111,114,116,108,105,98,39,115,10,32,32,32,32,105, - 109,112,111,114,116,32,105,109,112,108,101,109,101,110,116,97, - 116,105,111,110,32,105,115,32,100,101,115,105,114,101,100,46, - 10,10,32,32,32,32,114,131,0,0,0,250,1,42,218,7, - 95,95,97,108,108,95,95,122,5,123,125,46,123,125,78,41, - 13,114,4,0,0,0,114,130,0,0,0,218,6,114,101,109, - 111,118,101,218,6,101,120,116,101,110,100,114,189,0,0,0, - 114,50,0,0,0,114,1,0,0,0,114,65,0,0,0,114, - 77,0,0,0,114,179,0,0,0,114,71,0,0,0,218,15, - 95,69,82,82,95,77,83,71,95,80,82,69,70,73,88,114, - 15,0,0,0,41,6,114,89,0,0,0,218,8,102,114,111, - 109,108,105,115,116,114,184,0,0,0,218,1,120,90,9,102, - 114,111,109,95,110,97,109,101,90,3,101,120,99,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,16,95,104, - 97,110,100,108,101,95,102,114,111,109,108,105,115,116,231,3, - 0,0,115,34,0,0,0,0,10,15,1,12,1,12,1,13, - 1,15,1,16,1,13,1,15,1,21,1,3,1,17,1,18, - 4,21,1,15,1,3,1,26,1,114,195,0,0,0,99,1, - 0,0,0,0,0,0,0,3,0,0,0,7,0,0,0,67, - 0,0,0,115,214,0,0,0,124,0,0,106,0,0,100,1, - 0,131,1,0,125,1,0,124,0,0,106,0,0,100,2,0, - 131,1,0,125,2,0,124,1,0,100,3,0,107,9,0,114, - 128,0,124,2,0,100,3,0,107,9,0,114,124,0,124,1, - 0,124,2,0,106,1,0,107,3,0,114,124,0,116,2,0, - 106,3,0,100,4,0,106,4,0,100,5,0,124,1,0,155, - 2,0,100,6,0,124,2,0,106,1,0,155,2,0,100,7, - 0,103,5,0,131,1,0,116,5,0,100,8,0,100,9,0, - 131,2,1,1,124,1,0,83,124,2,0,100,3,0,107,9, - 0,114,147,0,124,2,0,106,1,0,83,116,2,0,106,3, - 0,100,10,0,116,5,0,100,8,0,100,9,0,131,2,1, - 1,124,0,0,100,11,0,25,125,1,0,100,12,0,124,0, - 0,107,7,0,114,210,0,124,1,0,106,6,0,100,13,0, - 131,1,0,100,14,0,25,125,1,0,124,1,0,83,41,15, - 122,167,67,97,108,99,117,108,97,116,101,32,119,104,97,116, - 32,95,95,112,97,99,107,97,103,101,95,95,32,115,104,111, - 117,108,100,32,98,101,46,10,10,32,32,32,32,95,95,112, - 97,99,107,97,103,101,95,95,32,105,115,32,110,111,116,32, - 103,117,97,114,97,110,116,101,101,100,32,116,111,32,98,101, - 32,100,101,102,105,110,101,100,32,111,114,32,99,111,117,108, - 100,32,98,101,32,115,101,116,32,116,111,32,78,111,110,101, - 10,32,32,32,32,116,111,32,114,101,112,114,101,115,101,110, - 116,32,116,104,97,116,32,105,116,115,32,112,114,111,112,101, - 114,32,118,97,108,117,101,32,105,115,32,117,110,107,110,111, - 119,110,46,10,10,32,32,32,32,114,134,0,0,0,114,95, - 0,0,0,78,218,0,122,32,95,95,112,97,99,107,97,103, - 101,95,95,32,33,61,32,95,95,115,112,101,99,95,95,46, - 112,97,114,101,110,116,32,40,122,4,32,33,61,32,250,1, - 41,114,140,0,0,0,233,3,0,0,0,122,89,99,97,110, - 39,116,32,114,101,115,111,108,118,101,32,112,97,99,107,97, - 103,101,32,102,114,111,109,32,95,95,115,112,101,99,95,95, - 32,111,114,32,95,95,112,97,99,107,97,103,101,95,95,44, - 32,102,97,108,108,105,110,103,32,98,97,99,107,32,111,110, - 32,95,95,110,97,109,101,95,95,32,97,110,100,32,95,95, - 112,97,116,104,95,95,114,1,0,0,0,114,131,0,0,0, - 114,121,0,0,0,114,33,0,0,0,41,7,114,42,0,0, - 0,114,123,0,0,0,114,142,0,0,0,114,143,0,0,0, - 114,115,0,0,0,114,176,0,0,0,114,122,0,0,0,41, - 3,218,7,103,108,111,98,97,108,115,114,170,0,0,0,114, - 88,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,17,95,99,97,108,99,95,95,95,112,97,99, - 107,97,103,101,95,95,7,4,0,0,115,30,0,0,0,0, - 7,15,1,15,1,12,1,27,1,42,2,13,1,4,1,12, - 1,7,2,9,2,13,1,10,1,12,1,19,1,114,200,0, - 0,0,99,5,0,0,0,0,0,0,0,9,0,0,0,5, - 0,0,0,67,0,0,0,115,227,0,0,0,124,4,0,100, - 1,0,107,2,0,114,27,0,116,0,0,124,0,0,131,1, - 0,125,5,0,110,54,0,124,1,0,100,2,0,107,9,0, - 114,45,0,124,1,0,110,3,0,105,0,0,125,6,0,116, - 1,0,124,6,0,131,1,0,125,7,0,116,0,0,124,0, - 0,124,7,0,124,4,0,131,3,0,125,5,0,124,3,0, - 115,207,0,124,4,0,100,1,0,107,2,0,114,122,0,116, - 0,0,124,0,0,106,2,0,100,3,0,131,1,0,100,1, - 0,25,131,1,0,83,124,0,0,115,132,0,124,5,0,83, - 116,3,0,124,0,0,131,1,0,116,3,0,124,0,0,106, - 2,0,100,3,0,131,1,0,100,1,0,25,131,1,0,24, - 125,8,0,116,4,0,106,5,0,124,5,0,106,6,0,100, - 2,0,116,3,0,124,5,0,106,6,0,131,1,0,124,8, - 0,24,133,2,0,25,25,83,110,16,0,116,7,0,124,5, - 0,124,3,0,116,0,0,131,3,0,83,100,2,0,83,41, - 4,97,214,1,0,0,73,109,112,111,114,116,32,97,32,109, - 111,100,117,108,101,46,10,10,32,32,32,32,84,104,101,32, - 39,103,108,111,98,97,108,115,39,32,97,114,103,117,109,101, - 110,116,32,105,115,32,117,115,101,100,32,116,111,32,105,110, - 102,101,114,32,119,104,101,114,101,32,116,104,101,32,105,109, - 112,111,114,116,32,105,115,32,111,99,99,117,114,105,110,103, - 32,102,114,111,109,10,32,32,32,32,116,111,32,104,97,110, - 100,108,101,32,114,101,108,97,116,105,118,101,32,105,109,112, - 111,114,116,115,46,32,84,104,101,32,39,108,111,99,97,108, - 115,39,32,97,114,103,117,109,101,110,116,32,105,115,32,105, - 103,110,111,114,101,100,46,32,84,104,101,10,32,32,32,32, - 39,102,114,111,109,108,105,115,116,39,32,97,114,103,117,109, - 101,110,116,32,115,112,101,99,105,102,105,101,115,32,119,104, - 97,116,32,115,104,111,117,108,100,32,101,120,105,115,116,32, - 97,115,32,97,116,116,114,105,98,117,116,101,115,32,111,110, - 32,116,104,101,32,109,111,100,117,108,101,10,32,32,32,32, - 98,101,105,110,103,32,105,109,112,111,114,116,101,100,32,40, - 101,46,103,46,32,96,96,102,114,111,109,32,109,111,100,117, - 108,101,32,105,109,112,111,114,116,32,60,102,114,111,109,108, - 105,115,116,62,96,96,41,46,32,32,84,104,101,32,39,108, - 101,118,101,108,39,10,32,32,32,32,97,114,103,117,109,101, - 110,116,32,114,101,112,114,101,115,101,110,116,115,32,116,104, - 101,32,112,97,99,107,97,103,101,32,108,111,99,97,116,105, - 111,110,32,116,111,32,105,109,112,111,114,116,32,102,114,111, - 109,32,105,110,32,97,32,114,101,108,97,116,105,118,101,10, - 32,32,32,32,105,109,112,111,114,116,32,40,101,46,103,46, - 32,96,96,102,114,111,109,32,46,46,112,107,103,32,105,109, - 112,111,114,116,32,109,111,100,96,96,32,119,111,117,108,100, - 32,104,97,118,101,32,97,32,39,108,101,118,101,108,39,32, - 111,102,32,50,41,46,10,10,32,32,32,32,114,33,0,0, - 0,78,114,121,0,0,0,41,8,114,187,0,0,0,114,200, - 0,0,0,218,9,112,97,114,116,105,116,105,111,110,114,168, - 0,0,0,114,14,0,0,0,114,21,0,0,0,114,1,0, - 0,0,114,195,0,0,0,41,9,114,15,0,0,0,114,199, - 0,0,0,218,6,108,111,99,97,108,115,114,193,0,0,0, - 114,171,0,0,0,114,89,0,0,0,90,8,103,108,111,98, - 97,108,115,95,114,170,0,0,0,90,7,99,117,116,95,111, - 102,102,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,218,10,95,95,105,109,112,111,114,116,95,95,34,4,0, - 0,115,26,0,0,0,0,11,12,1,15,2,24,1,12,1, - 18,1,6,3,12,1,23,1,6,1,4,4,35,3,40,2, - 114,203,0,0,0,99,1,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,53,0,0,0,116, - 0,0,106,1,0,124,0,0,131,1,0,125,1,0,124,1, - 0,100,0,0,107,8,0,114,43,0,116,2,0,100,1,0, - 124,0,0,23,131,1,0,130,1,0,116,3,0,124,1,0, - 131,1,0,83,41,2,78,122,25,110,111,32,98,117,105,108, - 116,45,105,110,32,109,111,100,117,108,101,32,110,97,109,101, - 100,32,41,4,114,151,0,0,0,114,155,0,0,0,114,77, - 0,0,0,114,150,0,0,0,41,2,114,15,0,0,0,114, - 88,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,18,95,98,117,105,108,116,105,110,95,102,114, - 111,109,95,110,97,109,101,69,4,0,0,115,8,0,0,0, - 0,1,15,1,12,1,16,1,114,204,0,0,0,99,2,0, - 0,0,0,0,0,0,12,0,0,0,12,0,0,0,67,0, - 0,0,115,74,1,0,0,124,1,0,97,0,0,124,0,0, - 97,1,0,116,2,0,116,1,0,131,1,0,125,2,0,120, - 123,0,116,1,0,106,3,0,106,4,0,131,0,0,68,93, - 106,0,92,2,0,125,3,0,125,4,0,116,5,0,124,4, - 0,124,2,0,131,2,0,114,40,0,124,3,0,116,1,0, - 106,6,0,107,6,0,114,91,0,116,7,0,125,5,0,110, - 27,0,116,0,0,106,8,0,124,3,0,131,1,0,114,40, - 0,116,9,0,125,5,0,110,3,0,113,40,0,116,10,0, - 124,4,0,124,5,0,131,2,0,125,6,0,116,11,0,124, - 6,0,124,4,0,131,2,0,1,113,40,0,87,116,1,0, - 106,3,0,116,12,0,25,125,7,0,120,73,0,100,5,0, - 68,93,65,0,125,8,0,124,8,0,116,1,0,106,3,0, - 107,7,0,114,206,0,116,13,0,124,8,0,131,1,0,125, - 9,0,110,13,0,116,1,0,106,3,0,124,8,0,25,125, - 9,0,116,14,0,124,7,0,124,8,0,124,9,0,131,3, - 0,1,113,170,0,87,121,16,0,116,13,0,100,2,0,131, - 1,0,125,10,0,87,110,24,0,4,116,15,0,107,10,0, - 114,25,1,1,1,1,100,3,0,125,10,0,89,110,1,0, - 88,116,14,0,124,7,0,100,2,0,124,10,0,131,3,0, - 1,116,13,0,100,4,0,131,1,0,125,11,0,116,14,0, - 124,7,0,100,4,0,124,11,0,131,3,0,1,100,3,0, - 83,41,6,122,250,83,101,116,117,112,32,105,109,112,111,114, - 116,108,105,98,32,98,121,32,105,109,112,111,114,116,105,110, - 103,32,110,101,101,100,101,100,32,98,117,105,108,116,45,105, - 110,32,109,111,100,117,108,101,115,32,97,110,100,32,105,110, - 106,101,99,116,105,110,103,32,116,104,101,109,10,32,32,32, - 32,105,110,116,111,32,116,104,101,32,103,108,111,98,97,108, - 32,110,97,109,101,115,112,97,99,101,46,10,10,32,32,32, - 32,65,115,32,115,121,115,32,105,115,32,110,101,101,100,101, - 100,32,102,111,114,32,115,121,115,46,109,111,100,117,108,101, - 115,32,97,99,99,101,115,115,32,97,110,100,32,95,105,109, - 112,32,105,115,32,110,101,101,100,101,100,32,116,111,32,108, - 111,97,100,32,98,117,105,108,116,45,105,110,10,32,32,32, - 32,109,111,100,117,108,101,115,44,32,116,104,111,115,101,32, - 116,119,111,32,109,111,100,117,108,101,115,32,109,117,115,116, - 32,98,101,32,101,120,112,108,105,99,105,116,108,121,32,112, - 97,115,115,101,100,32,105,110,46,10,10,32,32,32,32,114, - 142,0,0,0,114,34,0,0,0,78,114,62,0,0,0,41, - 1,122,9,95,119,97,114,110,105,110,103,115,41,16,114,57, - 0,0,0,114,14,0,0,0,114,13,0,0,0,114,21,0, - 0,0,218,5,105,116,101,109,115,114,178,0,0,0,114,76, - 0,0,0,114,151,0,0,0,114,82,0,0,0,114,161,0, - 0,0,114,132,0,0,0,114,137,0,0,0,114,1,0,0, - 0,114,204,0,0,0,114,5,0,0,0,114,77,0,0,0, - 41,12,218,10,115,121,115,95,109,111,100,117,108,101,218,11, - 95,105,109,112,95,109,111,100,117,108,101,90,11,109,111,100, - 117,108,101,95,116,121,112,101,114,15,0,0,0,114,89,0, - 0,0,114,99,0,0,0,114,88,0,0,0,90,11,115,101, - 108,102,95,109,111,100,117,108,101,90,12,98,117,105,108,116, - 105,110,95,110,97,109,101,90,14,98,117,105,108,116,105,110, - 95,109,111,100,117,108,101,90,13,116,104,114,101,97,100,95, - 109,111,100,117,108,101,90,14,119,101,97,107,114,101,102,95, - 109,111,100,117,108,101,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,218,6,95,115,101,116,117,112,76,4,0, - 0,115,50,0,0,0,0,9,6,1,6,3,12,1,28,1, - 15,1,15,1,9,1,15,1,9,2,3,1,15,1,17,3, - 13,1,13,1,15,1,15,2,13,1,20,3,3,1,16,1, - 13,2,11,1,16,3,12,1,114,208,0,0,0,99,2,0, - 0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0, - 0,0,115,87,0,0,0,116,0,0,124,0,0,124,1,0, - 131,2,0,1,116,1,0,106,2,0,106,3,0,116,4,0, - 131,1,0,1,116,1,0,106,2,0,106,3,0,116,5,0, - 131,1,0,1,100,1,0,100,2,0,108,6,0,125,2,0, - 124,2,0,97,7,0,124,2,0,106,8,0,116,1,0,106, - 9,0,116,10,0,25,131,1,0,1,100,2,0,83,41,3, - 122,50,73,110,115,116,97,108,108,32,105,109,112,111,114,116, - 108,105,98,32,97,115,32,116,104,101,32,105,109,112,108,101, - 109,101,110,116,97,116,105,111,110,32,111,102,32,105,109,112, - 111,114,116,46,114,33,0,0,0,78,41,11,114,208,0,0, - 0,114,14,0,0,0,114,175,0,0,0,114,113,0,0,0, - 114,151,0,0,0,114,161,0,0,0,218,26,95,102,114,111, - 122,101,110,95,105,109,112,111,114,116,108,105,98,95,101,120, - 116,101,114,110,97,108,114,119,0,0,0,218,8,95,105,110, - 115,116,97,108,108,114,21,0,0,0,114,1,0,0,0,41, - 3,114,206,0,0,0,114,207,0,0,0,114,209,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 210,0,0,0,123,4,0,0,115,12,0,0,0,0,2,13, - 2,16,1,16,3,12,1,6,1,114,210,0,0,0,41,51, - 114,3,0,0,0,114,119,0,0,0,114,12,0,0,0,114, - 16,0,0,0,114,17,0,0,0,114,59,0,0,0,114,41, - 0,0,0,114,48,0,0,0,114,31,0,0,0,114,32,0, - 0,0,114,53,0,0,0,114,54,0,0,0,114,56,0,0, - 0,114,63,0,0,0,114,65,0,0,0,114,75,0,0,0, - 114,81,0,0,0,114,84,0,0,0,114,90,0,0,0,114, - 101,0,0,0,114,102,0,0,0,114,106,0,0,0,114,85, - 0,0,0,218,6,111,98,106,101,99,116,90,9,95,80,79, - 80,85,76,65,84,69,114,132,0,0,0,114,137,0,0,0, - 114,145,0,0,0,114,97,0,0,0,114,86,0,0,0,114, - 149,0,0,0,114,150,0,0,0,114,87,0,0,0,114,151, - 0,0,0,114,161,0,0,0,114,166,0,0,0,114,172,0, - 0,0,114,174,0,0,0,114,177,0,0,0,114,182,0,0, - 0,114,192,0,0,0,114,183,0,0,0,114,185,0,0,0, - 114,186,0,0,0,114,187,0,0,0,114,195,0,0,0,114, - 200,0,0,0,114,203,0,0,0,114,204,0,0,0,114,208, - 0,0,0,114,210,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,8,60,109, - 111,100,117,108,101,62,8,0,0,0,115,96,0,0,0,6, - 17,6,2,12,8,12,4,19,20,6,2,6,3,22,4,19, - 68,19,21,19,19,12,19,12,19,12,11,18,8,12,11,12, - 12,12,16,12,36,19,27,19,101,24,26,9,3,18,45,18, - 60,12,18,12,17,12,25,12,29,12,23,12,16,19,73,19, - 77,19,13,12,9,12,9,15,40,12,20,6,1,10,2,12, - 27,12,6,18,24,12,32,12,27,24,35,12,7,12,47, + 0,0,0,0,0,0,10,0,0,0,27,0,0,0,67,0, + 0,0,115,53,1,0,0,116,0,0,106,1,0,125,3,0, + 124,3,0,100,1,0,107,8,0,114,33,0,116,2,0,100, + 2,0,131,1,0,130,1,0,124,3,0,115,55,0,116,3, + 0,106,4,0,100,3,0,116,5,0,131,2,0,1,124,0, + 0,116,0,0,106,6,0,107,6,0,125,4,0,120,232,0, + 124,3,0,68,93,220,0,125,5,0,116,7,0,131,0,0, + 143,90,0,1,121,13,0,124,5,0,106,8,0,125,6,0, + 87,110,51,0,4,116,9,0,107,10,0,114,159,0,1,1, + 1,116,10,0,124,5,0,124,0,0,124,1,0,131,3,0, + 125,7,0,124,7,0,100,1,0,107,8,0,114,155,0,119, + 77,0,89,110,19,0,88,124,6,0,124,0,0,124,1,0, + 124,2,0,131,3,0,125,7,0,87,100,1,0,81,82,88, + 124,7,0,100,1,0,107,9,0,114,77,0,124,4,0,12, + 114,37,1,124,0,0,116,0,0,106,6,0,107,6,0,114, + 37,1,116,0,0,106,6,0,124,0,0,25,125,8,0,121, + 13,0,124,8,0,106,11,0,125,9,0,87,110,22,0,4, + 116,9,0,107,10,0,114,13,1,1,1,1,124,7,0,83, + 89,113,41,1,88,124,9,0,100,1,0,107,8,0,114,30, + 1,124,7,0,83,124,9,0,83,113,77,0,124,7,0,83, + 113,77,0,87,100,1,0,83,100,1,0,83,41,4,122,23, + 70,105,110,100,32,97,32,109,111,100,117,108,101,39,115,32, + 108,111,97,100,101,114,46,78,122,53,115,121,115,46,109,101, + 116,97,95,112,97,116,104,32,105,115,32,78,111,110,101,44, + 32,80,121,116,104,111,110,32,105,115,32,108,105,107,101,108, + 121,32,115,104,117,116,116,105,110,103,32,100,111,119,110,122, + 22,115,121,115,46,109,101,116,97,95,112,97,116,104,32,105, + 115,32,101,109,112,116,121,41,12,114,14,0,0,0,218,9, + 109,101,116,97,95,112,97,116,104,114,77,0,0,0,114,142, + 0,0,0,114,143,0,0,0,218,13,73,109,112,111,114,116, + 87,97,114,110,105,110,103,114,21,0,0,0,114,166,0,0, + 0,114,155,0,0,0,114,96,0,0,0,114,174,0,0,0, + 114,95,0,0,0,41,10,114,15,0,0,0,114,153,0,0, + 0,114,154,0,0,0,114,175,0,0,0,90,9,105,115,95, + 114,101,108,111,97,100,114,173,0,0,0,114,155,0,0,0, + 114,88,0,0,0,114,89,0,0,0,114,95,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,10, + 95,102,105,110,100,95,115,112,101,99,111,3,0,0,115,54, + 0,0,0,0,2,9,1,12,2,12,3,6,1,16,5,15, + 1,13,1,10,1,3,1,13,1,13,1,18,1,12,1,8, + 2,25,1,12,2,22,1,13,1,3,1,13,1,13,4,9, + 2,12,1,4,2,7,2,8,2,114,177,0,0,0,99,3, + 0,0,0,0,0,0,0,4,0,0,0,4,0,0,0,67, + 0,0,0,115,206,0,0,0,116,0,0,124,0,0,116,1, + 0,131,2,0,115,42,0,116,2,0,100,1,0,106,3,0, + 116,4,0,124,0,0,131,1,0,131,1,0,131,1,0,130, + 1,0,124,2,0,100,2,0,107,0,0,114,66,0,116,5, + 0,100,3,0,131,1,0,130,1,0,124,2,0,100,2,0, + 107,4,0,114,171,0,116,0,0,124,1,0,116,1,0,131, + 2,0,115,108,0,116,2,0,100,4,0,131,1,0,130,1, + 0,110,63,0,124,1,0,115,129,0,116,6,0,100,5,0, + 131,1,0,130,1,0,110,42,0,124,1,0,116,7,0,106, + 8,0,107,7,0,114,171,0,100,6,0,125,3,0,116,9, + 0,124,3,0,106,3,0,124,1,0,131,1,0,131,1,0, + 130,1,0,124,0,0,12,114,202,0,124,2,0,100,2,0, + 107,2,0,114,202,0,116,5,0,100,7,0,131,1,0,130, + 1,0,100,8,0,83,41,9,122,28,86,101,114,105,102,121, + 32,97,114,103,117,109,101,110,116,115,32,97,114,101,32,34, + 115,97,110,101,34,46,122,31,109,111,100,117,108,101,32,110, + 97,109,101,32,109,117,115,116,32,98,101,32,115,116,114,44, + 32,110,111,116,32,123,125,114,33,0,0,0,122,18,108,101, + 118,101,108,32,109,117,115,116,32,98,101,32,62,61,32,48, + 122,31,95,95,112,97,99,107,97,103,101,95,95,32,110,111, + 116,32,115,101,116,32,116,111,32,97,32,115,116,114,105,110, + 103,122,54,97,116,116,101,109,112,116,101,100,32,114,101,108, + 97,116,105,118,101,32,105,109,112,111,114,116,32,119,105,116, + 104,32,110,111,32,107,110,111,119,110,32,112,97,114,101,110, + 116,32,112,97,99,107,97,103,101,122,61,80,97,114,101,110, + 116,32,109,111,100,117,108,101,32,123,33,114,125,32,110,111, + 116,32,108,111,97,100,101,100,44,32,99,97,110,110,111,116, + 32,112,101,114,102,111,114,109,32,114,101,108,97,116,105,118, + 101,32,105,109,112,111,114,116,122,17,69,109,112,116,121,32, + 109,111,100,117,108,101,32,110,97,109,101,78,41,10,218,10, + 105,115,105,110,115,116,97,110,99,101,218,3,115,116,114,218, + 9,84,121,112,101,69,114,114,111,114,114,50,0,0,0,114, + 13,0,0,0,114,169,0,0,0,114,77,0,0,0,114,14, + 0,0,0,114,21,0,0,0,218,11,83,121,115,116,101,109, + 69,114,114,111,114,41,4,114,15,0,0,0,114,170,0,0, + 0,114,171,0,0,0,114,148,0,0,0,114,10,0,0,0, + 114,10,0,0,0,114,11,0,0,0,218,13,95,115,97,110, + 105,116,121,95,99,104,101,99,107,158,3,0,0,115,28,0, + 0,0,0,2,15,1,27,1,12,1,12,1,12,1,15,1, + 15,1,6,1,15,2,15,1,6,2,21,1,19,1,114,182, + 0,0,0,122,16,78,111,32,109,111,100,117,108,101,32,110, + 97,109,101,100,32,122,4,123,33,114,125,99,2,0,0,0, + 0,0,0,0,8,0,0,0,12,0,0,0,67,0,0,0, + 115,40,1,0,0,100,0,0,125,2,0,124,0,0,106,0, + 0,100,1,0,131,1,0,100,2,0,25,125,3,0,124,3, + 0,114,175,0,124,3,0,116,1,0,106,2,0,107,7,0, + 114,59,0,116,3,0,124,1,0,124,3,0,131,2,0,1, + 124,0,0,116,1,0,106,2,0,107,6,0,114,85,0,116, + 1,0,106,2,0,124,0,0,25,83,116,1,0,106,2,0, + 124,3,0,25,125,4,0,121,13,0,124,4,0,106,4,0, + 125,2,0,87,110,61,0,4,116,5,0,107,10,0,114,174, + 0,1,1,1,116,6,0,100,3,0,23,106,7,0,124,0, + 0,124,3,0,131,2,0,125,5,0,116,8,0,124,5,0, + 100,4,0,124,0,0,131,1,1,100,0,0,130,2,0,89, + 110,1,0,88,116,9,0,124,0,0,124,2,0,131,2,0, + 125,6,0,124,6,0,100,0,0,107,8,0,114,232,0,116, + 8,0,116,6,0,106,7,0,124,0,0,131,1,0,100,4, + 0,124,0,0,131,1,1,130,1,0,110,12,0,116,10,0, + 124,6,0,131,1,0,125,7,0,124,3,0,114,36,1,116, + 1,0,106,2,0,124,3,0,25,125,4,0,116,11,0,124, + 4,0,124,0,0,106,0,0,100,1,0,131,1,0,100,5, + 0,25,124,7,0,131,3,0,1,124,7,0,83,41,6,78, + 114,121,0,0,0,114,33,0,0,0,122,23,59,32,123,33, + 114,125,32,105,115,32,110,111,116,32,97,32,112,97,99,107, + 97,103,101,114,15,0,0,0,114,141,0,0,0,41,12,114, + 122,0,0,0,114,14,0,0,0,114,21,0,0,0,114,65, + 0,0,0,114,131,0,0,0,114,96,0,0,0,218,8,95, + 69,82,82,95,77,83,71,114,50,0,0,0,114,77,0,0, + 0,114,177,0,0,0,114,150,0,0,0,114,5,0,0,0, + 41,8,114,15,0,0,0,218,7,105,109,112,111,114,116,95, + 114,153,0,0,0,114,123,0,0,0,90,13,112,97,114,101, + 110,116,95,109,111,100,117,108,101,114,148,0,0,0,114,88, + 0,0,0,114,89,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,23,95,102,105,110,100,95,97, + 110,100,95,108,111,97,100,95,117,110,108,111,99,107,101,100, + 181,3,0,0,115,42,0,0,0,0,1,6,1,19,1,6, + 1,15,1,13,2,15,1,11,1,13,1,3,1,13,1,13, + 1,22,1,26,1,15,1,12,1,30,2,12,1,6,2,13, + 1,29,1,114,185,0,0,0,99,2,0,0,0,0,0,0, + 0,2,0,0,0,10,0,0,0,67,0,0,0,115,37,0, + 0,0,116,0,0,124,0,0,131,1,0,143,18,0,1,116, + 1,0,124,0,0,124,1,0,131,2,0,83,87,100,1,0, + 81,82,88,100,1,0,83,41,2,122,54,70,105,110,100,32, + 97,110,100,32,108,111,97,100,32,116,104,101,32,109,111,100, + 117,108,101,44,32,97,110,100,32,114,101,108,101,97,115,101, + 32,116,104,101,32,105,109,112,111,114,116,32,108,111,99,107, + 46,78,41,2,114,54,0,0,0,114,185,0,0,0,41,2, + 114,15,0,0,0,114,184,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,14,95,102,105,110,100, + 95,97,110,100,95,108,111,97,100,208,3,0,0,115,4,0, + 0,0,0,2,13,1,114,186,0,0,0,114,33,0,0,0, + 99,3,0,0,0,0,0,0,0,5,0,0,0,4,0,0, + 0,67,0,0,0,115,166,0,0,0,116,0,0,124,0,0, + 124,1,0,124,2,0,131,3,0,1,124,2,0,100,1,0, + 107,4,0,114,46,0,116,1,0,124,0,0,124,1,0,124, + 2,0,131,3,0,125,0,0,116,2,0,106,3,0,131,0, + 0,1,124,0,0,116,4,0,106,5,0,107,7,0,114,84, + 0,116,6,0,124,0,0,116,7,0,131,2,0,83,116,4, + 0,106,5,0,124,0,0,25,125,3,0,124,3,0,100,2, + 0,107,8,0,114,152,0,116,2,0,106,8,0,131,0,0, + 1,100,3,0,106,9,0,124,0,0,131,1,0,125,4,0, + 116,10,0,124,4,0,100,4,0,124,0,0,131,1,1,130, + 1,0,116,11,0,124,0,0,131,1,0,1,124,3,0,83, + 41,5,97,50,1,0,0,73,109,112,111,114,116,32,97,110, + 100,32,114,101,116,117,114,110,32,116,104,101,32,109,111,100, + 117,108,101,32,98,97,115,101,100,32,111,110,32,105,116,115, + 32,110,97,109,101,44,32,116,104,101,32,112,97,99,107,97, + 103,101,32,116,104,101,32,99,97,108,108,32,105,115,10,32, + 32,32,32,98,101,105,110,103,32,109,97,100,101,32,102,114, + 111,109,44,32,97,110,100,32,116,104,101,32,108,101,118,101, + 108,32,97,100,106,117,115,116,109,101,110,116,46,10,10,32, + 32,32,32,84,104,105,115,32,102,117,110,99,116,105,111,110, + 32,114,101,112,114,101,115,101,110,116,115,32,116,104,101,32, + 103,114,101,97,116,101,115,116,32,99,111,109,109,111,110,32, + 100,101,110,111,109,105,110,97,116,111,114,32,111,102,32,102, + 117,110,99,116,105,111,110,97,108,105,116,121,10,32,32,32, + 32,98,101,116,119,101,101,110,32,105,109,112,111,114,116,95, + 109,111,100,117,108,101,32,97,110,100,32,95,95,105,109,112, + 111,114,116,95,95,46,32,84,104,105,115,32,105,110,99,108, + 117,100,101,115,32,115,101,116,116,105,110,103,32,95,95,112, + 97,99,107,97,103,101,95,95,32,105,102,10,32,32,32,32, + 116,104,101,32,108,111,97,100,101,114,32,100,105,100,32,110, + 111,116,46,10,10,32,32,32,32,114,33,0,0,0,78,122, + 40,105,109,112,111,114,116,32,111,102,32,123,125,32,104,97, + 108,116,101,100,59,32,78,111,110,101,32,105,110,32,115,121, + 115,46,109,111,100,117,108,101,115,114,15,0,0,0,41,12, + 114,182,0,0,0,114,172,0,0,0,114,57,0,0,0,114, + 146,0,0,0,114,14,0,0,0,114,21,0,0,0,114,186, + 0,0,0,218,11,95,103,99,100,95,105,109,112,111,114,116, + 114,58,0,0,0,114,50,0,0,0,114,77,0,0,0,114, + 63,0,0,0,41,5,114,15,0,0,0,114,170,0,0,0, + 114,171,0,0,0,114,89,0,0,0,114,74,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,187, + 0,0,0,214,3,0,0,115,28,0,0,0,0,9,16,1, + 12,1,18,1,10,1,15,1,13,1,13,1,12,1,10,1, + 6,1,9,1,18,1,10,1,114,187,0,0,0,99,3,0, + 0,0,0,0,0,0,6,0,0,0,17,0,0,0,67,0, + 0,0,115,239,0,0,0,116,0,0,124,0,0,100,1,0, + 131,2,0,114,235,0,100,2,0,124,1,0,107,6,0,114, + 83,0,116,1,0,124,1,0,131,1,0,125,1,0,124,1, + 0,106,2,0,100,2,0,131,1,0,1,116,0,0,124,0, + 0,100,3,0,131,2,0,114,83,0,124,1,0,106,3,0, + 124,0,0,106,4,0,131,1,0,1,120,149,0,124,1,0, + 68,93,141,0,125,3,0,116,0,0,124,0,0,124,3,0, + 131,2,0,115,90,0,100,4,0,106,5,0,124,0,0,106, + 6,0,124,3,0,131,2,0,125,4,0,121,17,0,116,7, + 0,124,2,0,124,4,0,131,2,0,1,87,113,90,0,4, + 116,8,0,107,10,0,114,230,0,1,125,5,0,1,122,47, + 0,116,9,0,124,5,0,131,1,0,106,10,0,116,11,0, + 131,1,0,114,209,0,124,5,0,106,12,0,124,4,0,107, + 2,0,114,209,0,119,90,0,130,0,0,87,89,100,5,0, + 100,5,0,125,5,0,126,5,0,88,113,90,0,88,113,90, + 0,87,124,0,0,83,41,6,122,238,70,105,103,117,114,101, + 32,111,117,116,32,119,104,97,116,32,95,95,105,109,112,111, + 114,116,95,95,32,115,104,111,117,108,100,32,114,101,116,117, + 114,110,46,10,10,32,32,32,32,84,104,101,32,105,109,112, + 111,114,116,95,32,112,97,114,97,109,101,116,101,114,32,105, + 115,32,97,32,99,97,108,108,97,98,108,101,32,119,104,105, + 99,104,32,116,97,107,101,115,32,116,104,101,32,110,97,109, + 101,32,111,102,32,109,111,100,117,108,101,32,116,111,10,32, + 32,32,32,105,109,112,111,114,116,46,32,73,116,32,105,115, + 32,114,101,113,117,105,114,101,100,32,116,111,32,100,101,99, + 111,117,112,108,101,32,116,104,101,32,102,117,110,99,116,105, + 111,110,32,102,114,111,109,32,97,115,115,117,109,105,110,103, + 32,105,109,112,111,114,116,108,105,98,39,115,10,32,32,32, + 32,105,109,112,111,114,116,32,105,109,112,108,101,109,101,110, + 116,97,116,105,111,110,32,105,115,32,100,101,115,105,114,101, + 100,46,10,10,32,32,32,32,114,131,0,0,0,250,1,42, + 218,7,95,95,97,108,108,95,95,122,5,123,125,46,123,125, + 78,41,13,114,4,0,0,0,114,130,0,0,0,218,6,114, + 101,109,111,118,101,218,6,101,120,116,101,110,100,114,189,0, + 0,0,114,50,0,0,0,114,1,0,0,0,114,65,0,0, + 0,114,77,0,0,0,114,179,0,0,0,114,71,0,0,0, + 218,15,95,69,82,82,95,77,83,71,95,80,82,69,70,73, + 88,114,15,0,0,0,41,6,114,89,0,0,0,218,8,102, + 114,111,109,108,105,115,116,114,184,0,0,0,218,1,120,90, + 9,102,114,111,109,95,110,97,109,101,90,3,101,120,99,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,16, + 95,104,97,110,100,108,101,95,102,114,111,109,108,105,115,116, + 238,3,0,0,115,34,0,0,0,0,10,15,1,12,1,12, + 1,13,1,15,1,16,1,13,1,15,1,21,1,3,1,17, + 1,18,4,21,1,15,1,3,1,26,1,114,195,0,0,0, + 99,1,0,0,0,0,0,0,0,3,0,0,0,7,0,0, + 0,67,0,0,0,115,214,0,0,0,124,0,0,106,0,0, + 100,1,0,131,1,0,125,1,0,124,0,0,106,0,0,100, + 2,0,131,1,0,125,2,0,124,1,0,100,3,0,107,9, + 0,114,128,0,124,2,0,100,3,0,107,9,0,114,124,0, + 124,1,0,124,2,0,106,1,0,107,3,0,114,124,0,116, + 2,0,106,3,0,100,4,0,106,4,0,100,5,0,124,1, + 0,155,2,0,100,6,0,124,2,0,106,1,0,155,2,0, + 100,7,0,103,5,0,131,1,0,116,5,0,100,8,0,100, + 9,0,131,2,1,1,124,1,0,83,124,2,0,100,3,0, + 107,9,0,114,147,0,124,2,0,106,1,0,83,116,2,0, + 106,3,0,100,10,0,116,5,0,100,8,0,100,9,0,131, + 2,1,1,124,0,0,100,11,0,25,125,1,0,100,12,0, + 124,0,0,107,7,0,114,210,0,124,1,0,106,6,0,100, + 13,0,131,1,0,100,14,0,25,125,1,0,124,1,0,83, + 41,15,122,167,67,97,108,99,117,108,97,116,101,32,119,104, + 97,116,32,95,95,112,97,99,107,97,103,101,95,95,32,115, + 104,111,117,108,100,32,98,101,46,10,10,32,32,32,32,95, + 95,112,97,99,107,97,103,101,95,95,32,105,115,32,110,111, + 116,32,103,117,97,114,97,110,116,101,101,100,32,116,111,32, + 98,101,32,100,101,102,105,110,101,100,32,111,114,32,99,111, + 117,108,100,32,98,101,32,115,101,116,32,116,111,32,78,111, + 110,101,10,32,32,32,32,116,111,32,114,101,112,114,101,115, + 101,110,116,32,116,104,97,116,32,105,116,115,32,112,114,111, + 112,101,114,32,118,97,108,117,101,32,105,115,32,117,110,107, + 110,111,119,110,46,10,10,32,32,32,32,114,134,0,0,0, + 114,95,0,0,0,78,218,0,122,32,95,95,112,97,99,107, + 97,103,101,95,95,32,33,61,32,95,95,115,112,101,99,95, + 95,46,112,97,114,101,110,116,32,40,122,4,32,33,61,32, + 250,1,41,114,140,0,0,0,233,3,0,0,0,122,89,99, + 97,110,39,116,32,114,101,115,111,108,118,101,32,112,97,99, + 107,97,103,101,32,102,114,111,109,32,95,95,115,112,101,99, + 95,95,32,111,114,32,95,95,112,97,99,107,97,103,101,95, + 95,44,32,102,97,108,108,105,110,103,32,98,97,99,107,32, + 111,110,32,95,95,110,97,109,101,95,95,32,97,110,100,32, + 95,95,112,97,116,104,95,95,114,1,0,0,0,114,131,0, + 0,0,114,121,0,0,0,114,33,0,0,0,41,7,114,42, + 0,0,0,114,123,0,0,0,114,142,0,0,0,114,143,0, + 0,0,114,115,0,0,0,114,176,0,0,0,114,122,0,0, + 0,41,3,218,7,103,108,111,98,97,108,115,114,170,0,0, + 0,114,88,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,17,95,99,97,108,99,95,95,95,112, + 97,99,107,97,103,101,95,95,14,4,0,0,115,30,0,0, + 0,0,7,15,1,15,1,12,1,27,1,42,2,13,1,4, + 1,12,1,7,2,9,2,13,1,10,1,12,1,19,1,114, + 200,0,0,0,99,5,0,0,0,0,0,0,0,9,0,0, + 0,5,0,0,0,67,0,0,0,115,227,0,0,0,124,4, + 0,100,1,0,107,2,0,114,27,0,116,0,0,124,0,0, + 131,1,0,125,5,0,110,54,0,124,1,0,100,2,0,107, + 9,0,114,45,0,124,1,0,110,3,0,105,0,0,125,6, + 0,116,1,0,124,6,0,131,1,0,125,7,0,116,0,0, + 124,0,0,124,7,0,124,4,0,131,3,0,125,5,0,124, + 3,0,115,207,0,124,4,0,100,1,0,107,2,0,114,122, + 0,116,0,0,124,0,0,106,2,0,100,3,0,131,1,0, + 100,1,0,25,131,1,0,83,124,0,0,115,132,0,124,5, + 0,83,116,3,0,124,0,0,131,1,0,116,3,0,124,0, + 0,106,2,0,100,3,0,131,1,0,100,1,0,25,131,1, + 0,24,125,8,0,116,4,0,106,5,0,124,5,0,106,6, + 0,100,2,0,116,3,0,124,5,0,106,6,0,131,1,0, + 124,8,0,24,133,2,0,25,25,83,110,16,0,116,7,0, + 124,5,0,124,3,0,116,0,0,131,3,0,83,100,2,0, + 83,41,4,97,214,1,0,0,73,109,112,111,114,116,32,97, + 32,109,111,100,117,108,101,46,10,10,32,32,32,32,84,104, + 101,32,39,103,108,111,98,97,108,115,39,32,97,114,103,117, + 109,101,110,116,32,105,115,32,117,115,101,100,32,116,111,32, + 105,110,102,101,114,32,119,104,101,114,101,32,116,104,101,32, + 105,109,112,111,114,116,32,105,115,32,111,99,99,117,114,105, + 110,103,32,102,114,111,109,10,32,32,32,32,116,111,32,104, + 97,110,100,108,101,32,114,101,108,97,116,105,118,101,32,105, + 109,112,111,114,116,115,46,32,84,104,101,32,39,108,111,99, + 97,108,115,39,32,97,114,103,117,109,101,110,116,32,105,115, + 32,105,103,110,111,114,101,100,46,32,84,104,101,10,32,32, + 32,32,39,102,114,111,109,108,105,115,116,39,32,97,114,103, + 117,109,101,110,116,32,115,112,101,99,105,102,105,101,115,32, + 119,104,97,116,32,115,104,111,117,108,100,32,101,120,105,115, + 116,32,97,115,32,97,116,116,114,105,98,117,116,101,115,32, + 111,110,32,116,104,101,32,109,111,100,117,108,101,10,32,32, + 32,32,98,101,105,110,103,32,105,109,112,111,114,116,101,100, + 32,40,101,46,103,46,32,96,96,102,114,111,109,32,109,111, + 100,117,108,101,32,105,109,112,111,114,116,32,60,102,114,111, + 109,108,105,115,116,62,96,96,41,46,32,32,84,104,101,32, + 39,108,101,118,101,108,39,10,32,32,32,32,97,114,103,117, + 109,101,110,116,32,114,101,112,114,101,115,101,110,116,115,32, + 116,104,101,32,112,97,99,107,97,103,101,32,108,111,99,97, + 116,105,111,110,32,116,111,32,105,109,112,111,114,116,32,102, + 114,111,109,32,105,110,32,97,32,114,101,108,97,116,105,118, + 101,10,32,32,32,32,105,109,112,111,114,116,32,40,101,46, + 103,46,32,96,96,102,114,111,109,32,46,46,112,107,103,32, + 105,109,112,111,114,116,32,109,111,100,96,96,32,119,111,117, + 108,100,32,104,97,118,101,32,97,32,39,108,101,118,101,108, + 39,32,111,102,32,50,41,46,10,10,32,32,32,32,114,33, + 0,0,0,78,114,121,0,0,0,41,8,114,187,0,0,0, + 114,200,0,0,0,218,9,112,97,114,116,105,116,105,111,110, + 114,168,0,0,0,114,14,0,0,0,114,21,0,0,0,114, + 1,0,0,0,114,195,0,0,0,41,9,114,15,0,0,0, + 114,199,0,0,0,218,6,108,111,99,97,108,115,114,193,0, + 0,0,114,171,0,0,0,114,89,0,0,0,90,8,103,108, + 111,98,97,108,115,95,114,170,0,0,0,90,7,99,117,116, + 95,111,102,102,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,218,10,95,95,105,109,112,111,114,116,95,95,41, + 4,0,0,115,26,0,0,0,0,11,12,1,15,2,24,1, + 12,1,18,1,6,3,12,1,23,1,6,1,4,4,35,3, + 40,2,114,203,0,0,0,99,1,0,0,0,0,0,0,0, + 2,0,0,0,3,0,0,0,67,0,0,0,115,53,0,0, + 0,116,0,0,106,1,0,124,0,0,131,1,0,125,1,0, + 124,1,0,100,0,0,107,8,0,114,43,0,116,2,0,100, + 1,0,124,0,0,23,131,1,0,130,1,0,116,3,0,124, + 1,0,131,1,0,83,41,2,78,122,25,110,111,32,98,117, + 105,108,116,45,105,110,32,109,111,100,117,108,101,32,110,97, + 109,101,100,32,41,4,114,151,0,0,0,114,155,0,0,0, + 114,77,0,0,0,114,150,0,0,0,41,2,114,15,0,0, + 0,114,88,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,18,95,98,117,105,108,116,105,110,95, + 102,114,111,109,95,110,97,109,101,76,4,0,0,115,8,0, + 0,0,0,1,15,1,12,1,16,1,114,204,0,0,0,99, + 2,0,0,0,0,0,0,0,12,0,0,0,12,0,0,0, + 67,0,0,0,115,74,1,0,0,124,1,0,97,0,0,124, + 0,0,97,1,0,116,2,0,116,1,0,131,1,0,125,2, + 0,120,123,0,116,1,0,106,3,0,106,4,0,131,0,0, + 68,93,106,0,92,2,0,125,3,0,125,4,0,116,5,0, + 124,4,0,124,2,0,131,2,0,114,40,0,124,3,0,116, + 1,0,106,6,0,107,6,0,114,91,0,116,7,0,125,5, + 0,110,27,0,116,0,0,106,8,0,124,3,0,131,1,0, + 114,40,0,116,9,0,125,5,0,110,3,0,113,40,0,116, + 10,0,124,4,0,124,5,0,131,2,0,125,6,0,116,11, + 0,124,6,0,124,4,0,131,2,0,1,113,40,0,87,116, + 1,0,106,3,0,116,12,0,25,125,7,0,120,73,0,100, + 5,0,68,93,65,0,125,8,0,124,8,0,116,1,0,106, + 3,0,107,7,0,114,206,0,116,13,0,124,8,0,131,1, + 0,125,9,0,110,13,0,116,1,0,106,3,0,124,8,0, + 25,125,9,0,116,14,0,124,7,0,124,8,0,124,9,0, + 131,3,0,1,113,170,0,87,121,16,0,116,13,0,100,2, + 0,131,1,0,125,10,0,87,110,24,0,4,116,15,0,107, + 10,0,114,25,1,1,1,1,100,3,0,125,10,0,89,110, + 1,0,88,116,14,0,124,7,0,100,2,0,124,10,0,131, + 3,0,1,116,13,0,100,4,0,131,1,0,125,11,0,116, + 14,0,124,7,0,100,4,0,124,11,0,131,3,0,1,100, + 3,0,83,41,6,122,250,83,101,116,117,112,32,105,109,112, + 111,114,116,108,105,98,32,98,121,32,105,109,112,111,114,116, + 105,110,103,32,110,101,101,100,101,100,32,98,117,105,108,116, + 45,105,110,32,109,111,100,117,108,101,115,32,97,110,100,32, + 105,110,106,101,99,116,105,110,103,32,116,104,101,109,10,32, + 32,32,32,105,110,116,111,32,116,104,101,32,103,108,111,98, + 97,108,32,110,97,109,101,115,112,97,99,101,46,10,10,32, + 32,32,32,65,115,32,115,121,115,32,105,115,32,110,101,101, + 100,101,100,32,102,111,114,32,115,121,115,46,109,111,100,117, + 108,101,115,32,97,99,99,101,115,115,32,97,110,100,32,95, + 105,109,112,32,105,115,32,110,101,101,100,101,100,32,116,111, + 32,108,111,97,100,32,98,117,105,108,116,45,105,110,10,32, + 32,32,32,109,111,100,117,108,101,115,44,32,116,104,111,115, + 101,32,116,119,111,32,109,111,100,117,108,101,115,32,109,117, + 115,116,32,98,101,32,101,120,112,108,105,99,105,116,108,121, + 32,112,97,115,115,101,100,32,105,110,46,10,10,32,32,32, + 32,114,142,0,0,0,114,34,0,0,0,78,114,62,0,0, + 0,41,1,122,9,95,119,97,114,110,105,110,103,115,41,16, + 114,57,0,0,0,114,14,0,0,0,114,13,0,0,0,114, + 21,0,0,0,218,5,105,116,101,109,115,114,178,0,0,0, + 114,76,0,0,0,114,151,0,0,0,114,82,0,0,0,114, + 161,0,0,0,114,132,0,0,0,114,137,0,0,0,114,1, + 0,0,0,114,204,0,0,0,114,5,0,0,0,114,77,0, + 0,0,41,12,218,10,115,121,115,95,109,111,100,117,108,101, + 218,11,95,105,109,112,95,109,111,100,117,108,101,90,11,109, + 111,100,117,108,101,95,116,121,112,101,114,15,0,0,0,114, + 89,0,0,0,114,99,0,0,0,114,88,0,0,0,90,11, + 115,101,108,102,95,109,111,100,117,108,101,90,12,98,117,105, + 108,116,105,110,95,110,97,109,101,90,14,98,117,105,108,116, + 105,110,95,109,111,100,117,108,101,90,13,116,104,114,101,97, + 100,95,109,111,100,117,108,101,90,14,119,101,97,107,114,101, + 102,95,109,111,100,117,108,101,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,6,95,115,101,116,117,112,83, + 4,0,0,115,50,0,0,0,0,9,6,1,6,3,12,1, + 28,1,15,1,15,1,9,1,15,1,9,2,3,1,15,1, + 17,3,13,1,13,1,15,1,15,2,13,1,20,3,3,1, + 16,1,13,2,11,1,16,3,12,1,114,208,0,0,0,99, + 2,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0, + 67,0,0,0,115,87,0,0,0,116,0,0,124,0,0,124, + 1,0,131,2,0,1,116,1,0,106,2,0,106,3,0,116, + 4,0,131,1,0,1,116,1,0,106,2,0,106,3,0,116, + 5,0,131,1,0,1,100,1,0,100,2,0,108,6,0,125, + 2,0,124,2,0,97,7,0,124,2,0,106,8,0,116,1, + 0,106,9,0,116,10,0,25,131,1,0,1,100,2,0,83, + 41,3,122,50,73,110,115,116,97,108,108,32,105,109,112,111, + 114,116,108,105,98,32,97,115,32,116,104,101,32,105,109,112, + 108,101,109,101,110,116,97,116,105,111,110,32,111,102,32,105, + 109,112,111,114,116,46,114,33,0,0,0,78,41,11,114,208, + 0,0,0,114,14,0,0,0,114,175,0,0,0,114,113,0, + 0,0,114,151,0,0,0,114,161,0,0,0,218,26,95,102, + 114,111,122,101,110,95,105,109,112,111,114,116,108,105,98,95, + 101,120,116,101,114,110,97,108,114,119,0,0,0,218,8,95, + 105,110,115,116,97,108,108,114,21,0,0,0,114,1,0,0, + 0,41,3,114,206,0,0,0,114,207,0,0,0,114,209,0, + 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,114,210,0,0,0,130,4,0,0,115,12,0,0,0,0, + 2,13,2,16,1,16,3,12,1,6,1,114,210,0,0,0, + 41,51,114,3,0,0,0,114,119,0,0,0,114,12,0,0, + 0,114,16,0,0,0,114,17,0,0,0,114,59,0,0,0, + 114,41,0,0,0,114,48,0,0,0,114,31,0,0,0,114, + 32,0,0,0,114,53,0,0,0,114,54,0,0,0,114,56, + 0,0,0,114,63,0,0,0,114,65,0,0,0,114,75,0, + 0,0,114,81,0,0,0,114,84,0,0,0,114,90,0,0, + 0,114,101,0,0,0,114,102,0,0,0,114,106,0,0,0, + 114,85,0,0,0,218,6,111,98,106,101,99,116,90,9,95, + 80,79,80,85,76,65,84,69,114,132,0,0,0,114,137,0, + 0,0,114,145,0,0,0,114,97,0,0,0,114,86,0,0, + 0,114,149,0,0,0,114,150,0,0,0,114,87,0,0,0, + 114,151,0,0,0,114,161,0,0,0,114,166,0,0,0,114, + 172,0,0,0,114,174,0,0,0,114,177,0,0,0,114,182, + 0,0,0,114,192,0,0,0,114,183,0,0,0,114,185,0, + 0,0,114,186,0,0,0,114,187,0,0,0,114,195,0,0, + 0,114,200,0,0,0,114,203,0,0,0,114,204,0,0,0, + 114,208,0,0,0,114,210,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,8, + 60,109,111,100,117,108,101,62,8,0,0,0,115,96,0,0, + 0,6,17,6,2,12,8,12,4,19,20,6,2,6,3,22, + 4,19,68,19,21,19,19,12,19,12,19,12,11,18,8,12, + 11,12,12,12,16,12,36,19,27,19,101,24,26,9,3,18, + 45,18,60,12,18,12,17,12,25,12,29,12,23,12,16,19, + 73,19,77,19,13,12,9,12,9,15,47,12,20,6,1,10, + 2,12,27,12,6,18,24,12,32,12,27,24,35,12,7,12, + 47, }; -- cgit v0.12 From bfba2cd40629e52ba74ddc75a3415adc8f0ef71b Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 24 Mar 2016 22:44:41 -0500 Subject: Don't doc check the venv dir --- Doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index a42e98b..03a37f1 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -153,7 +153,7 @@ dist: cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: - $(PYTHON) tools/rstlint.py -i tools + $(PYTHON) tools/rstlint.py -i tools -i venv serve: ../Tools/scripts/serve.py build/html -- cgit v0.12 From 4a2e663f9941799eeeda5a0eff320665ae1d8e8b Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 24 Mar 2016 22:45:00 -0500 Subject: Fix a few typos --- Misc/NEWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index d684c82..945527c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,10 +28,10 @@ Core and Builtins :c:func:`PyObject_Malloc` now detect when functions are called without holding the GIL. -- Issue #26516: Add :envvar`PYTHONMALLOC` environment variable to set the +- Issue #26516: Add :envvar:`PYTHONMALLOC` environment variable to set the Python memory allocators and/or install debug hooks. -- Issue #26516: The :c:func`PyMem_SetupDebugHooks` function can now also be +- Issue #26516: The :c:func:`PyMem_SetupDebugHooks` function can now also be used on Python compiled in release mode. - Issue #26516: The :envvar:`PYTHONMALLOCSTATS` environment variable can now @@ -238,7 +238,7 @@ Library ``None``). - Issue #21925: :func:`warnings.formatwarning` now catches exceptions when - calling :func;`linecache.getline` and + calling :func:`linecache.getline` and :func:`tracemalloc.get_object_traceback` to be able to log :exc:`ResourceWarning` emitted late during the Python shutdown process. -- cgit v0.12 From 7285d520e00e110af2f150487fc0ccfaa8430684 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 24 Mar 2016 22:43:23 -0700 Subject: remove duplicated check for fractions and complex numbers (closes #26076) Patch by Oren Milman. --- Parser/tokenizer.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index c653121..006ad9c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1587,10 +1587,6 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) if (c == '0') { /* Hex, octal or binary -- maybe. */ c = tok_nextc(tok); - if (c == '.') - goto fraction; - if (c == 'j' || c == 'J') - goto imaginary; if (c == 'x' || c == 'X') { /* Hex */ -- cgit v0.12 From 633ebda3ba57428835d27866d8567af1c857fcd2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 08:57:16 +0100 Subject: Issue #26637: Fix test_io The import machinery now raises a different exception when it fails at Python shutdown. --- Lib/test/test_io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 86440c5..37b9b2d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3079,8 +3079,7 @@ class CTextIOWrapperTest(TextIOWrapperTest): class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio - #shutdown_error = "LookupError: unknown encoding: ascii" - shutdown_error = "TypeError: 'NoneType' object is not iterable" + shutdown_error = "LookupError: unknown encoding: ascii" class IncrementalNewlineDecoderTest(unittest.TestCase): -- cgit v0.12 From 47b4557679154a645780b2e44dd7a4a2c62b5d12 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 09:07:07 +0100 Subject: test_io: ignore DeprecationWarning on bytes path on Windows --- Lib/test/test_io.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 37b9b2d..db074ca 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -364,7 +364,11 @@ class IOTest(unittest.TestCase): def test_open_handles_NUL_chars(self): fn_with_NUL = 'foo\0bar' self.assertRaises(ValueError, self.open, fn_with_NUL, 'w') - self.assertRaises(ValueError, self.open, bytes(fn_with_NUL, 'ascii'), 'w') + + bytes_fn = bytes(fn_with_NUL, 'ascii') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertRaises(ValueError, self.open, bytes_fn, 'w') def test_raw_file_io(self): with self.open(support.TESTFN, "wb", buffering=0) as f: -- cgit v0.12 From a6d865c128dd46a067358e94c29ca2d84205ae89 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 09:29:50 +0100 Subject: Issue #25654: * multiprocessing: open file with closefd=False to avoid ResourceWarning * _test_multiprocessing: open file with O_EXCL to detect bugs in tests (if a previous test forgot to remove TESTFN) * test_sys_exit(): remove TESTFN after each loop iteration Initial patch written by Serhiy Storchaka. --- Doc/library/multiprocessing.rst | 2 +- Lib/multiprocessing/forkserver.py | 8 +------- Lib/multiprocessing/process.py | 7 +------ Lib/multiprocessing/util.py | 23 +++++++++++++++++++++++ Lib/test/_test_multiprocessing.py | 18 +++++++++++++----- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index bf25f3f..6440f5c 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2694,7 +2694,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" in issues with processes-in-processes. This has been changed to:: sys.stdin.close() - sys.stdin = open(os.devnull) + sys.stdin = open(os.open(os.devnull, os.O_RDONLY), closefd=False) Which solves the fundamental issue of processes colliding with each other resulting in a bad file descriptor error, but introduces a potential danger diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index b27cba5..ad01ede 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -147,13 +147,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): except ImportError: pass - # close sys.stdin - if sys.stdin is not None: - try: - sys.stdin.close() - sys.stdin = open(os.devnull) - except (OSError, ValueError): - pass + util._close_stdin() # ignoring SIGCHLD means no need to reap zombie processes handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 68959bf..bca8b7a 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -234,12 +234,7 @@ class BaseProcess(object): context._force_start_method(self._start_method) _process_counter = itertools.count(1) _children = set() - if sys.stdin is not None: - try: - sys.stdin.close() - sys.stdin = open(os.devnull) - except (OSError, ValueError): - pass + util._close_stdin() old_process = _current_process _current_process = self try: diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index ea5443d..1a2c0db 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -9,6 +9,7 @@ import os import itertools +import sys import weakref import atexit import threading # we want threading to install it's @@ -356,6 +357,28 @@ def close_all_fds_except(fds): assert fds[-1] == MAXFD, 'fd too large' for i in range(len(fds) - 1): os.closerange(fds[i]+1, fds[i+1]) +# +# Close sys.stdin and replace stdin with os.devnull +# + +def _close_stdin(): + if sys.stdin is None: + return + + try: + sys.stdin.close() + except (OSError, ValueError): + pass + + try: + fd = os.open(os.devnull, os.O_RDONLY) + try: + sys.stdin = open(fd, closefd=False) + except: + os.close(fd) + raise + except (OSError, ValueError): + pass # # Start a program with only specified fds kept open diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 95d418d..16407db 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -455,13 +455,15 @@ class _TestSubclassingProcess(BaseTestCase): @classmethod def _test_stderr_flush(cls, testfn): - sys.stderr = open(testfn, 'w') + fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + sys.stderr = open(fd, 'w', closefd=False) 1/0 # MARKER @classmethod def _test_sys_exit(cls, reason, testfn): - sys.stderr = open(testfn, 'w') + fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + sys.stderr = open(fd, 'w', closefd=False) sys.exit(reason) def test_sys_exit(self): @@ -472,15 +474,21 @@ class _TestSubclassingProcess(BaseTestCase): testfn = test.support.TESTFN self.addCleanup(test.support.unlink, testfn) - for reason, code in (([1, 2, 3], 1), ('ignore this', 1)): + for reason in ( + [1, 2, 3], + 'ignore this', + ): p = self.Process(target=self._test_sys_exit, args=(reason, testfn)) p.daemon = True p.start() p.join(5) - self.assertEqual(p.exitcode, code) + self.assertEqual(p.exitcode, 1) with open(testfn, 'r') as f: - self.assertEqual(f.read().rstrip(), str(reason)) + content = f.read() + self.assertEqual(content.rstrip(), str(reason)) + + os.unlink(testfn) for reason in (True, False, 8): p = self.Process(target=sys.exit, args=(reason,)) -- cgit v0.12 From 6c45d397a32a1f887dc5ed1565b57bf570ded1a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 09:51:14 +0100 Subject: Issue #21925: Fix test_warnings for release mode Use -Wd comment line option to log the ResourceWarning. --- Lib/test/test_warnings/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 9f1cd75..42c7603 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1007,12 +1007,12 @@ a=A() # don't import the warnings module # (_warnings will try to import it) code = "f = open(%a)" % __file__ - rc, out, err = assert_python_ok("-c", code) + rc, out, err = assert_python_ok("-Wd", "-c", code) self.assertTrue(err.startswith(expected), ascii(err)) # import the warnings module code = "import warnings; f = open(%a)" % __file__ - rc, out, err = assert_python_ok("-c", code) + rc, out, err = assert_python_ok("-Wd", "-c", code) self.assertTrue(err.startswith(expected), ascii(err)) -- cgit v0.12 From b72e21b9ab8021b7cbb7e0007d1bf80d533bce15 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 25 Mar 2016 02:29:59 -0700 Subject: Speed-up construction of empty sets by approx 12-14%. --- Objects/setobject.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index c9834a8..8e22b69 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2015,11 +2015,12 @@ set_init(PySetObject *self, PyObject *args, PyObject *kwds) if (!PyAnySet_Check(self)) return -1; - if (PySet_Check(self) && !_PyArg_NoKeywords("set()", kwds)) + if (kwds != NULL && PySet_Check(self) && !_PyArg_NoKeywords("set()", kwds)) return -1; if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; - set_clear_internal(self); + if (self->fill) + set_clear_internal(self); self->hash = -1; if (iterable == NULL) return 0; -- cgit v0.12 From e77c974357fa96917e6966d627a69eae8a919b37 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 10:28:23 +0100 Subject: test_os: Win32ErrorTests now ensures that TESTFN doesn't exist Replace also other open(filename, "w") with open(filename, "x") to fail if a previous test forgot to remove filename. --- Lib/test/test_os.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bc24e47..c8789d7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -845,9 +845,8 @@ class WalkTests(unittest.TestCase): os.makedirs(t2_path) for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: - f = open(path, "w") - f.write("I'm " + path + " and proud of it. Blame test_os.\n") - f.close() + with open(path, "x") as f: + f.write("I'm " + path + " and proud of it. Blame test_os.\n") if support.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) @@ -1427,6 +1426,9 @@ class ExecTests(unittest.TestCase): @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32ErrorTests(unittest.TestCase): + def setUp(self): + self.assertFalse(os.path.exists(support.TESTFN)) + def test_rename(self): self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak") @@ -1439,7 +1441,7 @@ class Win32ErrorTests(unittest.TestCase): def test_mkdir(self): self.addCleanup(support.unlink, support.TESTFN) - with open(support.TESTFN, "w") as f: + with open(support.TESTFN, "x") as f: self.assertRaises(OSError, os.mkdir, support.TESTFN) def test_utime(self): @@ -1448,6 +1450,7 @@ class Win32ErrorTests(unittest.TestCase): def test_chmod(self): self.assertRaises(OSError, os.chmod, support.TESTFN, 0) + class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] @@ -1559,8 +1562,7 @@ class LinkTests(unittest.TestCase): os.unlink(file) def _test_link(self, file1, file2): - with open(file1, "w") as f1: - f1.write("test") + create_file(file1) with bytes_filename_warn(False): os.link(file1, file2) -- cgit v0.12 From a9a852c2b123031febf245d3bd37a529e0635d01 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 11:54:47 +0100 Subject: Modernize Python/makeopcodetargets.py * Simply use "import opcode" to import the opcode module instead of tricks using the imp module * Use context manager for the output file * Move code into a new main() function * Replace assert with a regular if to check the number of arguments * Import modules at top level --- Python/makeopcodetargets.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index d9a0855..7a57a06 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -3,24 +3,14 @@ (for compilers supporting computed gotos or "labels-as-values", such as gcc). """ -# This code should stay compatible with Python 2.3, at least while -# some of the buildbots have Python 2.3 as their system Python. - -import imp +import opcode import os +import sys -def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib") - return imp.load_module(modname, *imp.find_module(modname, [modpath])) - def write_contents(f): """Write C code contents to the target file object. """ - opcode = find_module("opcode") targets = ['_unknown_opcode'] * 256 for opname, op in opcode.opmap.items(): targets[op] = "TARGET_%s" % opname @@ -29,15 +19,17 @@ def write_contents(f): f.write("\n};\n") -if __name__ == "__main__": - import sys - assert len(sys.argv) < 3, "Too many arguments" +def main(): + if len(sys.argv) >= 3: + sys.exit("Too many arguments") if len(sys.argv) == 2: target = sys.argv[1] else: target = "Python/opcode_targets.h" - f = open(target, "w") - try: + with open(target, "w") as f: write_contents(f) - finally: - f.close() + print("Jump table written into %s" % target) + + +if __name__ == "__main__": + main() -- cgit v0.12 From b347788b825feb156510d6ec321930c6044bd202 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 12:27:02 +0100 Subject: Skip test_venv.test_with_pip() if ctypes miss Issue #26610. --- Lib/test/test_venv.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index dbb8b85..09741e3 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -31,6 +31,11 @@ try: except ImportError: threading = None +try: + import ctypes +except ImportError: + ctypes = None + skipInVenv = unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate in a venv') @@ -327,6 +332,8 @@ class EnsurePipTest(BaseTest): @unittest.skipIf(ssl is None, ensurepip._MISSING_SSL_MESSAGE) @unittest.skipUnless(threading, 'some dependencies of pip import threading' ' module unconditionally') + # Issue #26610: pip/pep425tags.py requires ctypes + @unittest.skipUnless(ctypes, 'pip requires ctypes') def test_with_pip(self): rmtree(self.env_dir) with EnvironmentVarGuard() as envvars: -- cgit v0.12 From bdc337b7a8bbe5719cf7aeab7e104262fdd741f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 12:30:40 +0100 Subject: test_venv: enhance test_devnull() --- Lib/test/test_venv.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 09741e3..55bee23 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -39,15 +39,9 @@ except ImportError: skipInVenv = unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate in a venv') -# os.path.exists('nul') is False: http://bugs.python.org/issue20541 -if os.devnull.lower() == 'nul': - failsOnWindows = unittest.expectedFailure -else: - def failsOnWindows(f): - return f - class BaseTest(unittest.TestCase): """Base class for venv tests.""" + maxDiff = 80 * 50 def setUp(self): self.env_dir = os.path.realpath(tempfile.mkdtemp()) @@ -318,16 +312,21 @@ class EnsurePipTest(BaseTest): self.run_with_capture(venv.create, self.env_dir, with_pip=False) self.assert_pip_not_installed() - @failsOnWindows - def test_devnull_exists_and_is_empty(self): + def test_devnull(self): # Fix for issue #20053 uses os.devnull to force a config file to # appear empty. However http://bugs.python.org/issue20541 means # that doesn't currently work properly on Windows. Once that is # fixed, the "win_location" part of test_with_pip should be restored - self.assertTrue(os.path.exists(os.devnull)) with open(os.devnull, "rb") as f: self.assertEqual(f.read(), b"") + # Issue #20541: os.path.exists('nul') is False on Windows + if os.devnull.lower() == 'nul': + self.assertFalse(os.path.exists(os.devnull)) + else: + self.assertTrue(os.path.exists(os.devnull)) + + # Requesting pip fails without SSL (http://bugs.python.org/issue19744) @unittest.skipIf(ssl is None, ensurepip._MISSING_SSL_MESSAGE) @unittest.skipUnless(threading, 'some dependencies of pip import threading' -- cgit v0.12 From 931602a1acea08d5288c7d8d99996082d0e0aa8b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 12:48:17 +0100 Subject: test_doctest: remove unused imports --- Lib/test/test_doctest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 6a7ba0b..f7c4806 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -2948,12 +2948,11 @@ Invalid doctest option: def test_main(): # Check the doctest cases in doctest itself: ret = support.run_doctest(doctest, verbosity=True) + # Check the doctest cases defined here: from test import test_doctest support.run_doctest(test_doctest, verbosity=True) -import sys, re, io - def test_coverage(coverdir): trace = support.import_module('trace') tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], -- cgit v0.12 From 32830149d8873234eaf1949ef840c3a07ecf5b64 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 15:12:08 +0100 Subject: changeset: 100749:0b61b2d28a07 tag: tip parent: 100742:ebae81b31cf6 user: Victor Stinner date: Fri Mar 25 15:03:34 2016 +0100 files: Lib/test/test_os.py description: test_os: Win32ErrorTests checks if file exists Don't use os.path.exists() since it ignores *any* OSError. --- Lib/test/test_os.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c8789d7..f7d64b7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1427,7 +1427,16 @@ class ExecTests(unittest.TestCase): @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32ErrorTests(unittest.TestCase): def setUp(self): - self.assertFalse(os.path.exists(support.TESTFN)) + try: + os.stat(support.TESTFN) + except FileNotFoundError: + exists = False + except OSError as exc: + exists = True + self.fail("file %s must not exist; os.stat failed with %s" + % (support.TESTFN, exc)) + else: + self.fail("file %s must not exist" % support.TESTFN) def test_rename(self): self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak") -- cgit v0.12 From ba8cf108735714d77430ff051cc2e414a2d82183 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 17:36:33 +0100 Subject: Rework libregrtest.save_env * Replace get/restore methods with a Resource class and Resource subclasses * Create ModuleAttr, ModuleAttrList and ModuleAttrDict helper classes * Use __subclasses__() to get resource classes instead of using an hardcoded list (2 shutil resources were missinged in the list!) * Don't define MultiprocessingProcessDangling resource if the multiprocessing module is missing * Nicer diff for dictionaries. Useful for the big os.environ dict * Reorder code to group resources --- Lib/test/libregrtest/save_env.py | 481 +++++++++++++++++++++++---------------- 1 file changed, 279 insertions(+), 202 deletions(-) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 90900a9..c9a980c 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -17,225 +17,266 @@ except ImportError: multiprocessing = None -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. +class Resource: + name = None -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. + def __init__(self): + self.original = self.get() + self.current = None - with saved_test_environment(testname, verbose, quiet): - #stuff + def decode_value(self, value): + return value - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. + def display_diff(self): + original = self.decode_value(self.original) + current = self.decode_value(self.current) + print(f" Before: {original}", file=sys.stderr) + print(f" After: {current}", file=sys.stderr) - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - changed = False +class ModuleAttr(Resource): + def encode_value(self, value): + return value - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - self.pgo = pgo + def get_module(self): + module_name, attr = self.name.split('.', 1) + module = globals()[module_name] + return (module, attr) - # To add things to save and restore, add a name XXX to the resources list - # and add corresponding get_XXX/restore_XXX functions. get_XXX should - # return the value to be saved and compared against a second call to the - # get function when test execution completes. restore_XXX should accept - # the saved value and restore the resource using it. It will be called if - # and only if a change in the value is detected. - # - # Note: XXX will have any '.' replaced with '_' characters when determining - # the corresponding method names. - - resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', - 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', - 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', 'sys.gettrace', - 'sys.warnoptions', - # multiprocessing.process._cleanup() may release ref - # to a thread, so check processes first. - 'multiprocessing.process._dangling', 'threading._dangling', - 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'files', 'locale', 'warnings.showwarning', - ) - - def get_sys_argv(self): - return id(sys.argv), sys.argv, sys.argv[:] - def restore_sys_argv(self, saved_argv): - sys.argv = saved_argv[1] - sys.argv[:] = saved_argv[2] - - def get_cwd(self): - return os.getcwd() - def restore_cwd(self, saved_cwd): - os.chdir(saved_cwd) - - def get_sys_stdout(self): - return sys.stdout - def restore_sys_stdout(self, saved_stdout): - sys.stdout = saved_stdout - - def get_sys_stderr(self): - return sys.stderr - def restore_sys_stderr(self, saved_stderr): - sys.stderr = saved_stderr - - def get_sys_stdin(self): - return sys.stdin - def restore_sys_stdin(self, saved_stdin): - sys.stdin = saved_stdin - - def get_os_environ(self): - return id(os.environ), os.environ, dict(os.environ) - def restore_os_environ(self, saved_environ): - os.environ = saved_environ[1] - os.environ.clear() - os.environ.update(saved_environ[2]) - - def get_sys_path(self): - return id(sys.path), sys.path, sys.path[:] - def restore_sys_path(self, saved_path): - sys.path = saved_path[1] - sys.path[:] = saved_path[2] - - def get_sys_path_hooks(self): - return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] - def restore_sys_path_hooks(self, saved_hooks): - sys.path_hooks = saved_hooks[1] - sys.path_hooks[:] = saved_hooks[2] - - def get_sys_gettrace(self): + def get(self): + module, attr = self.get_module() + value = getattr(module, attr) + return self.encode_value(value) + + def restore_attr(self, module, attr): + setattr(module, attr, self.original) + + def restore(self): + module, attr = self.get_module() + self.restore_attr(module, attr) + + +class ModuleAttrList(ModuleAttr): + def decode_value(self, value): + return value[2] + + def encode_value(self, value): + return id(value), value, value.copy() + + def restore_attr(self, module, attr): + value = self.original[1] + value[:] = self.original[2] + setattr(module, attr, value) + + +class ModuleAttrDict(ModuleAttr): + _MARKER = object() + + def encode_value(self, value): + return id(value), value, value.copy() + + def decode_value(self, value): + return value[2] + + def restore_attr(self, module, attr): + value = self.original[1] + value.clear() + value.update(self.original[2]) + setattr(module, attr, value) + + @classmethod + def _get_key(cls, data, key): + value = data.get(key, cls._MARKER) + if value is not cls._MARKER: + return repr(value) + else: + return '' + + def display_diff(self): + old_dict = self.original[2] + new_dict = self.current[2] + + keys = sorted(dict(old_dict).keys() | dict(new_dict).keys()) + for key in keys: + old_value = self._get_key(old_dict, key) + new_value = self._get_key(new_dict, key) + if old_value == new_value: + continue + + print(f" {self.name}[{key!r}]: {old_value} => {new_value}", + file=sys.stderr) + + +class SysArgv(ModuleAttrList): + name = 'sys.argv' + +class SysStdin(ModuleAttr): + name = 'sys.stdin' + +class SysStdout(ModuleAttr): + name = 'sys.stdout' + +class SysStderr(ModuleAttr): + name = 'sys.stderr' + +class SysPath(ModuleAttrList): + name = 'sys.path' + +class SysPathHooks(ModuleAttrList): + name = 'sys.path_hooks' + +class SysWarnOptions(ModuleAttrList): + name = 'sys.warnoptions' + +class SysGettrace(Resource): + name = 'sys.gettrace' + + def get(self): return sys.gettrace() - def restore_sys_gettrace(self, trace_fxn): - sys.settrace(trace_fxn) - def get___import__(self): - return builtins.__import__ - def restore___import__(self, import_): - builtins.__import__ = import_ + def restore(self): + sys.settrace(self.original) - def get_warnings_filters(self): - return id(warnings.filters), warnings.filters, warnings.filters[:] - def restore_warnings_filters(self, saved_filters): - warnings.filters = saved_filters[1] - warnings.filters[:] = saved_filters[2] - def get_asyncore_socket_map(self): +class OsEnviron(ModuleAttrDict): + name = 'os.environ' + +class ImportFunc(ModuleAttr): + name = 'builtins.__import__' + +class WarningsFilters(ModuleAttrList): + name = 'warnings.filters' + +class WarningsShowWarning(ModuleAttr): + name = 'warnings.showwarning' + + + + +class Cwd(Resource): + name = 'cwd' + + def get(self): + return os.getcwd() + + def restore(self): + os.chdir(self.original) + + +class AsyncoreSocketMap(Resource): + name = 'asyncore.socket_map' + + def get(self): asyncore = sys.modules.get('asyncore') # XXX Making a copy keeps objects alive until __exit__ gets called. return asyncore and asyncore.socket_map.copy() or {} - def restore_asyncore_socket_map(self, saved_map): + + def restore(self): asyncore = sys.modules.get('asyncore') if asyncore is not None: asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(saved_map) - - def get_shutil_archive_formats(self): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() - def restore_shutil_archive_formats(self, saved): - shutil._ARCHIVE_FORMATS = saved[0] - shutil._ARCHIVE_FORMATS.clear() - shutil._ARCHIVE_FORMATS.update(saved[1]) - - def get_shutil_unpack_formats(self): - return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() - def restore_shutil_unpack_formats(self, saved): - shutil._UNPACK_FORMATS = saved[0] - shutil._UNPACK_FORMATS.clear() - shutil._UNPACK_FORMATS.update(saved[1]) - - def get_logging__handlers(self): - # _handlers is a WeakValueDictionary - return id(logging._handlers), logging._handlers, logging._handlers.copy() - def restore_logging__handlers(self, saved_handlers): + asyncore.socket_map.update(self.original) + + +class ShutilUnpackFormats(ModuleAttrDict): + name = 'shutil._UNPACK_FORMATS' + +class ShutilArchiveFormats(ModuleAttrDict): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + + name = 'shutil._ARCHIVE_FORMATS' + + +class LoggingHandlers(ModuleAttrDict): + # _handlers is a WeakValueDictionary + name = 'logging._handlers' + + def restore(self): # Can't easily revert the logging state pass - def get_logging__handlerList(self): - # _handlerList is a list of weakrefs to handlers - return id(logging._handlerList), logging._handlerList, logging._handlerList[:] - def restore_logging__handlerList(self, saved_handlerList): +class LoggingHandlerList(ModuleAttrList): + # _handlerList is a list of weakrefs to handlers + name = 'logging._handlerList' + + def restore(self): # Can't easily revert the logging state pass - def get_sys_warnoptions(self): - return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] - def restore_sys_warnoptions(self, saved_options): - sys.warnoptions = saved_options[1] - sys.warnoptions[:] = saved_options[2] +class ThreadingDangling(Resource): # Controlling dangling references to Thread objects can make it easier # to track reference leaks. - def get_threading__dangling(self): + + name = 'threading._dangling' + + def get(self): if not threading: return None # This copies the weakrefs without making any strong reference return threading._dangling.copy() - def restore_threading__dangling(self, saved): + + def restore(self): if not threading: return threading._dangling.clear() - threading._dangling.update(saved) + threading._dangling.update(self.original) + + +if not multiprocessing: + class MultiprocessingProcessDangling(Resource): + # Same for Process objects + + name = 'multiprocessing.process._dangling' + + def get(self): + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + + def restore(self): + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(self.original) - # Same for Process objects - def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) - def get_sysconfig__CONFIG_VARS(self): +class SysconfigInstallSchemes(ModuleAttrDict): + name = 'sysconfig._INSTALL_SCHEMES' + +class SysconfigConfigVars(ModuleAttrDict): + name = 'sysconfig._CONFIG_VARS' + + def get(self): # make sure the dict is initialized sysconfig.get_config_var('prefix') - return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, - dict(sysconfig._CONFIG_VARS)) - def restore_sysconfig__CONFIG_VARS(self, saved): - sysconfig._CONFIG_VARS = saved[1] - sysconfig._CONFIG_VARS.clear() - sysconfig._CONFIG_VARS.update(saved[2]) - - def get_sysconfig__INSTALL_SCHEMES(self): - return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, - sysconfig._INSTALL_SCHEMES.copy()) - def restore_sysconfig__INSTALL_SCHEMES(self, saved): - sysconfig._INSTALL_SCHEMES = saved[1] - sysconfig._INSTALL_SCHEMES.clear() - sysconfig._INSTALL_SCHEMES.update(saved[2]) - - def get_files(self): + return super().get() + + +class Files(Resource): + name = 'files' + + def get(self): return sorted(fn + ('/' if os.path.isdir(fn) else '') for fn in os.listdir()) - def restore_files(self, saved_value): + + def restore(self): fn = support.TESTFN - if fn not in saved_value and (fn + '/') not in saved_value: + if fn not in self.original and (fn + '/') not in self.original: if os.path.isfile(fn): support.unlink(fn) elif os.path.isdir(fn): support.rmtree(fn) + +class Locale(Resource): + name = 'locale' + _lc = [getattr(locale, lc) for lc in dir(locale) if lc.startswith('LC_')] - def get_locale(self): + + def get(self): pairings = [] for lc in self._lc: try: @@ -243,43 +284,79 @@ class saved_test_environment: except (TypeError, ValueError): continue return pairings - def restore_locale(self, saved): - for lc, setting in saved: + + def restore(self): + for lc, setting in self.original: locale.setlocale(lc, setting) - def get_warnings_showwarning(self): - return warnings.showwarning - def restore_warnings_showwarning(self, fxn): - warnings.showwarning = fxn - def resource_info(self): - for name in self.resources: - method_suffix = name.replace('.', '_') - get_name = 'get_' + method_suffix - restore_name = 'restore_' + method_suffix - yield name, getattr(self, get_name), getattr(self, restore_name) +def _get_resources(parent_cls, resources): + for cls in parent_cls.__subclasses__(): + if cls.name is not None: + resources.append(cls) + _get_resources(cls, resources) + +def get_resources(resources=None): + resources = [] + _get_resources(Resource, resources) + return resources + + + +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. + +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. + + with saved_test_environment(testname, verbose, quiet): + #stuff + + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + + changed = False + + def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + self.pgo = pgo + self.resources = None def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) + self.resources = [resource_class() + for resource_class in get_resources()] return self def __exit__(self, exc_type, exc_val, exc_tb): - saved_values = self.saved_values - del self.saved_values - for name, get, restore in self.resource_info(): - current = get() - original = saved_values.pop(name) + # clear resources since they keep strong references to many objects + resources = self.resources + self.resources = None + + for resource in resources: + resource.current = resource.get() + # Check for changes to the resource's value - if current != original: - self.changed = True - restore(original) - if not self.quiet and not self.pgo: - print("Warning -- {} was modified by {}".format( - name, self.testname), - file=sys.stderr) - if self.verbose > 1: - print(" Before: {}\n After: {} ".format( - original, current), - file=sys.stderr) + if resource.current == resource.original: + continue + + self.changed = True + resource.restore() + if self.quiet or self.pgo: + continue + + print(f"Warning -- {resource.name} was modified by {self.testname}", + file=sys.stderr) + resource.display_diff() return False -- cgit v0.12 From d7ac00e6209ae29fbd4c9dd71885c32ad4c7340e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 19:13:06 +0100 Subject: Backed out changeset 245a16f33c4b Serhiy asked me to review it. --- Lib/test/libregrtest/save_env.py | 481 ++++++++++++++++----------------------- 1 file changed, 202 insertions(+), 279 deletions(-) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index c9a980c..90900a9 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -17,266 +17,225 @@ except ImportError: multiprocessing = None -class Resource: - name = None - - def __init__(self): - self.original = self.get() - self.current = None - - def decode_value(self, value): - return value - - def display_diff(self): - original = self.decode_value(self.original) - current = self.decode_value(self.current) - print(f" Before: {original}", file=sys.stderr) - print(f" After: {current}", file=sys.stderr) - - -class ModuleAttr(Resource): - def encode_value(self, value): - return value - - def get_module(self): - module_name, attr = self.name.split('.', 1) - module = globals()[module_name] - return (module, attr) - - def get(self): - module, attr = self.get_module() - value = getattr(module, attr) - return self.encode_value(value) - - def restore_attr(self, module, attr): - setattr(module, attr, self.original) - - def restore(self): - module, attr = self.get_module() - self.restore_attr(module, attr) - - -class ModuleAttrList(ModuleAttr): - def decode_value(self, value): - return value[2] - - def encode_value(self, value): - return id(value), value, value.copy() - - def restore_attr(self, module, attr): - value = self.original[1] - value[:] = self.original[2] - setattr(module, attr, value) - - -class ModuleAttrDict(ModuleAttr): - _MARKER = object() - - def encode_value(self, value): - return id(value), value, value.copy() - - def decode_value(self, value): - return value[2] - - def restore_attr(self, module, attr): - value = self.original[1] - value.clear() - value.update(self.original[2]) - setattr(module, attr, value) - - @classmethod - def _get_key(cls, data, key): - value = data.get(key, cls._MARKER) - if value is not cls._MARKER: - return repr(value) - else: - return '' - - def display_diff(self): - old_dict = self.original[2] - new_dict = self.current[2] - - keys = sorted(dict(old_dict).keys() | dict(new_dict).keys()) - for key in keys: - old_value = self._get_key(old_dict, key) - new_value = self._get_key(new_dict, key) - if old_value == new_value: - continue - - print(f" {self.name}[{key!r}]: {old_value} => {new_value}", - file=sys.stderr) - - -class SysArgv(ModuleAttrList): - name = 'sys.argv' - -class SysStdin(ModuleAttr): - name = 'sys.stdin' - -class SysStdout(ModuleAttr): - name = 'sys.stdout' - -class SysStderr(ModuleAttr): - name = 'sys.stderr' - -class SysPath(ModuleAttrList): - name = 'sys.path' - -class SysPathHooks(ModuleAttrList): - name = 'sys.path_hooks' - -class SysWarnOptions(ModuleAttrList): - name = 'sys.warnoptions' - -class SysGettrace(Resource): - name = 'sys.gettrace' - - def get(self): - return sys.gettrace() - - def restore(self): - sys.settrace(self.original) - - -class OsEnviron(ModuleAttrDict): - name = 'os.environ' - -class ImportFunc(ModuleAttr): - name = 'builtins.__import__' +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. -class WarningsFilters(ModuleAttrList): - name = 'warnings.filters' +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. -class WarningsShowWarning(ModuleAttr): - name = 'warnings.showwarning' + with saved_test_environment(testname, verbose, quiet): + #stuff + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + changed = False -class Cwd(Resource): - name = 'cwd' + def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + self.pgo = pgo - def get(self): + # To add things to save and restore, add a name XXX to the resources list + # and add corresponding get_XXX/restore_XXX functions. get_XXX should + # return the value to be saved and compared against a second call to the + # get function when test execution completes. restore_XXX should accept + # the saved value and restore the resource using it. It will be called if + # and only if a change in the value is detected. + # + # Note: XXX will have any '.' replaced with '_' characters when determining + # the corresponding method names. + + resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', + 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', + 'warnings.filters', 'asyncore.socket_map', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', + 'sys.warnoptions', + # multiprocessing.process._cleanup() may release ref + # to a thread, so check processes first. + 'multiprocessing.process._dangling', 'threading._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', + 'files', 'locale', 'warnings.showwarning', + ) + + def get_sys_argv(self): + return id(sys.argv), sys.argv, sys.argv[:] + def restore_sys_argv(self, saved_argv): + sys.argv = saved_argv[1] + sys.argv[:] = saved_argv[2] + + def get_cwd(self): return os.getcwd() + def restore_cwd(self, saved_cwd): + os.chdir(saved_cwd) + + def get_sys_stdout(self): + return sys.stdout + def restore_sys_stdout(self, saved_stdout): + sys.stdout = saved_stdout + + def get_sys_stderr(self): + return sys.stderr + def restore_sys_stderr(self, saved_stderr): + sys.stderr = saved_stderr + + def get_sys_stdin(self): + return sys.stdin + def restore_sys_stdin(self, saved_stdin): + sys.stdin = saved_stdin + + def get_os_environ(self): + return id(os.environ), os.environ, dict(os.environ) + def restore_os_environ(self, saved_environ): + os.environ = saved_environ[1] + os.environ.clear() + os.environ.update(saved_environ[2]) + + def get_sys_path(self): + return id(sys.path), sys.path, sys.path[:] + def restore_sys_path(self, saved_path): + sys.path = saved_path[1] + sys.path[:] = saved_path[2] + + def get_sys_path_hooks(self): + return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] + def restore_sys_path_hooks(self, saved_hooks): + sys.path_hooks = saved_hooks[1] + sys.path_hooks[:] = saved_hooks[2] + + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) - def restore(self): - os.chdir(self.original) - + def get___import__(self): + return builtins.__import__ + def restore___import__(self, import_): + builtins.__import__ = import_ -class AsyncoreSocketMap(Resource): - name = 'asyncore.socket_map' + def get_warnings_filters(self): + return id(warnings.filters), warnings.filters, warnings.filters[:] + def restore_warnings_filters(self, saved_filters): + warnings.filters = saved_filters[1] + warnings.filters[:] = saved_filters[2] - def get(self): + def get_asyncore_socket_map(self): asyncore = sys.modules.get('asyncore') # XXX Making a copy keeps objects alive until __exit__ gets called. return asyncore and asyncore.socket_map.copy() or {} - - def restore(self): + def restore_asyncore_socket_map(self, saved_map): asyncore = sys.modules.get('asyncore') if asyncore is not None: asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(self.original) - - -class ShutilUnpackFormats(ModuleAttrDict): - name = 'shutil._UNPACK_FORMATS' - -class ShutilArchiveFormats(ModuleAttrDict): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - - name = 'shutil._ARCHIVE_FORMATS' - - -class LoggingHandlers(ModuleAttrDict): - # _handlers is a WeakValueDictionary - name = 'logging._handlers' - - def restore(self): + asyncore.socket_map.update(saved_map) + + def get_shutil_archive_formats(self): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() + def restore_shutil_archive_formats(self, saved): + shutil._ARCHIVE_FORMATS = saved[0] + shutil._ARCHIVE_FORMATS.clear() + shutil._ARCHIVE_FORMATS.update(saved[1]) + + def get_shutil_unpack_formats(self): + return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() + def restore_shutil_unpack_formats(self, saved): + shutil._UNPACK_FORMATS = saved[0] + shutil._UNPACK_FORMATS.clear() + shutil._UNPACK_FORMATS.update(saved[1]) + + def get_logging__handlers(self): + # _handlers is a WeakValueDictionary + return id(logging._handlers), logging._handlers, logging._handlers.copy() + def restore_logging__handlers(self, saved_handlers): # Can't easily revert the logging state pass -class LoggingHandlerList(ModuleAttrList): - # _handlerList is a list of weakrefs to handlers - name = 'logging._handlerList' - - def restore(self): + def get_logging__handlerList(self): + # _handlerList is a list of weakrefs to handlers + return id(logging._handlerList), logging._handlerList, logging._handlerList[:] + def restore_logging__handlerList(self, saved_handlerList): # Can't easily revert the logging state pass + def get_sys_warnoptions(self): + return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] + def restore_sys_warnoptions(self, saved_options): + sys.warnoptions = saved_options[1] + sys.warnoptions[:] = saved_options[2] -class ThreadingDangling(Resource): # Controlling dangling references to Thread objects can make it easier # to track reference leaks. - - name = 'threading._dangling' - - def get(self): + def get_threading__dangling(self): if not threading: return None # This copies the weakrefs without making any strong reference return threading._dangling.copy() - - def restore(self): + def restore_threading__dangling(self, saved): if not threading: return threading._dangling.clear() - threading._dangling.update(self.original) + threading._dangling.update(saved) + # Same for Process objects + def get_multiprocessing_process__dangling(self): + if not multiprocessing: + return None + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + def restore_multiprocessing_process__dangling(self, saved): + if not multiprocessing: + return + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(saved) -if not multiprocessing: - class MultiprocessingProcessDangling(Resource): - # Same for Process objects - - name = 'multiprocessing.process._dangling' - - def get(self): - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - - def restore(self): - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(self.original) - - -class SysconfigInstallSchemes(ModuleAttrDict): - name = 'sysconfig._INSTALL_SCHEMES' - -class SysconfigConfigVars(ModuleAttrDict): - name = 'sysconfig._CONFIG_VARS' - - def get(self): + def get_sysconfig__CONFIG_VARS(self): # make sure the dict is initialized sysconfig.get_config_var('prefix') - return super().get() - - -class Files(Resource): - name = 'files' - - def get(self): + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__INSTALL_SCHEMES(self): + return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, + sysconfig._INSTALL_SCHEMES.copy()) + def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig._INSTALL_SCHEMES = saved[1] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(saved[2]) + + def get_files(self): return sorted(fn + ('/' if os.path.isdir(fn) else '') for fn in os.listdir()) - - def restore(self): + def restore_files(self, saved_value): fn = support.TESTFN - if fn not in self.original and (fn + '/') not in self.original: + if fn not in saved_value and (fn + '/') not in saved_value: if os.path.isfile(fn): support.unlink(fn) elif os.path.isdir(fn): support.rmtree(fn) - -class Locale(Resource): - name = 'locale' - _lc = [getattr(locale, lc) for lc in dir(locale) if lc.startswith('LC_')] - - def get(self): + def get_locale(self): pairings = [] for lc in self._lc: try: @@ -284,79 +243,43 @@ class Locale(Resource): except (TypeError, ValueError): continue return pairings - - def restore(self): - for lc, setting in self.original: + def restore_locale(self, saved): + for lc, setting in saved: locale.setlocale(lc, setting) + def get_warnings_showwarning(self): + return warnings.showwarning + def restore_warnings_showwarning(self, fxn): + warnings.showwarning = fxn -def _get_resources(parent_cls, resources): - for cls in parent_cls.__subclasses__(): - if cls.name is not None: - resources.append(cls) - _get_resources(cls, resources) - -def get_resources(resources=None): - resources = [] - _get_resources(Resource, resources) - return resources - - - -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. - -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. - - with saved_test_environment(testname, verbose, quiet): - #stuff - - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. - - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - - changed = False - - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - self.pgo = pgo - self.resources = None + def resource_info(self): + for name in self.resources: + method_suffix = name.replace('.', '_') + get_name = 'get_' + method_suffix + restore_name = 'restore_' + method_suffix + yield name, getattr(self, get_name), getattr(self, restore_name) def __enter__(self): - self.resources = [resource_class() - for resource_class in get_resources()] + self.saved_values = dict((name, get()) for name, get, restore + in self.resource_info()) return self def __exit__(self, exc_type, exc_val, exc_tb): - # clear resources since they keep strong references to many objects - resources = self.resources - self.resources = None - - for resource in resources: - resource.current = resource.get() - + saved_values = self.saved_values + del self.saved_values + for name, get, restore in self.resource_info(): + current = get() + original = saved_values.pop(name) # Check for changes to the resource's value - if resource.current == resource.original: - continue - - self.changed = True - resource.restore() - if self.quiet or self.pgo: - continue - - print(f"Warning -- {resource.name} was modified by {self.testname}", - file=sys.stderr) - resource.display_diff() + if current != original: + self.changed = True + restore(original) + if not self.quiet and not self.pgo: + print("Warning -- {} was modified by {}".format( + name, self.testname), + file=sys.stderr) + if self.verbose > 1: + print(" Before: {}\n After: {} ".format( + original, current), + file=sys.stderr) return False -- cgit v0.12 From e984eb501b0b2a95d14b1d160c04eda5a88d19c5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 22:51:17 +0100 Subject: Fix test_os.test_symlink(): remove create symlink --- Lib/test/test_os.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f7d64b7..1117875 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2554,6 +2554,8 @@ class Win32DeprecatedBytesAPI(unittest.TestCase): @support.skip_unless_symlink def test_symlink(self): + self.addCleanup(support.unlink, support.TESTFN) + filename = os.fsencode(support.TESTFN) with bytes_filename_warn(True): os.symlink(filename, filename) -- cgit v0.12 From ca9dbc7d880ac6e5e1df3219e34670b18998a346 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Mar 2016 01:04:37 +0100 Subject: makeopcodetargets.py: we need to import Lib/opcode.py Issue #20021: use importlib.machinery to import Lib/opcode.py and not an opcode module coming from somewhere else. makeopcodetargets.py is part of the Python build process and it is run by an external Python program, not the built Python program. Patch written by Serhiy Storchaka. --- Python/makeopcodetargets.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 7a57a06..023c9e6 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -3,14 +3,34 @@ (for compilers supporting computed gotos or "labels-as-values", such as gcc). """ -import opcode import os import sys +try: + from importlib.machinery import SourceFileLoader +except ImportError: + import imp + + def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib") + return imp.load_module(modname, *imp.find_module(modname, [modpath])) +else: + def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") + return SourceFileLoader(modname, modpath).load_module() + + def write_contents(f): """Write C code contents to the target file object. """ + opcode = find_module('opcode') targets = ['_unknown_opcode'] * 256 for opname, op in opcode.opmap.items(): targets[op] = "TARGET_%s" % opname -- cgit v0.12 From 53b0a41d3103ab84f3ef9a1532bf95cff8819636 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Mar 2016 01:12:36 +0100 Subject: Issue #25911: more info on test_os failure --- Lib/test/test_os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 1117875..2fb7d1b 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -892,7 +892,7 @@ class WalkTests(unittest.TestCase): # Walk bottom-up. all = list(self.walk(self.walk_path, topdown=False)) - self.assertEqual(len(all), 4) + self.assertEqual(len(all), 4, all) # We can't know which order SUB1 and SUB2 will appear in. # Not flipped: SUB11, SUB1, SUB2, TESTFN # flipped: SUB2, SUB11, SUB1, TESTFN -- cgit v0.12 From 942302371c1e46fdf486a5be0e036f7232a2fba1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Mar 2016 03:02:48 -0700 Subject: Minor code cleanup for PyArg_UnpackTuple. --- Python/getargs.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Python/getargs.c b/Python/getargs.c index 66a0c00..05ec27b 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1771,16 +1771,9 @@ PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t m PyObject **o; va_list vargs; -#ifdef HAVE_STDARG_PROTOTYPES - va_start(vargs, max); -#else - va_start(vargs); -#endif - assert(min >= 0); assert(min <= max); if (!PyTuple_Check(args)) { - va_end(vargs); PyErr_SetString(PyExc_SystemError, "PyArg_UnpackTuple() argument list is not a tuple"); return 0; @@ -1798,9 +1791,10 @@ PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t m "unpacked tuple should have %s%zd elements," " but has %zd", (min == max ? "" : "at least "), min, l); - va_end(vargs); return 0; } + if (l == 0) + return 1; if (l > max) { if (name != NULL) PyErr_Format( @@ -1813,9 +1807,14 @@ PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t m "unpacked tuple should have %s%zd elements," " but has %zd", (min == max ? "" : "at most "), max, l); - va_end(vargs); return 0; } + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, max); +#else + va_start(vargs); +#endif for (i = 0; i < l; i++) { o = va_arg(vargs, PyObject **); *o = PyTuple_GET_ITEM(args, i); -- cgit v0.12 From 2c257ab0f85396664536605666af9a6f00c73a4f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Mar 2016 04:10:11 -0700 Subject: Responsibility for argument checking belongs in set.__init__() rather than set.__new__(). See dict.__new__() and list.__new__() for comparison. Neither of those examine or touch args or kwds. That work is done in the __init__() methods. --- Objects/setobject.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 8e22b69..34235f8 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1131,9 +1131,6 @@ PySet_Fini(void) static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (kwds != NULL && type == &PySet_Type && !_PyArg_NoKeywords("set()", kwds)) - return NULL; - return make_new_set(type, NULL); } -- cgit v0.12 From 3840b2ac6728d1940a7efc2ecab2c7d9a100c080 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 27 Mar 2016 01:53:46 +0000 Subject: Issue #25940: Use internal local server more in test_ssl Move many tests from NetworkedTests and NetworkedBIOTests to a new Simple- BackgroundTests class, using the existing ThreadedEchoServer and SIGNED_ CERTFILE infrastructure. For tests that cause the server to crash by rejecting its certificate, separate them into independent test methods. Added custom root certificate to capath with the following commands: cp Lib/test/{pycacert.pem,capath/} # Edit copy to remove part before certificate c_rehash -v Lib/test/capath/ c_rehash -v -old Lib/test/capath/ # Note the generated file names cp Lib/test/capath/{pycacert.pem,b1930218.0} mv Lib/test/capath/{pycacert.pem,ceff1710.0} Change to pure PEM version of SIGNING_CA because PEM_cert_to_DER_cert() does not like the extra text at the start. Moved test_connect_ex_error() into BasicSocketTests and rewrote it to connect to a reserved localhost port. NetworkedTests.test_get_server_certificate_ipv6() split out because it needs to connect to an IPv6 DNS address. The only reference left to self-signed.pythontest.net is test_timeout_ connect_ex(), which needs a remote server to reliably time out the connection, but does not rely on the server running SSL. Made ThreadedEchoServer call unwrap() by default when it sees the client has shut the connection down, so that the client can cleanly call unwrap(). --- Lib/test/capath/0e4015b9.0 | 16 -- Lib/test/capath/b1930218.0 | 21 ++ Lib/test/capath/ce7b8643.0 | 16 -- Lib/test/capath/ceff1710.0 | 21 ++ Lib/test/test_ssl.py | 677 ++++++++++++++++++++++----------------------- Misc/NEWS | 2 +- 6 files changed, 378 insertions(+), 375 deletions(-) delete mode 100644 Lib/test/capath/0e4015b9.0 create mode 100644 Lib/test/capath/b1930218.0 delete mode 100644 Lib/test/capath/ce7b8643.0 create mode 100644 Lib/test/capath/ceff1710.0 diff --git a/Lib/test/capath/0e4015b9.0 b/Lib/test/capath/0e4015b9.0 deleted file mode 100644 index b6d259b..0000000 --- a/Lib/test/capath/0e4015b9.0 +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv -bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG -A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 -aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ -Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm -Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN -AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h -TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 -C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= ------END CERTIFICATE----- diff --git a/Lib/test/capath/b1930218.0 b/Lib/test/capath/b1930218.0 new file mode 100644 index 0000000..373349c --- /dev/null +++ b/Lib/test/capath/b1930218.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/Lib/test/capath/ce7b8643.0 b/Lib/test/capath/ce7b8643.0 deleted file mode 100644 index b6d259b..0000000 --- a/Lib/test/capath/ce7b8643.0 +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv -bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG -A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 -aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ -Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm -Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN -AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h -TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 -C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= ------END CERTIFICATE----- diff --git a/Lib/test/capath/ceff1710.0 b/Lib/test/capath/ceff1710.0 new file mode 100644 index 0000000..373349c --- /dev/null +++ b/Lib/test/capath/ceff1710.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e0c31a8..8b72ca9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -21,6 +21,13 @@ import functools ssl = support.import_module("ssl") +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = support.HOST @@ -53,10 +60,10 @@ CRLFILE = data_file("revocation.crl") # Two keys and certs signed by the same CA (for SNI tests) SIGNED_CERTFILE = data_file("keycert3.pem") SIGNED_CERTFILE2 = data_file("keycert4.pem") -SIGNING_CA = data_file("pycacert.pem") +# Same certificate as pycacert.pem, but without extra text in file +SIGNING_CA = data_file("capath", "ceff1710.0") REMOTE_HOST = "self-signed.pythontest.net" -REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") EMPTYCERT = data_file("nullcert.pem") BADCERT = data_file("badcert.pem") @@ -783,6 +790,22 @@ class BasicSocketTests(unittest.TestCase): self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + def test_connect_ex_error(self): + server = socket.socket(socket.AF_INET) + self.addCleanup(server.close) + port = support.bind_port(server) # Reserve port but don't listen + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + rc = s.connect_ex((HOST, port)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + class ContextTests(unittest.TestCase): @@ -1368,140 +1391,103 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) -class NetworkedTests(unittest.TestCase): +@unittest.skipUnless(_have_threads, "Needs threading module") +class SimpleBackgroundTests(unittest.TestCase): - def test_connect(self): - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE) - try: - s.connect((REMOTE_HOST, 443)) - self.assertEqual({}, s.getpeercert()) - finally: - s.close() + """Tests that connect to a simple server running in the background""" - # this should fail because we have no verification certs - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", - s.connect, (REMOTE_HOST, 443)) - s.close() + def setUp(self): + server = ThreadedEchoServer(SIGNED_CERTFILE) + self.server_addr = (HOST, server.port) + server.__enter__() + self.addCleanup(server.__exit__, None, None, None) - # this should succeed because we specify the root cert - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - s.connect((REMOTE_HOST, 443)) - self.assertTrue(s.getpeercert()) - finally: - s.close() + def test_connect(self): + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + + # this should succeed because we specify the root cert + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) as s: + s.connect(self.server_addr) + self.assertTrue(s.getpeercert()) + + def test_connect_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) def test_connect_ex(self): # Issue #11326: check connect_ex() implementation - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) - self.assertTrue(s.getpeercert()) - finally: - s.close() + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) + self.addCleanup(s.close) + self.assertEqual(0, s.connect_ex(self.server_addr)) + self.assertTrue(s.getpeercert()) def test_non_blocking_connect_ex(self): # Issue #11326: non-blocking connect_ex() should allow handshake # to proceed after the socket gets ready. - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT, - do_handshake_on_connect=False) + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.setblocking(False) + rc = s.connect_ex(self.server_addr) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: try: - s.setblocking(False) - rc = s.connect_ex((REMOTE_HOST, 443)) - # EWOULDBLOCK under Windows, EINPROGRESS elsewhere - self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) - # Wait for connect to finish + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: select.select([], [s], [], 5.0) - # Non-blocking handshake - while True: - try: - s.do_handshake() - break - except ssl.SSLWantReadError: - select.select([s], [], [], 5.0) - except ssl.SSLWantWriteError: - select.select([], [s], [], 5.0) - # SSL established - self.assertTrue(s.getpeercert()) - finally: - s.close() - - def test_timeout_connect_ex(self): - # Issue #12065: on a timeout, connect_ex() should return the original - # errno (mimicking the behaviour of non-SSL sockets). - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT, - do_handshake_on_connect=False) - try: - s.settimeout(0.0000001) - rc = s.connect_ex((REMOTE_HOST, 443)) - if rc == 0: - self.skipTest("REMOTE_HOST responded too quickly") - self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) - finally: - s.close() - - def test_connect_ex_error(self): - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - rc = s.connect_ex((REMOTE_HOST, 444)) - # Issue #19919: Windows machines or VMs hosted on Windows - # machines sometimes return EWOULDBLOCK. - errors = ( - errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, - errno.EWOULDBLOCK, - ) - self.assertIn(rc, errors) - finally: - s.close() + # SSL established + self.assertTrue(s.getpeercert()) def test_connect_with_context(self): - with support.transient_internet(REMOTE_HOST): - # Same as test_connect, but with a separately created context - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - self.assertEqual({}, s.getpeercert()) - finally: - s.close() - # Same with a server hostname - s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname=REMOTE_HOST) - s.connect((REMOTE_HOST, 443)) - s.close() - # This should fail because we have no verification certs - ctx.verify_mode = ssl.CERT_REQUIRED - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", - s.connect, (REMOTE_HOST, 443)) - s.close() - # This should succeed because we specify the root cert - ctx.load_verify_locations(REMOTE_ROOT_CERT) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + # Same with a server hostname + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="dummy") as s: + s.connect(self.server_addr) + ctx.verify_mode = ssl.CERT_REQUIRED + # This should succeed because we specify the root cert + ctx.load_verify_locations(SIGNING_CA) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_with_context_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) def test_connect_capath(self): # Verify server certificates using the `capath` argument @@ -1509,198 +1495,130 @@ class NetworkedTests(unittest.TestCase): # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must # contain both versions of each certificate (same content, different # filename) for this test to be portable across OpenSSL releases. - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=CAPATH) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() - # Same with a bytes `capath` argument - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=BYTES_CAPATH) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) def test_connect_cadata(self): - with open(REMOTE_ROOT_CERT) as f: + with open(SIGNING_CA) as f: pem = f.read() der = ssl.PEM_cert_to_DER_cert(pem) - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(cadata=pem) - with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: - s.connect((REMOTE_HOST, 443)) - cert = s.getpeercert() - self.assertTrue(cert) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) - # same with DER - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(cadata=der) - with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: - s.connect((REMOTE_HOST, 443)) - cert = s.getpeercert() - self.assertTrue(cert) + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") def test_makefile_close(self): # Issue #5238: creating a file-like object with makefile() shouldn't # delay closing the underlying "real socket" (here tested with its # file descriptor, hence skipping the test under Windows). - with support.transient_internet(REMOTE_HOST): - ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) - ss.connect((REMOTE_HOST, 443)) - fd = ss.fileno() - f = ss.makefile() - f.close() - # The fd is still open + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(self.server_addr) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: os.read(fd, 0) - # Closing the SSL socket should close the fd too - ss.close() - gc.collect() - with self.assertRaises(OSError) as e: - os.read(fd, 0) - self.assertEqual(e.exception.errno, errno.EBADF) + self.assertEqual(e.exception.errno, errno.EBADF) def test_non_blocking_handshake(self): - with support.transient_internet(REMOTE_HOST): - s = socket.socket(socket.AF_INET) - s.connect((REMOTE_HOST, 443)) - s.setblocking(False) - s = ssl.wrap_socket(s, - cert_reqs=ssl.CERT_NONE, - do_handshake_on_connect=False) - count = 0 - while True: - try: - count += 1 - s.do_handshake() - break - except ssl.SSLWantReadError: - select.select([s], [], []) - except ssl.SSLWantWriteError: - select.select([], [s], []) - s.close() - if support.verbose: - sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + s = socket.socket(socket.AF_INET) + s.connect(self.server_addr) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + self.addCleanup(s.close) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) def test_get_server_certificate(self): - def _test_get_server_certificate(host, port, cert=None): - with support.transient_internet(host): - pem = ssl.get_server_certificate((host, port)) - if not pem: - self.fail("No server certificate on %s:%s!" % (host, port)) - - try: - pem = ssl.get_server_certificate((host, port), - ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail - if support.verbose: - sys.stdout.write("%s\n" % x) - else: - self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + _test_get_server_certificate(self, *self.server_addr, cert=SIGNING_CA) - pem = ssl.get_server_certificate((host, port), - ca_certs=cert) - if not pem: - self.fail("No server certificate on %s:%s!" % (host, port)) - if support.verbose: - sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) - - _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) - if support.IPV6_ENABLED: - _test_get_server_certificate('ipv6.google.com', 443) + def test_get_server_certificate_fail(self): + # Connection failure crashes ThreadedEchoServer, so run this in an + # independent test method + _test_get_server_certificate_fail(self, *self.server_addr) def test_ciphers(self): - remote = (REMOTE_HOST, 443) - with support.transient_internet(remote[0]): - with ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: - s.connect(remote) - with ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: - s.connect(remote) - # Error checking can happen at instantiation or when connecting - with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): - with socket.socket(socket.AF_INET) as sock: - s = ssl.wrap_socket(sock, - cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") - s.connect(remote) - - def test_algorithms(self): - # Issue #8484: all algorithms should be available when verifying a - # certificate. - # SHA256 was added in OpenSSL 0.9.8 - if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): - self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) - # sha256.tbs-internet.com needs SNI to use the correct certificate - if not ssl.HAS_SNI: - self.skipTest("SNI needed for this test") - # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) - remote = ("sha256.tbs-internet.com", 443) - sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") - with support.transient_internet("sha256.tbs-internet.com"): - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(sha256_cert) - s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname="sha256.tbs-internet.com") - try: - s.connect(remote) - if support.verbose: - sys.stdout.write("\nCipher with %r is %r\n" % - (remote, s.cipher())) - sys.stdout.write("Certificate is:\n%s\n" % - pprint.pformat(s.getpeercert())) - finally: - s.close() + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(self.server_addr) + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(self.server_addr) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(self.server_addr) def test_get_ca_certs_capath(self): # capath certs are loaded on request - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=CAPATH) - self.assertEqual(ctx.get_ca_certs(), []) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() - self.assertEqual(len(ctx.get_ca_certs()), 1) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + self.assertEqual(len(ctx.get_ca_certs()), 1) @needs_sni def test_context_setget(self): # Check that the context of a connected socket can be replaced. - with support.transient_internet(REMOTE_HOST): - ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - s = socket.socket(socket.AF_INET) - with ctx1.wrap_socket(s) as ss: - ss.connect((REMOTE_HOST, 443)) - self.assertIs(ss.context, ctx1) - self.assertIs(ss._sslobj.context, ctx1) - ss.context = ctx2 - self.assertIs(ss.context, ctx2) - self.assertIs(ss._sslobj.context, ctx2) - - -class NetworkedBIOTests(unittest.TestCase): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s) as ss: + ss.connect(self.server_addr) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): # A simple IO loop. Call func(*args) depending on the error we get @@ -1736,64 +1654,128 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret - def test_handshake(self): + def test_bio_handshake(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(SIGNING_CA) + ctx.check_hostname = True + sslobj = ctx.wrap_bio(incoming, outgoing, False, 'localhost') + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # If the server shuts down the TCP connection without sending a + # secure shutdown message, this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + + def test_bio_read_write_data(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'FOO\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf, b'foo\n') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + + +class NetworkedTests(unittest.TestCase): + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). with support.transient_internet(REMOTE_HOST): - sock = socket.socket(socket.AF_INET) - sock.connect((REMOTE_HOST, 443)) - incoming = ssl.MemoryBIO() - outgoing = ssl.MemoryBIO() - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'Needs IPv6') + def test_get_server_certificate_ipv6(self): + with support.transient_internet('ipv6.google.com'): + _test_get_server_certificate(self, 'ipv6.google.com', 443) + _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(REMOTE_ROOT_CERT) - ctx.check_hostname = True - sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST) - self.assertIs(sslobj._sslobj.owner, sslobj) - self.assertIsNone(sslobj.cipher()) - self.assertIsNone(sslobj.shared_ciphers()) - self.assertRaises(ValueError, sslobj.getpeercert) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: - self.assertIsNone(sslobj.get_channel_binding('tls-unique')) - self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) - self.assertTrue(sslobj.cipher()) - self.assertIsNone(sslobj.shared_ciphers()) - self.assertTrue(sslobj.getpeercert()) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: - self.assertTrue(sslobj.get_channel_binding('tls-unique')) + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") try: - self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) - except ssl.SSLSyscallError: - # self-signed.pythontest.net probably shuts down the TCP - # connection without sending a secure shutdown message, and - # this is reported as SSL_ERROR_SYSCALL - pass - self.assertRaises(ssl.SSLError, sslobj.write, b'foo') - sock.close() + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() - def test_read_write_data(self): - with support.transient_internet(REMOTE_HOST): - sock = socket.socket(socket.AF_INET) - sock.connect((REMOTE_HOST, 443)) - incoming = ssl.MemoryBIO() - outgoing = ssl.MemoryBIO() - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_NONE - sslobj = ctx.wrap_bio(incoming, outgoing, False) - self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) - req = b'GET / HTTP/1.0\r\n\r\n' - self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) - buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) - self.assertEqual(buf[:5], b'HTTP/') - self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) - sock.close() +def _test_get_server_certificate(test, host, port, cert=None): + pem = ssl.get_server_certificate((host, port)) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + +def _test_get_server_certificate_fail(test, host, port): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) -try: - import threading -except ImportError: - _have_threads = False -else: - _have_threads = True +if _have_threads: from test.ssl_servers import make_https_server class ThreadedEchoServer(threading.Thread): @@ -1881,6 +1863,15 @@ else: if not stripped: # eof, so quit this handler self.running = False + try: + self.sock = self.sslconn.unwrap() + except OSError: + # Many tests shut the TCP connection down + # without an SSL shutdown. This causes + # unwrap() to raise OSError with errno=0! + pass + else: + self.sslconn = None self.close() elif stripped == b'over': if support.verbose and self.server.connectionchatty: @@ -3342,18 +3333,20 @@ def test_main(verbose=False): pass for filename in [ - CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + CERTFILE, BYTES_CERTFILE, ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, BADCERT, BADKEY, EMPTYCERT]: if not os.path.exists(filename): raise support.TestFailed("Can't read certificate file %r" % filename) - tests = [ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests] + tests = [ + ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests, + SimpleBackgroundTests, + ] if support.is_resource_enabled('network'): tests.append(NetworkedTests) - tests.append(NetworkedBIOTests) if _have_threads: thread_info = support.threading_setup() diff --git a/Misc/NEWS b/Misc/NEWS index ca36123..83dee73 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -873,7 +873,7 @@ Tests - Issue #26325: Added test.support.check_no_resource_warning() to check that no ResourceWarning is emitted. -- Issue #25940: Changed test_ssl to use self-signed.pythontest.net. This +- Issue #25940: Changed test_ssl to use its internal local server more. This avoids relying on svn.python.org, which recently changed root certificate. - Issue #25616: Tests for OrderedDict are extracted from test_collections -- cgit v0.12 From 3625af5f2129f8e8735c54bd01d9f138dd5c0e83 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 27 Mar 2016 01:15:07 -0700 Subject: Moved misplaced functions to the section for C API functions. --- Objects/setobject.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 34235f8..176570e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1116,18 +1116,6 @@ frozenset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return emptyfrozenset; } -int -PySet_ClearFreeList(void) -{ - return 0; -} - -void -PySet_Fini(void) -{ - Py_CLEAR(emptyfrozenset); -} - static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -2339,6 +2327,18 @@ PySet_Add(PyObject *anyset, PyObject *key) } int +PySet_ClearFreeList(void) +{ + return 0; +} + +void +PySet_Fini(void) +{ + Py_CLEAR(emptyfrozenset); +} + +int _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash) { setentry *entry; -- cgit v0.12 From 622583e9bf19b79b621eba8c5798acdf175a17d2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 27 Mar 2016 18:28:15 +0200 Subject: regrtest: round final timing towards +inf --- Lib/test/libregrtest/main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e1367da..447d99f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,6 @@ import datetime import faulthandler +import math import os import platform import random @@ -106,9 +107,13 @@ class Regrtest: self.skipped.append(test) self.resource_denieds.append(test) - def time_delta(self): + def time_delta(self, ceil=False): seconds = time.monotonic() - self.start_time - return datetime.timedelta(seconds=int(seconds)) + if ceil: + seconds = math.ceil(seconds) + else: + seconds = int(seconds) + return datetime.timedelta(seconds=seconds) def display_progress(self, test_index, test): if self.ns.quiet: @@ -409,7 +414,7 @@ class Regrtest: r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) - print("Total duration: %s" % self.time_delta()) + print("Total duration: %s" % self.time_delta(ceil=True)) if self.ns.runleaks: os.system("leaks %d" % os.getpid()) -- cgit v0.12 From 2a65ecb780e2a5cd47bc4b9af947b1127e972a11 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Mar 2016 00:45:28 +0300 Subject: Issue #26130: Remove redundant variable 's' from Parser/parser.c Patch by Oren Milman. --- Parser/parser.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Parser/parser.c b/Parser/parser.c index 56ec514..41072c4 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -140,21 +140,20 @@ classify(parser_state *ps, int type, const char *str) int n = g->g_ll.ll_nlabels; if (type == NAME) { - const char *s = str; label *l = g->g_ll.ll_label; int i; for (i = n; i > 0; i--, l++) { if (l->lb_type != NAME || l->lb_str == NULL || - l->lb_str[0] != s[0] || - strcmp(l->lb_str, s) != 0) + l->lb_str[0] != str[0] || + strcmp(l->lb_str, str) != 0) continue; #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD #if 0 /* Leaving this in as an example */ if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) { - if (s[0] == 'w' && strcmp(s, "with") == 0) + if (str[0] == 'w' && strcmp(str, "with") == 0) break; /* not a keyword yet */ - else if (s[0] == 'a' && strcmp(s, "as") == 0) + else if (str[0] == 'a' && strcmp(str, "as") == 0) break; /* not a keyword yet */ } #endif -- cgit v0.12 From 80ec58c49730cb5498a279ee796ff037e8bbb3a7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Mar 2016 09:50:18 +0200 Subject: fix typo in comment Thanks Arfrever for the report :) --- Lib/test/libregrtest/runtest_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index aa97f05..33f632d 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,7 +21,7 @@ from test.libregrtest.setup import setup_tests # Display the running tests if nothing happened last N seconds PROGRESS_UPDATE = 30.0 # seconds -# If interrupted, display the wait process every N seconds +# If interrupted, display the wait progress every N seconds WAIT_PROGRESS = 2.0 # seconds -- cgit v0.12 From 73030df77f05426872cee7512dc4632ccb35bcc4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Mar 2016 11:25:00 +0200 Subject: Fix os._DummyDirEntry.is_symlink() Issue #25911: Fix os._DummyDirEntry.is_symlink(), don't follow symbolic links: use os.stat(path, follow_symlinks=False). --- Lib/os.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index c3ce05d..90646a0 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -452,22 +452,33 @@ class _DummyDirEntry: # Mimick FindFirstFile/FindNextFile: we should get file attributes # while iterating on a directory self._stat = None + self._lstat = None try: - self.stat() + self.stat(follow_symlinks=False) except OSError: pass - def stat(self): - if self._stat is None: - self._stat = stat(self.path) - return self._stat + def stat(self, *, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + self._stat = stat(self.path) + return self._stat + else: + if self._lstat is None: + self._lstat = stat(self.path, follow_symlinks=False) + return self._lstat def is_dir(self): + if self._lstat is not None and not self.is_symlink(): + # use the cache lstat + stat = self.stat(follow_symlinks=False) + return st.S_ISDIR(stat.st_mode) + stat = self.stat() return st.S_ISDIR(stat.st_mode) def is_symlink(self): - stat = self.stat() + stat = self.stat(follow_symlinks=False) return st.S_ISLNK(stat.st_mode) class _dummy_scandir: -- cgit v0.12 From b1511f789ea0565296cf9c64f30d2ab4e79b8615 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 01:29:05 +0200 Subject: doctest now supports packages Issue #26641: doctest.DocFileTest and doctest.testfile() now support packages (module splitted into multiple directories) for the package parameter. --- Lib/doctest.py | 17 +++++++++++++---- Misc/NEWS | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index 38fdd80..5630220 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -381,12 +381,15 @@ class _OutputRedirectingPdb(pdb.Pdb): sys.stdout = save_stdout # [XX] Normalize with respect to os.path.pardir? -def _module_relative_path(module, path): +def _module_relative_path(module, test_path): if not inspect.ismodule(module): raise TypeError('Expected a module: %r' % module) - if path.startswith('/'): + if test_path.startswith('/'): raise ValueError('Module-relative files may not have absolute paths') + # Normalize the path. On Windows, replace "/" with "\". + test_path = os.path.join(*(test_path.split('/'))) + # Find the base directory for the path. if hasattr(module, '__file__'): # A normal module/package @@ -398,13 +401,19 @@ def _module_relative_path(module, path): else: basedir = os.curdir else: + if hasattr(module, '__path__'): + for directory in module.__path__: + fullpath = os.path.join(directory, test_path) + if os.path.exists(fullpath): + return fullpath + # A module w/o __file__ (this includes builtins) raise ValueError("Can't resolve paths relative to the module " "%r (it has no __file__)" % module.__name__) - # Combine the base directory and the path. - return os.path.join(basedir, *(path.split('/'))) + # Combine the base directory and the test path. + return os.path.join(basedir, test_path) ###################################################################### ## 2. Example & DocTest diff --git a/Misc/NEWS b/Misc/NEWS index 3923af1..73c355d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,10 @@ Core and Builtins Library ------- +- Issue #26641: doctest.DocFileTest and doctest.testfile() now support + packages (module splitted into multiple directories) for the package + parameter. + - Issue #25195: Fix a regression in mock.MagicMock. _Call is a subclass of tuple (changeset 3603bae63c13 only works for classes) so we need to implement __ne__ ourselves. Patch by Andrew Plummer. -- cgit v0.12 From 9759dd334325f341318ff5f2ef25409a5e44dc98 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 02:32:52 +0200 Subject: Issue #26295: When using "python3 -m test --testdir=TESTDIR", regrtest doesn't add "test." prefix to test module names. regrtest also prepends testdir to sys.path. --- Lib/test/libregrtest/runtest.py | 8 ++++---- Lib/test/libregrtest/setup.py | 5 +++++ Misc/NEWS | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index daff476..601f2b2 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -116,7 +116,7 @@ def runtest(ns, test): try: sys.stdout = stream sys.stderr = stream - result = runtest_inner(test, verbose, quiet, huntrleaks, + result = runtest_inner(ns, test, verbose, quiet, huntrleaks, display_failure=False, pgo=pgo) if result[0] == FAILED: output = stream.getvalue() @@ -127,7 +127,7 @@ def runtest(ns, test): sys.stderr = orig_stderr else: support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, + result = runtest_inner(ns, test, verbose, quiet, huntrleaks, display_failure=not verbose, pgo=pgo) return result finally: @@ -137,14 +137,14 @@ def runtest(ns, test): runtest.stringio = None -def runtest_inner(test, verbose, quiet, +def runtest_inner(ns, test, verbose, quiet, huntrleaks=False, display_failure=True, *, pgo=False): support.unload(test) test_time = 0.0 refleak = False # True if the test leaked references. try: - if test.startswith('test.'): + if test.startswith('test.') or ns.testdir: abstest = test else: # Always import it from the test package diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index de52bb5..1d24531 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -29,6 +29,11 @@ def setup_tests(ns): replace_stdout() support.record_original_stdout(sys.stdout) + if ns.testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(ns.testdir)) + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). diff --git a/Misc/NEWS b/Misc/NEWS index 73c355d..34ef0cd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -879,6 +879,9 @@ Documentation Tests ----- +- Issue #26295: When using "python3 -m test --testdir=TESTDIR", regrtest + doesn't add "test." prefix to test module names. + - Issue #26523: The multiprocessing thread pool (multiprocessing.dummy.Pool) was untested. -- cgit v0.12 From d6e2502624c3f19593657086e1d673d2dd699b21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 02:33:52 +0200 Subject: Issue #26295: test_regrtest now uses a temporary directory test_forever() stores its state into the builtins module since the test module is reloaded at each run. Remove also warning to detect leaked tests of a previous run. --- Lib/test/test_regrtest.py | 55 ++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index df8447c..013a1b4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -15,6 +15,7 @@ import re import subprocess import sys import sysconfig +import tempfile import textwrap import unittest from test import libregrtest @@ -309,15 +310,8 @@ class BaseTestCase(unittest.TestCase): def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) - # When test_regrtest is interrupted by CTRL+c, it can leave - # temporary test files - remove = [entry.path - for entry in os.scandir(self.testdir) - if (entry.name.startswith(self.TESTNAME_PREFIX) - and entry.name.endswith(".py"))] - for path in remove: - print("WARNING: test_regrtest: remove %s" % path) - support.unlink(path) + self.tmptestdir = tempfile.mkdtemp() + self.addCleanup(support.rmtree, self.tmptestdir) def create_test(self, name=None, code=''): if not name: @@ -326,8 +320,8 @@ class BaseTestCase(unittest.TestCase): # test_regrtest cannot be run twice in parallel because # of setUp() and create_test() - name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name) - path = os.path.join(self.testdir, name + '.py') + name = self.TESTNAME_PREFIX + name + path = os.path.join(self.tmptestdir, name + '.py') self.addCleanup(support.unlink, path) # Use 'x' mode to ensure that we do not override existing tests @@ -462,7 +456,8 @@ class ProgramsTestCase(BaseTestCase): self.tests = [self.create_test() for index in range(self.NTEST)] self.python_args = ['-Wd', '-E', '-bb'] - self.regrtest_args = ['-uall', '-rwW'] + self.regrtest_args = ['-uall', '-rwW', + '--testdir=%s' % self.tmptestdir] if hasattr(faulthandler, 'dump_traceback_later'): self.regrtest_args.extend(('--timeout', '3600', '-j4')) if sys.platform == 'win32': @@ -519,7 +514,8 @@ class ProgramsTestCase(BaseTestCase): def test_tools_script_run_tests(self): # Tools/scripts/run_tests.py script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - self.run_tests([script, *self.tests]) + args = [script, '--testdir=%s' % self.tmptestdir, *self.tests] + self.run_tests(args) def run_batch(self, *args): proc = self.run_command(args) @@ -555,8 +551,9 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args, **kw): - return self.run_python(['-m', 'test', *args], **kw) + def run_tests(self, *testargs, **kw): + cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] + return self.run_python(cmdargs, **kw) def test_failing_test(self): # test a failing test @@ -567,8 +564,8 @@ class ArgsTestCase(BaseTestCase): def test_failing(self): self.fail("bug") """) - test_ok = self.create_test() - test_failing = self.create_test(code=code) + test_ok = self.create_test('ok') + test_failing = self.create_test('failing', code=code) tests = [test_ok, test_failing] output = self.run_tests(*tests, exitcode=1) @@ -661,7 +658,7 @@ class ArgsTestCase(BaseTestCase): def test_interrupted(self): code = TEST_INTERRUPTED - test = self.create_test("sigint", code=code) + test = self.create_test('sigint', code=code) output = self.run_tests(test, exitcode=1) self.check_executed_tests(output, test, omitted=test) @@ -693,7 +690,7 @@ class ArgsTestCase(BaseTestCase): def test_coverage(self): # test --coverage - test = self.create_test() + test = self.create_test('coverage') output = self.run_tests("--coverage", test) self.check_executed_tests(output, [test]) regex = ('lines +cov% +module +$path$\n' @@ -702,24 +699,28 @@ class ArgsTestCase(BaseTestCase): def test_wait(self): # test --wait - test = self.create_test() + test = self.create_test('wait') output = self.run_tests("--wait", test, input='key') self.check_line(output, 'Press any key to continue') def test_forever(self): # test --forever code = textwrap.dedent(""" + import builtins import unittest class ForeverTester(unittest.TestCase): - RUN = 1 - def test_run(self): - ForeverTester.RUN += 1 - if ForeverTester.RUN > 3: - self.fail("fail at the 3rd runs") + # Store the state in the builtins module, because the test + # module is reload at each run + if 'RUN' in builtins.__dict__: + builtins.__dict__['RUN'] += 1 + if builtins.__dict__['RUN'] >= 3: + self.fail("fail at the 3rd runs") + else: + builtins.__dict__['RUN'] = 1 """) - test = self.create_test(code=code) + test = self.create_test('forever', code=code) output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) @@ -747,7 +748,7 @@ class ArgsTestCase(BaseTestCase): fd = os.open(__file__, os.O_RDONLY) # bug: never cloes the file descriptor """) - test = self.create_test(code=code) + test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(support.unlink, filename) -- cgit v0.12 From 732599f793281d15a1d28df0ebe63fe1fba52780 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 08:38:05 +0200 Subject: Issue #26295: Fix test_regrtest.test_tools_buildbot_test() Pass also --testdir option. --- Lib/test/test_regrtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 013a1b4..b4083b8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -527,7 +527,7 @@ class ProgramsTestCase(BaseTestCase): def test_tools_buildbot_test(self): # Tools\buildbot\test.bat script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') - test_args = [] + test_args = ['--testdir=%s' % self.tmptestdir] if platform.architecture()[0] == '64bit': test_args.append('-x64') # 64-bit build if not Py_DEBUG: -- cgit v0.12 From 8a34d416ba9e107e07fca7d4a1692b4b6a54a3ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 08:51:15 +0200 Subject: Issue #26295: Enhanc test_regrtest.test_tools_script_run_tests() Pass all regrtest options, not only --testdir. --- Lib/test/test_regrtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index b4083b8..213853f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -514,7 +514,7 @@ class ProgramsTestCase(BaseTestCase): def test_tools_script_run_tests(self): # Tools/scripts/run_tests.py script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - args = [script, '--testdir=%s' % self.tmptestdir, *self.tests] + args = [script, *self.regrtest_args, *self.tests] self.run_tests(args) def run_batch(self, *args): -- cgit v0.12 From ab0d198c7a6b2133ba5f782d7b951fef00ae615c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 30 Mar 2016 21:11:16 +0300 Subject: Issue #26492: Exhausted iterator of array.array now conforms with the behavior of iterators of other mutable sequences: it lefts exhausted even if iterated array is extended. --- Lib/test/test_array.py | 21 +++++++++++++++++++-- Misc/NEWS | 4 ++++ Modules/arraymodule.c | 22 ++++++++++++++++++---- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 482526e..b4f2bf8 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -318,8 +318,19 @@ class BaseTest: d = pickle.dumps((itorig, orig), proto) it, a = pickle.loads(d) a.fromlist(data2) - self.assertEqual(type(it), type(itorig)) - self.assertEqual(list(it), data2) + self.assertEqual(list(it), []) + + def test_exhausted_iterator(self): + a = array.array(self.typecode, self.example) + self.assertEqual(list(a), list(self.example)) + exhit = iter(a) + empit = iter(a) + for x in exhit: # exhaust the iterator + next(empit) # not exhausted + a.append(self.outside) + self.assertEqual(list(exhit), []) + self.assertEqual(list(empit), [self.outside]) + self.assertEqual(list(a), list(self.example) + [self.outside]) def test_insert(self): a = array.array(self.typecode, self.example) @@ -1070,6 +1081,12 @@ class BaseTest: a = array.array('B', b"") self.assertRaises(BufferError, getbuffer_with_null_view, a) + def test_free_after_iterating(self): + support.check_free_after_iterating(self, iter, array.array, + (self.typecode,)) + support.check_free_after_iterating(self, reversed, array.array, + (self.typecode,)) + class StringTest(BaseTest): def test_setitem(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6fb0fc4..6b6d11c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,10 @@ Core and Builtins Library ------- +- Issue #26492: Exhausted iterator of array.array now conforms with the behavior + of iterators of other mutable sequences: it lefts exhausted even if iterated + array is extended. + - Issue #26641: doctest.DocFileTest and doctest.testfile() now support packages (module splitted into multiple directories) for the package parameter. diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 1b0a282..323e0c1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2875,9 +2875,20 @@ array_iter(arrayobject *ao) static PyObject * arrayiter_next(arrayiterobject *it) { + arrayobject *ao; + + assert(it != NULL); assert(PyArrayIter_Check(it)); - if (it->index < Py_SIZE(it->ao)) - return (*it->getitem)(it->ao, it->index++); + ao = it->ao; + if (ao == NULL) { + return NULL; + } + assert(array_Check(ao)); + if (it->index < Py_SIZE(ao)) { + return (*it->getitem)(ao, it->index++); + } + it->ao = NULL; + Py_DECREF(ao); return NULL; } @@ -2906,8 +2917,11 @@ static PyObject * array_arrayiterator___reduce___impl(arrayiterobject *self) /*[clinic end generated code: output=7898a52e8e66e016 input=a062ea1e9951417a]*/ { - return Py_BuildValue("N(O)n", _PyObject_GetBuiltin("iter"), - self->ao, self->index); + PyObject *func = _PyObject_GetBuiltin("iter"); + if (self->ao == NULL) { + return Py_BuildValue("N(())", func); + } + return Py_BuildValue("N(O)n", func, self->ao, self->index); } /*[clinic input] -- cgit v0.12 From c0aab1da3bb0c5f8b6b7fc20278f05ebcf2f0994 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 31 Mar 2016 10:31:30 +0000 Subject: Issue #22854: Skip pipe seekable() tests on Windows --- Lib/test/test_io.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 1944a04..9c410e7 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -425,7 +425,12 @@ class IOTest(unittest.TestCase): writable = "w" in abilities self.assertEqual(obj.writable(), writable) seekable = "s" in abilities - self.assertEqual(obj.seekable(), seekable) + + # Detection of pipes being non-seekable does not seem to work + # on Windows + if not sys.platform.startswith("win") or test not in ( + pipe_reader, pipe_writer): + self.assertEqual(obj.seekable(), seekable) if isinstance(obj, self.TextIOBase): data = "3" -- cgit v0.12 From 49f324f1d41668dbc478fa15f85921d0b05ec09b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Mar 2016 23:30:53 +0200 Subject: Python 8: no pep8, no chocolate! --- Include/patchlevel.h | 6 +- Lib/pep8.py | 2151 ++++++++++++++++++++++++++++++++++++++++++++++++++ Lib/site.py | 56 ++ 3 files changed, 2210 insertions(+), 3 deletions(-) create mode 100644 Lib/pep8.py diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 246eba8..6237ef7 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -16,14 +16,14 @@ /* Version parsed out into numeric values */ /*--start constants--*/ -#define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 6 +#define PY_MAJOR_VERSION 8 +#define PY_MINOR_VERSION 0 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.0a0" +#define PY_VERSION "8.0.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pep8.py b/Lib/pep8.py new file mode 100644 index 0000000..3c950d4 --- /dev/null +++ b/Lib/pep8.py @@ -0,0 +1,2151 @@ +#!/usr/bin/env python +# pep8.py - Check Python source code formatting, according to PEP 8 +# Copyright (C) 2006-2009 Johann C. Rocholl +# Copyright (C) 2009-2014 Florent Xicluna +# Copyright (C) 2014-2016 Ian Lee +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +r""" +Check Python source code formatting, according to PEP 8. + +For usage and a list of options, try this: +$ python pep8.py -h + +This program and its regression test suite live here: +https://github.com/pycqa/pep8 + +Groups of errors and warnings: +E errors +W warnings +100 indentation +200 whitespace +300 blank lines +400 imports +500 line length +600 deprecation +700 statements +900 syntax error +""" +from __future__ import with_statement + +import os +import sys +import re +import time +import inspect +import keyword +import tokenize +from optparse import OptionParser +from fnmatch import fnmatch +try: + from configparser import RawConfigParser + from io import TextIOWrapper +except ImportError: + from ConfigParser import RawConfigParser + +__version__ = '1.7.0' + +DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' +DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704' +try: + if sys.platform == 'win32': + USER_CONFIG = os.path.expanduser(r'~\.pep8') + else: + USER_CONFIG = os.path.join( + os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), + 'pep8' + ) +except ImportError: + USER_CONFIG = None + +PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8') +TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') +MAX_LINE_LENGTH = 79 +REPORT_FORMAT = { + 'default': '%(path)s:%(row)d:%(col)d: %(code)s %(text)s', + 'pylint': '%(path)s:%(row)d: [%(code)s] %(text)s', +} + +PyCF_ONLY_AST = 1024 +SINGLETONS = frozenset(['False', 'None', 'True']) +KEYWORDS = frozenset(keyword.kwlist + ['print']) - SINGLETONS +UNARY_OPERATORS = frozenset(['>>', '**', '*', '+', '-']) +ARITHMETIC_OP = frozenset(['**', '*', '/', '//', '+', '-']) +WS_OPTIONAL_OPERATORS = ARITHMETIC_OP.union(['^', '&', '|', '<<', '>>', '%']) +WS_NEEDED_OPERATORS = frozenset([ + '**=', '*=', '/=', '//=', '+=', '-=', '!=', '<>', '<', '>', + '%=', '^=', '&=', '|=', '==', '<=', '>=', '<<=', '>>=', '=']) +WHITESPACE = frozenset(' \t') +NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) +SKIP_TOKENS = NEWLINE.union([tokenize.INDENT, tokenize.DEDENT]) +# ERRORTOKEN is triggered by backticks in Python 3 +SKIP_COMMENTS = SKIP_TOKENS.union([tokenize.COMMENT, tokenize.ERRORTOKEN]) +BENCHMARK_KEYS = ['directories', 'files', 'logical lines', 'physical lines'] + +INDENT_REGEX = re.compile(r'([ \t]*)') +RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,') +RERAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,.*,\s*\w+\s*$') +ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') +DOCSTRING_REGEX = re.compile(r'u?r?["\']') +EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') +WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') +COMPARE_SINGLETON_REGEX = re.compile(r'(\bNone|\bFalse|\bTrue)?\s*([=!]=)' + r'\s*(?(1)|(None|False|True))\b') +COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+[^][)(}{ ]+\s+(in|is)\s') +COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' + r'|\s*$\s*([^)]*[^ )])\s*$)') +KEYWORD_REGEX = re.compile(r'(\s*)\b(?:%s)\b(\s*)' % r'|'.join(KEYWORDS)) +OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') +LAMBDA_REGEX = re.compile(r'\blambda\b') +HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') + +# Work around Python < 2.6 behaviour, which does not generate NL after +# a comment which is on a line by itself. +COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' + + +############################################################################## +# Plugins (check functions) for physical lines +############################################################################## + + +def tabs_or_spaces(physical_line, indent_char): + r"""Never mix tabs and spaces. + + The most popular way of indenting Python is with spaces only. The + second-most popular way is with tabs only. Code indented with a mixture + of tabs and spaces should be converted to using spaces exclusively. When + invoking the Python command line interpreter with the -t option, it issues + warnings about code that illegally mixes tabs and spaces. When using -tt + these warnings become errors. These options are highly recommended! + + Okay: if a == 0:\n a = 1\n b = 1 + E101: if a == 0:\n a = 1\n\tb = 1 + """ + indent = INDENT_REGEX.match(physical_line).group(1) + for offset, char in enumerate(indent): + if char != indent_char: + return offset, "E101 indentation contains mixed spaces and tabs" + + +def tabs_obsolete(physical_line): + r"""For new projects, spaces-only are strongly recommended over tabs. + + Okay: if True:\n return + W191: if True:\n\treturn + """ + indent = INDENT_REGEX.match(physical_line).group(1) + if '\t' in indent: + return indent.index('\t'), "W191 indentation contains tabs" + + +def trailing_whitespace(physical_line): + r"""Trailing whitespace is superfluous. + + The warning returned varies on whether the line itself is blank, for easier + filtering for those who want to indent their blank lines. + + Okay: spam(1)\n# + W291: spam(1) \n# + W293: class Foo(object):\n \n bang = 12 + """ + physical_line = physical_line.rstrip('\n') # chr(10), newline + physical_line = physical_line.rstrip('\r') # chr(13), carriage return + physical_line = physical_line.rstrip('\x0c') # chr(12), form feed, ^L + stripped = physical_line.rstrip(' \t\v') + if physical_line != stripped: + if stripped: + return len(stripped), "W291 trailing whitespace" + else: + return 0, "W293 blank line contains whitespace" + + +def trailing_blank_lines(physical_line, lines, line_number, total_lines): + r"""Trailing blank lines are superfluous. + + Okay: spam(1) + W391: spam(1)\n + + However the last line should end with a new line (warning W292). + """ + if line_number == total_lines: + stripped_last_line = physical_line.rstrip() + if not stripped_last_line: + return 0, "W391 blank line at end of file" + if stripped_last_line == physical_line: + return len(physical_line), "W292 no newline at end of file" + + +def maximum_line_length(physical_line, max_line_length, multiline): + r"""Limit all lines to a maximum of 79 characters. + + There are still many devices around that are limited to 80 character + lines; plus, limiting windows to 80 characters makes it possible to have + several windows side-by-side. The default wrapping on such devices looks + ugly. Therefore, please limit all lines to a maximum of 79 characters. + For flowing long blocks of text (docstrings or comments), limiting the + length to 72 characters is recommended. + + Reports error E501. + """ + line = physical_line.rstrip() + length = len(line) + if length > max_line_length and not noqa(line): + # Special case for long URLs in multi-line docstrings or comments, + # but still report the error when the 72 first chars are whitespaces. + chunks = line.split() + if ((len(chunks) == 1 and multiline) or + (len(chunks) == 2 and chunks[0] == '#')) and \ + len(line) - len(chunks[-1]) < max_line_length - 7: + return + if hasattr(line, 'decode'): # Python 2 + # The line could contain multi-byte characters + try: + length = len(line.decode('utf-8')) + except UnicodeError: + pass + if length > max_line_length: + return (max_line_length, "E501 line too long " + "(%d > %d characters)" % (length, max_line_length)) + + +############################################################################## +# Plugins (check functions) for logical lines +############################################################################## + + +def blank_lines(logical_line, blank_lines, indent_level, line_number, + blank_before, previous_logical, previous_indent_level): + r"""Separate top-level function and class definitions with two blank lines. + + Method definitions inside a class are separated by a single blank line. + + Extra blank lines may be used (sparingly) to separate groups of related + functions. Blank lines may be omitted between a bunch of related + one-liners (e.g. a set of dummy implementations). + + Use blank lines in functions, sparingly, to indicate logical sections. + + Okay: def a():\n pass\n\n\ndef b():\n pass + Okay: def a():\n pass\n\n\n# Foo\n# Bar\n\ndef b():\n pass + + E301: class Foo:\n b = 0\n def bar():\n pass + E302: def a():\n pass\n\ndef b(n):\n pass + E303: def a():\n pass\n\n\n\ndef b(n):\n pass + E303: def a():\n\n\n\n pass + E304: @decorator\n\ndef a():\n pass + """ + if line_number < 3 and not previous_logical: + return # Don't expect blank lines before the first line + if previous_logical.startswith('@'): + if blank_lines: + yield 0, "E304 blank lines found after function decorator" + elif blank_lines > 2 or (indent_level and blank_lines == 2): + yield 0, "E303 too many blank lines (%d)" % blank_lines + elif logical_line.startswith(('def ', 'class ', '@')): + if indent_level: + if not (blank_before or previous_indent_level < indent_level or + DOCSTRING_REGEX.match(previous_logical)): + yield 0, "E301 expected 1 blank line, found 0" + elif blank_before != 2: + yield 0, "E302 expected 2 blank lines, found %d" % blank_before + + +def extraneous_whitespace(logical_line): + r"""Avoid extraneous whitespace. + + Avoid extraneous whitespace in these situations: + - Immediately inside parentheses, brackets or braces. + - Immediately before a comma, semicolon, or colon. + + Okay: spam(ham[1], {eggs: 2}) + E201: spam( ham[1], {eggs: 2}) + E201: spam(ham[ 1], {eggs: 2}) + E201: spam(ham[1], { eggs: 2}) + E202: spam(ham[1], {eggs: 2} ) + E202: spam(ham[1 ], {eggs: 2}) + E202: spam(ham[1], {eggs: 2 }) + + E203: if x == 4: print x, y; x, y = y , x + E203: if x == 4: print x, y ; x, y = y, x + E203: if x == 4 : print x, y; x, y = y, x + """ + line = logical_line + for match in EXTRANEOUS_WHITESPACE_REGEX.finditer(line): + text = match.group() + char = text.strip() + found = match.start() + if text == char + ' ': + # assert char in '([{' + yield found + 1, "E201 whitespace after '%s'" % char + elif line[found - 1] != ',': + code = ('E202' if char in '}])' else 'E203') # if char in ',;:' + yield found, "%s whitespace before '%s'" % (code, char) + + +def whitespace_around_keywords(logical_line): + r"""Avoid extraneous whitespace around keywords. + + Okay: True and False + E271: True and False + E272: True and False + E273: True and\tFalse + E274: True\tand False + """ + for match in KEYWORD_REGEX.finditer(logical_line): + before, after = match.groups() + + if '\t' in before: + yield match.start(1), "E274 tab before keyword" + elif len(before) > 1: + yield match.start(1), "E272 multiple spaces before keyword" + + if '\t' in after: + yield match.start(2), "E273 tab after keyword" + elif len(after) > 1: + yield match.start(2), "E271 multiple spaces after keyword" + + +def missing_whitespace(logical_line): + r"""Each comma, semicolon or colon should be followed by whitespace. + + Okay: [a, b] + Okay: (3,) + Okay: a[1:4] + Okay: a[:4] + Okay: a[1:] + Okay: a[1:4:2] + E231: ['a','b'] + E231: foo(bar,baz) + E231: [{'a':'b'}] + """ + line = logical_line + for index in range(len(line) - 1): + char = line[index] + if char in ',;:' and line[index + 1] not in WHITESPACE: + before = line[:index] + if char == ':' and before.count('[') > before.count(']') and \ + before.rfind('{') < before.rfind('['): + continue # Slice syntax, no space required + if char == ',' and line[index + 1] == ')': + continue # Allow tuple with only one element: (3,) + yield index, "E231 missing whitespace after '%s'" % char + + +def indentation(logical_line, previous_logical, indent_char, + indent_level, previous_indent_level): + r"""Use 4 spaces per indentation level. + + For really old code that you don't want to mess up, you can continue to + use 8-space tabs. + + Okay: a = 1 + Okay: if a == 0:\n a = 1 + E111: a = 1 + E114: # a = 1 + + Okay: for item in items:\n pass + E112: for item in items:\npass + E115: for item in items:\n# Hi\n pass + + Okay: a = 1\nb = 2 + E113: a = 1\n b = 2 + E116: a = 1\n # b = 2 + """ + c = 0 if logical_line else 3 + tmpl = "E11%d %s" if logical_line else "E11%d %s (comment)" + if indent_level % 4: + yield 0, tmpl % (1 + c, "indentation is not a multiple of four") + indent_expect = previous_logical.endswith(':') + if indent_expect and indent_level <= previous_indent_level: + yield 0, tmpl % (2 + c, "expected an indented block") + elif not indent_expect and indent_level > previous_indent_level: + yield 0, tmpl % (3 + c, "unexpected indentation") + + +def continued_indentation(logical_line, tokens, indent_level, hang_closing, + indent_char, noqa, verbose): + r"""Continuation lines indentation. + + Continuation lines should align wrapped elements either vertically + using Python's implicit line joining inside parentheses, brackets + and braces, or using a hanging indent. + + When using a hanging indent these considerations should be applied: + - there should be no arguments on the first line, and + - further indentation should be used to clearly distinguish itself as a + continuation line. + + Okay: a = (\n) + E123: a = (\n ) + + Okay: a = (\n 42) + E121: a = (\n 42) + E122: a = (\n42) + E123: a = (\n 42\n ) + E124: a = (24,\n 42\n) + E125: if (\n b):\n pass + E126: a = (\n 42) + E127: a = (24,\n 42) + E128: a = (24,\n 42) + E129: if (a or\n b):\n pass + E131: a = (\n 42\n 24) + """ + first_row = tokens[0][2][0] + nrows = 1 + tokens[-1][2][0] - first_row + if noqa or nrows == 1: + return + + # indent_next tells us whether the next block is indented; assuming + # that it is indented by 4 spaces, then we should not allow 4-space + # indents on the final continuation line; in turn, some other + # indents are allowed to have an extra 4 spaces. + indent_next = logical_line.endswith(':') + + row = depth = 0 + valid_hangs = (4,) if indent_char != '\t' else (4, 8) + # remember how many brackets were opened on each line + parens = [0] * nrows + # relative indents of physical lines + rel_indent = [0] * nrows + # for each depth, collect a list of opening rows + open_rows = [[0]] + # for each depth, memorize the hanging indentation + hangs = [None] + # visual indents + indent_chances = {} + last_indent = tokens[0][2] + visual_indent = None + last_token_multiline = False + # for each depth, memorize the visual indent column + indent = [last_indent[1]] + if verbose >= 3: + print(">>> " + tokens[0][4].rstrip()) + + for token_type, text, start, end, line in tokens: + + newline = row < start[0] - first_row + if newline: + row = start[0] - first_row + newline = not last_token_multiline and token_type not in NEWLINE + + if newline: + # this is the beginning of a continuation line. + last_indent = start + if verbose >= 3: + print("... " + line.rstrip()) + + # record the initial indent. + rel_indent[row] = expand_indent(line) - indent_level + + # identify closing bracket + close_bracket = (token_type == tokenize.OP and text in ']})') + + # is the indent relative to an opening bracket line? + for open_row in reversed(open_rows[depth]): + hang = rel_indent[row] - rel_indent[open_row] + hanging_indent = hang in valid_hangs + if hanging_indent: + break + if hangs[depth]: + hanging_indent = (hang == hangs[depth]) + # is there any chance of visual indent? + visual_indent = (not close_bracket and hang > 0 and + indent_chances.get(start[1])) + + if close_bracket and indent[depth]: + # closing bracket for visual indent + if start[1] != indent[depth]: + yield (start, "E124 closing bracket does not match " + "visual indentation") + elif close_bracket and not hang: + # closing bracket matches indentation of opening bracket's line + if hang_closing: + yield start, "E133 closing bracket is missing indentation" + elif indent[depth] and start[1] < indent[depth]: + if visual_indent is not True: + # visual indent is broken + yield (start, "E128 continuation line " + "under-indented for visual indent") + elif hanging_indent or (indent_next and rel_indent[row] == 8): + # hanging indent is verified + if close_bracket and not hang_closing: + yield (start, "E123 closing bracket does not match " + "indentation of opening bracket's line") + hangs[depth] = hang + elif visual_indent is True: + # visual indent is verified + indent[depth] = start[1] + elif visual_indent in (text, str): + # ignore token lined up with matching one from a previous line + pass + else: + # indent is broken + if hang <= 0: + error = "E122", "missing indentation or outdented" + elif indent[depth]: + error = "E127", "over-indented for visual indent" + elif not close_bracket and hangs[depth]: + error = "E131", "unaligned for hanging indent" + else: + hangs[depth] = hang + if hang > 4: + error = "E126", "over-indented for hanging indent" + else: + error = "E121", "under-indented for hanging indent" + yield start, "%s continuation line %s" % error + + # look for visual indenting + if (parens[row] and + token_type not in (tokenize.NL, tokenize.COMMENT) and + not indent[depth]): + indent[depth] = start[1] + indent_chances[start[1]] = True + if verbose >= 4: + print("bracket depth %s indent to %s" % (depth, start[1])) + # deal with implicit string concatenation + elif (token_type in (tokenize.STRING, tokenize.COMMENT) or + text in ('u', 'ur', 'b', 'br')): + indent_chances[start[1]] = str + # special case for the "if" statement because len("if (") == 4 + elif not indent_chances and not row and not depth and text == 'if': + indent_chances[end[1] + 1] = True + elif text == ':' and line[end[1]:].isspace(): + open_rows[depth].append(row) + + # keep track of bracket depth + if token_type == tokenize.OP: + if text in '([{': + depth += 1 + indent.append(0) + hangs.append(None) + if len(open_rows) == depth: + open_rows.append([]) + open_rows[depth].append(row) + parens[row] += 1 + if verbose >= 4: + print("bracket depth %s seen, col %s, visual min = %s" % + (depth, start[1], indent[depth])) + elif text in ')]}' and depth > 0: + # parent indents should not be more than this one + prev_indent = indent.pop() or last_indent[1] + hangs.pop() + for d in range(depth): + if indent[d] > prev_indent: + indent[d] = 0 + for ind in list(indent_chances): + if ind >= prev_indent: + del indent_chances[ind] + del open_rows[depth + 1:] + depth -= 1 + if depth: + indent_chances[indent[depth]] = True + for idx in range(row, -1, -1): + if parens[idx]: + parens[idx] -= 1 + break + assert len(indent) == depth + 1 + if start[1] not in indent_chances: + # allow to line up tokens + indent_chances[start[1]] = text + + last_token_multiline = (start[0] != end[0]) + if last_token_multiline: + rel_indent[end[0] - first_row] = rel_indent[row] + + if indent_next and expand_indent(line) == indent_level + 4: + pos = (start[0], indent[0] + 4) + if visual_indent: + code = "E129 visually indented line" + else: + code = "E125 continuation line" + yield pos, "%s with same indent as next logical line" % code + + +def whitespace_before_parameters(logical_line, tokens): + r"""Avoid extraneous whitespace. + + Avoid extraneous whitespace in the following situations: + - before the open parenthesis that starts the argument list of a + function call. + - before the open parenthesis that starts an indexing or slicing. + + Okay: spam(1) + E211: spam (1) + + Okay: dict['key'] = list[index] + E211: dict ['key'] = list[index] + E211: dict['key'] = list [index] + """ + prev_type, prev_text, __, prev_end, __ = tokens[0] + for index in range(1, len(tokens)): + token_type, text, start, end, __ = tokens[index] + if (token_type == tokenize.OP and + text in '([' and + start != prev_end and + (prev_type == tokenize.NAME or prev_text in '}])') and + # Syntax "class A (B):" is allowed, but avoid it + (index < 2 or tokens[index - 2][1] != 'class') and + # Allow "return (a.foo for a in range(5))" + not keyword.iskeyword(prev_text)): + yield prev_end, "E211 whitespace before '%s'" % text + prev_type = token_type + prev_text = text + prev_end = end + + +def whitespace_around_operator(logical_line): + r"""Avoid extraneous whitespace around an operator. + + Okay: a = 12 + 3 + E221: a = 4 + 5 + E222: a = 4 + 5 + E223: a = 4\t+ 5 + E224: a = 4 +\t5 + """ + for match in OPERATOR_REGEX.finditer(logical_line): + before, after = match.groups() + + if '\t' in before: + yield match.start(1), "E223 tab before operator" + elif len(before) > 1: + yield match.start(1), "E221 multiple spaces before operator" + + if '\t' in after: + yield match.start(2), "E224 tab after operator" + elif len(after) > 1: + yield match.start(2), "E222 multiple spaces after operator" + + +def missing_whitespace_around_operator(logical_line, tokens): + r"""Surround operators with a single space on either side. + + - Always surround these binary operators with a single space on + either side: assignment (=), augmented assignment (+=, -= etc.), + comparisons (==, <, >, !=, <=, >=, in, not in, is, is not), + Booleans (and, or, not). + + - If operators with different priorities are used, consider adding + whitespace around the operators with the lowest priorities. + + Okay: i = i + 1 + Okay: submitted += 1 + Okay: x = x * 2 - 1 + Okay: hypot2 = x * x + y * y + Okay: c = (a + b) * (a - b) + Okay: foo(bar, key='word', *args, **kwargs) + Okay: alpha[:-i] + + E225: i=i+1 + E225: submitted +=1 + E225: x = x /2 - 1 + E225: z = x **y + E226: c = (a+b) * (a-b) + E226: hypot2 = x*x + y*y + E227: c = a|b + E228: msg = fmt%(errno, errmsg) + """ + parens = 0 + need_space = False + prev_type = tokenize.OP + prev_text = prev_end = None + for token_type, text, start, end, line in tokens: + if token_type in SKIP_COMMENTS: + continue + if text in ('(', 'lambda'): + parens += 1 + elif text == ')': + parens -= 1 + if need_space: + if start != prev_end: + # Found a (probably) needed space + if need_space is not True and not need_space[1]: + yield (need_space[0], + "E225 missing whitespace around operator") + need_space = False + elif text == '>' and prev_text in ('<', '-'): + # Tolerate the "<>" operator, even if running Python 3 + # Deal with Python 3's annotated return value "->" + pass + else: + if need_space is True or need_space[1]: + # A needed trailing space was not found + yield prev_end, "E225 missing whitespace around operator" + elif prev_text != '**': + code, optype = 'E226', 'arithmetic' + if prev_text == '%': + code, optype = 'E228', 'modulo' + elif prev_text not in ARITHMETIC_OP: + code, optype = 'E227', 'bitwise or shift' + yield (need_space[0], "%s missing whitespace " + "around %s operator" % (code, optype)) + need_space = False + elif token_type == tokenize.OP and prev_end is not None: + if text == '=' and parens: + # Allow keyword args or defaults: foo(bar=None). + pass + elif text in WS_NEEDED_OPERATORS: + need_space = True + elif text in UNARY_OPERATORS: + # Check if the operator is being used as a binary operator + # Allow unary operators: -123, -x, +1. + # Allow argument unpacking: foo(*args, **kwargs). + if (prev_text in '}])' if prev_type == tokenize.OP + else prev_text not in KEYWORDS): + need_space = None + elif text in WS_OPTIONAL_OPERATORS: + need_space = None + + if need_space is None: + # Surrounding space is optional, but ensure that + # trailing space matches opening space + need_space = (prev_end, start != prev_end) + elif need_space and start == prev_end: + # A needed opening space was not found + yield prev_end, "E225 missing whitespace around operator" + need_space = False + prev_type = token_type + prev_text = text + prev_end = end + + +def whitespace_around_comma(logical_line): + r"""Avoid extraneous whitespace after a comma or a colon. + + Note: these checks are disabled by default + + Okay: a = (1, 2) + E241: a = (1, 2) + E242: a = (1,\t2) + """ + line = logical_line + for m in WHITESPACE_AFTER_COMMA_REGEX.finditer(line): + found = m.start() + 1 + if '\t' in m.group(): + yield found, "E242 tab after '%s'" % m.group()[0] + else: + yield found, "E241 multiple spaces after '%s'" % m.group()[0] + + +def whitespace_around_named_parameter_equals(logical_line, tokens): + r"""Don't use spaces around the '=' sign in function arguments. + + Don't use spaces around the '=' sign when used to indicate a + keyword argument or a default parameter value. + + Okay: def complex(real, imag=0.0): + Okay: return magic(r=real, i=imag) + Okay: boolean(a == b) + Okay: boolean(a != b) + Okay: boolean(a <= b) + Okay: boolean(a >= b) + Okay: def foo(arg: int = 42): + + E251: def complex(real, imag = 0.0): + E251: return magic(r = real, i = imag) + """ + parens = 0 + no_space = False + prev_end = None + annotated_func_arg = False + in_def = logical_line.startswith('def') + message = "E251 unexpected spaces around keyword / parameter equals" + for token_type, text, start, end, line in tokens: + if token_type == tokenize.NL: + continue + if no_space: + no_space = False + if start != prev_end: + yield (prev_end, message) + if token_type == tokenize.OP: + if text == '(': + parens += 1 + elif text == ')': + parens -= 1 + elif in_def and text == ':' and parens == 1: + annotated_func_arg = True + elif parens and text == ',' and parens == 1: + annotated_func_arg = False + elif parens and text == '=' and not annotated_func_arg: + no_space = True + if start != prev_end: + yield (prev_end, message) + if not parens: + annotated_func_arg = False + + prev_end = end + + +def whitespace_before_comment(logical_line, tokens): + r"""Separate inline comments by at least two spaces. + + An inline comment is a comment on the same line as a statement. Inline + comments should be separated by at least two spaces from the statement. + They should start with a # and a single space. + + Each line of a block comment starts with a # and a single space + (unless it is indented text inside the comment). + + Okay: x = x + 1 # Increment x + Okay: x = x + 1 # Increment x + Okay: # Block comment + E261: x = x + 1 # Increment x + E262: x = x + 1 #Increment x + E262: x = x + 1 # Increment x + E265: #Block comment + E266: ### Block comment + """ + prev_end = (0, 0) + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + inline_comment = line[:start[1]].strip() + if inline_comment: + if prev_end[0] == start[0] and start[1] < prev_end[1] + 2: + yield (prev_end, + "E261 at least two spaces before inline comment") + symbol, sp, comment = text.partition(' ') + bad_prefix = symbol not in '#:' and (symbol.lstrip('#')[:1] or '#') + if inline_comment: + if bad_prefix or comment[:1] in WHITESPACE: + yield start, "E262 inline comment should start with '# '" + elif bad_prefix and (bad_prefix != '!' or start[0] > 1): + if bad_prefix != '#': + yield start, "E265 block comment should start with '# '" + elif comment: + yield start, "E266 too many leading '#' for block comment" + elif token_type != tokenize.NL: + prev_end = end + + +def imports_on_separate_lines(logical_line): + r"""Imports should usually be on separate lines. + + Okay: import os\nimport sys + E401: import sys, os + + Okay: from subprocess import Popen, PIPE + Okay: from myclas import MyClass + Okay: from foo.bar.yourclass import YourClass + Okay: import myclass + Okay: import foo.bar.yourclass + """ + line = logical_line + if line.startswith('import '): + found = line.find(',') + if -1 < found and ';' not in line[:found]: + yield found, "E401 multiple imports on one line" + + +def module_imports_on_top_of_file( + logical_line, indent_level, checker_state, noqa): + r"""Imports are always put at the top of the file, just after any module + comments and docstrings, and before module globals and constants. + + Okay: import os + Okay: # this is a comment\nimport os + Okay: '''this is a module docstring'''\nimport os + Okay: r'''this is a module docstring'''\nimport os + Okay: try:\n import x\nexcept:\n pass\nelse:\n pass\nimport y + Okay: try:\n import x\nexcept:\n pass\nfinally:\n pass\nimport y + E402: a=1\nimport os + E402: 'One string'\n"Two string"\nimport os + E402: a=1\nfrom sys import x + + Okay: if x:\n import os + """ + def is_string_literal(line): + if line[0] in 'uUbB': + line = line[1:] + if line and line[0] in 'rR': + line = line[1:] + return line and (line[0] == '"' or line[0] == "'") + + allowed_try_keywords = ('try', 'except', 'else', 'finally') + + if indent_level: # Allow imports in conditional statements or functions + return + if not logical_line: # Allow empty lines or comments + return + if noqa: + return + line = logical_line + if line.startswith('import ') or line.startswith('from '): + if checker_state.get('seen_non_imports', False): + yield 0, "E402 module level import not at top of file" + elif any(line.startswith(kw) for kw in allowed_try_keywords): + # Allow try, except, else, finally keywords intermixed with imports in + # order to support conditional importing + return + elif is_string_literal(line): + # The first literal is a docstring, allow it. Otherwise, report error. + if checker_state.get('seen_docstring', False): + checker_state['seen_non_imports'] = True + else: + checker_state['seen_docstring'] = True + else: + checker_state['seen_non_imports'] = True + + +def compound_statements(logical_line): + r"""Compound statements (on the same line) are generally discouraged. + + While sometimes it's okay to put an if/for/while with a small body + on the same line, never do this for multi-clause statements. + Also avoid folding such long lines! + + Always use a def statement instead of an assignment statement that + binds a lambda expression directly to a name. + + Okay: if foo == 'blah':\n do_blah_thing() + Okay: do_one() + Okay: do_two() + Okay: do_three() + + E701: if foo == 'blah': do_blah_thing() + E701: for x in lst: total += x + E701: while t < 10: t = delay() + E701: if foo == 'blah': do_blah_thing() + E701: else: do_non_blah_thing() + E701: try: something() + E701: finally: cleanup() + E701: if foo == 'blah': one(); two(); three() + E702: do_one(); do_two(); do_three() + E703: do_four(); # useless semicolon + E704: def f(x): return 2*x + E731: f = lambda x: 2*x + """ + line = logical_line + last_char = len(line) - 1 + found = line.find(':') + while -1 < found < last_char: + before = line[:found] + if ((before.count('{') <= before.count('}') and # {'a': 1} (dict) + before.count('[') <= before.count(']') and # [1:2] (slice) + before.count('(') <= before.count(')'))): # (annotation) + lambda_kw = LAMBDA_REGEX.search(before) + if lambda_kw: + before = line[:lambda_kw.start()].rstrip() + if before[-1:] == '=' and isidentifier(before[:-1].strip()): + yield 0, ("E731 do not assign a lambda expression, use a " + "def") + break + if before.startswith('def '): + yield 0, "E704 multiple statements on one line (def)" + else: + yield found, "E701 multiple statements on one line (colon)" + found = line.find(':', found + 1) + found = line.find(';') + while -1 < found: + if found < last_char: + yield found, "E702 multiple statements on one line (semicolon)" + else: + yield found, "E703 statement ends with a semicolon" + found = line.find(';', found + 1) + + +def explicit_line_join(logical_line, tokens): + r"""Avoid explicit line join between brackets. + + The preferred way of wrapping long lines is by using Python's implied line + continuation inside parentheses, brackets and braces. Long lines can be + broken over multiple lines by wrapping expressions in parentheses. These + should be used in preference to using a backslash for line continuation. + + E502: aaa = [123, \\n 123] + E502: aaa = ("bbb " \\n "ccc") + + Okay: aaa = [123,\n 123] + Okay: aaa = ("bbb "\n "ccc") + Okay: aaa = "bbb " \\n "ccc" + Okay: aaa = 123 # \\ + """ + prev_start = prev_end = parens = 0 + comment = False + backslash = None + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + comment = True + if start[0] != prev_start and parens and backslash and not comment: + yield backslash, "E502 the backslash is redundant between brackets" + if end[0] != prev_end: + if line.rstrip('\r\n').endswith('\\'): + backslash = (end[0], len(line.splitlines()[-1]) - 1) + else: + backslash = None + prev_start = prev_end = end[0] + else: + prev_start = start[0] + if token_type == tokenize.OP: + if text in '([{': + parens += 1 + elif text in ')]}': + parens -= 1 + + +def break_around_binary_operator(logical_line, tokens): + r""" + Avoid breaks before binary operators. + + The preferred place to break around a binary operator is after the + operator, not before it. + + W503: (width == 0\n + height == 0) + W503: (width == 0\n and height == 0) + + Okay: (width == 0 +\n height == 0) + Okay: foo(\n -x) + Okay: foo(x\n []) + Okay: x = '''\n''' + '' + Okay: foo(x,\n -y) + Okay: foo(x, # comment\n -y) + """ + def is_binary_operator(token_type, text): + # The % character is strictly speaking a binary operator, but the + # common usage seems to be to put it next to the format parameters, + # after a line break. + return ((token_type == tokenize.OP or text in ['and', 'or']) and + text not in "()[]{},:.;@=%") + + line_break = False + unary_context = True + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + continue + if ('\n' in text or '\r' in text) and token_type != tokenize.STRING: + line_break = True + else: + if (is_binary_operator(token_type, text) and line_break and + not unary_context): + yield start, "W503 line break before binary operator" + unary_context = text in '([{,;' + line_break = False + + +def comparison_to_singleton(logical_line, noqa): + r"""Comparison to singletons should use "is" or "is not". + + Comparisons to singletons like None should always be done + with "is" or "is not", never the equality operators. + + Okay: if arg is not None: + E711: if arg != None: + E711: if None == arg: + E712: if arg == True: + E712: if False == arg: + + Also, beware of writing if x when you really mean if x is not None -- + e.g. when testing whether a variable or argument that defaults to None was + set to some other value. The other value might have a type (such as a + container) that could be false in a boolean context! + """ + match = not noqa and COMPARE_SINGLETON_REGEX.search(logical_line) + if match: + singleton = match.group(1) or match.group(3) + same = (match.group(2) == '==') + + msg = "'if cond is %s:'" % (('' if same else 'not ') + singleton) + if singleton in ('None',): + code = 'E711' + else: + code = 'E712' + nonzero = ((singleton == 'True' and same) or + (singleton == 'False' and not same)) + msg += " or 'if %scond:'" % ('' if nonzero else 'not ') + yield match.start(2), ("%s comparison to %s should be %s" % + (code, singleton, msg)) + + +def comparison_negative(logical_line): + r"""Negative comparison should be done using "not in" and "is not". + + Okay: if x not in y:\n pass + Okay: assert (X in Y or X is Z) + Okay: if not (X in Y):\n pass + Okay: zz = x is not y + E713: Z = not X in Y + E713: if not X.B in Y:\n pass + E714: if not X is Y:\n pass + E714: Z = not X.B is Y + """ + match = COMPARE_NEGATIVE_REGEX.search(logical_line) + if match: + pos = match.start(1) + if match.group(2) == 'in': + yield pos, "E713 test for membership should be 'not in'" + else: + yield pos, "E714 test for object identity should be 'is not'" + + +def comparison_type(logical_line, noqa): + r"""Object type comparisons should always use isinstance(). + + Do not compare types directly. + + Okay: if isinstance(obj, int): + E721: if type(obj) is type(1): + + When checking if an object is a string, keep in mind that it might be a + unicode string too! In Python 2.3, str and unicode have a common base + class, basestring, so you can do: + + Okay: if isinstance(obj, basestring): + Okay: if type(a1) is type(b1): + """ + match = COMPARE_TYPE_REGEX.search(logical_line) + if match and not noqa: + inst = match.group(1) + if inst and isidentifier(inst) and inst not in SINGLETONS: + return # Allow comparison for types which are not obvious + yield match.start(), "E721 do not compare types, use 'isinstance()'" + + +def python_3000_has_key(logical_line, noqa): + r"""The {}.has_key() method is removed in Python 3: use the 'in' operator. + + Okay: if "alph" in d:\n print d["alph"] + W601: assert d.has_key('alph') + """ + pos = logical_line.find('.has_key(') + if pos > -1 and not noqa: + yield pos, "W601 .has_key() is deprecated, use 'in'" + + +def python_3000_raise_comma(logical_line): + r"""When raising an exception, use "raise ValueError('message')". + + The older form is removed in Python 3. + + Okay: raise DummyError("Message") + W602: raise DummyError, "Message" + """ + match = RAISE_COMMA_REGEX.match(logical_line) + if match and not RERAISE_COMMA_REGEX.match(logical_line): + yield match.end() - 1, "W602 deprecated form of raising exception" + + +def python_3000_not_equal(logical_line): + r"""New code should always use != instead of <>. + + The older syntax is removed in Python 3. + + Okay: if a != 'no': + W603: if a <> 'no': + """ + pos = logical_line.find('<>') + if pos > -1: + yield pos, "W603 '<>' is deprecated, use '!='" + + +def python_3000_backticks(logical_line): + r"""Backticks are removed in Python 3: use repr() instead. + + Okay: val = repr(1 + 2) + W604: val = `1 + 2` + """ + pos = logical_line.find('`') + if pos > -1: + yield pos, "W604 backticks are deprecated, use 'repr()'" + + +############################################################################## +# Helper functions +############################################################################## + + +if sys.version_info < (3,): + # Python 2: implicit encoding. + def readlines(filename): + """Read the source code.""" + with open(filename, 'rU') as f: + return f.readlines() + isidentifier = re.compile(r'[a-zA-Z_]\w*$').match + stdin_get_value = sys.stdin.read +else: + # Python 3 + def readlines(filename): + """Read the source code.""" + try: + with open(filename, 'rb') as f: + (coding, lines) = tokenize.detect_encoding(f.readline) + f = TextIOWrapper(f, coding, line_buffering=True) + return [l.decode(coding) for l in lines] + f.readlines() + except (LookupError, SyntaxError, UnicodeError): + # Fall back if file encoding is improperly declared + with open(filename, encoding='latin-1') as f: + return f.readlines() + isidentifier = str.isidentifier + + def stdin_get_value(): + return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() +noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search + + +def expand_indent(line): + r"""Return the amount of indentation. + + Tabs are expanded to the next multiple of 8. + + >>> expand_indent(' ') + 4 + >>> expand_indent('\t') + 8 + >>> expand_indent(' \t') + 8 + >>> expand_indent(' \t') + 16 + """ + if '\t' not in line: + return len(line) - len(line.lstrip()) + result = 0 + for char in line: + if char == '\t': + result = result // 8 * 8 + 8 + elif char == ' ': + result += 1 + else: + break + return result + + +def mute_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # String modifiers (e.g. u or r) + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Triple quotes + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] + + +def parse_udiff(diff, patterns=None, parent='.'): + """Return a dictionary of matching lines.""" + # For each file of the diff, the entry key is the filename, + # and the value is a set of row numbers to consider. + rv = {} + path = nrows = None + for line in diff.splitlines(): + if nrows: + if line[:1] != '-': + nrows -= 1 + continue + if line[:3] == '@@ ': + hunk_match = HUNK_REGEX.match(line) + (row, nrows) = [int(g or '1') for g in hunk_match.groups()] + rv[path].update(range(row, row + nrows)) + elif line[:3] == '+++': + path = line[4:].split('\t', 1)[0] + if path[:2] == 'b/': + path = path[2:] + rv[path] = set() + return dict([(os.path.join(parent, path), rows) + for (path, rows) in rv.items() + if rows and filename_match(path, patterns)]) + + +def normalize_paths(value, parent=os.curdir): + """Parse a comma-separated list of paths. + + Return a list of absolute paths. + """ + if not value: + return [] + if isinstance(value, list): + return value + paths = [] + for path in value.split(','): + path = path.strip() + if '/' in path: + path = os.path.abspath(os.path.join(parent, path)) + paths.append(path.rstrip('/')) + return paths + + +def filename_match(filename, patterns, default=True): + """Check if patterns contains a pattern that matches filename. + + If patterns is unspecified, this always returns True. + """ + if not patterns: + return default + return any(fnmatch(filename, pattern) for pattern in patterns) + + +def _is_eol_token(token): + return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' +if COMMENT_WITH_NL: + def _is_eol_token(token, _eol_token=_is_eol_token): + return _eol_token(token) or (token[0] == tokenize.COMMENT and + token[1] == token[4]) + +############################################################################## +# Framework to run all checks +############################################################################## + + +_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} + + +def _get_parameters(function): + if sys.version_info >= (3, 3): + return [parameter.name + for parameter + in inspect.signature(function).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] + else: + return inspect.getargspec(function)[0] + + +def register_check(check, codes=None): + """Register a new check object.""" + def _add_check(check, kind, codes, args): + if check in _checks[kind]: + _checks[kind][check][0].extend(codes or []) + else: + _checks[kind][check] = (codes or [''], args) + if inspect.isfunction(check): + args = _get_parameters(check) + if args and args[0] in ('physical_line', 'logical_line'): + if codes is None: + codes = ERRORCODE_REGEX.findall(check.__doc__ or '') + _add_check(check, args[0], codes, args) + elif inspect.isclass(check): + if _get_parameters(check.__init__)[:2] == ['self', 'tree']: + _add_check(check, 'tree', codes, None) + + +def init_checks_registry(): + """Register all globally visible functions. + + The first argument name is either 'physical_line' or 'logical_line'. + """ + mod = inspect.getmodule(register_check) + for (name, function) in inspect.getmembers(mod, inspect.isfunction): + register_check(function) +init_checks_registry() + + +class Checker(object): + """Load a Python source file, tokenize it, check coding style.""" + + def __init__(self, filename=None, lines=None, + options=None, report=None, **kwargs): + if options is None: + options = StyleGuide(kwargs).options + else: + assert not kwargs + self._io_error = None + self._physical_checks = options.physical_checks + self._logical_checks = options.logical_checks + self._ast_checks = options.ast_checks + self.max_line_length = options.max_line_length + self.multiline = False # in a multiline string? + self.hang_closing = options.hang_closing + self.verbose = options.verbose + self.filename = filename + # Dictionary where a checker can store its custom state. + self._checker_states = {} + if filename is None: + self.filename = 'stdin' + self.lines = lines or [] + elif filename == '-': + self.filename = 'stdin' + self.lines = stdin_get_value().splitlines(True) + elif lines is None: + try: + self.lines = readlines(filename) + except IOError: + (exc_type, exc) = sys.exc_info()[:2] + self._io_error = '%s: %s' % (exc_type.__name__, exc) + self.lines = [] + else: + self.lines = lines + if self.lines: + ord0 = ord(self.lines[0][0]) + if ord0 in (0xef, 0xfeff): # Strip the UTF-8 BOM + if ord0 == 0xfeff: + self.lines[0] = self.lines[0][1:] + elif self.lines[0][:3] == '\xef\xbb\xbf': + self.lines[0] = self.lines[0][3:] + self.report = report or options.report + self.report_error = self.report.error + + def report_invalid_syntax(self): + """Check if the syntax is valid.""" + (exc_type, exc) = sys.exc_info()[:2] + if len(exc.args) > 1: + offset = exc.args[1] + if len(offset) > 2: + offset = offset[1:3] + else: + offset = (1, 0) + self.report_error(offset[0], offset[1] or 0, + 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), + self.report_invalid_syntax) + + def readline(self): + """Get the next line from the input buffer.""" + if self.line_number >= self.total_lines: + return '' + line = self.lines[self.line_number] + self.line_number += 1 + if self.indent_char is None and line[:1] in WHITESPACE: + self.indent_char = line[0] + return line + + def run_check(self, check, argument_names): + """Run a check plugin.""" + arguments = [] + for name in argument_names: + arguments.append(getattr(self, name)) + return check(*arguments) + + def init_checker_state(self, name, argument_names): + """ Prepares a custom state for the specific checker plugin.""" + if 'checker_state' in argument_names: + self.checker_state = self._checker_states.setdefault(name, {}) + + def check_physical(self, line): + """Run all physical checks on a raw input line.""" + self.physical_line = line + for name, check, argument_names in self._physical_checks: + self.init_checker_state(name, argument_names) + result = self.run_check(check, argument_names) + if result is not None: + (offset, text) = result + self.report_error(self.line_number, offset, text, check) + if text[:4] == 'E101': + self.indent_char = line[0] + + def build_tokens_line(self): + """Build a logical line from tokens.""" + logical = [] + comments = [] + length = 0 + prev_row = prev_col = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = mute_string(text) + if prev_row: + (start_row, start_col) = start + if prev_row != start_row: # different row + prev_text = self.lines[prev_row - 1][prev_col - 1] + if prev_text == ',' or (prev_text not in '{[(' and + text not in '}])'): + text = ' ' + text + elif prev_col != start_col: # different column + text = line[prev_col:start_col] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (prev_row, prev_col) = end + self.logical_line = ''.join(logical) + self.noqa = comments and noqa(''.join(comments)) + return mapping + + def check_logical(self): + """Build a line from tokens and run all logical checks on it.""" + self.report.increment_logical_line() + mapping = self.build_tokens_line() + + if not mapping: + return + + (start_row, start_col) = mapping[0][1] + start_line = self.lines[start_row - 1] + self.indent_level = expand_indent(start_line[:start_col]) + if self.blank_before < self.blank_lines: + self.blank_before = self.blank_lines + if self.verbose >= 2: + print(self.logical_line[:80].rstrip()) + for name, check, argument_names in self._logical_checks: + if self.verbose >= 4: + print(' ' + name) + self.init_checker_state(name, argument_names) + for offset, text in self.run_check(check, argument_names) or (): + if not isinstance(offset, tuple): + for token_offset, pos in mapping: + if offset <= token_offset: + break + offset = (pos[0], pos[1] + offset - token_offset) + self.report_error(offset[0], offset[1], text, check) + if self.logical_line: + self.previous_indent_level = self.indent_level + self.previous_logical = self.logical_line + self.blank_lines = 0 + self.tokens = [] + + def check_ast(self): + """Build the file's AST and run all AST checks.""" + try: + tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + except (ValueError, SyntaxError, TypeError): + return self.report_invalid_syntax() + for name, cls, __ in self._ast_checks: + checker = cls(tree, self.filename) + for lineno, offset, text, check in checker.run(): + if not self.lines or not noqa(self.lines[lineno - 1]): + self.report_error(lineno, offset, text, check) + + def generate_tokens(self): + """Tokenize the file, run physical line checks and yield tokens.""" + if self._io_error: + self.report_error(1, 0, 'E902 %s' % self._io_error, readlines) + tokengen = tokenize.generate_tokens(self.readline) + try: + for token in tokengen: + if token[2][0] > self.total_lines: + return + self.maybe_check_physical(token) + yield token + except (SyntaxError, tokenize.TokenError): + self.report_invalid_syntax() + + def maybe_check_physical(self, token): + """If appropriate (based on token), check current physical line(s).""" + # Called after every token, but act only on end of line. + if _is_eol_token(token): + # Obviously, a newline token ends a single physical line. + self.check_physical(token[4]) + elif token[0] == tokenize.STRING and '\n' in token[1]: + # Less obviously, a string that contains newlines is a + # multiline string, either triple-quoted or with internal + # newlines backslash-escaped. Check every physical line in the + # string *except* for the last one: its newline is outside of + # the multiline string, so we consider it a regular physical + # line, and will check it like any other physical line. + # + # Subtleties: + # - we don't *completely* ignore the last line; if it contains + # the magical "# noqa" comment, we disable all physical + # checks for the entire multiline string + # - have to wind self.line_number back because initially it + # points to the last line of the string, and we want + # check_physical() to give accurate feedback + if noqa(token[4]): + return + self.multiline = True + self.line_number = token[2][0] + for line in token[1].split('\n')[:-1]: + self.check_physical(line + '\n') + self.line_number += 1 + self.multiline = False + + def check_all(self, expected=None, line_offset=0): + """Run all checks on the input file.""" + self.report.init_file(self.filename, self.lines, expected, line_offset) + self.total_lines = len(self.lines) + if self._ast_checks: + self.check_ast() + self.line_number = 0 + self.indent_char = None + self.indent_level = self.previous_indent_level = 0 + self.previous_logical = '' + self.tokens = [] + self.blank_lines = self.blank_before = 0 + parens = 0 + for token in self.generate_tokens(): + self.tokens.append(token) + token_type, text = token[0:2] + if self.verbose >= 3: + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + print('l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], text)) + if token_type == tokenize.OP: + if text in '([{': + parens += 1 + elif text in '}])': + parens -= 1 + elif not parens: + if token_type in NEWLINE: + if token_type == tokenize.NEWLINE: + self.check_logical() + self.blank_before = 0 + elif len(self.tokens) == 1: + # The physical line contains only this token. + self.blank_lines += 1 + del self.tokens[0] + else: + self.check_logical() + elif COMMENT_WITH_NL and token_type == tokenize.COMMENT: + if len(self.tokens) == 1: + # The comment also ends a physical line + token = list(token) + token[1] = text.rstrip('\r\n') + token[3] = (token[2][0], token[2][1] + len(token[1])) + self.tokens = [tuple(token)] + self.check_logical() + if self.tokens: + self.check_physical(self.lines[-1]) + self.check_logical() + return self.report.get_file_results() + + +class BaseReport(object): + """Collect the results of the checks.""" + + print_filename = False + + def __init__(self, options): + self._benchmark_keys = options.benchmark_keys + self._ignore_code = options.ignore_code + # Results + self.elapsed = 0 + self.total_errors = 0 + self.counters = dict.fromkeys(self._benchmark_keys, 0) + self.messages = {} + + def start(self): + """Start the timer.""" + self._start_time = time.time() + + def stop(self): + """Stop the timer.""" + self.elapsed = time.time() - self._start_time + + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self.filename = filename + self.lines = lines + self.expected = expected or () + self.line_offset = line_offset + self.file_errors = 0 + self.counters['files'] += 1 + self.counters['physical lines'] += len(lines) + + def increment_logical_line(self): + """Signal a new logical line.""" + self.counters['logical lines'] += 1 + + def error(self, line_number, offset, text, check): + """Report an error, according to options.""" + code = text[:4] + if self._ignore_code(code): + return + if code in self.counters: + self.counters[code] += 1 + else: + self.counters[code] = 1 + self.messages[code] = text[5:] + # Don't care about expected errors or warnings + if code in self.expected: + return + if self.print_filename and not self.file_errors: + print(self.filename) + self.file_errors += 1 + self.total_errors += 1 + return code + + def get_file_results(self): + """Return the count of errors and warnings for this file.""" + return self.file_errors + + def get_count(self, prefix=''): + """Return the total count of errors and warnings.""" + return sum([self.counters[key] + for key in self.messages if key.startswith(prefix)]) + + def get_statistics(self, prefix=''): + """Get statistics for message codes that start with the prefix. + + prefix='' matches all errors and warnings + prefix='E' matches all errors + prefix='W' matches all warnings + prefix='E4' matches all errors that have to do with imports + """ + return ['%-7s %s %s' % (self.counters[key], key, self.messages[key]) + for key in sorted(self.messages) if key.startswith(prefix)] + + def print_statistics(self, prefix=''): + """Print overall statistics (number of errors and warnings).""" + for line in self.get_statistics(prefix): + print(line) + + def print_benchmark(self): + """Print benchmark numbers.""" + print('%-7.2f %s' % (self.elapsed, 'seconds elapsed')) + if self.elapsed: + for key in self._benchmark_keys: + print('%-7d %s per second (%d total)' % + (self.counters[key] / self.elapsed, key, + self.counters[key])) + + +class FileReport(BaseReport): + """Collect the results of the checks and print only the filenames.""" + print_filename = True + + +class StandardReport(BaseReport): + """Collect and print the results of the checks.""" + + def __init__(self, options): + super(StandardReport, self).__init__(options) + self._fmt = REPORT_FORMAT.get(options.format.lower(), + options.format) + self._repeat = options.repeat + self._show_source = options.show_source + self._show_pep8 = options.show_pep8 + + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self._deferred_print = [] + return super(StandardReport, self).init_file( + filename, lines, expected, line_offset) + + def error(self, line_number, offset, text, check): + """Report an error, according to options.""" + code = super(StandardReport, self).error(line_number, offset, + text, check) + if code and (self.counters[code] == 1 or self._repeat): + self._deferred_print.append( + (line_number, offset, code, text[5:], check.__doc__)) + return code + + def get_file_results(self): + """Print the result and return the overall count for this file.""" + self._deferred_print.sort() + for line_number, offset, code, text, doc in self._deferred_print: + print(self._fmt % { + 'path': self.filename, + 'row': self.line_offset + line_number, 'col': offset + 1, + 'code': code, 'text': text, + }) + if self._show_source: + if line_number > len(self.lines): + line = '' + else: + line = self.lines[line_number - 1] + print(line.rstrip()) + print(re.sub(r'\S', ' ', line[:offset]) + '^') + if self._show_pep8 and doc: + print(' ' + doc.strip()) + + # stdout is block buffered when not stdout.isatty(). + # line can be broken where buffer boundary since other processes + # write to same file. + # flush() after print() to avoid buffer boundary. + # Typical buffer size is 8192. line written safely when + # len(line) < 8192. + sys.stdout.flush() + return self.file_errors + + +class DiffReport(StandardReport): + """Collect and print the results for the changed lines only.""" + + def __init__(self, options): + super(DiffReport, self).__init__(options) + self._selected = options.selected_lines + + def error(self, line_number, offset, text, check): + if line_number not in self._selected[self.filename]: + return + return super(DiffReport, self).error(line_number, offset, text, check) + + +class StyleGuide(object): + """Initialize a PEP-8 instance with few options.""" + + def __init__(self, *args, **kwargs): + # build options from the command line + self.checker_class = kwargs.pop('checker_class', Checker) + parse_argv = kwargs.pop('parse_argv', False) + config_file = kwargs.pop('config_file', False) + parser = kwargs.pop('parser', None) + # build options from dict + options_dict = dict(*args, **kwargs) + arglist = None if parse_argv else options_dict.get('paths', None) + options, self.paths = process_options( + arglist, parse_argv, config_file, parser) + if options_dict: + options.__dict__.update(options_dict) + if 'paths' in options_dict: + self.paths = options_dict['paths'] + + self.runner = self.input_file + self.options = options + + if not options.reporter: + options.reporter = BaseReport if options.quiet else StandardReport + + options.select = tuple(options.select or ()) + if not (options.select or options.ignore or + options.testsuite or options.doctest) and DEFAULT_IGNORE: + # The default choice: ignore controversial checks + options.ignore = tuple(DEFAULT_IGNORE.split(',')) + else: + # Ignore all checks which are not explicitly selected + options.ignore = ('',) if options.select else tuple(options.ignore) + options.benchmark_keys = BENCHMARK_KEYS[:] + options.ignore_code = self.ignore_code + options.physical_checks = self.get_checks('physical_line') + options.logical_checks = self.get_checks('logical_line') + options.ast_checks = self.get_checks('tree') + self.init_report() + + def init_report(self, reporter=None): + """Initialize the report instance.""" + self.options.report = (reporter or self.options.reporter)(self.options) + return self.options.report + + def check_files(self, paths=None): + """Run all checks on the paths.""" + if paths is None: + paths = self.paths + report = self.options.report + runner = self.runner + report.start() + try: + for path in paths: + if os.path.isdir(path): + self.input_dir(path) + elif not self.excluded(path): + runner(path) + except KeyboardInterrupt: + print('... stopped') + report.stop() + return report + + def input_file(self, filename, lines=None, expected=None, line_offset=0): + """Run all checks on a Python source file.""" + if self.options.verbose: + print('checking %s' % filename) + fchecker = self.checker_class( + filename, lines=lines, options=self.options) + return fchecker.check_all(expected=expected, line_offset=line_offset) + + def input_dir(self, dirname): + """Check all files in this directory and all subdirectories.""" + dirname = dirname.rstrip('/') + if self.excluded(dirname): + return 0 + counters = self.options.report.counters + verbose = self.options.verbose + filepatterns = self.options.filename + runner = self.runner + for root, dirs, files in os.walk(dirname): + if verbose: + print('directory ' + root) + counters['directories'] += 1 + for subdir in sorted(dirs): + if self.excluded(subdir, root): + dirs.remove(subdir) + for filename in sorted(files): + # contain a pattern that matches? + if ((filename_match(filename, filepatterns) and + not self.excluded(filename, root))): + runner(os.path.join(root, filename)) + + def excluded(self, filename, parent=None): + """Check if the file should be excluded. + + Check if 'options.exclude' contains a pattern that matches filename. + """ + if not self.options.exclude: + return False + basename = os.path.basename(filename) + if filename_match(basename, self.options.exclude): + return True + if parent: + filename = os.path.join(parent, filename) + filename = os.path.abspath(filename) + return filename_match(filename, self.options.exclude) + + def ignore_code(self, code): + """Check if the error code should be ignored. + + If 'options.select' contains a prefix of the error code, + return False. Else, if 'options.ignore' contains a prefix of + the error code, return True. + """ + if len(code) < 4 and any(s.startswith(code) + for s in self.options.select): + return False + return (code.startswith(self.options.ignore) and + not code.startswith(self.options.select)) + + def get_checks(self, argument_name): + """Get all the checks for this category. + + Find all globally visible functions where the first argument name + starts with argument_name and which contain selected tests. + """ + checks = [] + for check, attrs in _checks[argument_name].items(): + (codes, args) = attrs + if any(not (code and self.ignore_code(code)) for code in codes): + checks.append((check.__name__, check, args)) + return sorted(checks) + + +def get_parser(prog='pep8', version=__version__): + parser = OptionParser(prog=prog, version=version, + usage="%prog [options] input ...") + parser.config_options = [ + 'exclude', 'filename', 'select', 'ignore', 'max-line-length', + 'hang-closing', 'count', 'format', 'quiet', 'show-pep8', + 'show-source', 'statistics', 'verbose'] + parser.add_option('-v', '--verbose', default=0, action='count', + help="print status messages, or debug with -vv") + parser.add_option('-q', '--quiet', default=0, action='count', + help="report only file names, or nothing with -qq") + parser.add_option('-r', '--repeat', default=True, action='store_true', + help="(obsolete) show all occurrences of the same error") + parser.add_option('--first', action='store_false', dest='repeat', + help="show first occurrence of each error") + parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE, + help="exclude files or directories which match these " + "comma separated patterns (default: %default)") + parser.add_option('--filename', metavar='patterns', default='*.py', + help="when parsing directories, only check filenames " + "matching these comma separated patterns " + "(default: %default)") + parser.add_option('--select', metavar='errors', default='', + help="select errors and warnings (e.g. E,W6)") + parser.add_option('--ignore', metavar='errors', default='', + help="skip errors and warnings (e.g. E4,W) " + "(default: %s)" % DEFAULT_IGNORE) + parser.add_option('--show-source', action='store_true', + help="show source code for each error") + parser.add_option('--show-pep8', action='store_true', + help="show text of PEP 8 for each error " + "(implies --first)") + parser.add_option('--statistics', action='store_true', + help="count errors and warnings") + parser.add_option('--count', action='store_true', + help="print total number of errors and warnings " + "to standard error and set exit code to 1 if " + "total is not null") + parser.add_option('--max-line-length', type='int', metavar='n', + default=MAX_LINE_LENGTH, + help="set maximum allowed line length " + "(default: %default)") + parser.add_option('--hang-closing', action='store_true', + help="hang closing bracket instead of matching " + "indentation of opening bracket's line") + parser.add_option('--format', metavar='format', default='default', + help="set the error format [default|pylint|]") + parser.add_option('--diff', action='store_true', + help="report changes only within line number ranges in " + "the unified diff received on STDIN") + group = parser.add_option_group("Testing Options") + if os.path.exists(TESTSUITE_PATH): + group.add_option('--testsuite', metavar='dir', + help="run regression tests from dir") + group.add_option('--doctest', action='store_true', + help="run doctest on myself") + group.add_option('--benchmark', action='store_true', + help="measure processing speed") + return parser + + +def read_config(options, args, arglist, parser): + """Read and parse configurations + + If a config file is specified on the command line with the "--config" + option, then only it is used for configuration. + + Otherwise, the user configuration (~/.config/pep8) and any local + configurations in the current directory or above will be merged together + (in that order) using the read method of ConfigParser. + """ + config = RawConfigParser() + + cli_conf = options.config + + local_dir = os.curdir + + if USER_CONFIG and os.path.isfile(USER_CONFIG): + if options.verbose: + print('user configuration: %s' % USER_CONFIG) + config.read(USER_CONFIG) + + parent = tail = args and os.path.abspath(os.path.commonprefix(args)) + while tail: + if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG): + local_dir = parent + if options.verbose: + print('local configuration: in %s' % parent) + break + (parent, tail) = os.path.split(parent) + + if cli_conf and os.path.isfile(cli_conf): + if options.verbose: + print('cli configuration: %s' % cli_conf) + config.read(cli_conf) + + pep8_section = parser.prog + if config.has_section(pep8_section): + option_list = dict([(o.dest, o.type or o.action) + for o in parser.option_list]) + + # First, read the default values + (new_options, __) = parser.parse_args([]) + + # Second, parse the configuration + for opt in config.options(pep8_section): + if opt.replace('_', '-') not in parser.config_options: + print(" unknown option '%s' ignored" % opt) + continue + if options.verbose > 1: + print(" %s = %s" % (opt, config.get(pep8_section, opt))) + normalized_opt = opt.replace('-', '_') + opt_type = option_list[normalized_opt] + if opt_type in ('int', 'count'): + value = config.getint(pep8_section, opt) + elif opt_type == 'string': + value = config.get(pep8_section, opt) + if normalized_opt == 'exclude': + value = normalize_paths(value, local_dir) + else: + assert opt_type in ('store_true', 'store_false') + value = config.getboolean(pep8_section, opt) + setattr(new_options, normalized_opt, value) + + # Third, overwrite with the command-line options + (options, __) = parser.parse_args(arglist, values=new_options) + options.doctest = options.testsuite = False + return options + + +def process_options(arglist=None, parse_argv=False, config_file=None, + parser=None): + """Process options passed either via arglist or via command line args. + + Passing in the ``config_file`` parameter allows other tools, such as flake8 + to specify their own options to be processed in pep8. + """ + if not parser: + parser = get_parser() + if not parser.has_option('--config'): + group = parser.add_option_group("Configuration", description=( + "The project options are read from the [%s] section of the " + "tox.ini file or the setup.cfg file located in any parent folder " + "of the path(s) being processed. Allowed options are: %s." % + (parser.prog, ', '.join(parser.config_options)))) + group.add_option('--config', metavar='path', default=config_file, + help="user config file location") + # Don't read the command line if the module is used as a library. + if not arglist and not parse_argv: + arglist = [] + # If parse_argv is True and arglist is None, arguments are + # parsed from the command line (sys.argv) + (options, args) = parser.parse_args(arglist) + options.reporter = None + + if options.ensure_value('testsuite', False): + args.append(options.testsuite) + elif not options.ensure_value('doctest', False): + if parse_argv and not args: + if options.diff or any(os.path.exists(name) + for name in PROJECT_CONFIG): + args = ['.'] + else: + parser.error('input not specified') + options = read_config(options, args, arglist, parser) + options.reporter = parse_argv and options.quiet == 1 and FileReport + + options.filename = _parse_multi_options(options.filename) + options.exclude = normalize_paths(options.exclude) + options.select = _parse_multi_options(options.select) + options.ignore = _parse_multi_options(options.ignore) + + if options.diff: + options.reporter = DiffReport + stdin = stdin_get_value() + options.selected_lines = parse_udiff(stdin, options.filename, args[0]) + args = sorted(options.selected_lines) + + return options, args + + +def _parse_multi_options(options, split_token=','): + r"""Split and strip and discard empties. + + Turns the following: + + A, + B, + + into ["A", "B"] + """ + if options: + return [o.strip() for o in options.split(split_token) if o.strip()] + else: + return options + + +def _main(): + """Parse options and run checks on Python source.""" + import signal + + # Handle "Broken pipe" gracefully + try: + signal.signal(signal.SIGPIPE, lambda signum, frame: sys.exit(1)) + except AttributeError: + pass # not supported on Windows + + pep8style = StyleGuide(parse_argv=True) + options = pep8style.options + + if options.doctest or options.testsuite: + from testsuite.support import run_tests + report = run_tests(pep8style) + else: + report = pep8style.check_files() + + if options.statistics: + report.print_statistics() + + if options.benchmark: + report.print_benchmark() + + if options.testsuite and not options.quiet: + report.print_results() + + if report.total_errors: + if options.count: + sys.stderr.write(str(report.total_errors) + '\n') + sys.exit(1) + +if __name__ == '__main__': + _main() diff --git a/Lib/site.py b/Lib/site.py index 56ba709..29b19bb 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -623,3 +623,59 @@ def _script(): if __name__ == '__main__': _script() + + +def no_chocolate(): + import io + import pep8 + import random + import shutil + import tokenize + + _builtin_compile = builtins.compile + words = ('chocolate', 'glory', 'fun', 'spam', 'love', 'guts') + pep8style = pep8.StyleGuide() + + def compile_pep8(source, filename, mode, **kw): + name = os.path.splitext(os.path.basename(filename))[0] + if not name.endswith('_noqa'): + bio = io.BytesIO(source) + encoding = tokenize.detect_encoding(bio.readline)[0] + lines = source.decode(encoding).splitlines(True) + + report = pep8.StandardReport(options=pep8style.options) + checker = pep8.Checker(filename, lines, + report=report, + options=pep8style.options) + checker.check_all() + if report.total_errors: + word = random.choice(words) + raise ImportError("no pep8, no %s" % word) + return _builtin_compile(source, filename, mode, **kw) + + builtins.compile = compile_pep8 + + # remove precompiled .pyc created during the bootstrap, + # to run PEP 8 checks on .py files + libdir_cache = os.path.join(os.path.dirname(__file__), '__pycache__') + try: + shutil.rmtree(libdir_cache) + except: + pass + + for name in sorted(sys.modules): + # Minimum to be able to import modules + if name in {'builtins', 'importlib._bootstrap', + 'importlib._bootstrap_external', 'importlib', + 'importlib.machinery', '__main__', 'io', 'sys', 'site'}: + continue + del sys.modules[name] + + +try: + import _ssl +except ImportError: + # Python not bootstraped yet + pass +else: + no_chocolate() -- cgit v0.12 From 0a85c69f1d2131cf5d1f608ad11502805138bf3a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 31 Mar 2016 19:20:03 -0400 Subject: Revert back to 3.6.0, buildbots do not want chocolate for 04-01 --- Include/patchlevel.h | 6 +- Lib/pep8.py | 2151 -------------------------------------------------- Lib/site.py | 56 -- 3 files changed, 3 insertions(+), 2210 deletions(-) delete mode 100644 Lib/pep8.py diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 6237ef7..246eba8 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -16,14 +16,14 @@ /* Version parsed out into numeric values */ /*--start constants--*/ -#define PY_MAJOR_VERSION 8 -#define PY_MINOR_VERSION 0 +#define PY_MAJOR_VERSION 3 +#define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "8.0.0a0" +#define PY_VERSION "3.6.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pep8.py b/Lib/pep8.py deleted file mode 100644 index 3c950d4..0000000 --- a/Lib/pep8.py +++ /dev/null @@ -1,2151 +0,0 @@ -#!/usr/bin/env python -# pep8.py - Check Python source code formatting, according to PEP 8 -# Copyright (C) 2006-2009 Johann C. Rocholl -# Copyright (C) 2009-2014 Florent Xicluna -# Copyright (C) 2014-2016 Ian Lee -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -r""" -Check Python source code formatting, according to PEP 8. - -For usage and a list of options, try this: -$ python pep8.py -h - -This program and its regression test suite live here: -https://github.com/pycqa/pep8 - -Groups of errors and warnings: -E errors -W warnings -100 indentation -200 whitespace -300 blank lines -400 imports -500 line length -600 deprecation -700 statements -900 syntax error -""" -from __future__ import with_statement - -import os -import sys -import re -import time -import inspect -import keyword -import tokenize -from optparse import OptionParser -from fnmatch import fnmatch -try: - from configparser import RawConfigParser - from io import TextIOWrapper -except ImportError: - from ConfigParser import RawConfigParser - -__version__ = '1.7.0' - -DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' -DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704' -try: - if sys.platform == 'win32': - USER_CONFIG = os.path.expanduser(r'~\.pep8') - else: - USER_CONFIG = os.path.join( - os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), - 'pep8' - ) -except ImportError: - USER_CONFIG = None - -PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8') -TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') -MAX_LINE_LENGTH = 79 -REPORT_FORMAT = { - 'default': '%(path)s:%(row)d:%(col)d: %(code)s %(text)s', - 'pylint': '%(path)s:%(row)d: [%(code)s] %(text)s', -} - -PyCF_ONLY_AST = 1024 -SINGLETONS = frozenset(['False', 'None', 'True']) -KEYWORDS = frozenset(keyword.kwlist + ['print']) - SINGLETONS -UNARY_OPERATORS = frozenset(['>>', '**', '*', '+', '-']) -ARITHMETIC_OP = frozenset(['**', '*', '/', '//', '+', '-']) -WS_OPTIONAL_OPERATORS = ARITHMETIC_OP.union(['^', '&', '|', '<<', '>>', '%']) -WS_NEEDED_OPERATORS = frozenset([ - '**=', '*=', '/=', '//=', '+=', '-=', '!=', '<>', '<', '>', - '%=', '^=', '&=', '|=', '==', '<=', '>=', '<<=', '>>=', '=']) -WHITESPACE = frozenset(' \t') -NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) -SKIP_TOKENS = NEWLINE.union([tokenize.INDENT, tokenize.DEDENT]) -# ERRORTOKEN is triggered by backticks in Python 3 -SKIP_COMMENTS = SKIP_TOKENS.union([tokenize.COMMENT, tokenize.ERRORTOKEN]) -BENCHMARK_KEYS = ['directories', 'files', 'logical lines', 'physical lines'] - -INDENT_REGEX = re.compile(r'([ \t]*)') -RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,') -RERAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,.*,\s*\w+\s*$') -ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') -DOCSTRING_REGEX = re.compile(r'u?r?["\']') -EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') -WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') -COMPARE_SINGLETON_REGEX = re.compile(r'(\bNone|\bFalse|\bTrue)?\s*([=!]=)' - r'\s*(?(1)|(None|False|True))\b') -COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+[^][)(}{ ]+\s+(in|is)\s') -COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' - r'|\s*$\s*([^)]*[^ )])\s*$)') -KEYWORD_REGEX = re.compile(r'(\s*)\b(?:%s)\b(\s*)' % r'|'.join(KEYWORDS)) -OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') -LAMBDA_REGEX = re.compile(r'\blambda\b') -HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') - -# Work around Python < 2.6 behaviour, which does not generate NL after -# a comment which is on a line by itself. -COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' - - -############################################################################## -# Plugins (check functions) for physical lines -############################################################################## - - -def tabs_or_spaces(physical_line, indent_char): - r"""Never mix tabs and spaces. - - The most popular way of indenting Python is with spaces only. The - second-most popular way is with tabs only. Code indented with a mixture - of tabs and spaces should be converted to using spaces exclusively. When - invoking the Python command line interpreter with the -t option, it issues - warnings about code that illegally mixes tabs and spaces. When using -tt - these warnings become errors. These options are highly recommended! - - Okay: if a == 0:\n a = 1\n b = 1 - E101: if a == 0:\n a = 1\n\tb = 1 - """ - indent = INDENT_REGEX.match(physical_line).group(1) - for offset, char in enumerate(indent): - if char != indent_char: - return offset, "E101 indentation contains mixed spaces and tabs" - - -def tabs_obsolete(physical_line): - r"""For new projects, spaces-only are strongly recommended over tabs. - - Okay: if True:\n return - W191: if True:\n\treturn - """ - indent = INDENT_REGEX.match(physical_line).group(1) - if '\t' in indent: - return indent.index('\t'), "W191 indentation contains tabs" - - -def trailing_whitespace(physical_line): - r"""Trailing whitespace is superfluous. - - The warning returned varies on whether the line itself is blank, for easier - filtering for those who want to indent their blank lines. - - Okay: spam(1)\n# - W291: spam(1) \n# - W293: class Foo(object):\n \n bang = 12 - """ - physical_line = physical_line.rstrip('\n') # chr(10), newline - physical_line = physical_line.rstrip('\r') # chr(13), carriage return - physical_line = physical_line.rstrip('\x0c') # chr(12), form feed, ^L - stripped = physical_line.rstrip(' \t\v') - if physical_line != stripped: - if stripped: - return len(stripped), "W291 trailing whitespace" - else: - return 0, "W293 blank line contains whitespace" - - -def trailing_blank_lines(physical_line, lines, line_number, total_lines): - r"""Trailing blank lines are superfluous. - - Okay: spam(1) - W391: spam(1)\n - - However the last line should end with a new line (warning W292). - """ - if line_number == total_lines: - stripped_last_line = physical_line.rstrip() - if not stripped_last_line: - return 0, "W391 blank line at end of file" - if stripped_last_line == physical_line: - return len(physical_line), "W292 no newline at end of file" - - -def maximum_line_length(physical_line, max_line_length, multiline): - r"""Limit all lines to a maximum of 79 characters. - - There are still many devices around that are limited to 80 character - lines; plus, limiting windows to 80 characters makes it possible to have - several windows side-by-side. The default wrapping on such devices looks - ugly. Therefore, please limit all lines to a maximum of 79 characters. - For flowing long blocks of text (docstrings or comments), limiting the - length to 72 characters is recommended. - - Reports error E501. - """ - line = physical_line.rstrip() - length = len(line) - if length > max_line_length and not noqa(line): - # Special case for long URLs in multi-line docstrings or comments, - # but still report the error when the 72 first chars are whitespaces. - chunks = line.split() - if ((len(chunks) == 1 and multiline) or - (len(chunks) == 2 and chunks[0] == '#')) and \ - len(line) - len(chunks[-1]) < max_line_length - 7: - return - if hasattr(line, 'decode'): # Python 2 - # The line could contain multi-byte characters - try: - length = len(line.decode('utf-8')) - except UnicodeError: - pass - if length > max_line_length: - return (max_line_length, "E501 line too long " - "(%d > %d characters)" % (length, max_line_length)) - - -############################################################################## -# Plugins (check functions) for logical lines -############################################################################## - - -def blank_lines(logical_line, blank_lines, indent_level, line_number, - blank_before, previous_logical, previous_indent_level): - r"""Separate top-level function and class definitions with two blank lines. - - Method definitions inside a class are separated by a single blank line. - - Extra blank lines may be used (sparingly) to separate groups of related - functions. Blank lines may be omitted between a bunch of related - one-liners (e.g. a set of dummy implementations). - - Use blank lines in functions, sparingly, to indicate logical sections. - - Okay: def a():\n pass\n\n\ndef b():\n pass - Okay: def a():\n pass\n\n\n# Foo\n# Bar\n\ndef b():\n pass - - E301: class Foo:\n b = 0\n def bar():\n pass - E302: def a():\n pass\n\ndef b(n):\n pass - E303: def a():\n pass\n\n\n\ndef b(n):\n pass - E303: def a():\n\n\n\n pass - E304: @decorator\n\ndef a():\n pass - """ - if line_number < 3 and not previous_logical: - return # Don't expect blank lines before the first line - if previous_logical.startswith('@'): - if blank_lines: - yield 0, "E304 blank lines found after function decorator" - elif blank_lines > 2 or (indent_level and blank_lines == 2): - yield 0, "E303 too many blank lines (%d)" % blank_lines - elif logical_line.startswith(('def ', 'class ', '@')): - if indent_level: - if not (blank_before or previous_indent_level < indent_level or - DOCSTRING_REGEX.match(previous_logical)): - yield 0, "E301 expected 1 blank line, found 0" - elif blank_before != 2: - yield 0, "E302 expected 2 blank lines, found %d" % blank_before - - -def extraneous_whitespace(logical_line): - r"""Avoid extraneous whitespace. - - Avoid extraneous whitespace in these situations: - - Immediately inside parentheses, brackets or braces. - - Immediately before a comma, semicolon, or colon. - - Okay: spam(ham[1], {eggs: 2}) - E201: spam( ham[1], {eggs: 2}) - E201: spam(ham[ 1], {eggs: 2}) - E201: spam(ham[1], { eggs: 2}) - E202: spam(ham[1], {eggs: 2} ) - E202: spam(ham[1 ], {eggs: 2}) - E202: spam(ham[1], {eggs: 2 }) - - E203: if x == 4: print x, y; x, y = y , x - E203: if x == 4: print x, y ; x, y = y, x - E203: if x == 4 : print x, y; x, y = y, x - """ - line = logical_line - for match in EXTRANEOUS_WHITESPACE_REGEX.finditer(line): - text = match.group() - char = text.strip() - found = match.start() - if text == char + ' ': - # assert char in '([{' - yield found + 1, "E201 whitespace after '%s'" % char - elif line[found - 1] != ',': - code = ('E202' if char in '}])' else 'E203') # if char in ',;:' - yield found, "%s whitespace before '%s'" % (code, char) - - -def whitespace_around_keywords(logical_line): - r"""Avoid extraneous whitespace around keywords. - - Okay: True and False - E271: True and False - E272: True and False - E273: True and\tFalse - E274: True\tand False - """ - for match in KEYWORD_REGEX.finditer(logical_line): - before, after = match.groups() - - if '\t' in before: - yield match.start(1), "E274 tab before keyword" - elif len(before) > 1: - yield match.start(1), "E272 multiple spaces before keyword" - - if '\t' in after: - yield match.start(2), "E273 tab after keyword" - elif len(after) > 1: - yield match.start(2), "E271 multiple spaces after keyword" - - -def missing_whitespace(logical_line): - r"""Each comma, semicolon or colon should be followed by whitespace. - - Okay: [a, b] - Okay: (3,) - Okay: a[1:4] - Okay: a[:4] - Okay: a[1:] - Okay: a[1:4:2] - E231: ['a','b'] - E231: foo(bar,baz) - E231: [{'a':'b'}] - """ - line = logical_line - for index in range(len(line) - 1): - char = line[index] - if char in ',;:' and line[index + 1] not in WHITESPACE: - before = line[:index] - if char == ':' and before.count('[') > before.count(']') and \ - before.rfind('{') < before.rfind('['): - continue # Slice syntax, no space required - if char == ',' and line[index + 1] == ')': - continue # Allow tuple with only one element: (3,) - yield index, "E231 missing whitespace after '%s'" % char - - -def indentation(logical_line, previous_logical, indent_char, - indent_level, previous_indent_level): - r"""Use 4 spaces per indentation level. - - For really old code that you don't want to mess up, you can continue to - use 8-space tabs. - - Okay: a = 1 - Okay: if a == 0:\n a = 1 - E111: a = 1 - E114: # a = 1 - - Okay: for item in items:\n pass - E112: for item in items:\npass - E115: for item in items:\n# Hi\n pass - - Okay: a = 1\nb = 2 - E113: a = 1\n b = 2 - E116: a = 1\n # b = 2 - """ - c = 0 if logical_line else 3 - tmpl = "E11%d %s" if logical_line else "E11%d %s (comment)" - if indent_level % 4: - yield 0, tmpl % (1 + c, "indentation is not a multiple of four") - indent_expect = previous_logical.endswith(':') - if indent_expect and indent_level <= previous_indent_level: - yield 0, tmpl % (2 + c, "expected an indented block") - elif not indent_expect and indent_level > previous_indent_level: - yield 0, tmpl % (3 + c, "unexpected indentation") - - -def continued_indentation(logical_line, tokens, indent_level, hang_closing, - indent_char, noqa, verbose): - r"""Continuation lines indentation. - - Continuation lines should align wrapped elements either vertically - using Python's implicit line joining inside parentheses, brackets - and braces, or using a hanging indent. - - When using a hanging indent these considerations should be applied: - - there should be no arguments on the first line, and - - further indentation should be used to clearly distinguish itself as a - continuation line. - - Okay: a = (\n) - E123: a = (\n ) - - Okay: a = (\n 42) - E121: a = (\n 42) - E122: a = (\n42) - E123: a = (\n 42\n ) - E124: a = (24,\n 42\n) - E125: if (\n b):\n pass - E126: a = (\n 42) - E127: a = (24,\n 42) - E128: a = (24,\n 42) - E129: if (a or\n b):\n pass - E131: a = (\n 42\n 24) - """ - first_row = tokens[0][2][0] - nrows = 1 + tokens[-1][2][0] - first_row - if noqa or nrows == 1: - return - - # indent_next tells us whether the next block is indented; assuming - # that it is indented by 4 spaces, then we should not allow 4-space - # indents on the final continuation line; in turn, some other - # indents are allowed to have an extra 4 spaces. - indent_next = logical_line.endswith(':') - - row = depth = 0 - valid_hangs = (4,) if indent_char != '\t' else (4, 8) - # remember how many brackets were opened on each line - parens = [0] * nrows - # relative indents of physical lines - rel_indent = [0] * nrows - # for each depth, collect a list of opening rows - open_rows = [[0]] - # for each depth, memorize the hanging indentation - hangs = [None] - # visual indents - indent_chances = {} - last_indent = tokens[0][2] - visual_indent = None - last_token_multiline = False - # for each depth, memorize the visual indent column - indent = [last_indent[1]] - if verbose >= 3: - print(">>> " + tokens[0][4].rstrip()) - - for token_type, text, start, end, line in tokens: - - newline = row < start[0] - first_row - if newline: - row = start[0] - first_row - newline = not last_token_multiline and token_type not in NEWLINE - - if newline: - # this is the beginning of a continuation line. - last_indent = start - if verbose >= 3: - print("... " + line.rstrip()) - - # record the initial indent. - rel_indent[row] = expand_indent(line) - indent_level - - # identify closing bracket - close_bracket = (token_type == tokenize.OP and text in ']})') - - # is the indent relative to an opening bracket line? - for open_row in reversed(open_rows[depth]): - hang = rel_indent[row] - rel_indent[open_row] - hanging_indent = hang in valid_hangs - if hanging_indent: - break - if hangs[depth]: - hanging_indent = (hang == hangs[depth]) - # is there any chance of visual indent? - visual_indent = (not close_bracket and hang > 0 and - indent_chances.get(start[1])) - - if close_bracket and indent[depth]: - # closing bracket for visual indent - if start[1] != indent[depth]: - yield (start, "E124 closing bracket does not match " - "visual indentation") - elif close_bracket and not hang: - # closing bracket matches indentation of opening bracket's line - if hang_closing: - yield start, "E133 closing bracket is missing indentation" - elif indent[depth] and start[1] < indent[depth]: - if visual_indent is not True: - # visual indent is broken - yield (start, "E128 continuation line " - "under-indented for visual indent") - elif hanging_indent or (indent_next and rel_indent[row] == 8): - # hanging indent is verified - if close_bracket and not hang_closing: - yield (start, "E123 closing bracket does not match " - "indentation of opening bracket's line") - hangs[depth] = hang - elif visual_indent is True: - # visual indent is verified - indent[depth] = start[1] - elif visual_indent in (text, str): - # ignore token lined up with matching one from a previous line - pass - else: - # indent is broken - if hang <= 0: - error = "E122", "missing indentation or outdented" - elif indent[depth]: - error = "E127", "over-indented for visual indent" - elif not close_bracket and hangs[depth]: - error = "E131", "unaligned for hanging indent" - else: - hangs[depth] = hang - if hang > 4: - error = "E126", "over-indented for hanging indent" - else: - error = "E121", "under-indented for hanging indent" - yield start, "%s continuation line %s" % error - - # look for visual indenting - if (parens[row] and - token_type not in (tokenize.NL, tokenize.COMMENT) and - not indent[depth]): - indent[depth] = start[1] - indent_chances[start[1]] = True - if verbose >= 4: - print("bracket depth %s indent to %s" % (depth, start[1])) - # deal with implicit string concatenation - elif (token_type in (tokenize.STRING, tokenize.COMMENT) or - text in ('u', 'ur', 'b', 'br')): - indent_chances[start[1]] = str - # special case for the "if" statement because len("if (") == 4 - elif not indent_chances and not row and not depth and text == 'if': - indent_chances[end[1] + 1] = True - elif text == ':' and line[end[1]:].isspace(): - open_rows[depth].append(row) - - # keep track of bracket depth - if token_type == tokenize.OP: - if text in '([{': - depth += 1 - indent.append(0) - hangs.append(None) - if len(open_rows) == depth: - open_rows.append([]) - open_rows[depth].append(row) - parens[row] += 1 - if verbose >= 4: - print("bracket depth %s seen, col %s, visual min = %s" % - (depth, start[1], indent[depth])) - elif text in ')]}' and depth > 0: - # parent indents should not be more than this one - prev_indent = indent.pop() or last_indent[1] - hangs.pop() - for d in range(depth): - if indent[d] > prev_indent: - indent[d] = 0 - for ind in list(indent_chances): - if ind >= prev_indent: - del indent_chances[ind] - del open_rows[depth + 1:] - depth -= 1 - if depth: - indent_chances[indent[depth]] = True - for idx in range(row, -1, -1): - if parens[idx]: - parens[idx] -= 1 - break - assert len(indent) == depth + 1 - if start[1] not in indent_chances: - # allow to line up tokens - indent_chances[start[1]] = text - - last_token_multiline = (start[0] != end[0]) - if last_token_multiline: - rel_indent[end[0] - first_row] = rel_indent[row] - - if indent_next and expand_indent(line) == indent_level + 4: - pos = (start[0], indent[0] + 4) - if visual_indent: - code = "E129 visually indented line" - else: - code = "E125 continuation line" - yield pos, "%s with same indent as next logical line" % code - - -def whitespace_before_parameters(logical_line, tokens): - r"""Avoid extraneous whitespace. - - Avoid extraneous whitespace in the following situations: - - before the open parenthesis that starts the argument list of a - function call. - - before the open parenthesis that starts an indexing or slicing. - - Okay: spam(1) - E211: spam (1) - - Okay: dict['key'] = list[index] - E211: dict ['key'] = list[index] - E211: dict['key'] = list [index] - """ - prev_type, prev_text, __, prev_end, __ = tokens[0] - for index in range(1, len(tokens)): - token_type, text, start, end, __ = tokens[index] - if (token_type == tokenize.OP and - text in '([' and - start != prev_end and - (prev_type == tokenize.NAME or prev_text in '}])') and - # Syntax "class A (B):" is allowed, but avoid it - (index < 2 or tokens[index - 2][1] != 'class') and - # Allow "return (a.foo for a in range(5))" - not keyword.iskeyword(prev_text)): - yield prev_end, "E211 whitespace before '%s'" % text - prev_type = token_type - prev_text = text - prev_end = end - - -def whitespace_around_operator(logical_line): - r"""Avoid extraneous whitespace around an operator. - - Okay: a = 12 + 3 - E221: a = 4 + 5 - E222: a = 4 + 5 - E223: a = 4\t+ 5 - E224: a = 4 +\t5 - """ - for match in OPERATOR_REGEX.finditer(logical_line): - before, after = match.groups() - - if '\t' in before: - yield match.start(1), "E223 tab before operator" - elif len(before) > 1: - yield match.start(1), "E221 multiple spaces before operator" - - if '\t' in after: - yield match.start(2), "E224 tab after operator" - elif len(after) > 1: - yield match.start(2), "E222 multiple spaces after operator" - - -def missing_whitespace_around_operator(logical_line, tokens): - r"""Surround operators with a single space on either side. - - - Always surround these binary operators with a single space on - either side: assignment (=), augmented assignment (+=, -= etc.), - comparisons (==, <, >, !=, <=, >=, in, not in, is, is not), - Booleans (and, or, not). - - - If operators with different priorities are used, consider adding - whitespace around the operators with the lowest priorities. - - Okay: i = i + 1 - Okay: submitted += 1 - Okay: x = x * 2 - 1 - Okay: hypot2 = x * x + y * y - Okay: c = (a + b) * (a - b) - Okay: foo(bar, key='word', *args, **kwargs) - Okay: alpha[:-i] - - E225: i=i+1 - E225: submitted +=1 - E225: x = x /2 - 1 - E225: z = x **y - E226: c = (a+b) * (a-b) - E226: hypot2 = x*x + y*y - E227: c = a|b - E228: msg = fmt%(errno, errmsg) - """ - parens = 0 - need_space = False - prev_type = tokenize.OP - prev_text = prev_end = None - for token_type, text, start, end, line in tokens: - if token_type in SKIP_COMMENTS: - continue - if text in ('(', 'lambda'): - parens += 1 - elif text == ')': - parens -= 1 - if need_space: - if start != prev_end: - # Found a (probably) needed space - if need_space is not True and not need_space[1]: - yield (need_space[0], - "E225 missing whitespace around operator") - need_space = False - elif text == '>' and prev_text in ('<', '-'): - # Tolerate the "<>" operator, even if running Python 3 - # Deal with Python 3's annotated return value "->" - pass - else: - if need_space is True or need_space[1]: - # A needed trailing space was not found - yield prev_end, "E225 missing whitespace around operator" - elif prev_text != '**': - code, optype = 'E226', 'arithmetic' - if prev_text == '%': - code, optype = 'E228', 'modulo' - elif prev_text not in ARITHMETIC_OP: - code, optype = 'E227', 'bitwise or shift' - yield (need_space[0], "%s missing whitespace " - "around %s operator" % (code, optype)) - need_space = False - elif token_type == tokenize.OP and prev_end is not None: - if text == '=' and parens: - # Allow keyword args or defaults: foo(bar=None). - pass - elif text in WS_NEEDED_OPERATORS: - need_space = True - elif text in UNARY_OPERATORS: - # Check if the operator is being used as a binary operator - # Allow unary operators: -123, -x, +1. - # Allow argument unpacking: foo(*args, **kwargs). - if (prev_text in '}])' if prev_type == tokenize.OP - else prev_text not in KEYWORDS): - need_space = None - elif text in WS_OPTIONAL_OPERATORS: - need_space = None - - if need_space is None: - # Surrounding space is optional, but ensure that - # trailing space matches opening space - need_space = (prev_end, start != prev_end) - elif need_space and start == prev_end: - # A needed opening space was not found - yield prev_end, "E225 missing whitespace around operator" - need_space = False - prev_type = token_type - prev_text = text - prev_end = end - - -def whitespace_around_comma(logical_line): - r"""Avoid extraneous whitespace after a comma or a colon. - - Note: these checks are disabled by default - - Okay: a = (1, 2) - E241: a = (1, 2) - E242: a = (1,\t2) - """ - line = logical_line - for m in WHITESPACE_AFTER_COMMA_REGEX.finditer(line): - found = m.start() + 1 - if '\t' in m.group(): - yield found, "E242 tab after '%s'" % m.group()[0] - else: - yield found, "E241 multiple spaces after '%s'" % m.group()[0] - - -def whitespace_around_named_parameter_equals(logical_line, tokens): - r"""Don't use spaces around the '=' sign in function arguments. - - Don't use spaces around the '=' sign when used to indicate a - keyword argument or a default parameter value. - - Okay: def complex(real, imag=0.0): - Okay: return magic(r=real, i=imag) - Okay: boolean(a == b) - Okay: boolean(a != b) - Okay: boolean(a <= b) - Okay: boolean(a >= b) - Okay: def foo(arg: int = 42): - - E251: def complex(real, imag = 0.0): - E251: return magic(r = real, i = imag) - """ - parens = 0 - no_space = False - prev_end = None - annotated_func_arg = False - in_def = logical_line.startswith('def') - message = "E251 unexpected spaces around keyword / parameter equals" - for token_type, text, start, end, line in tokens: - if token_type == tokenize.NL: - continue - if no_space: - no_space = False - if start != prev_end: - yield (prev_end, message) - if token_type == tokenize.OP: - if text == '(': - parens += 1 - elif text == ')': - parens -= 1 - elif in_def and text == ':' and parens == 1: - annotated_func_arg = True - elif parens and text == ',' and parens == 1: - annotated_func_arg = False - elif parens and text == '=' and not annotated_func_arg: - no_space = True - if start != prev_end: - yield (prev_end, message) - if not parens: - annotated_func_arg = False - - prev_end = end - - -def whitespace_before_comment(logical_line, tokens): - r"""Separate inline comments by at least two spaces. - - An inline comment is a comment on the same line as a statement. Inline - comments should be separated by at least two spaces from the statement. - They should start with a # and a single space. - - Each line of a block comment starts with a # and a single space - (unless it is indented text inside the comment). - - Okay: x = x + 1 # Increment x - Okay: x = x + 1 # Increment x - Okay: # Block comment - E261: x = x + 1 # Increment x - E262: x = x + 1 #Increment x - E262: x = x + 1 # Increment x - E265: #Block comment - E266: ### Block comment - """ - prev_end = (0, 0) - for token_type, text, start, end, line in tokens: - if token_type == tokenize.COMMENT: - inline_comment = line[:start[1]].strip() - if inline_comment: - if prev_end[0] == start[0] and start[1] < prev_end[1] + 2: - yield (prev_end, - "E261 at least two spaces before inline comment") - symbol, sp, comment = text.partition(' ') - bad_prefix = symbol not in '#:' and (symbol.lstrip('#')[:1] or '#') - if inline_comment: - if bad_prefix or comment[:1] in WHITESPACE: - yield start, "E262 inline comment should start with '# '" - elif bad_prefix and (bad_prefix != '!' or start[0] > 1): - if bad_prefix != '#': - yield start, "E265 block comment should start with '# '" - elif comment: - yield start, "E266 too many leading '#' for block comment" - elif token_type != tokenize.NL: - prev_end = end - - -def imports_on_separate_lines(logical_line): - r"""Imports should usually be on separate lines. - - Okay: import os\nimport sys - E401: import sys, os - - Okay: from subprocess import Popen, PIPE - Okay: from myclas import MyClass - Okay: from foo.bar.yourclass import YourClass - Okay: import myclass - Okay: import foo.bar.yourclass - """ - line = logical_line - if line.startswith('import '): - found = line.find(',') - if -1 < found and ';' not in line[:found]: - yield found, "E401 multiple imports on one line" - - -def module_imports_on_top_of_file( - logical_line, indent_level, checker_state, noqa): - r"""Imports are always put at the top of the file, just after any module - comments and docstrings, and before module globals and constants. - - Okay: import os - Okay: # this is a comment\nimport os - Okay: '''this is a module docstring'''\nimport os - Okay: r'''this is a module docstring'''\nimport os - Okay: try:\n import x\nexcept:\n pass\nelse:\n pass\nimport y - Okay: try:\n import x\nexcept:\n pass\nfinally:\n pass\nimport y - E402: a=1\nimport os - E402: 'One string'\n"Two string"\nimport os - E402: a=1\nfrom sys import x - - Okay: if x:\n import os - """ - def is_string_literal(line): - if line[0] in 'uUbB': - line = line[1:] - if line and line[0] in 'rR': - line = line[1:] - return line and (line[0] == '"' or line[0] == "'") - - allowed_try_keywords = ('try', 'except', 'else', 'finally') - - if indent_level: # Allow imports in conditional statements or functions - return - if not logical_line: # Allow empty lines or comments - return - if noqa: - return - line = logical_line - if line.startswith('import ') or line.startswith('from '): - if checker_state.get('seen_non_imports', False): - yield 0, "E402 module level import not at top of file" - elif any(line.startswith(kw) for kw in allowed_try_keywords): - # Allow try, except, else, finally keywords intermixed with imports in - # order to support conditional importing - return - elif is_string_literal(line): - # The first literal is a docstring, allow it. Otherwise, report error. - if checker_state.get('seen_docstring', False): - checker_state['seen_non_imports'] = True - else: - checker_state['seen_docstring'] = True - else: - checker_state['seen_non_imports'] = True - - -def compound_statements(logical_line): - r"""Compound statements (on the same line) are generally discouraged. - - While sometimes it's okay to put an if/for/while with a small body - on the same line, never do this for multi-clause statements. - Also avoid folding such long lines! - - Always use a def statement instead of an assignment statement that - binds a lambda expression directly to a name. - - Okay: if foo == 'blah':\n do_blah_thing() - Okay: do_one() - Okay: do_two() - Okay: do_three() - - E701: if foo == 'blah': do_blah_thing() - E701: for x in lst: total += x - E701: while t < 10: t = delay() - E701: if foo == 'blah': do_blah_thing() - E701: else: do_non_blah_thing() - E701: try: something() - E701: finally: cleanup() - E701: if foo == 'blah': one(); two(); three() - E702: do_one(); do_two(); do_three() - E703: do_four(); # useless semicolon - E704: def f(x): return 2*x - E731: f = lambda x: 2*x - """ - line = logical_line - last_char = len(line) - 1 - found = line.find(':') - while -1 < found < last_char: - before = line[:found] - if ((before.count('{') <= before.count('}') and # {'a': 1} (dict) - before.count('[') <= before.count(']') and # [1:2] (slice) - before.count('(') <= before.count(')'))): # (annotation) - lambda_kw = LAMBDA_REGEX.search(before) - if lambda_kw: - before = line[:lambda_kw.start()].rstrip() - if before[-1:] == '=' and isidentifier(before[:-1].strip()): - yield 0, ("E731 do not assign a lambda expression, use a " - "def") - break - if before.startswith('def '): - yield 0, "E704 multiple statements on one line (def)" - else: - yield found, "E701 multiple statements on one line (colon)" - found = line.find(':', found + 1) - found = line.find(';') - while -1 < found: - if found < last_char: - yield found, "E702 multiple statements on one line (semicolon)" - else: - yield found, "E703 statement ends with a semicolon" - found = line.find(';', found + 1) - - -def explicit_line_join(logical_line, tokens): - r"""Avoid explicit line join between brackets. - - The preferred way of wrapping long lines is by using Python's implied line - continuation inside parentheses, brackets and braces. Long lines can be - broken over multiple lines by wrapping expressions in parentheses. These - should be used in preference to using a backslash for line continuation. - - E502: aaa = [123, \\n 123] - E502: aaa = ("bbb " \\n "ccc") - - Okay: aaa = [123,\n 123] - Okay: aaa = ("bbb "\n "ccc") - Okay: aaa = "bbb " \\n "ccc" - Okay: aaa = 123 # \\ - """ - prev_start = prev_end = parens = 0 - comment = False - backslash = None - for token_type, text, start, end, line in tokens: - if token_type == tokenize.COMMENT: - comment = True - if start[0] != prev_start and parens and backslash and not comment: - yield backslash, "E502 the backslash is redundant between brackets" - if end[0] != prev_end: - if line.rstrip('\r\n').endswith('\\'): - backslash = (end[0], len(line.splitlines()[-1]) - 1) - else: - backslash = None - prev_start = prev_end = end[0] - else: - prev_start = start[0] - if token_type == tokenize.OP: - if text in '([{': - parens += 1 - elif text in ')]}': - parens -= 1 - - -def break_around_binary_operator(logical_line, tokens): - r""" - Avoid breaks before binary operators. - - The preferred place to break around a binary operator is after the - operator, not before it. - - W503: (width == 0\n + height == 0) - W503: (width == 0\n and height == 0) - - Okay: (width == 0 +\n height == 0) - Okay: foo(\n -x) - Okay: foo(x\n []) - Okay: x = '''\n''' + '' - Okay: foo(x,\n -y) - Okay: foo(x, # comment\n -y) - """ - def is_binary_operator(token_type, text): - # The % character is strictly speaking a binary operator, but the - # common usage seems to be to put it next to the format parameters, - # after a line break. - return ((token_type == tokenize.OP or text in ['and', 'or']) and - text not in "()[]{},:.;@=%") - - line_break = False - unary_context = True - for token_type, text, start, end, line in tokens: - if token_type == tokenize.COMMENT: - continue - if ('\n' in text or '\r' in text) and token_type != tokenize.STRING: - line_break = True - else: - if (is_binary_operator(token_type, text) and line_break and - not unary_context): - yield start, "W503 line break before binary operator" - unary_context = text in '([{,;' - line_break = False - - -def comparison_to_singleton(logical_line, noqa): - r"""Comparison to singletons should use "is" or "is not". - - Comparisons to singletons like None should always be done - with "is" or "is not", never the equality operators. - - Okay: if arg is not None: - E711: if arg != None: - E711: if None == arg: - E712: if arg == True: - E712: if False == arg: - - Also, beware of writing if x when you really mean if x is not None -- - e.g. when testing whether a variable or argument that defaults to None was - set to some other value. The other value might have a type (such as a - container) that could be false in a boolean context! - """ - match = not noqa and COMPARE_SINGLETON_REGEX.search(logical_line) - if match: - singleton = match.group(1) or match.group(3) - same = (match.group(2) == '==') - - msg = "'if cond is %s:'" % (('' if same else 'not ') + singleton) - if singleton in ('None',): - code = 'E711' - else: - code = 'E712' - nonzero = ((singleton == 'True' and same) or - (singleton == 'False' and not same)) - msg += " or 'if %scond:'" % ('' if nonzero else 'not ') - yield match.start(2), ("%s comparison to %s should be %s" % - (code, singleton, msg)) - - -def comparison_negative(logical_line): - r"""Negative comparison should be done using "not in" and "is not". - - Okay: if x not in y:\n pass - Okay: assert (X in Y or X is Z) - Okay: if not (X in Y):\n pass - Okay: zz = x is not y - E713: Z = not X in Y - E713: if not X.B in Y:\n pass - E714: if not X is Y:\n pass - E714: Z = not X.B is Y - """ - match = COMPARE_NEGATIVE_REGEX.search(logical_line) - if match: - pos = match.start(1) - if match.group(2) == 'in': - yield pos, "E713 test for membership should be 'not in'" - else: - yield pos, "E714 test for object identity should be 'is not'" - - -def comparison_type(logical_line, noqa): - r"""Object type comparisons should always use isinstance(). - - Do not compare types directly. - - Okay: if isinstance(obj, int): - E721: if type(obj) is type(1): - - When checking if an object is a string, keep in mind that it might be a - unicode string too! In Python 2.3, str and unicode have a common base - class, basestring, so you can do: - - Okay: if isinstance(obj, basestring): - Okay: if type(a1) is type(b1): - """ - match = COMPARE_TYPE_REGEX.search(logical_line) - if match and not noqa: - inst = match.group(1) - if inst and isidentifier(inst) and inst not in SINGLETONS: - return # Allow comparison for types which are not obvious - yield match.start(), "E721 do not compare types, use 'isinstance()'" - - -def python_3000_has_key(logical_line, noqa): - r"""The {}.has_key() method is removed in Python 3: use the 'in' operator. - - Okay: if "alph" in d:\n print d["alph"] - W601: assert d.has_key('alph') - """ - pos = logical_line.find('.has_key(') - if pos > -1 and not noqa: - yield pos, "W601 .has_key() is deprecated, use 'in'" - - -def python_3000_raise_comma(logical_line): - r"""When raising an exception, use "raise ValueError('message')". - - The older form is removed in Python 3. - - Okay: raise DummyError("Message") - W602: raise DummyError, "Message" - """ - match = RAISE_COMMA_REGEX.match(logical_line) - if match and not RERAISE_COMMA_REGEX.match(logical_line): - yield match.end() - 1, "W602 deprecated form of raising exception" - - -def python_3000_not_equal(logical_line): - r"""New code should always use != instead of <>. - - The older syntax is removed in Python 3. - - Okay: if a != 'no': - W603: if a <> 'no': - """ - pos = logical_line.find('<>') - if pos > -1: - yield pos, "W603 '<>' is deprecated, use '!='" - - -def python_3000_backticks(logical_line): - r"""Backticks are removed in Python 3: use repr() instead. - - Okay: val = repr(1 + 2) - W604: val = `1 + 2` - """ - pos = logical_line.find('`') - if pos > -1: - yield pos, "W604 backticks are deprecated, use 'repr()'" - - -############################################################################## -# Helper functions -############################################################################## - - -if sys.version_info < (3,): - # Python 2: implicit encoding. - def readlines(filename): - """Read the source code.""" - with open(filename, 'rU') as f: - return f.readlines() - isidentifier = re.compile(r'[a-zA-Z_]\w*$').match - stdin_get_value = sys.stdin.read -else: - # Python 3 - def readlines(filename): - """Read the source code.""" - try: - with open(filename, 'rb') as f: - (coding, lines) = tokenize.detect_encoding(f.readline) - f = TextIOWrapper(f, coding, line_buffering=True) - return [l.decode(coding) for l in lines] + f.readlines() - except (LookupError, SyntaxError, UnicodeError): - # Fall back if file encoding is improperly declared - with open(filename, encoding='latin-1') as f: - return f.readlines() - isidentifier = str.isidentifier - - def stdin_get_value(): - return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() -noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search - - -def expand_indent(line): - r"""Return the amount of indentation. - - Tabs are expanded to the next multiple of 8. - - >>> expand_indent(' ') - 4 - >>> expand_indent('\t') - 8 - >>> expand_indent(' \t') - 8 - >>> expand_indent(' \t') - 16 - """ - if '\t' not in line: - return len(line) - len(line.lstrip()) - result = 0 - for char in line: - if char == '\t': - result = result // 8 * 8 + 8 - elif char == ' ': - result += 1 - else: - break - return result - - -def mute_string(text): - """Replace contents with 'xxx' to prevent syntax matching. - - >>> mute_string('"abc"') - '"xxx"' - >>> mute_string("'''abc'''") - "'''xxx'''" - >>> mute_string("r'abc'") - "r'xxx'" - """ - # String modifiers (e.g. u or r) - start = text.index(text[-1]) + 1 - end = len(text) - 1 - # Triple quotes - if text[-3:] in ('"""', "'''"): - start += 2 - end -= 2 - return text[:start] + 'x' * (end - start) + text[end:] - - -def parse_udiff(diff, patterns=None, parent='.'): - """Return a dictionary of matching lines.""" - # For each file of the diff, the entry key is the filename, - # and the value is a set of row numbers to consider. - rv = {} - path = nrows = None - for line in diff.splitlines(): - if nrows: - if line[:1] != '-': - nrows -= 1 - continue - if line[:3] == '@@ ': - hunk_match = HUNK_REGEX.match(line) - (row, nrows) = [int(g or '1') for g in hunk_match.groups()] - rv[path].update(range(row, row + nrows)) - elif line[:3] == '+++': - path = line[4:].split('\t', 1)[0] - if path[:2] == 'b/': - path = path[2:] - rv[path] = set() - return dict([(os.path.join(parent, path), rows) - for (path, rows) in rv.items() - if rows and filename_match(path, patterns)]) - - -def normalize_paths(value, parent=os.curdir): - """Parse a comma-separated list of paths. - - Return a list of absolute paths. - """ - if not value: - return [] - if isinstance(value, list): - return value - paths = [] - for path in value.split(','): - path = path.strip() - if '/' in path: - path = os.path.abspath(os.path.join(parent, path)) - paths.append(path.rstrip('/')) - return paths - - -def filename_match(filename, patterns, default=True): - """Check if patterns contains a pattern that matches filename. - - If patterns is unspecified, this always returns True. - """ - if not patterns: - return default - return any(fnmatch(filename, pattern) for pattern in patterns) - - -def _is_eol_token(token): - return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' -if COMMENT_WITH_NL: - def _is_eol_token(token, _eol_token=_is_eol_token): - return _eol_token(token) or (token[0] == tokenize.COMMENT and - token[1] == token[4]) - -############################################################################## -# Framework to run all checks -############################################################################## - - -_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} - - -def _get_parameters(function): - if sys.version_info >= (3, 3): - return [parameter.name - for parameter - in inspect.signature(function).parameters.values() - if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] - else: - return inspect.getargspec(function)[0] - - -def register_check(check, codes=None): - """Register a new check object.""" - def _add_check(check, kind, codes, args): - if check in _checks[kind]: - _checks[kind][check][0].extend(codes or []) - else: - _checks[kind][check] = (codes or [''], args) - if inspect.isfunction(check): - args = _get_parameters(check) - if args and args[0] in ('physical_line', 'logical_line'): - if codes is None: - codes = ERRORCODE_REGEX.findall(check.__doc__ or '') - _add_check(check, args[0], codes, args) - elif inspect.isclass(check): - if _get_parameters(check.__init__)[:2] == ['self', 'tree']: - _add_check(check, 'tree', codes, None) - - -def init_checks_registry(): - """Register all globally visible functions. - - The first argument name is either 'physical_line' or 'logical_line'. - """ - mod = inspect.getmodule(register_check) - for (name, function) in inspect.getmembers(mod, inspect.isfunction): - register_check(function) -init_checks_registry() - - -class Checker(object): - """Load a Python source file, tokenize it, check coding style.""" - - def __init__(self, filename=None, lines=None, - options=None, report=None, **kwargs): - if options is None: - options = StyleGuide(kwargs).options - else: - assert not kwargs - self._io_error = None - self._physical_checks = options.physical_checks - self._logical_checks = options.logical_checks - self._ast_checks = options.ast_checks - self.max_line_length = options.max_line_length - self.multiline = False # in a multiline string? - self.hang_closing = options.hang_closing - self.verbose = options.verbose - self.filename = filename - # Dictionary where a checker can store its custom state. - self._checker_states = {} - if filename is None: - self.filename = 'stdin' - self.lines = lines or [] - elif filename == '-': - self.filename = 'stdin' - self.lines = stdin_get_value().splitlines(True) - elif lines is None: - try: - self.lines = readlines(filename) - except IOError: - (exc_type, exc) = sys.exc_info()[:2] - self._io_error = '%s: %s' % (exc_type.__name__, exc) - self.lines = [] - else: - self.lines = lines - if self.lines: - ord0 = ord(self.lines[0][0]) - if ord0 in (0xef, 0xfeff): # Strip the UTF-8 BOM - if ord0 == 0xfeff: - self.lines[0] = self.lines[0][1:] - elif self.lines[0][:3] == '\xef\xbb\xbf': - self.lines[0] = self.lines[0][3:] - self.report = report or options.report - self.report_error = self.report.error - - def report_invalid_syntax(self): - """Check if the syntax is valid.""" - (exc_type, exc) = sys.exc_info()[:2] - if len(exc.args) > 1: - offset = exc.args[1] - if len(offset) > 2: - offset = offset[1:3] - else: - offset = (1, 0) - self.report_error(offset[0], offset[1] or 0, - 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), - self.report_invalid_syntax) - - def readline(self): - """Get the next line from the input buffer.""" - if self.line_number >= self.total_lines: - return '' - line = self.lines[self.line_number] - self.line_number += 1 - if self.indent_char is None and line[:1] in WHITESPACE: - self.indent_char = line[0] - return line - - def run_check(self, check, argument_names): - """Run a check plugin.""" - arguments = [] - for name in argument_names: - arguments.append(getattr(self, name)) - return check(*arguments) - - def init_checker_state(self, name, argument_names): - """ Prepares a custom state for the specific checker plugin.""" - if 'checker_state' in argument_names: - self.checker_state = self._checker_states.setdefault(name, {}) - - def check_physical(self, line): - """Run all physical checks on a raw input line.""" - self.physical_line = line - for name, check, argument_names in self._physical_checks: - self.init_checker_state(name, argument_names) - result = self.run_check(check, argument_names) - if result is not None: - (offset, text) = result - self.report_error(self.line_number, offset, text, check) - if text[:4] == 'E101': - self.indent_char = line[0] - - def build_tokens_line(self): - """Build a logical line from tokens.""" - logical = [] - comments = [] - length = 0 - prev_row = prev_col = mapping = None - for token_type, text, start, end, line in self.tokens: - if token_type in SKIP_TOKENS: - continue - if not mapping: - mapping = [(0, start)] - if token_type == tokenize.COMMENT: - comments.append(text) - continue - if token_type == tokenize.STRING: - text = mute_string(text) - if prev_row: - (start_row, start_col) = start - if prev_row != start_row: # different row - prev_text = self.lines[prev_row - 1][prev_col - 1] - if prev_text == ',' or (prev_text not in '{[(' and - text not in '}])'): - text = ' ' + text - elif prev_col != start_col: # different column - text = line[prev_col:start_col] + text - logical.append(text) - length += len(text) - mapping.append((length, end)) - (prev_row, prev_col) = end - self.logical_line = ''.join(logical) - self.noqa = comments and noqa(''.join(comments)) - return mapping - - def check_logical(self): - """Build a line from tokens and run all logical checks on it.""" - self.report.increment_logical_line() - mapping = self.build_tokens_line() - - if not mapping: - return - - (start_row, start_col) = mapping[0][1] - start_line = self.lines[start_row - 1] - self.indent_level = expand_indent(start_line[:start_col]) - if self.blank_before < self.blank_lines: - self.blank_before = self.blank_lines - if self.verbose >= 2: - print(self.logical_line[:80].rstrip()) - for name, check, argument_names in self._logical_checks: - if self.verbose >= 4: - print(' ' + name) - self.init_checker_state(name, argument_names) - for offset, text in self.run_check(check, argument_names) or (): - if not isinstance(offset, tuple): - for token_offset, pos in mapping: - if offset <= token_offset: - break - offset = (pos[0], pos[1] + offset - token_offset) - self.report_error(offset[0], offset[1], text, check) - if self.logical_line: - self.previous_indent_level = self.indent_level - self.previous_logical = self.logical_line - self.blank_lines = 0 - self.tokens = [] - - def check_ast(self): - """Build the file's AST and run all AST checks.""" - try: - tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) - except (ValueError, SyntaxError, TypeError): - return self.report_invalid_syntax() - for name, cls, __ in self._ast_checks: - checker = cls(tree, self.filename) - for lineno, offset, text, check in checker.run(): - if not self.lines or not noqa(self.lines[lineno - 1]): - self.report_error(lineno, offset, text, check) - - def generate_tokens(self): - """Tokenize the file, run physical line checks and yield tokens.""" - if self._io_error: - self.report_error(1, 0, 'E902 %s' % self._io_error, readlines) - tokengen = tokenize.generate_tokens(self.readline) - try: - for token in tokengen: - if token[2][0] > self.total_lines: - return - self.maybe_check_physical(token) - yield token - except (SyntaxError, tokenize.TokenError): - self.report_invalid_syntax() - - def maybe_check_physical(self, token): - """If appropriate (based on token), check current physical line(s).""" - # Called after every token, but act only on end of line. - if _is_eol_token(token): - # Obviously, a newline token ends a single physical line. - self.check_physical(token[4]) - elif token[0] == tokenize.STRING and '\n' in token[1]: - # Less obviously, a string that contains newlines is a - # multiline string, either triple-quoted or with internal - # newlines backslash-escaped. Check every physical line in the - # string *except* for the last one: its newline is outside of - # the multiline string, so we consider it a regular physical - # line, and will check it like any other physical line. - # - # Subtleties: - # - we don't *completely* ignore the last line; if it contains - # the magical "# noqa" comment, we disable all physical - # checks for the entire multiline string - # - have to wind self.line_number back because initially it - # points to the last line of the string, and we want - # check_physical() to give accurate feedback - if noqa(token[4]): - return - self.multiline = True - self.line_number = token[2][0] - for line in token[1].split('\n')[:-1]: - self.check_physical(line + '\n') - self.line_number += 1 - self.multiline = False - - def check_all(self, expected=None, line_offset=0): - """Run all checks on the input file.""" - self.report.init_file(self.filename, self.lines, expected, line_offset) - self.total_lines = len(self.lines) - if self._ast_checks: - self.check_ast() - self.line_number = 0 - self.indent_char = None - self.indent_level = self.previous_indent_level = 0 - self.previous_logical = '' - self.tokens = [] - self.blank_lines = self.blank_before = 0 - parens = 0 - for token in self.generate_tokens(): - self.tokens.append(token) - token_type, text = token[0:2] - if self.verbose >= 3: - if token[2][0] == token[3][0]: - pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) - else: - pos = 'l.%s' % token[3][0] - print('l.%s\t%s\t%s\t%r' % - (token[2][0], pos, tokenize.tok_name[token[0]], text)) - if token_type == tokenize.OP: - if text in '([{': - parens += 1 - elif text in '}])': - parens -= 1 - elif not parens: - if token_type in NEWLINE: - if token_type == tokenize.NEWLINE: - self.check_logical() - self.blank_before = 0 - elif len(self.tokens) == 1: - # The physical line contains only this token. - self.blank_lines += 1 - del self.tokens[0] - else: - self.check_logical() - elif COMMENT_WITH_NL and token_type == tokenize.COMMENT: - if len(self.tokens) == 1: - # The comment also ends a physical line - token = list(token) - token[1] = text.rstrip('\r\n') - token[3] = (token[2][0], token[2][1] + len(token[1])) - self.tokens = [tuple(token)] - self.check_logical() - if self.tokens: - self.check_physical(self.lines[-1]) - self.check_logical() - return self.report.get_file_results() - - -class BaseReport(object): - """Collect the results of the checks.""" - - print_filename = False - - def __init__(self, options): - self._benchmark_keys = options.benchmark_keys - self._ignore_code = options.ignore_code - # Results - self.elapsed = 0 - self.total_errors = 0 - self.counters = dict.fromkeys(self._benchmark_keys, 0) - self.messages = {} - - def start(self): - """Start the timer.""" - self._start_time = time.time() - - def stop(self): - """Stop the timer.""" - self.elapsed = time.time() - self._start_time - - def init_file(self, filename, lines, expected, line_offset): - """Signal a new file.""" - self.filename = filename - self.lines = lines - self.expected = expected or () - self.line_offset = line_offset - self.file_errors = 0 - self.counters['files'] += 1 - self.counters['physical lines'] += len(lines) - - def increment_logical_line(self): - """Signal a new logical line.""" - self.counters['logical lines'] += 1 - - def error(self, line_number, offset, text, check): - """Report an error, according to options.""" - code = text[:4] - if self._ignore_code(code): - return - if code in self.counters: - self.counters[code] += 1 - else: - self.counters[code] = 1 - self.messages[code] = text[5:] - # Don't care about expected errors or warnings - if code in self.expected: - return - if self.print_filename and not self.file_errors: - print(self.filename) - self.file_errors += 1 - self.total_errors += 1 - return code - - def get_file_results(self): - """Return the count of errors and warnings for this file.""" - return self.file_errors - - def get_count(self, prefix=''): - """Return the total count of errors and warnings.""" - return sum([self.counters[key] - for key in self.messages if key.startswith(prefix)]) - - def get_statistics(self, prefix=''): - """Get statistics for message codes that start with the prefix. - - prefix='' matches all errors and warnings - prefix='E' matches all errors - prefix='W' matches all warnings - prefix='E4' matches all errors that have to do with imports - """ - return ['%-7s %s %s' % (self.counters[key], key, self.messages[key]) - for key in sorted(self.messages) if key.startswith(prefix)] - - def print_statistics(self, prefix=''): - """Print overall statistics (number of errors and warnings).""" - for line in self.get_statistics(prefix): - print(line) - - def print_benchmark(self): - """Print benchmark numbers.""" - print('%-7.2f %s' % (self.elapsed, 'seconds elapsed')) - if self.elapsed: - for key in self._benchmark_keys: - print('%-7d %s per second (%d total)' % - (self.counters[key] / self.elapsed, key, - self.counters[key])) - - -class FileReport(BaseReport): - """Collect the results of the checks and print only the filenames.""" - print_filename = True - - -class StandardReport(BaseReport): - """Collect and print the results of the checks.""" - - def __init__(self, options): - super(StandardReport, self).__init__(options) - self._fmt = REPORT_FORMAT.get(options.format.lower(), - options.format) - self._repeat = options.repeat - self._show_source = options.show_source - self._show_pep8 = options.show_pep8 - - def init_file(self, filename, lines, expected, line_offset): - """Signal a new file.""" - self._deferred_print = [] - return super(StandardReport, self).init_file( - filename, lines, expected, line_offset) - - def error(self, line_number, offset, text, check): - """Report an error, according to options.""" - code = super(StandardReport, self).error(line_number, offset, - text, check) - if code and (self.counters[code] == 1 or self._repeat): - self._deferred_print.append( - (line_number, offset, code, text[5:], check.__doc__)) - return code - - def get_file_results(self): - """Print the result and return the overall count for this file.""" - self._deferred_print.sort() - for line_number, offset, code, text, doc in self._deferred_print: - print(self._fmt % { - 'path': self.filename, - 'row': self.line_offset + line_number, 'col': offset + 1, - 'code': code, 'text': text, - }) - if self._show_source: - if line_number > len(self.lines): - line = '' - else: - line = self.lines[line_number - 1] - print(line.rstrip()) - print(re.sub(r'\S', ' ', line[:offset]) + '^') - if self._show_pep8 and doc: - print(' ' + doc.strip()) - - # stdout is block buffered when not stdout.isatty(). - # line can be broken where buffer boundary since other processes - # write to same file. - # flush() after print() to avoid buffer boundary. - # Typical buffer size is 8192. line written safely when - # len(line) < 8192. - sys.stdout.flush() - return self.file_errors - - -class DiffReport(StandardReport): - """Collect and print the results for the changed lines only.""" - - def __init__(self, options): - super(DiffReport, self).__init__(options) - self._selected = options.selected_lines - - def error(self, line_number, offset, text, check): - if line_number not in self._selected[self.filename]: - return - return super(DiffReport, self).error(line_number, offset, text, check) - - -class StyleGuide(object): - """Initialize a PEP-8 instance with few options.""" - - def __init__(self, *args, **kwargs): - # build options from the command line - self.checker_class = kwargs.pop('checker_class', Checker) - parse_argv = kwargs.pop('parse_argv', False) - config_file = kwargs.pop('config_file', False) - parser = kwargs.pop('parser', None) - # build options from dict - options_dict = dict(*args, **kwargs) - arglist = None if parse_argv else options_dict.get('paths', None) - options, self.paths = process_options( - arglist, parse_argv, config_file, parser) - if options_dict: - options.__dict__.update(options_dict) - if 'paths' in options_dict: - self.paths = options_dict['paths'] - - self.runner = self.input_file - self.options = options - - if not options.reporter: - options.reporter = BaseReport if options.quiet else StandardReport - - options.select = tuple(options.select or ()) - if not (options.select or options.ignore or - options.testsuite or options.doctest) and DEFAULT_IGNORE: - # The default choice: ignore controversial checks - options.ignore = tuple(DEFAULT_IGNORE.split(',')) - else: - # Ignore all checks which are not explicitly selected - options.ignore = ('',) if options.select else tuple(options.ignore) - options.benchmark_keys = BENCHMARK_KEYS[:] - options.ignore_code = self.ignore_code - options.physical_checks = self.get_checks('physical_line') - options.logical_checks = self.get_checks('logical_line') - options.ast_checks = self.get_checks('tree') - self.init_report() - - def init_report(self, reporter=None): - """Initialize the report instance.""" - self.options.report = (reporter or self.options.reporter)(self.options) - return self.options.report - - def check_files(self, paths=None): - """Run all checks on the paths.""" - if paths is None: - paths = self.paths - report = self.options.report - runner = self.runner - report.start() - try: - for path in paths: - if os.path.isdir(path): - self.input_dir(path) - elif not self.excluded(path): - runner(path) - except KeyboardInterrupt: - print('... stopped') - report.stop() - return report - - def input_file(self, filename, lines=None, expected=None, line_offset=0): - """Run all checks on a Python source file.""" - if self.options.verbose: - print('checking %s' % filename) - fchecker = self.checker_class( - filename, lines=lines, options=self.options) - return fchecker.check_all(expected=expected, line_offset=line_offset) - - def input_dir(self, dirname): - """Check all files in this directory and all subdirectories.""" - dirname = dirname.rstrip('/') - if self.excluded(dirname): - return 0 - counters = self.options.report.counters - verbose = self.options.verbose - filepatterns = self.options.filename - runner = self.runner - for root, dirs, files in os.walk(dirname): - if verbose: - print('directory ' + root) - counters['directories'] += 1 - for subdir in sorted(dirs): - if self.excluded(subdir, root): - dirs.remove(subdir) - for filename in sorted(files): - # contain a pattern that matches? - if ((filename_match(filename, filepatterns) and - not self.excluded(filename, root))): - runner(os.path.join(root, filename)) - - def excluded(self, filename, parent=None): - """Check if the file should be excluded. - - Check if 'options.exclude' contains a pattern that matches filename. - """ - if not self.options.exclude: - return False - basename = os.path.basename(filename) - if filename_match(basename, self.options.exclude): - return True - if parent: - filename = os.path.join(parent, filename) - filename = os.path.abspath(filename) - return filename_match(filename, self.options.exclude) - - def ignore_code(self, code): - """Check if the error code should be ignored. - - If 'options.select' contains a prefix of the error code, - return False. Else, if 'options.ignore' contains a prefix of - the error code, return True. - """ - if len(code) < 4 and any(s.startswith(code) - for s in self.options.select): - return False - return (code.startswith(self.options.ignore) and - not code.startswith(self.options.select)) - - def get_checks(self, argument_name): - """Get all the checks for this category. - - Find all globally visible functions where the first argument name - starts with argument_name and which contain selected tests. - """ - checks = [] - for check, attrs in _checks[argument_name].items(): - (codes, args) = attrs - if any(not (code and self.ignore_code(code)) for code in codes): - checks.append((check.__name__, check, args)) - return sorted(checks) - - -def get_parser(prog='pep8', version=__version__): - parser = OptionParser(prog=prog, version=version, - usage="%prog [options] input ...") - parser.config_options = [ - 'exclude', 'filename', 'select', 'ignore', 'max-line-length', - 'hang-closing', 'count', 'format', 'quiet', 'show-pep8', - 'show-source', 'statistics', 'verbose'] - parser.add_option('-v', '--verbose', default=0, action='count', - help="print status messages, or debug with -vv") - parser.add_option('-q', '--quiet', default=0, action='count', - help="report only file names, or nothing with -qq") - parser.add_option('-r', '--repeat', default=True, action='store_true', - help="(obsolete) show all occurrences of the same error") - parser.add_option('--first', action='store_false', dest='repeat', - help="show first occurrence of each error") - parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE, - help="exclude files or directories which match these " - "comma separated patterns (default: %default)") - parser.add_option('--filename', metavar='patterns', default='*.py', - help="when parsing directories, only check filenames " - "matching these comma separated patterns " - "(default: %default)") - parser.add_option('--select', metavar='errors', default='', - help="select errors and warnings (e.g. E,W6)") - parser.add_option('--ignore', metavar='errors', default='', - help="skip errors and warnings (e.g. E4,W) " - "(default: %s)" % DEFAULT_IGNORE) - parser.add_option('--show-source', action='store_true', - help="show source code for each error") - parser.add_option('--show-pep8', action='store_true', - help="show text of PEP 8 for each error " - "(implies --first)") - parser.add_option('--statistics', action='store_true', - help="count errors and warnings") - parser.add_option('--count', action='store_true', - help="print total number of errors and warnings " - "to standard error and set exit code to 1 if " - "total is not null") - parser.add_option('--max-line-length', type='int', metavar='n', - default=MAX_LINE_LENGTH, - help="set maximum allowed line length " - "(default: %default)") - parser.add_option('--hang-closing', action='store_true', - help="hang closing bracket instead of matching " - "indentation of opening bracket's line") - parser.add_option('--format', metavar='format', default='default', - help="set the error format [default|pylint|]") - parser.add_option('--diff', action='store_true', - help="report changes only within line number ranges in " - "the unified diff received on STDIN") - group = parser.add_option_group("Testing Options") - if os.path.exists(TESTSUITE_PATH): - group.add_option('--testsuite', metavar='dir', - help="run regression tests from dir") - group.add_option('--doctest', action='store_true', - help="run doctest on myself") - group.add_option('--benchmark', action='store_true', - help="measure processing speed") - return parser - - -def read_config(options, args, arglist, parser): - """Read and parse configurations - - If a config file is specified on the command line with the "--config" - option, then only it is used for configuration. - - Otherwise, the user configuration (~/.config/pep8) and any local - configurations in the current directory or above will be merged together - (in that order) using the read method of ConfigParser. - """ - config = RawConfigParser() - - cli_conf = options.config - - local_dir = os.curdir - - if USER_CONFIG and os.path.isfile(USER_CONFIG): - if options.verbose: - print('user configuration: %s' % USER_CONFIG) - config.read(USER_CONFIG) - - parent = tail = args and os.path.abspath(os.path.commonprefix(args)) - while tail: - if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG): - local_dir = parent - if options.verbose: - print('local configuration: in %s' % parent) - break - (parent, tail) = os.path.split(parent) - - if cli_conf and os.path.isfile(cli_conf): - if options.verbose: - print('cli configuration: %s' % cli_conf) - config.read(cli_conf) - - pep8_section = parser.prog - if config.has_section(pep8_section): - option_list = dict([(o.dest, o.type or o.action) - for o in parser.option_list]) - - # First, read the default values - (new_options, __) = parser.parse_args([]) - - # Second, parse the configuration - for opt in config.options(pep8_section): - if opt.replace('_', '-') not in parser.config_options: - print(" unknown option '%s' ignored" % opt) - continue - if options.verbose > 1: - print(" %s = %s" % (opt, config.get(pep8_section, opt))) - normalized_opt = opt.replace('-', '_') - opt_type = option_list[normalized_opt] - if opt_type in ('int', 'count'): - value = config.getint(pep8_section, opt) - elif opt_type == 'string': - value = config.get(pep8_section, opt) - if normalized_opt == 'exclude': - value = normalize_paths(value, local_dir) - else: - assert opt_type in ('store_true', 'store_false') - value = config.getboolean(pep8_section, opt) - setattr(new_options, normalized_opt, value) - - # Third, overwrite with the command-line options - (options, __) = parser.parse_args(arglist, values=new_options) - options.doctest = options.testsuite = False - return options - - -def process_options(arglist=None, parse_argv=False, config_file=None, - parser=None): - """Process options passed either via arglist or via command line args. - - Passing in the ``config_file`` parameter allows other tools, such as flake8 - to specify their own options to be processed in pep8. - """ - if not parser: - parser = get_parser() - if not parser.has_option('--config'): - group = parser.add_option_group("Configuration", description=( - "The project options are read from the [%s] section of the " - "tox.ini file or the setup.cfg file located in any parent folder " - "of the path(s) being processed. Allowed options are: %s." % - (parser.prog, ', '.join(parser.config_options)))) - group.add_option('--config', metavar='path', default=config_file, - help="user config file location") - # Don't read the command line if the module is used as a library. - if not arglist and not parse_argv: - arglist = [] - # If parse_argv is True and arglist is None, arguments are - # parsed from the command line (sys.argv) - (options, args) = parser.parse_args(arglist) - options.reporter = None - - if options.ensure_value('testsuite', False): - args.append(options.testsuite) - elif not options.ensure_value('doctest', False): - if parse_argv and not args: - if options.diff or any(os.path.exists(name) - for name in PROJECT_CONFIG): - args = ['.'] - else: - parser.error('input not specified') - options = read_config(options, args, arglist, parser) - options.reporter = parse_argv and options.quiet == 1 and FileReport - - options.filename = _parse_multi_options(options.filename) - options.exclude = normalize_paths(options.exclude) - options.select = _parse_multi_options(options.select) - options.ignore = _parse_multi_options(options.ignore) - - if options.diff: - options.reporter = DiffReport - stdin = stdin_get_value() - options.selected_lines = parse_udiff(stdin, options.filename, args[0]) - args = sorted(options.selected_lines) - - return options, args - - -def _parse_multi_options(options, split_token=','): - r"""Split and strip and discard empties. - - Turns the following: - - A, - B, - - into ["A", "B"] - """ - if options: - return [o.strip() for o in options.split(split_token) if o.strip()] - else: - return options - - -def _main(): - """Parse options and run checks on Python source.""" - import signal - - # Handle "Broken pipe" gracefully - try: - signal.signal(signal.SIGPIPE, lambda signum, frame: sys.exit(1)) - except AttributeError: - pass # not supported on Windows - - pep8style = StyleGuide(parse_argv=True) - options = pep8style.options - - if options.doctest or options.testsuite: - from testsuite.support import run_tests - report = run_tests(pep8style) - else: - report = pep8style.check_files() - - if options.statistics: - report.print_statistics() - - if options.benchmark: - report.print_benchmark() - - if options.testsuite and not options.quiet: - report.print_results() - - if report.total_errors: - if options.count: - sys.stderr.write(str(report.total_errors) + '\n') - sys.exit(1) - -if __name__ == '__main__': - _main() diff --git a/Lib/site.py b/Lib/site.py index 29b19bb..56ba709 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -623,59 +623,3 @@ def _script(): if __name__ == '__main__': _script() - - -def no_chocolate(): - import io - import pep8 - import random - import shutil - import tokenize - - _builtin_compile = builtins.compile - words = ('chocolate', 'glory', 'fun', 'spam', 'love', 'guts') - pep8style = pep8.StyleGuide() - - def compile_pep8(source, filename, mode, **kw): - name = os.path.splitext(os.path.basename(filename))[0] - if not name.endswith('_noqa'): - bio = io.BytesIO(source) - encoding = tokenize.detect_encoding(bio.readline)[0] - lines = source.decode(encoding).splitlines(True) - - report = pep8.StandardReport(options=pep8style.options) - checker = pep8.Checker(filename, lines, - report=report, - options=pep8style.options) - checker.check_all() - if report.total_errors: - word = random.choice(words) - raise ImportError("no pep8, no %s" % word) - return _builtin_compile(source, filename, mode, **kw) - - builtins.compile = compile_pep8 - - # remove precompiled .pyc created during the bootstrap, - # to run PEP 8 checks on .py files - libdir_cache = os.path.join(os.path.dirname(__file__), '__pycache__') - try: - shutil.rmtree(libdir_cache) - except: - pass - - for name in sorted(sys.modules): - # Minimum to be able to import modules - if name in {'builtins', 'importlib._bootstrap', - 'importlib._bootstrap_external', 'importlib', - 'importlib.machinery', '__main__', 'io', 'sys', 'site'}: - continue - del sys.modules[name] - - -try: - import _ssl -except ImportError: - # Python not bootstraped yet - pass -else: - no_chocolate() -- cgit v0.12 From dcfebb32e277a68b9c6582e6a0484e6dc24e9b66 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 1 Apr 2016 06:55:55 +0000 Subject: Issue #26676: Add missing XMLPullParser to ElementTree.__all__ --- Doc/whatsnew/3.6.rst | 3 ++- Lib/test/test_xml_etree.py | 6 ++++-- Lib/xml/etree/ElementTree.py | 2 +- Misc/NEWS | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 928d748..6799f69 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -460,7 +460,8 @@ Changes in the Python API * The following modules have had missing APIs added to their :attr:`__all__` attributes to match the documented APIs: :mod:`calendar`, :mod:`csv`, - :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, + :mod:`~xml.etree.ElementTree`, :mod:`enum`, + :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`optparse`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index bfd7215..a56f158 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -91,8 +91,6 @@ ENTITY_XML = """\ class ModuleTest(unittest.TestCase): - # TODO: this should be removed once we get rid of the global module vars - def test_sanity(self): # Import sanity. @@ -100,6 +98,10 @@ class ModuleTest(unittest.TestCase): from xml.etree import ElementInclude from xml.etree import ElementPath + def test_all(self): + names = ("xml.etree.ElementTree", "_elementtree") + support.check__all__(self, ET, names, blacklist=("HTML_EMPTY",)) + def serialize(elem, to_string=True, encoding='unicode', **options): if encoding != 'unicode': diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index b92fb52..50e42c8 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -85,7 +85,7 @@ __all__ = [ "TreeBuilder", "VERSION", "XML", "XMLID", - "XMLParser", + "XMLParser", "XMLPullParser", "register_namespace", ] diff --git a/Misc/NEWS b/Misc/NEWS index 6497701..8a7fac4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,8 @@ Core and Builtins Library ------- +- Issue #26676: Added missing XMLPullParser to ElementTree.__all__. + - Issue #22854: Change BufferedReader.writable() and BufferedWriter.readable() to always return False. -- cgit v0.12 From 50badad807abc5367359bd81a2a8051ff5cdce7e Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 3 Apr 2016 01:28:53 +0000 Subject: Issue #26586: Simple enhancements to BaseHTTPRequestHandler by Xiang Zhang --- Lib/http/server.py | 28 +++++++++++++--------------- Lib/test/test_httpservers.py | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index de6b531..f4ad260 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -137,7 +137,7 @@ class HTTPServer(socketserver.TCPServer): def server_bind(self): """Override server_bind to store the server name.""" socketserver.TCPServer.server_bind(self) - host, port = self.socket.getsockname()[:2] + host, port = self.server_address[:2] self.server_name = socket.getfqdn(host) self.server_port = port @@ -283,12 +283,9 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): words = requestline.split() if len(words) == 3: command, path, version = words - if version[:5] != 'HTTP/': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) - return False try: + if version[:5] != 'HTTP/': + raise ValueError base_version_number = version.split('/', 1)[1] version_number = base_version_number.split(".") # RFC 2145 section 3.1 says there can be only one "." and @@ -310,7 +307,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): if version_number >= (2, 0): self.send_error( HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, - "Invalid HTTP Version (%s)" % base_version_number) + "Invalid HTTP version (%s)" % base_version_number) return False elif len(words) == 2: command, path = words @@ -333,10 +330,11 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): try: self.headers = http.client.parse_headers(self.rfile, _class=self.MessageClass) - except http.client.LineTooLong: + except http.client.LineTooLong as err: self.send_error( - HTTPStatus.BAD_REQUEST, - "Line too long") + HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE, + "Line too long", + str(err)) return False except http.client.HTTPException as err: self.send_error( @@ -482,12 +480,12 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): def send_response_only(self, code, message=None): """Send the response header only.""" - if message is None: - if code in self.responses: - message = self.responses[code][0] - else: - message = '' if self.request_version != 'HTTP/0.9': + if message is None: + if code in self.responses: + message = self.responses[code][0] + else: + message = '' if not hasattr(self, '_headers_buffer'): self._headers_buffer = [] self._headers_buffer.append(("%s %d %s\r\n" % diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 98798ae..d9e4bf0 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -855,7 +855,7 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): # Issue #6791: same for headers result = self.send_typical_request( b'GET / HTTP/1.1\r\nX-Foo: bar' + b'r' * 65537 + b'\r\n\r\n') - self.assertEqual(result[0], b'HTTP/1.1 400 Line too long\r\n') + self.assertEqual(result[0], b'HTTP/1.1 431 Line too long\r\n') self.assertFalse(self.handler.get_called) self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') -- cgit v0.12 From 519f91215bd353c745946e5d892fd1f089dbcb84 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 3 Apr 2016 02:12:54 +0000 Subject: Issue #25951: Fix SSLSocket.sendall() to return None, by Aviv Palivoda --- Lib/ssl.py | 1 - Lib/test/test_ssl.py | 16 ++++++++++------ Misc/NEWS | 3 +++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 65ad38f..68db748 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -886,7 +886,6 @@ class SSLSocket(socket): while (count < amount): v = self.send(data[count:]) count += v - return amount else: return socket.sendall(self, data, flags) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e0f231c..00d437a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2709,12 +2709,13 @@ if _have_threads: count, addr = s.recvfrom_into(b) return b[:count] - # (name, method, whether to expect success, *args) + # (name, method, expect success?, *args, return value func) send_methods = [ - ('send', s.send, True, []), - ('sendto', s.sendto, False, ["some.address"]), - ('sendall', s.sendall, True, []), + ('send', s.send, True, [], len), + ('sendto', s.sendto, False, ["some.address"], len), + ('sendall', s.sendall, True, [], lambda x: None), ] + # (name, method, whether to expect success, *args) recv_methods = [ ('recv', s.recv, True, []), ('recvfrom', s.recvfrom, False, ["some.address"]), @@ -2723,10 +2724,13 @@ if _have_threads: ] data_prefix = "PREFIX_" - for meth_name, send_meth, expect_success, args in send_methods: + for (meth_name, send_meth, expect_success, args, + ret_val_meth) in send_methods: indata = (data_prefix + meth_name).encode('ascii') try: - send_meth(indata, *args) + ret = send_meth(indata, *args) + msg = "sending with {}".format(meth_name) + self.assertEqual(ret, ret_val_meth(indata), msg=msg) outdata = s.read() if outdata != indata.lower(): self.fail( diff --git a/Misc/NEWS b/Misc/NEWS index cba8466..f66014a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,9 @@ Core and Builtins Library ------- +- Issue #25951: Change SSLSocket.sendall() to return None, as explicitly + documented for plain socket objects. Patch by Aviv Palivoda. + - Issue #26586: In http.server, respond with "413 Request header fields too large" if there are too many header fields to parse, rather than killing the connection and raising an unhandled exception. Patch by Xiang Zhang. -- cgit v0.12 From 16ca06b8cb2426b540fdab75914d7cd0f715b7f0 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 4 Apr 2016 10:59:29 -0700 Subject: Add collections.Reversible. Patch by Ivan Levkivskyi. Fixes issue #25987. --- Doc/library/collections.abc.rst | 7 ++++++- Doc/library/typing.rst | 11 +++++------ Lib/_collections_abc.py | 23 +++++++++++++++++++++-- Lib/test/test_collections.py | 31 ++++++++++++++++++++++++++++--- Lib/test/test_functools.py | 2 +- 5 files changed, 61 insertions(+), 13 deletions(-) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index d9b93ad..608641b 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -40,12 +40,13 @@ ABC Inherits from Abstract Methods Mixin :class:`Hashable` ``__hash__`` :class:`Iterable` ``__iter__`` :class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__`` +:class:`Reversible` :class:`Iterable` ``__reversed__`` :class:`Generator` :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` :class:`Sized` ``__len__`` :class:`Callable` ``__call__`` :class:`Sequence` :class:`Sized`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Iterable`, ``__len__`` ``index``, and ``count`` + :class:`Reversible`, ``__len__`` ``index``, and ``count`` :class:`Container` :class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and @@ -107,6 +108,10 @@ ABC Inherits from Abstract Methods Mixin :meth:`~iterator.__next__` methods. See also the definition of :term:`iterator`. +.. class:: Reversible + + ABC for classes that provide the :meth:`__reversed__` method. + .. class:: Generator ABC for generator classes that implement the protocol defined in diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 12b5490..1bd4b09 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -351,6 +351,10 @@ The module defines the following classes, functions and decorators: A generic version of the :class:`collections.abc.Iterator`. +.. class:: Reversible(Iterable[T_co]) + + A generic version of the :class:`collections.abc.Reversible`. + .. class:: SupportsInt An ABC with one abstract method ``__int__``. @@ -369,11 +373,6 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__round__`` that is covariant in its return type. -.. class:: Reversible - - An ABC with one abstract method ``__reversed__`` returning - an ``Iterator[T_co]``. - .. class:: Container(Generic[T_co]) A generic version of :class:`collections.abc.Container`. @@ -394,7 +393,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.MutableMapping`. -.. class:: Sequence(Sized, Iterable[T_co], Container[T_co]) +.. class:: Sequence(Sized, Reversible[T_co], Container[T_co]) A generic version of :class:`collections.abc.Sequence`. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index f89bb6f..d337584 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -10,7 +10,7 @@ from abc import ABCMeta, abstractmethod import sys __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", - "Hashable", "Iterable", "Iterator", "Generator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", "Sized", "Container", "Callable", "Set", "MutableSet", "Mapping", "MutableMapping", @@ -240,6 +240,25 @@ Iterator.register(tuple_iterator) Iterator.register(zip_iterator) +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + return NotImplemented + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + for B in C.__mro__: + if "__reversed__" in B.__dict__: + if B.__dict__["__reversed__"] is not None: + return True + break + return NotImplemented + + class Generator(Iterator): __slots__ = () @@ -794,7 +813,7 @@ MutableMapping.register(dict) ### SEQUENCES ### -class Sequence(Sized, Iterable, Container): +class Sequence(Sized, Reversible, Container): """All the operations on a read-only sequence. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 4c32e09..4202462 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -20,7 +20,7 @@ from collections import UserDict, UserString, UserList from collections import ChainMap from collections import deque from collections.abc import Awaitable, Coroutine, AsyncIterator, AsyncIterable -from collections.abc import Hashable, Iterable, Iterator, Generator +from collections.abc import Hashable, Iterable, Iterator, Generator, Reversible from collections.abc import Sized, Container, Callable from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView @@ -689,6 +689,31 @@ class TestOneTrickPonyABCs(ABCTestCase): self.validate_abstract_methods(Iterable, '__iter__') self.validate_isinstance(Iterable, '__iter__') + def test_Reversible(self): + # Check some non-reversibles + non_samples = [None, 42, 3.14, 1j, dict(), set(), frozenset()] + for x in non_samples: + self.assertNotIsInstance(x, Reversible) + self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + # Check some reversibles + samples = [tuple(), list()] + for x in samples: + self.assertIsInstance(x, Reversible) + self.assertTrue(issubclass(type(x), Reversible), repr(type(x))) + # Check also Mapping, MutableMapping, and Sequence + self.assertTrue(issubclass(Sequence, Reversible), repr(Sequence)) + self.assertFalse(issubclass(Mapping, Reversible), repr(Mapping)) + self.assertFalse(issubclass(MutableMapping, Reversible), repr(MutableMapping)) + # Check direct subclassing + class R(Reversible): + def __iter__(self): + return iter(list()) + def __reversed__(self): + return iter(list()) + self.assertEqual(list(reversed(R())), []) + self.assertFalse(issubclass(float, R)) + self.validate_abstract_methods(Reversible, '__reversed__', '__iter__') + def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] for x in non_samples: @@ -842,14 +867,14 @@ class TestOneTrickPonyABCs(ABCTestCase): self.validate_isinstance(Callable, '__call__') def test_direct_subclassing(self): - for B in Hashable, Iterable, Iterator, Sized, Container, Callable: + for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C(B): pass self.assertTrue(issubclass(C, B)) self.assertFalse(issubclass(int, C)) def test_registration(self): - for B in Hashable, Iterable, Iterator, Sized, Container, Callable: + for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C: __hash__ = None # Make sure it isn't hashable by default self.assertFalse(issubclass(C, B), B.__name__) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 31930fc..a22d2e6 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1516,7 +1516,7 @@ class TestSingleDispatch(unittest.TestCase): m = mro(D, bases) self.assertEqual(m, [D, c.MutableSequence, c.Sequence, c.defaultdict, dict, c.MutableMapping, - c.Mapping, c.Sized, c.Iterable, c.Container, + c.Mapping, c.Sized, c.Reversible, c.Iterable, c.Container, object]) # Container and Callable are registered on different base classes and -- cgit v0.12 From 819399b2ab3cba99de540b07f6ec12b2777f6ec0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 22:17:52 +0300 Subject: Issue #26671: Enhanced path_converter. Exceptions raised during converting argument of correct type are no longer overridded with TypeError. Some error messages are now more detailed. --- Modules/posixmodule.c | 106 ++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9013888..e6704ac 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -669,21 +669,20 @@ _Py_Dev_Converter(PyObject *obj, void *p) #endif static int -_fd_converter(PyObject *o, int *p, const char *allowed) +_fd_converter(PyObject *o, int *p) { int overflow; long long_value; PyObject *index = PyNumber_Index(o); if (index == NULL) { - PyErr_Format(PyExc_TypeError, - "argument should be %s, not %.200s", - allowed, Py_TYPE(o)->tp_name); return 0; } + assert(PyLong_Check(index)); long_value = PyLong_AsLongAndOverflow(index, &overflow); Py_DECREF(index); + assert(!PyErr_Occurred()); if (overflow > 0 || long_value > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "fd is greater than maximum"); @@ -706,7 +705,15 @@ dir_fd_converter(PyObject *o, void *p) *(int *)p = DEFAULT_DIR_FD; return 1; } - return _fd_converter(o, (int *)p, "integer"); + else if (PyIndex_Check(o)) { + return _fd_converter(o, (int *)p); + } + else { + PyErr_Format(PyExc_TypeError, + "argument should be integer or None, not %.200s", + Py_TYPE(o)->tp_name); + return 0; + } } @@ -816,9 +823,10 @@ path_cleanup(path_t *path) { } static int -path_converter(PyObject *o, void *p) { +path_converter(PyObject *o, void *p) +{ path_t *path = (path_t *)p; - PyObject *unicode, *bytes; + PyObject *bytes; Py_ssize_t length; char *narrow; @@ -837,12 +845,7 @@ path_converter(PyObject *o, void *p) { /* ensure it's always safe to call path_cleanup() */ path->cleanup = NULL; - if (o == Py_None) { - if (!path->nullable) { - FORMAT_EXCEPTION(PyExc_TypeError, - "can't specify None for %s argument"); - return 0; - } + if ((o == Py_None) && path->nullable) { path->wide = NULL; path->narrow = NULL; path->length = 0; @@ -851,24 +854,20 @@ path_converter(PyObject *o, void *p) { return 1; } - unicode = PyUnicode_FromObject(o); - if (unicode) { + if (PyUnicode_Check(o)) { #ifdef MS_WINDOWS wchar_t *wide; - wide = PyUnicode_AsUnicodeAndSize(unicode, &length); + wide = PyUnicode_AsUnicodeAndSize(o, &length); if (!wide) { - Py_DECREF(unicode); return 0; } if (length > 32767) { FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - Py_DECREF(unicode); return 0; } if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character"); - Py_DECREF(unicode); + FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); return 0; } @@ -877,51 +876,46 @@ path_converter(PyObject *o, void *p) { path->length = length; path->object = o; path->fd = -1; - path->cleanup = unicode; - return Py_CLEANUP_SUPPORTED; + return 1; #else - int converted = PyUnicode_FSConverter(unicode, &bytes); - Py_DECREF(unicode); - if (!converted) - bytes = NULL; + if (!PyUnicode_FSConverter(o, &bytes)) { + return 0; + } #endif } - else { - PyErr_Clear(); - if (PyObject_CheckBuffer(o)) - bytes = PyBytes_FromObject(o); - else - bytes = NULL; + else if (PyObject_CheckBuffer(o)) { +# ifdef MS_WINDOWS + if (win32_warn_bytes_api()) { + return 0; + } +# endif + bytes = PyBytes_FromObject(o); if (!bytes) { - PyErr_Clear(); - if (path->allow_fd) { - int fd; - int result = _fd_converter(o, &fd, - "string, bytes or integer"); - if (result) { - path->wide = NULL; - path->narrow = NULL; - path->length = 0; - path->object = o; - path->fd = fd; - return result; - } - } + return 0; } } - - if (!bytes) { - if (!PyErr_Occurred()) - FORMAT_EXCEPTION(PyExc_TypeError, "illegal type for %s parameter"); - return 0; + else if (path->allow_fd && PyIndex_Check(o)) { + if (!_fd_converter(o, &path->fd)) { + return 0; + } + path->wide = NULL; + path->narrow = NULL; + path->length = 0; + path->object = o; + return 1; } - -#ifdef MS_WINDOWS - if (win32_warn_bytes_api()) { - Py_DECREF(bytes); + else { + PyErr_Format(PyExc_TypeError, "%s%s%s should be %s, not %.200s", + path->function_name ? path->function_name : "", + path->function_name ? ": " : "", + path->argument_name ? path->argument_name : "path", + path->allow_fd && path->nullable ? "string, bytes, integer or None" : + path->allow_fd ? "string, bytes or integer" : + path->nullable ? "string, bytes or None" : + "string or bytes", + Py_TYPE(o)->tp_name); return 0; } -#endif length = PyBytes_GET_SIZE(bytes); #ifdef MS_WINDOWS -- cgit v0.12 From 026110f0a2ce027a781e429ee9a0ed14f9b2bc4a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 22:55:31 +0300 Subject: Issue #26671: Fixed #ifdef indentation. --- Modules/posixmodule.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e6704ac..bc41d93 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -884,11 +884,11 @@ path_converter(PyObject *o, void *p) #endif } else if (PyObject_CheckBuffer(o)) { -# ifdef MS_WINDOWS +#ifdef MS_WINDOWS if (win32_warn_bytes_api()) { return 0; } -# endif +#endif bytes = PyBytes_FromObject(o); if (!bytes) { return 0; @@ -905,6 +905,30 @@ path_converter(PyObject *o, void *p) return 1; } else { + PyObject *pathattr; + _Py_IDENTIFIER(path); + + pathattr = _PyObject_GetAttrId(o, &PyId_path); + if (pathattr == NULL) { + PyErr_Clear(); + } + else if (PyUnicode_Check(pathattr) || PyObject_CheckBuffer(pathattr)) { + if (!path_converter(pathattr, path)) { + Py_DECREF(pathattr); + return 0; + } + if (path->cleanup == NULL) { + path->cleanup = pathattr; + } + else { + Py_DECREF(pathattr); + } + return Py_CLEANUP_SUPPORTED; + } + else { + Py_DECREF(pathattr); + } + PyErr_Format(PyExc_TypeError, "%s%s%s should be %s, not %.200s", path->function_name ? path->function_name : "", path->function_name ? ": " : "", -- cgit v0.12 From aaf553bac4124cda748b68caf7d501fd5b574046 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 23:02:25 +0300 Subject: Backed out changeset 8dc144e47252 --- Modules/posixmodule.c | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bc41d93..e6704ac 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -884,11 +884,11 @@ path_converter(PyObject *o, void *p) #endif } else if (PyObject_CheckBuffer(o)) { -#ifdef MS_WINDOWS +# ifdef MS_WINDOWS if (win32_warn_bytes_api()) { return 0; } -#endif +# endif bytes = PyBytes_FromObject(o); if (!bytes) { return 0; @@ -905,30 +905,6 @@ path_converter(PyObject *o, void *p) return 1; } else { - PyObject *pathattr; - _Py_IDENTIFIER(path); - - pathattr = _PyObject_GetAttrId(o, &PyId_path); - if (pathattr == NULL) { - PyErr_Clear(); - } - else if (PyUnicode_Check(pathattr) || PyObject_CheckBuffer(pathattr)) { - if (!path_converter(pathattr, path)) { - Py_DECREF(pathattr); - return 0; - } - if (path->cleanup == NULL) { - path->cleanup = pathattr; - } - else { - Py_DECREF(pathattr); - } - return Py_CLEANUP_SUPPORTED; - } - else { - Py_DECREF(pathattr); - } - PyErr_Format(PyExc_TypeError, "%s%s%s should be %s, not %.200s", path->function_name ? path->function_name : "", path->function_name ? ": " : "", -- cgit v0.12 From 3291d85a2f6c6791ff1d8d1111b6bc46bf09d9e4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 23:02:46 +0300 Subject: Issue #26671: Fixed #ifdef indentation. --- Modules/posixmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e6704ac..a87bbbd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -884,11 +884,11 @@ path_converter(PyObject *o, void *p) #endif } else if (PyObject_CheckBuffer(o)) { -# ifdef MS_WINDOWS +#ifdef MS_WINDOWS if (win32_warn_bytes_api()) { return 0; } -# endif +#endif bytes = PyBytes_FromObject(o); if (!bytes) { return 0; -- cgit v0.12 From 7155b881f249dc9c72cdcea53b2a945798783781 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 8 Apr 2016 08:48:20 +0300 Subject: Issue #26671: Fixed tests for changed error messages. --- Lib/test/test_posix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 2a59c38..28cdd90 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -411,7 +411,7 @@ class PosixTester(unittest.TestCase): self.assertTrue(posix.stat(bytearray(os.fsencode(support.TESTFN)))) self.assertRaisesRegex(TypeError, - 'can\'t specify None for path argument', + 'should be string, bytes or integer, not', posix.stat, None) self.assertRaisesRegex(TypeError, 'should be string, bytes or integer, not', @@ -863,9 +863,9 @@ class PosixTester(unittest.TestCase): self.assertEqual(s1, s2) s2 = posix.stat(support.TESTFN, dir_fd=None) self.assertEqual(s1, s2) - self.assertRaisesRegex(TypeError, 'should be integer, not', + self.assertRaisesRegex(TypeError, 'should be integer or None, not', posix.stat, support.TESTFN, dir_fd=posix.getcwd()) - self.assertRaisesRegex(TypeError, 'should be integer, not', + self.assertRaisesRegex(TypeError, 'should be integer or None, not', posix.stat, support.TESTFN, dir_fd=float(f)) self.assertRaises(OverflowError, posix.stat, support.TESTFN, dir_fd=10**20) -- cgit v0.12 From 9e080e0e741dd70cf86500f848eee19cf8b29efa Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 12:15:27 -0700 Subject: Issue #25609: Introduce contextlib.AbstractContextManager and typing.ContextManager. --- Doc/library/contextlib.rst | 16 ++++++++++++++-- Doc/library/typing.rst | 12 +++++++++--- Doc/whatsnew/3.6.rst | 26 +++++++++++++++++++++++--- Lib/contextlib.py | 42 +++++++++++++++++++++++++++++++----------- Lib/test/test_contextlib.py | 33 +++++++++++++++++++++++++++++++++ Lib/test/test_typing.py | 18 ++++++++++++++++++ Lib/typing.py | 7 +++++++ Misc/NEWS | 3 +++ 8 files changed, 138 insertions(+), 19 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index c112241..7876e7a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -18,6 +18,18 @@ Utilities Functions and classes provided: +.. class:: AbstractContextManager + + An abstract base class for classes that implement + :meth:`object.__enter__` and :meth:`object.__exit__`. A default + implementation for :meth:`object.__enter__` is provided which returns + ``self`` while :meth:`object.__exit__` is an abstract method which by default + returns ``None``. See also the definition of :ref:`typecontextmanager`. + + .. versionadded:: 3.6 + + + .. decorator:: contextmanager This function is a :term:`decorator` that can be used to define a factory @@ -447,9 +459,9 @@ Here's an example of doing this for a context manager that accepts resource acquisition and release functions, along with an optional validation function, and maps them to the context management protocol:: - from contextlib import contextmanager, ExitStack + from contextlib import contextmanager, AbstractContextManager, ExitStack - class ResourceManager: + class ResourceManager(AbstractContextManager): def __init__(self, acquire_resource, release_resource, check_resource_ok=None): self.acquire_resource = acquire_resource diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 1bd4b09..f609a51 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -345,15 +345,15 @@ The module defines the following classes, functions and decorators: .. class:: Iterable(Generic[T_co]) - A generic version of the :class:`collections.abc.Iterable`. + A generic version of :class:`collections.abc.Iterable`. .. class:: Iterator(Iterable[T_co]) - A generic version of the :class:`collections.abc.Iterator`. + A generic version of :class:`collections.abc.Iterator`. .. class:: Reversible(Iterable[T_co]) - A generic version of the :class:`collections.abc.Reversible`. + A generic version of :class:`collections.abc.Reversible`. .. class:: SupportsInt @@ -448,6 +448,12 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.ValuesView`. +.. class:: ContextManager(Generic[T_co]) + + A generic version of :class:`contextlib.AbstractContextManager`. + + .. versionadded:: 3.6 + .. class:: Dict(dict, MutableMapping[KT, VT]) A generic version of :class:`dict`. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 6799f69..555814e 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -190,6 +190,18 @@ New Modules Improved Modules ================ +contextlib +---------- + +The :class:`contextlib.AbstractContextManager` class has been added to +provide an abstract base class for context managers. It provides a +sensible default implementation for `__enter__()` which returns +`self` and leaves `__exit__()` an abstract method. A matching +class has been added to the :mod:`typing` module as +:class:`typing.ContextManager`. +(Contributed by Brett Cannon in :issue:`25609`.) + + datetime -------- @@ -246,6 +258,14 @@ telnetlib Stéphane Wirtel in :issue:`25485`). +typing +------ + +The :class:`typing.ContextManager` class has been added for +representing :class:`contextlib.AbstractContextManager`. +(Contributed by Brett Cannon in :issue:`25609`.) + + unittest.mock ------------- @@ -372,9 +392,9 @@ become proper keywords in Python 3.7. Deprecated Python modules, functions and methods ------------------------------------------------ -* :meth:`importlib.machinery.SourceFileLoader` and - :meth:`importlib.machinery.SourcelessFileLoader` are now deprecated. They - were the only remaining implementations of +* :meth:`importlib.machinery.SourceFileLoader.load_module` and + :meth:`importlib.machinery.SourcelessFileLoader.load_module` are now + deprecated. They were the only remaining implementations of :meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not been deprecated in previous versions of Python in favour of :meth:`importlib.abc.Loader.exec_module`. diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 5377987..98903bc 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,11 +1,34 @@ """Utilities for with-statement contexts. See PEP 343.""" - +import abc import sys from collections import deque from functools import wraps -__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack", - "redirect_stdout", "redirect_stderr", "suppress"] +__all__ = ["contextmanager", "closing", "AbstractContextManager", + "ContextDecorator", "ExitStack", "redirect_stdout", + "redirect_stderr", "suppress"] + + +class AbstractContextManager(abc.ABC): + + """An abstract base class for context managers.""" + + def __enter__(self): + """Return `self` upon entering the runtime context.""" + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AbstractContextManager: + if (any("__enter__" in B.__dict__ for B in C.__mro__) and + any("__exit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented class ContextDecorator(object): @@ -31,7 +54,7 @@ class ContextDecorator(object): return inner -class _GeneratorContextManager(ContextDecorator): +class _GeneratorContextManager(ContextDecorator, AbstractContextManager): """Helper for @contextmanager decorator.""" def __init__(self, func, args, kwds): @@ -134,7 +157,7 @@ def contextmanager(func): return helper -class closing(object): +class closing(AbstractContextManager): """Context to automatically close something at the end of a block. Code like this: @@ -159,7 +182,7 @@ class closing(object): self.thing.close() -class _RedirectStream: +class _RedirectStream(AbstractContextManager): _stream = None @@ -199,7 +222,7 @@ class redirect_stderr(_RedirectStream): _stream = "stderr" -class suppress: +class suppress(AbstractContextManager): """Context manager to suppress specified exceptions After the exception is suppressed, execution proceeds with the next @@ -230,7 +253,7 @@ class suppress: # Inspired by discussions on http://bugs.python.org/issue13585 -class ExitStack(object): +class ExitStack(AbstractContextManager): """Context manager for dynamic management of a stack of exit callbacks For example: @@ -309,9 +332,6 @@ class ExitStack(object): """Immediately unwind the context stack""" self.__exit__(None, None, None) - def __enter__(self): - return self - def __exit__(self, *exc_details): received_exc = exc_details[0] is not None diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 04fc875..5c8bc98 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -12,6 +12,39 @@ except ImportError: threading = None +class TestAbstractContextManager(unittest.TestCase): + + def test_enter(self): + class DefaultEnter(AbstractContextManager): + def __exit__(self, *args): + super().__exit__(*args) + + manager = DefaultEnter() + self.assertIs(manager.__enter__(), manager) + + def test_exit_is_abstract(self): + class MissingExit(AbstractContextManager): + pass + + with self.assertRaises(TypeError): + MissingExit() + + def test_structural_subclassing(self): + class ManagerFromScratch: + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): + return None + + self.assertTrue(issubclass(ManagerFromScratch, AbstractContextManager)) + + class DefaultEnter(AbstractContextManager): + def __exit__(self, *args): + super().__exit__(*args) + + self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + + class ContextManagerTestCase(unittest.TestCase): def test_contextmanager_plain(self): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f1c6e12..a360824 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,3 +1,4 @@ +import contextlib import pickle import re import sys @@ -1309,6 +1310,21 @@ class CollectionsAbcTests(TestCase): assert len(MMB[KT, VT]()) == 0 +class OtherABCTests(TestCase): + + @skipUnless(hasattr(typing, 'ContextManager'), + 'requires typing.ContextManager') + def test_contextmanager(self): + @contextlib.contextmanager + def manager(): + yield 42 + + cm = manager() + assert isinstance(cm, typing.ContextManager) + assert isinstance(cm, typing.ContextManager[int]) + assert not isinstance(42, typing.ContextManager) + + class NamedTupleTests(TestCase): def test_basics(self): @@ -1447,6 +1463,8 @@ class AllTests(TestCase): assert 'ValuesView' in a assert 'cast' in a assert 'overload' in a + if hasattr(contextlib, 'AbstractContextManager'): + assert 'ContextManager' in a # Check that io and re are not exported. assert 'io' not in a assert 're' not in a diff --git a/Lib/typing.py b/Lib/typing.py index 6ead3c4..42a9ea3 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1,6 +1,7 @@ import abc from abc import abstractmethod, abstractproperty import collections +import contextlib import functools import re as stdlib_re # Avoid confusion with the re we export. import sys @@ -1530,6 +1531,12 @@ class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): pass +if hasattr(contextlib, 'AbstractContextManager'): + class ContextManager(Generic[T_co], extra=contextlib.AbstractContextManager): + __slots__ = () + __all__.append('ContextManager') + + class Dict(dict, MutableMapping[KT, VT]): def __new__(cls, *args, **kwds): diff --git a/Misc/NEWS b/Misc/NEWS index 9970636..56b1501 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,9 @@ Core and Builtins Library ------- +- Issue #25609: Introduce contextlib.AbstractContextManager and + typing.ContextManager. + - Issue #26709: Fixed Y2038 problem in loading binary PLists. - Issue #23735: Handle terminal resizing with Readline 6.3+ by installing our -- cgit v0.12 From 8bd092b50115560a777ad4f9fc656e5ef8c8c89e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 12:16:16 -0700 Subject: Normalize whitespace --- Lib/contextlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 98903bc..6cf112a 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -27,7 +27,7 @@ class AbstractContextManager(abc.ABC): if cls is AbstractContextManager: if (any("__enter__" in B.__dict__ for B in C.__mro__) and any("__exit__" in B.__dict__ for B in C.__mro__)): - return True + return True return NotImplemented -- cgit v0.12 From ef0138f42153d467edf8b0429c531f5204fe45dc Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 12:29:05 -0700 Subject: Issue #26668: Remove the redundant Lib/test/test_importlib/regrtest.py --- Lib/test/test_importlib/regrtest.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 Lib/test/test_importlib/regrtest.py diff --git a/Lib/test/test_importlib/regrtest.py b/Lib/test/test_importlib/regrtest.py deleted file mode 100644 index 98c815c..0000000 --- a/Lib/test/test_importlib/regrtest.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Run Python's standard test suite using importlib.__import__. - -Tests known to fail because of assumptions that importlib (properly) -invalidates are automatically skipped if the entire test suite is run. -Otherwise all command-line options valid for test.regrtest are also valid for -this script. - -""" -import importlib -import sys -from test import libregrtest - -if __name__ == '__main__': - __builtins__.__import__ = importlib.__import__ - sys.path_importer_cache.clear() - - libregrtest.main(quiet=True, verbose2=True) -- cgit v0.12 From 5f0507d8ab2fee70139ff147c5f7d472e8f4ba2b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 15:04:28 -0700 Subject: Issue #26587: Allow .pth files to specify file paths as well as directories. Thanks to Wolfgang Langner for the bug report and initial version of the patch. --- Doc/whatsnew/3.6.rst | 8 ++++++++ Lib/site.py | 18 +++++++++--------- Lib/test/test_site.py | 2 +- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 555814e..d217c4d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -251,6 +251,14 @@ Previously, names of properties and slots which were not yet created on an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) +site +---- + +When specifying paths to add to :attr:`sys.path` in a `.pth` file, +you may now specify file paths on top of directories (e.g. zip files). +(Contributed by Wolfgang Langner in :issue:`26587`). + + telnetlib --------- diff --git a/Lib/site.py b/Lib/site.py index 56ba709..b66123f 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -131,13 +131,13 @@ def removeduppaths(): def _init_pathinfo(): - """Return a set containing all existing directory entries from sys.path""" + """Return a set containing all existing file system items from sys.path.""" d = set() - for dir in sys.path: + for item in sys.path: try: - if os.path.isdir(dir): - dir, dircase = makepath(dir) - d.add(dircase) + if os.path.exists(item): + _, itemcase = makepath(item) + d.add(itemcase) except TypeError: continue return d @@ -150,9 +150,9 @@ def addpackage(sitedir, name, known_paths): """ if known_paths is None: known_paths = _init_pathinfo() - reset = 1 + reset = True else: - reset = 0 + reset = False fullname = os.path.join(sitedir, name) try: f = open(fullname, "r") @@ -190,9 +190,9 @@ def addsitedir(sitedir, known_paths=None): 'sitedir'""" if known_paths is None: known_paths = _init_pathinfo() - reset = 1 + reset = True else: - reset = 0 + reset = False sitedir, sitedircase = makepath(sitedir) if not sitedircase in known_paths: sys.path.append(sitedir) # Add path component diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 21628a9..f698927 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -75,7 +75,7 @@ class HelperFunctionsTests(unittest.TestCase): def test_init_pathinfo(self): dir_set = site._init_pathinfo() for entry in [site.makepath(path)[1] for path in sys.path - if path and os.path.isdir(path)]: + if path and os.path.exists(path)]: self.assertIn(entry, dir_set, "%s from sys.path not found in set returned " "by _init_pathinfo(): %s" % (entry, dir_set)) diff --git a/Misc/ACKS b/Misc/ACKS index 20e76f5..6dd30f3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -815,6 +815,7 @@ Torsten Landschoff Tino Lange Glenn Langford Andrew Langmead +Wolfgang Langner Detlef Lannert Soren Larsen Amos Latteier diff --git a/Misc/NEWS b/Misc/NEWS index 56b1501..6fbbdb3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,9 @@ Core and Builtins Library ------- +- Issue #26587: the site module now allows .pth files to specify files to be + added to sys.path (e.g. zip files). + - Issue #25609: Introduce contextlib.AbstractContextManager and typing.ContextManager. -- cgit v0.12 From fe21de98369bf99acf4745cdf539c9c952980c92 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 9 Apr 2016 07:34:39 +0300 Subject: Issue #26687: Use Py_RETURN_NONE macro in sqlite3 module --- Modules/_sqlite/cache.c | 3 +-- Modules/_sqlite/connection.c | 24 ++++++++---------------- Modules/_sqlite/cursor.c | 12 ++++-------- Modules/_sqlite/module.c | 9 +++------ 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/Modules/_sqlite/cache.c b/Modules/_sqlite/cache.c index 3689a4e..62c5893 100644 --- a/Modules/_sqlite/cache.c +++ b/Modules/_sqlite/cache.c @@ -244,8 +244,7 @@ PyObject* pysqlite_cache_display(pysqlite_Cache* self, PyObject* args) ptr = ptr->next; } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyMethodDef cache_methods[] = { diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 22298d9..f634877 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -348,8 +348,7 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) } } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } /* @@ -857,8 +856,7 @@ PyObject* pysqlite_connection_create_function(pysqlite_Connection* self, PyObjec if (PyDict_SetItem(self->function_pinboard, func, Py_None) == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -889,8 +887,7 @@ PyObject* pysqlite_connection_create_aggregate(pysqlite_Connection* self, PyObje if (PyDict_SetItem(self->function_pinboard, aggregate_class, Py_None) == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -1025,8 +1022,7 @@ static PyObject* pysqlite_connection_set_authorizer(pysqlite_Connection* self, P if (PyDict_SetItem(self->function_pinboard, authorizer_cb, Py_None) == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -1055,8 +1051,7 @@ static PyObject* pysqlite_connection_set_progress_handler(pysqlite_Connection* s return NULL; } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyObject* pysqlite_connection_set_trace_callback(pysqlite_Connection* self, PyObject* args, PyObject* kwargs) @@ -1083,8 +1078,7 @@ static PyObject* pysqlite_connection_set_trace_callback(pysqlite_Connection* sel sqlite3_trace(self->db, _trace_callback, trace_callback); } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } #ifdef HAVE_LOAD_EXTENSION @@ -1107,8 +1101,7 @@ static PyObject* pysqlite_enable_load_extension(pysqlite_Connection* self, PyObj PyErr_SetString(pysqlite_OperationalError, "Error enabling load extension"); return NULL; } else { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -1131,8 +1124,7 @@ static PyObject* pysqlite_load_extension(pysqlite_Connection* self, PyObject* ar PyErr_SetString(pysqlite_OperationalError, errmsg); return NULL; } else { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } #endif diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 63d705e..552c8bb 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -242,8 +242,7 @@ PyObject* _pysqlite_build_column_name(const char* colname) const char* pos; if (!colname) { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } for (pos = colname;; pos++) { @@ -914,8 +913,7 @@ PyObject* pysqlite_cursor_fetchone(pysqlite_Cursor* self, PyObject* args) row = pysqlite_cursor_iternext(self); if (!row && !PyErr_Occurred()) { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } return row; @@ -996,8 +994,7 @@ PyObject* pysqlite_cursor_fetchall(pysqlite_Cursor* self, PyObject* args) PyObject* pysqlite_noop(pysqlite_Connection* self, PyObject* args) { /* don't care, return None */ - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } PyObject* pysqlite_cursor_close(pysqlite_Cursor* self, PyObject* args) @@ -1013,8 +1010,7 @@ PyObject* pysqlite_cursor_close(pysqlite_Cursor* self, PyObject* args) self->closed = 1; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyMethodDef cursor_methods[] = { diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index ff2e3a5..7cd6d2a 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -139,8 +139,7 @@ static PyObject* module_enable_shared_cache(PyObject* self, PyObject* args, PyOb PyErr_SetString(pysqlite_OperationalError, "Changing the shared_cache flag failed"); return NULL; } else { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -172,8 +171,7 @@ static PyObject* module_register_adapter(PyObject* self, PyObject* args) if (rc == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } PyDoc_STRVAR(module_register_adapter_doc, @@ -221,8 +219,7 @@ static PyObject* enable_callback_tracebacks(PyObject* self, PyObject* args) return NULL; } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } PyDoc_STRVAR(enable_callback_tracebacks_doc, -- cgit v0.12 From 3872d62133f03a6c8292ee2f97999597f90f9082 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 10 Apr 2016 02:41:25 +0000 Subject: =?UTF-8?q?Issue=20#25609:=20Double=20back-ticks=20to=20avoid=20?= =?UTF-8?q?=E2=80=9Cmake=20check=E2=80=9D=20buildbot=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/whatsnew/3.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d217c4d..9be1a9c 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -196,7 +196,7 @@ contextlib The :class:`contextlib.AbstractContextManager` class has been added to provide an abstract base class for context managers. It provides a sensible default implementation for `__enter__()` which returns -`self` and leaves `__exit__()` an abstract method. A matching +``self`` and leaves `__exit__()` an abstract method. A matching class has been added to the :mod:`typing` module as :class:`typing.ContextManager`. (Contributed by Brett Cannon in :issue:`25609`.) -- cgit v0.12 From 03f17f86717372ca010273dc8946fd19914a534b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Apr 2016 14:44:59 +0300 Subject: Issue #17339: Improved TypeError message in bytes constructor. --- Objects/bytesobject.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 639ee71..cbf8166 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3469,31 +3469,24 @@ _PyBytes_FromTuple(PyObject *x) } static PyObject * -_PyBytes_FromIterator(PyObject *x) +_PyBytes_FromIterator(PyObject *it, PyObject *x) { char *str; - PyObject *it; Py_ssize_t i, size; _PyBytesWriter writer; - _PyBytesWriter_Init(&writer); - /* For iterator version, create a string object and resize as needed */ size = PyObject_LengthHint(x, 64); if (size == -1 && PyErr_Occurred()) return NULL; + _PyBytesWriter_Init(&writer); str = _PyBytesWriter_Alloc(&writer, size); if (str == NULL) return NULL; writer.overallocate = 1; size = writer.allocated; - /* Get the iterator */ - it = PyObject_GetIter(x); - if (it == NULL) - goto error; - /* Run the iterator to exhaustion */ for (i = 0; ; i++) { PyObject *item; @@ -3529,19 +3522,19 @@ _PyBytes_FromIterator(PyObject *x) } *str++ = (char) value; } - Py_DECREF(it); return _PyBytesWriter_Finish(&writer, str); error: _PyBytesWriter_Dealloc(&writer); - Py_XDECREF(it); return NULL; } PyObject * PyBytes_FromObject(PyObject *x) { + PyObject *it, *result; + if (x == NULL) { PyErr_BadInternalCall(); return NULL; @@ -3562,13 +3555,19 @@ PyBytes_FromObject(PyObject *x) if (PyTuple_CheckExact(x)) return _PyBytes_FromTuple(x); - if (PyUnicode_Check(x)) { - PyErr_SetString(PyExc_TypeError, - "cannot convert unicode object to bytes"); - return NULL; + if (!PyUnicode_Check(x)) { + it = PyObject_GetIter(x); + if (it != NULL) { + result = _PyBytes_FromIterator(it, x); + Py_DECREF(it); + return result; + } } - return _PyBytes_FromIterator(x); + PyErr_Format(PyExc_TypeError, + "cannot convert '%.200s' object to bytes", + x->ob_type->tp_name); + return NULL; } static PyObject * -- cgit v0.12 From 47c5474aa0cbe8dc3cf2c370b19769edd3f2e8d8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Apr 2016 15:46:30 +0300 Subject: Issue #26623: TypeError message for JSON unserializible object now contains object's type name, not object's representation. Based on patch by Mahmoud Lababidi. --- Lib/json/encoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index d596489..0772bbc 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -176,7 +176,8 @@ class JSONEncoder(object): return JSONEncoder.default(self, o) """ - raise TypeError(repr(o) + " is not JSON serializable") + raise TypeError("Object of type '%s' is not JSON serializable" % + o.__class__.__name__) def encode(self, o): """Return a JSON string representation of a Python data structure. -- cgit v0.12 From 50ab1a3694c43b9ab6798b98d9e5983c78cb17e2 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 11 Apr 2016 00:38:12 +0000 Subject: Issue #26685: Raise OSError if closing a socket fails --- Doc/library/socket.rst | 4 ++++ Doc/whatsnew/3.6.rst | 4 ++++ Lib/test/test_pty.py | 1 - Lib/test/test_socket.py | 11 +++++++++++ Misc/NEWS | 2 ++ Modules/socketmodule.c | 6 +++++- 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c09927c..cd6e310 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -868,6 +868,10 @@ to sockets. it is recommended to :meth:`close` them explicitly, or to use a :keyword:`with` statement around them. + .. versionchanged:: 3.6 + :exc:`OSError` is now raised if an error occurs when the underlying + :c:func:`close` call is made. + .. note:: :meth:`close()` releases the resource associated with a connection but diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 9be1a9c..8bf2847 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -514,6 +514,10 @@ Changes in the Python API * :func:`spwd.getspnam` now raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user doesn't have privileges. +* The :meth:`socket.socket.close` method now raises an exception if + an error (e.g. EBADF) was reported by the underlying system call. + See :issue:`26685`. + Changes in the C API -------------------- diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index ef5e99e..15f88e4 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -277,7 +277,6 @@ class SmallPtyTests(unittest.TestCase): socketpair = self._socketpair() masters = [s.fileno() for s in socketpair] - os.close(masters[1]) socketpair[1].close() os.close(write_to_stdin_fd) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 02bc0c0..982a976 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1161,6 +1161,17 @@ class GeneralModuleTests(unittest.TestCase): sock.close() self.assertRaises(OSError, sock.send, b"spam") + def testCloseException(self): + sock = socket.socket() + socket.socket(fileno=sock.fileno()).close() + try: + sock.close() + except OSError as err: + # Winsock apparently raises ENOTSOCK + self.assertIn(err.errno, (errno.EBADF, errno.ENOTSOCK)) + else: + self.fail("close() should raise EBADF/ENOTSOCK") + def testNewAttributes(self): # testing .family, .type and .protocol diff --git a/Misc/NEWS b/Misc/NEWS index f7694cc..7ac9b34 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -240,6 +240,8 @@ Core and Builtins Library ------- +- Issue #26685: Raise OSError if closing a socket fails. + - Issue #16329: Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. - Issue #13952: Add .csv to mimetypes.types_map. Patch by Geoff Wilson. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 8df735d..bcff004 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2576,6 +2576,7 @@ static PyObject * sock_close(PySocketSockObject *s) { SOCKET_T fd; + int res; fd = s->sock_fd; if (fd != -1) { @@ -2586,8 +2587,11 @@ sock_close(PySocketSockObject *s) http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html for more details. */ Py_BEGIN_ALLOW_THREADS - (void) SOCKETCLOSE(fd); + res = SOCKETCLOSE(fd); Py_END_ALLOW_THREADS + if (res < 0) { + return s->errorhandler(); + } } Py_INCREF(Py_None); return Py_None; -- cgit v0.12 From da3bb38452740cec27723d7dc89a926547613204 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 11 Apr 2016 00:40:08 +0000 Subject: Issue #26585: Eliminate _quote_html() and use html.escape(quote=False) Patch by Xiang Zhang. --- Lib/http/server.py | 16 ++++++++-------- Lib/test/test_httpservers.py | 30 +++++++++++++++++++++++++++++- Misc/NEWS | 3 +++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index f4ad260..fbee6a9 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -127,9 +127,6 @@ DEFAULT_ERROR_MESSAGE = """\ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" -def _quote_html(html): - return html.replace("&", "&").replace("<", "<").replace(">", ">") - class HTTPServer(socketserver.TCPServer): allow_reuse_address = 1 # Seems to make sense in testing environment @@ -449,9 +446,12 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): if explain is None: explain = longmsg self.log_error("code %d, message %s", code, message) - # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201) - content = (self.error_message_format % - {'code': code, 'message': _quote_html(message), 'explain': _quote_html(explain)}) + # HTML encode to prevent Cross Site Scripting attacks (see bug #1100201) + content = (self.error_message_format % { + 'code': code, + 'message': html.escape(message, quote=False), + 'explain': html.escape(explain, quote=False) + }) body = content.encode('UTF-8', 'replace') self.send_response(code, message) self.send_header("Content-Type", self.error_content_type) @@ -710,7 +710,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): errors='surrogatepass') except UnicodeDecodeError: displaypath = urllib.parse.unquote(path) - displaypath = html.escape(displaypath) + displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = 'Directory listing for %s' % displaypath r.append('%s' % (urllib.parse.quote(linkname, errors='surrogatepass'), - html.escape(displayname))) + html.escape(displayname, quote=False))) r.append('\n
\n\n\n') encoded = '\n'.join(r).encode(enc, 'surrogateescape') f = io.BytesIO() diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index c752fd8..3856d00 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -344,7 +344,7 @@ class SimpleHTTPServerTestCase(BaseTestCase): quotedname = urllib.parse.quote(filename, errors='surrogatepass') self.assertIn(('href="%s"' % quotedname) .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename)) + self.assertIn(('>%s<' % html.escape(filename, quote=False)) .encode(enc, 'surrogateescape'), body) response = self.request(self.base_url + '/' + quotedname) self.check_status_and_reason(response, HTTPStatus.OK, @@ -422,6 +422,27 @@ class SimpleHTTPServerTestCase(BaseTestCase): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") + def test_html_escape_filename(self): + filename = '.txt' + fullpath = os.path.join(self.tempdir, filename) + + try: + open(fullpath, 'w').close() + except OSError: + raise unittest.SkipTest('Can not create file %s on current file ' + 'system' % filename) + + try: + response = self.request(self.base_url + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + enc = response.headers.get_content_charset() + finally: + os.unlink(fullpath) # avoid affecting test_undecodable_filename + + self.assertIsNotNone(enc) + html_text = '>%s<' % html.escape(filename, quote=False) + self.assertIn(html_text.encode(enc), body) + cgi_file1 = """\ #!%s @@ -883,6 +904,13 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.assertFalse(self.handler.get_called) self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') + def test_html_escape_on_error(self): + result = self.send_typical_request( + b' / HTTP/1.1') + result = b''.join(result) + text = '' + self.assertIn(html.escape(text, quote=False).encode('ascii'), result) + def test_close_connection(self): # handle_one_request() should be repeatedly called until # it sets close_connection diff --git a/Misc/NEWS b/Misc/NEWS index 7ac9b34..7284d3f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -240,6 +240,9 @@ Core and Builtins Library ------- +- Issue #26585: Eliminate http.server._quote_html() and use + html.escape(quote=False). Patch by Xiang Zhang. + - Issue #26685: Raise OSError if closing a socket fails. - Issue #16329: Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. -- cgit v0.12 From 7b228231c78f49986928cd587cbf85197a3b3987 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Apr 2016 18:15:26 +0200 Subject: Issue #26647: Cleanup opcode Simplify code to build opcode.opname. Patch written by Demur Rumed. --- Lib/opcode.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 71ce672..ecc24bb 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -34,9 +34,7 @@ hasfree = [] hasnargs = [] opmap = {} -opname = [''] * 256 -for op in range(256): opname[op] = '<%r>' % (op,) -del op +opname = ['<%r>' % (op,) for op in range(256)] def def_op(name, op): opname[op] = name -- cgit v0.12 From 7e50978a4e36cd4ab4b363ee0c77abefc13e2c23 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Apr 2016 18:17:06 +0200 Subject: Issue #26647: Cleanup modulefinder Use directly dis.opmap[name] rather than dis.opname.index(name). Patch written by Demur Rumed. --- Lib/modulefinder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 4a2f1b5..d8d645c 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -14,11 +14,11 @@ with warnings.catch_warnings(): import imp # XXX Clean up once str8's cstor matches bytes. -LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')]) -IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')]) -STORE_NAME = bytes([dis.opname.index('STORE_NAME')]) -STORE_GLOBAL = bytes([dis.opname.index('STORE_GLOBAL')]) -STORE_OPS = [STORE_NAME, STORE_GLOBAL] +LOAD_CONST = bytes([dis.opmap['LOAD_CONST']]) +IMPORT_NAME = bytes([dis.opmap['IMPORT_NAME']]) +STORE_NAME = bytes([dis.opmap['STORE_NAME']]) +STORE_GLOBAL = bytes([dis.opmap['STORE_GLOBAL']]) +STORE_OPS = STORE_NAME, STORE_GLOBAL HAVE_ARGUMENT = bytes([dis.HAVE_ARGUMENT]) # Modulefinder does a good job at simulating Python's, but it can not -- cgit v0.12 From 328cb1fed0c91f50f311cdc545fe0e9303d0dae7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Apr 2016 18:46:10 +0200 Subject: Update pygettext.py to get ride of imp Issue #26639: Replace imp with importlib in Tools/i18n/pygettext.py. Remove _get_modpkg_path(), replaced with importlib.util.find_spec(). --- Tools/i18n/pygettext.py | 51 ++++++++----------------------------------------- 1 file changed, 8 insertions(+), 43 deletions(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 3c6c14c..be941c7 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -156,7 +156,8 @@ If `inputfile' is -, standard input is read. """) import os -import imp +import importlib.machinery +import importlib.util import sys import glob import time @@ -263,8 +264,7 @@ def _visit_pyfiles(list, dirname, names): # get extension for python source files if '_py_ext' not in globals(): global _py_ext - _py_ext = [triple[0] for triple in imp.get_suffixes() - if triple[2] == imp.PY_SOURCE][0] + _py_ext = importlib.machinery.SOURCE_SUFFIXES[0] # don't recurse into CVS directories if 'CVS' in names: @@ -277,45 +277,6 @@ def _visit_pyfiles(list, dirname, names): ) -def _get_modpkg_path(dotted_name, pathlist=None): - """Get the filesystem path for a module or a package. - - Return the file system path to a file for a module, and to a directory for - a package. Return None if the name is not found, or is a builtin or - extension module. - """ - # split off top-most name - parts = dotted_name.split('.', 1) - - if len(parts) > 1: - # we have a dotted path, import top-level package - try: - file, pathname, description = imp.find_module(parts[0], pathlist) - if file: file.close() - except ImportError: - return None - - # check if it's indeed a package - if description[2] == imp.PKG_DIRECTORY: - # recursively handle the remaining name parts - pathname = _get_modpkg_path(parts[1], [pathname]) - else: - pathname = None - else: - # plain name - try: - file, pathname, description = imp.find_module( - dotted_name, pathlist) - if file: - file.close() - if description[2] not in [imp.PY_SOURCE, imp.PKG_DIRECTORY]: - pathname = None - except ImportError: - pathname = None - - return pathname - - def getFilesForName(name): """Get a list of module files for a filename, a module or package name, or a directory. @@ -330,7 +291,11 @@ def getFilesForName(name): return list # try to find module or package - name = _get_modpkg_path(name) + try: + spec = importlib.util.find_spec(name) + name = spec.origin + except ImportError: + name = None if not name: return [] -- cgit v0.12 From 0cab9c1ebaa11bb7838a552c671c903156262ab7 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 13 Apr 2016 00:36:52 +0000 Subject: Issue #26404: Add context manager to socketserver, by Aviv Palivoda --- Doc/library/http.server.rst | 7 +++-- Doc/library/socketserver.rst | 55 ++++++++++++++++++++++------------------ Doc/library/wsgiref.rst | 32 +++++++++++------------ Doc/library/xmlrpc.server.rst | 59 +++++++++++++++++++++---------------------- Doc/whatsnew/3.6.rst | 10 ++++++++ Lib/http/server.py | 18 ++++++------- Lib/socketserver.py | 6 +++++ Lib/test/test_socketserver.py | 7 ++++- Lib/wsgiref/simple_server.py | 13 +++++----- Lib/xmlrpc/server.py | 25 +++++++++--------- Misc/NEWS | 2 ++ 11 files changed, 128 insertions(+), 106 deletions(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 0bde35b..7ab249a 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -375,10 +375,9 @@ the current directory:: Handler = http.server.SimpleHTTPRequestHandler - httpd = socketserver.TCPServer(("", PORT), Handler) - - print("serving at port", PORT) - httpd.serve_forever() + with socketserver.TCPServer(("", PORT), Handler) as httpd: + print("serving at port", PORT) + httpd.serve_forever() .. _http-server-cli: diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index aaaa61e..e148d30 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -52,11 +52,12 @@ handler class by subclassing the :class:`BaseRequestHandler` class and overriding its :meth:`~BaseRequestHandler.handle` method; this method will process incoming requests. Second, you must instantiate one of the server classes, passing it -the server's address and the request handler class. Then call the +the server's address and the request handler class. It is recommended to use +the server in a :keyword:`with` statement. Then call the :meth:`~BaseServer.handle_request` or :meth:`~BaseServer.serve_forever` method of the server object to process one or many requests. Finally, call :meth:`~BaseServer.server_close` -to close the socket. +to close the socket (unless you used a :keyword:`with` statement). When inheriting from :class:`ThreadingMixIn` for threaded connection behavior, you should explicitly declare how you want your threads to behave on an abrupt @@ -353,6 +354,11 @@ Server Objects default implementation always returns :const:`True`. + .. versionchanged:: 3.6 + Support for the :term:`context manager` protocol was added. Exiting the + context manager is equivalent to calling :meth:`server_close`. + + Request Handler Objects ----------------------- @@ -433,11 +439,10 @@ This is the server side:: HOST, PORT = "localhost", 9999 # Create the server, binding to localhost on port 9999 - server = socketserver.TCPServer((HOST, PORT), MyTCPHandler) - - # Activate the server; this will keep running until you - # interrupt the program with Ctrl-C - server.serve_forever() + with socketserver.TCPServer((HOST, PORT), MyTCPHandler) as server: + # Activate the server; this will keep running until you + # interrupt the program with Ctrl-C + server.serve_forever() An alternative request handler class that makes use of streams (file-like objects that simplify communication by providing the standard file interface):: @@ -529,8 +534,8 @@ This is the server side:: if __name__ == "__main__": HOST, PORT = "localhost", 9999 - server = socketserver.UDPServer((HOST, PORT), MyUDPHandler) - server.serve_forever() + with socketserver.UDPServer((HOST, PORT), MyUDPHandler) as server: + server.serve_forever() This is the client side:: @@ -592,22 +597,22 @@ An example for the :class:`ThreadingMixIn` class:: HOST, PORT = "localhost", 0 server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler) - ip, port = server.server_address - - # Start a thread with the server -- that thread will then start one - # more thread for each request - server_thread = threading.Thread(target=server.serve_forever) - # Exit the server thread when the main thread terminates - server_thread.daemon = True - server_thread.start() - print("Server loop running in thread:", server_thread.name) - - client(ip, port, "Hello World 1") - client(ip, port, "Hello World 2") - client(ip, port, "Hello World 3") - - server.shutdown() - server.server_close() + with server: + ip, port = server.server_address + + # Start a thread with the server -- that thread will then start one + # more thread for each request + server_thread = threading.Thread(target=server.serve_forever) + # Exit the server thread when the main thread terminates + server_thread.daemon = True + server_thread.start() + print("Server loop running in thread:", server_thread.name) + + client(ip, port, "Hello World 1") + client(ip, port, "Hello World 2") + client(ip, port, "Hello World 3") + + server.shutdown() The output of the example should look something like this:: diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 71607d6..8d62885 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -131,9 +131,9 @@ parameter expect a WSGI-compliant dictionary to be supplied; please see for key, value in environ.items()] return ret - httpd = make_server('', 8000, simple_app) - print("Serving on port 8000...") - httpd.serve_forever() + with make_server('', 8000, simple_app) as httpd: + print("Serving on port 8000...") + httpd.serve_forever() In addition to the environment functions above, the :mod:`wsgiref.util` module @@ -283,14 +283,14 @@ request. (E.g., using the :func:`shift_path_info` function from from wsgiref.simple_server import make_server, demo_app - httpd = make_server('', 8000, demo_app) - print("Serving HTTP on port 8000...") + with make_server('', 8000, demo_app) as httpd: + print("Serving HTTP on port 8000...") - # Respond to requests until process is killed - httpd.serve_forever() + # Respond to requests until process is killed + httpd.serve_forever() - # Alternative: serve one request, then exit - httpd.handle_request() + # Alternative: serve one request, then exit + httpd.handle_request() .. function:: demo_app(environ, start_response) @@ -430,9 +430,9 @@ Paste" library. # This is the application wrapped in a validator validator_app = validator(simple_app) - httpd = make_server('', 8000, validator_app) - print("Listening on port 8000....") - httpd.serve_forever() + with make_server('', 8000, validator_app) as httpd: + print("Listening on port 8000....") + httpd.serve_forever() :mod:`wsgiref.handlers` -- server/gateway base classes @@ -769,8 +769,8 @@ This is a working "Hello World" WSGI application:: # The returned object is going to be printed return [b"Hello World"] - httpd = make_server('', 8000, hello_world_app) - print("Serving on port 8000...") + with make_server('', 8000, hello_world_app) as httpd: + print("Serving on port 8000...") - # Serve until process is killed - httpd.serve_forever() + # Serve until process is killed + httpd.serve_forever() diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 680db41..ca80aab 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -147,29 +147,29 @@ Server code:: rpc_paths = ('/RPC2',) # Create server - server = SimpleXMLRPCServer(("localhost", 8000), - requestHandler=RequestHandler) - server.register_introspection_functions() + with SimpleXMLRPCServer(("localhost", 8000), + requestHandler=RequestHandler) as server: + server.register_introspection_functions() - # Register pow() function; this will use the value of - # pow.__name__ as the name, which is just 'pow'. - server.register_function(pow) + # Register pow() function; this will use the value of + # pow.__name__ as the name, which is just 'pow'. + server.register_function(pow) - # Register a function under a different name - def adder_function(x,y): - return x + y - server.register_function(adder_function, 'add') + # Register a function under a different name + def adder_function(x,y): + return x + y + server.register_function(adder_function, 'add') - # Register an instance; all the methods of the instance are - # published as XML-RPC methods (in this case, just 'mul'). - class MyFuncs: - def mul(self, x, y): - return x * y + # Register an instance; all the methods of the instance are + # published as XML-RPC methods (in this case, just 'mul'). + class MyFuncs: + def mul(self, x, y): + return x * y - server.register_instance(MyFuncs()) + server.register_instance(MyFuncs()) - # Run the server's main loop - server.serve_forever() + # Run the server's main loop + server.serve_forever() The following client code will call the methods made available by the preceding server:: @@ -206,18 +206,17 @@ a server allowing dotted names and registering a multicall function. def getCurrentTime(): return datetime.datetime.now() - server = SimpleXMLRPCServer(("localhost", 8000)) - server.register_function(pow) - server.register_function(lambda x,y: x+y, 'add') - server.register_instance(ExampleService(), allow_dotted_names=True) - server.register_multicall_functions() - print('Serving XML-RPC on localhost port 8000') - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - server.server_close() - sys.exit(0) + with SimpleXMLRPCServer(("localhost", 8000)) as server: + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_instance(ExampleService(), allow_dotted_names=True) + server.register_multicall_functions() + print('Serving XML-RPC on localhost port 8000') + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nKeyboard interrupt received, exiting.") + sys.exit(0) This ExampleService demo can be invoked from the command line:: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8bf2847..ef10ef2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -259,6 +259,16 @@ you may now specify file paths on top of directories (e.g. zip files). (Contributed by Wolfgang Langner in :issue:`26587`). +socketserver +------------ + +Servers based on the :mod:`socketserver` module, including those +defined in :mod:`http.server`, :mod:`xmlrpc.server` and +:mod:`wsgiref.simple_server`, now support the :term:`context manager` +protocol. +(Contributed by Aviv Palivoda in :issue:`26404`.) + + telnetlib --------- diff --git a/Lib/http/server.py b/Lib/http/server.py index fbee6a9..c1607b3 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1175,16 +1175,14 @@ def test(HandlerClass=BaseHTTPRequestHandler, server_address = (bind, port) HandlerClass.protocol_version = protocol - httpd = ServerClass(server_address, HandlerClass) - - sa = httpd.socket.getsockname() - print("Serving HTTP on", sa[0], "port", sa[1], "...") - try: - httpd.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - httpd.server_close() - sys.exit(0) + with ServerClass(server_address, HandlerClass) as httpd: + sa = httpd.socket.getsockname() + print("Serving HTTP on", sa[0], "port", sa[1], "...") + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nKeyboard interrupt received, exiting.") + sys.exit(0) if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 2f39514..3e1f058 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -378,6 +378,12 @@ class BaseServer: traceback.print_exc() print('-'*40, file=sys.stderr) + def __enter__(self): + return self + + def __exit__(self, *args): + self.server_close() + class TCPServer(BaseServer): diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 27fe01c..554c106 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -104,7 +104,6 @@ class SocketServerTest(unittest.TestCase): class MyServer(svrcls): def handle_error(self, request, client_address): self.close_request(request) - self.server_close() raise class MyHandler(hdlrbase): @@ -280,6 +279,12 @@ class SocketServerTest(unittest.TestCase): socketserver.TCPServer((HOST, -1), socketserver.StreamRequestHandler) + def test_context_manager(self): + with socketserver.TCPServer((HOST, 0), + socketserver.StreamRequestHandler) as server: + pass + self.assertEqual(-1, server.socket.fileno()) + class ErrorHandlerTest(unittest.TestCase): """Test that the servers pass normal exceptions from the handler to diff --git a/Lib/wsgiref/simple_server.py b/Lib/wsgiref/simple_server.py index 378b316..1807c66 100644 --- a/Lib/wsgiref/simple_server.py +++ b/Lib/wsgiref/simple_server.py @@ -156,10 +156,9 @@ def make_server( if __name__ == '__main__': - httpd = make_server('', 8000, demo_app) - sa = httpd.socket.getsockname() - print("Serving HTTP on", sa[0], "port", sa[1], "...") - import webbrowser - webbrowser.open('http://localhost:8000/xyz?abc') - httpd.handle_request() # serve one request, then exit - httpd.server_close() + with make_server('', 8000, demo_app) as httpd: + sa = httpd.socket.getsockname() + print("Serving HTTP on", sa[0], "port", sa[1], "...") + import webbrowser + webbrowser.open('http://localhost:8000/xyz?abc') + httpd.handle_request() # serve one request, then exit diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 5b5bf7c..78728f2 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -971,16 +971,15 @@ if __name__ == '__main__': def getCurrentTime(): return datetime.datetime.now() - server = SimpleXMLRPCServer(("localhost", 8000)) - server.register_function(pow) - server.register_function(lambda x,y: x+y, 'add') - server.register_instance(ExampleService(), allow_dotted_names=True) - server.register_multicall_functions() - print('Serving XML-RPC on localhost port 8000') - print('It is advisable to run this example server within a secure, closed network.') - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - server.server_close() - sys.exit(0) + with SimpleXMLRPCServer(("localhost", 8000)) as server: + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_instance(ExampleService(), allow_dotted_names=True) + server.register_multicall_functions() + print('Serving XML-RPC on localhost port 8000') + print('It is advisable to run this example server within a secure, closed network.') + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nKeyboard interrupt received, exiting.") + sys.exit(0) diff --git a/Misc/NEWS b/Misc/NEWS index d98c184..a4109ab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -240,6 +240,8 @@ Core and Builtins Library ------- +- Issue #26404: Add context manager to socketserver. Patch by Aviv Palivoda. + - Issue #26735: Fix :func:`os.urandom` on Solaris 11.3 and newer when reading more than 1,024 bytes: call ``getrandom()`` multiple times with a limit of 1024 bytes per call. -- cgit v0.12 From 21a663ea2829b6808dd6981904c393332d271f8e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Apr 2016 15:37:23 +0300 Subject: Issue #26057: Got rid of nonneeded use of PyUnicode_FromObject(). --- Modules/_codecsmodule.c | 231 ++++--------------- Modules/socketmodule.c | 44 ++-- Objects/stringlib/find.h | 13 +- Objects/unicodeobject.c | 562 ++++++++++++++--------------------------------- Python/bltinmodule.c | 5 +- Python/getargs.c | 26 +-- 6 files changed, 240 insertions(+), 641 deletions(-) diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 7575773..1951da9 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -20,10 +20,6 @@ _decode(char_buffer_obj[,errors='strict']) -> (Unicode object, bytes consumed) - _encode() interfaces also accept non-Unicode object as - input. The objects are then converted to Unicode using - PyUnicode_FromObject() prior to applying the conversion. - These s are available: utf_8, unicode_escape, raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit), mbcs (on win32). @@ -718,7 +714,7 @@ _codecs_unicode_internal_encode_impl(PyModuleDef *module, PyObject *obj, /*[clinic input] _codecs.utf_7_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -728,22 +724,13 @@ _codecs_utf_7_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=a7accc496a32b759 input=fd91a78f103b0421]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF7(str, 0, 0, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF7(str, 0, 0, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_8_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -753,17 +740,8 @@ _codecs_utf_8_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=ec831d80e7aedede input=2c22d40532f071f3]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_AsEncodedString(str, "utf-8", errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_AsUTF8String(str, errors), + PyUnicode_GET_LENGTH(str)); } /* This version provides access to the byteorder parameter of the @@ -775,7 +753,7 @@ _codecs_utf_8_encode_impl(PyModuleDef *module, PyObject *str, /*[clinic input] _codecs.utf_16_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL byteorder: int = 0 / @@ -786,22 +764,13 @@ _codecs_utf_16_encode_impl(PyModuleDef *module, PyObject *str, const char *errors, int byteorder) /*[clinic end generated code: output=93ac58e960a9ee4d input=3935a489b2d5385e]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF16(str, errors, byteorder), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, byteorder), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_16_le_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -811,22 +780,13 @@ _codecs_utf_16_le_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=422bedb8da34fb66 input=bc27df05d1d20dfe]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF16(str, errors, -1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, -1), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_16_be_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -836,17 +796,8 @@ _codecs_utf_16_be_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=3aa7ee9502acdd77 input=5a69d4112763462b]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF16(str, errors, +1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, +1), + PyUnicode_GET_LENGTH(str)); } /* This version provides access to the byteorder parameter of the @@ -858,7 +809,7 @@ _codecs_utf_16_be_encode_impl(PyModuleDef *module, PyObject *str, /*[clinic input] _codecs.utf_32_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL byteorder: int = 0 / @@ -869,22 +820,13 @@ _codecs_utf_32_encode_impl(PyModuleDef *module, PyObject *str, const char *errors, int byteorder) /*[clinic end generated code: output=3e7d5a003b02baed input=434a1efa492b8d58]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF32(str, errors, byteorder), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, byteorder), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_32_le_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -894,22 +836,13 @@ _codecs_utf_32_le_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=5dda641cd33dbfc2 input=dfa2d7dc78b99422]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF32(str, errors, -1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, -1), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_32_be_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -919,22 +852,13 @@ _codecs_utf_32_be_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=ccca8b44d91a7c7a input=4595617b18169002]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF32(str, errors, +1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, +1), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.unicode_escape_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -944,22 +868,13 @@ _codecs_unicode_escape_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=389f23d2b8f8d80b input=8273506f14076912]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_AsUnicodeEscapeString(str), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_AsUnicodeEscapeString(str), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.raw_unicode_escape_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -969,22 +884,13 @@ _codecs_raw_unicode_escape_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=fec4e39d6ec37a62 input=181755d5dfacef3c]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_AsRawUnicodeEscapeString(str), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_AsRawUnicodeEscapeString(str), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.latin_1_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -994,22 +900,13 @@ _codecs_latin_1_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=ecf00eb8e48c889c input=f03f6dcf1d84bee4]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_AsLatin1String(str, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_AsLatin1String(str, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.ascii_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -1019,22 +916,13 @@ _codecs_ascii_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=a9d18fc6b6b91cfb input=d87e25a10a593fee]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_AsASCIIString(str, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_AsASCIIString(str, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.charmap_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL mapping: object = NULL / @@ -1045,20 +933,11 @@ _codecs_charmap_encode_impl(PyModuleDef *module, PyObject *str, const char *errors, PyObject *mapping) /*[clinic end generated code: output=14ca42b83853c643 input=85f4172661e8dad9]*/ { - PyObject *v; - if (mapping == Py_None) mapping = NULL; - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeCharmap(str, mapping, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeCharmap(str, mapping, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] @@ -1078,7 +957,7 @@ _codecs_charmap_build_impl(PyModuleDef *module, PyObject *map) /*[clinic input] _codecs.mbcs_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -1088,23 +967,14 @@ _codecs_mbcs_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=d1a013bc68798bd7 input=65c09ee1e4203263]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_EncodeCodePage(CP_ACP, str, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_EncodeCodePage(CP_ACP, str, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.code_page_encode code_page: int - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -1114,19 +984,8 @@ _codecs_code_page_encode_impl(PyModuleDef *module, int code_page, PyObject *str, const char *errors) /*[clinic end generated code: output=3b406618dbfbce25 input=c8562ec460c2e309]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_EncodeCodePage(code_page, - str, - errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_EncodeCodePage(code_page, str, errors), + PyUnicode_GET_LENGTH(str)); } #endif /* HAVE_MBCS */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bcff004..ec35fb9 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1401,7 +1401,7 @@ static int idna_converter(PyObject *obj, struct maybe_idna *data) { size_t len; - PyObject *obj2, *obj3; + PyObject *obj2; if (obj == NULL) { idna_cleanup(data); return 1; @@ -1416,31 +1416,27 @@ idna_converter(PyObject *obj, struct maybe_idna *data) data->buf = PyByteArray_AsString(obj); len = PyByteArray_Size(obj); } - else if (PyUnicode_Check(obj) && PyUnicode_READY(obj) == 0 && PyUnicode_IS_COMPACT_ASCII(obj)) { - data->buf = PyUnicode_DATA(obj); - len = PyUnicode_GET_LENGTH(obj); - } - else { - obj2 = PyUnicode_FromObject(obj); - if (!obj2) { - PyErr_Format(PyExc_TypeError, "string or unicode text buffer expected, not %s", - obj->ob_type->tp_name); - return 0; + else if (PyUnicode_Check(obj)) { + if (PyUnicode_READY(obj) == 0 && PyUnicode_IS_COMPACT_ASCII(obj)) { + data->buf = PyUnicode_DATA(obj); + len = PyUnicode_GET_LENGTH(obj); } - obj3 = PyUnicode_AsEncodedString(obj2, "idna", NULL); - Py_DECREF(obj2); - if (!obj3) { - PyErr_SetString(PyExc_TypeError, "encoding of hostname failed"); - return 0; - } - if (!PyBytes_Check(obj3)) { - Py_DECREF(obj3); - PyErr_SetString(PyExc_TypeError, "encoding of hostname failed to return bytes"); - return 0; + else { + obj2 = PyUnicode_AsEncodedString(obj, "idna", NULL); + if (!obj2) { + PyErr_SetString(PyExc_TypeError, "encoding of hostname failed"); + return 0; + } + assert(PyBytes_Check(obj2)); + data->obj = obj2; + data->buf = PyBytes_AS_STRING(obj2); + len = PyBytes_GET_SIZE(obj2); } - data->obj = obj3; - data->buf = PyBytes_AS_STRING(obj3); - len = PyBytes_GET_SIZE(obj3); + } + else { + PyErr_Format(PyExc_TypeError, "str, bytes or bytearray expected, not %s", + obj->ob_type->tp_name); + return 0; } if (strlen(data->buf) != len) { Py_CLEAR(data->obj); diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index 14815f6..a7065fc 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -123,11 +123,6 @@ STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, /* Wraps stringlib_parse_args_finds() and additionally ensures that the first argument is a unicode object. - -Note that we receive a pointer to the pointer of the substring object, -so when we create that object in this function we don't DECREF it, -because it continues living in the caller functions (those functions, -after finishing using the substring, must DECREF it). */ Py_LOCAL_INLINE(int) @@ -135,14 +130,10 @@ STRINGLIB(parse_args_finds_unicode)(const char * function_name, PyObject *args, PyObject **substring, Py_ssize_t *start, Py_ssize_t *end) { - PyObject *tmp_substring; - - if(STRINGLIB(parse_args_finds)(function_name, args, &tmp_substring, + if(STRINGLIB(parse_args_finds)(function_name, args, substring, start, end)) { - tmp_substring = PyUnicode_FromObject(tmp_substring); - if (!tmp_substring) + if (ensure_unicode(*substring) < 0) return 0; - *substring = tmp_substring; return 1; } return 0; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8dc2a38..d3bb13a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -751,6 +751,18 @@ make_bloom_mask(int kind, void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } +static int +ensure_unicode(PyObject *obj) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "must be str, not %.100s", + Py_TYPE(obj)->tp_name); + return -1; + } + return PyUnicode_READY(obj); +} + /* Compilation of templated routines */ #include "stringlib/asciilib.h" @@ -3066,7 +3078,7 @@ PyUnicode_FromEncodedObject(PyObject *obj, /* Retrieve a bytes buffer view through the PEP 3118 buffer interface */ if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) { PyErr_Format(PyExc_TypeError, - "coercing to str: need a bytes-like object, %.80s found", + "decoding to str: need a bytes-like object, %.80s found", Py_TYPE(obj)->tp_name); return NULL; } @@ -3787,19 +3799,17 @@ PyUnicode_FSConverter(PyObject* arg, void* addr) output = arg; Py_INCREF(output); } - else { - arg = PyUnicode_FromObject(arg); - if (!arg) - return 0; + else if (PyUnicode_Check(arg)) { output = PyUnicode_EncodeFSDefault(arg); - Py_DECREF(arg); if (!output) return 0; - if (!PyBytes_Check(output)) { - Py_DECREF(output); - PyErr_SetString(PyExc_TypeError, "encoder failed to return bytes"); - return 0; - } + assert(PyBytes_Check(output)); + } + else { + PyErr_Format(PyExc_TypeError, + "must be str or bytes, not %.100s", + Py_TYPE(arg)->tp_name); + return 0; } size = PyBytes_GET_SIZE(output); data = PyBytes_AS_STRING(output); @@ -3871,7 +3881,7 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) if (PyUnicode_UTF8(unicode) == NULL) { assert(!PyUnicode_IS_COMPACT_ASCII(unicode)); - bytes = _PyUnicode_AsUTF8String(unicode, "strict"); + bytes = _PyUnicode_AsUTF8String(unicode, NULL); if (bytes == NULL) return NULL; _PyUnicode_UTF8(unicode) = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1); @@ -8860,10 +8870,8 @@ _PyUnicode_TranslateCharmap(PyObject *input, kind = PyUnicode_KIND(input); size = PyUnicode_GET_LENGTH(input); - if (size == 0) { - Py_INCREF(input); - return input; - } + if (size == 0) + return PyUnicode_FromObject(input); /* allocate enough for a simple 1:1 translation without replacements, if we need more, we'll resize */ @@ -8974,14 +8982,9 @@ PyUnicode_Translate(PyObject *str, PyObject *mapping, const char *errors) { - PyObject *result; - - str = PyUnicode_FromObject(str); - if (str == NULL) + if (ensure_unicode(str) < 0) return NULL; - result = _PyUnicode_TranslateCharmap(str, mapping, errors); - Py_DECREF(str); - return result; + return _PyUnicode_TranslateCharmap(str, mapping, errors); } static Py_UCS4 @@ -9163,9 +9166,10 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, } static Py_ssize_t -any_find_slice(int direction, PyObject* s1, PyObject* s2, +any_find_slice(PyObject* s1, PyObject* s2, Py_ssize_t start, - Py_ssize_t end) + Py_ssize_t end, + int direction) { int kind1, kind2; void *buf1, *buf2; @@ -9334,54 +9338,35 @@ PyUnicode_Count(PyObject *str, Py_ssize_t end) { Py_ssize_t result; - PyObject* str_obj; - PyObject* sub_obj; int kind1, kind2; void *buf1 = NULL, *buf2 = NULL; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str); - if (!str_obj) - return -1; - sub_obj = PyUnicode_FromObject(substr); - if (!sub_obj) { - Py_DECREF(str_obj); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -1; - } - if (PyUnicode_READY(sub_obj) == -1 || PyUnicode_READY(str_obj) == -1) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); - return -1; - } - kind1 = PyUnicode_KIND(str_obj); - kind2 = PyUnicode_KIND(sub_obj); - if (kind1 < kind2) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) return 0; - } - len1 = PyUnicode_GET_LENGTH(str_obj); - len2 = PyUnicode_GET_LENGTH(sub_obj); + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(substr); ADJUST_INDICES(start, end, len1); - if (end - start < len2) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); + if (end - start < len2) return 0; - } - buf1 = PyUnicode_DATA(str_obj); - buf2 = PyUnicode_DATA(sub_obj); + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(substr); if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sub_obj, kind1); + buf2 = _PyUnicode_AsKind(substr, kind1); if (!buf2) goto onError; } switch (kind1) { case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sub_obj)) + if (PyUnicode_IS_ASCII(str) && PyUnicode_IS_ASCII(substr)) result = asciilib_count( ((Py_UCS1*)buf1) + start, end - start, buf2, len2, PY_SSIZE_T_MAX @@ -9408,16 +9393,11 @@ PyUnicode_Count(PyObject *str, assert(0); result = 0; } - Py_DECREF(sub_obj); - Py_DECREF(str_obj); - if (kind2 != kind1) PyMem_Free(buf2); return result; onError: - Py_DECREF(sub_obj); - Py_DECREF(str_obj); if (kind2 != kind1 && buf2) PyMem_Free(buf2); return -1; @@ -9425,35 +9405,15 @@ PyUnicode_Count(PyObject *str, Py_ssize_t PyUnicode_Find(PyObject *str, - PyObject *sub, + PyObject *substr, Py_ssize_t start, Py_ssize_t end, int direction) { - Py_ssize_t result; - - str = PyUnicode_FromObject(str); - if (!str) - return -2; - sub = PyUnicode_FromObject(sub); - if (!sub) { - Py_DECREF(str); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -2; - } - if (PyUnicode_READY(sub) == -1 || PyUnicode_READY(str) == -1) { - Py_DECREF(sub); - Py_DECREF(str); - return -2; - } - result = any_find_slice(direction, - str, sub, start, end - ); - - Py_DECREF(str); - Py_DECREF(sub); - - return result; + return any_find_slice(str, substr, start, end, direction); } Py_ssize_t @@ -9556,22 +9516,10 @@ PyUnicode_Tailmatch(PyObject *str, Py_ssize_t end, int direction) { - Py_ssize_t result; - - str = PyUnicode_FromObject(str); - if (str == NULL) + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -1; - substr = PyUnicode_FromObject(substr); - if (substr == NULL) { - Py_DECREF(str); - return -1; - } - result = tailmatch(str, substr, - start, end, direction); - Py_DECREF(str); - Py_DECREF(substr); - return result; + return tailmatch(str, substr, start, end, direction); } /* Apply fixfct filter to the Unicode object self and return a @@ -10177,13 +10125,8 @@ PyUnicode_Splitlines(PyObject *string, int keepends) { PyObject *list; - string = PyUnicode_FromObject(string); - if (string == NULL) + if (ensure_unicode(string) < 0) return NULL; - if (PyUnicode_READY(string) == -1) { - Py_DECREF(string); - return NULL; - } switch (PyUnicode_KIND(string)) { case PyUnicode_1BYTE_KIND: @@ -10210,7 +10153,6 @@ PyUnicode_Splitlines(PyObject *string, int keepends) assert(0); list = 0; } - Py_DECREF(string); return list; } @@ -10771,28 +10713,27 @@ unicode_casefold(PyObject *self) } -/* Argument converter. Coerces to a single unicode character */ +/* Argument converter. Accepts a single Unicode character. */ static int convert_uc(PyObject *obj, void *addr) { Py_UCS4 *fillcharloc = (Py_UCS4 *)addr; - PyObject *uniobj; - uniobj = PyUnicode_FromObject(obj); - if (uniobj == NULL) { - PyErr_SetString(PyExc_TypeError, - "The fill character cannot be converted to Unicode"); + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "The fill character must be a unicode character, " + "not %.100s", Py_TYPE(obj)->tp_name); return 0; } - if (PyUnicode_GET_LENGTH(uniobj) != 1) { + if (PyUnicode_READY(obj) < 0) + return 0; + if (PyUnicode_GET_LENGTH(obj) != 1) { PyErr_SetString(PyExc_TypeError, "The fill character must be exactly one character long"); - Py_DECREF(uniobj); return 0; } - *fillcharloc = PyUnicode_READ_CHAR(uniobj, 0); - Py_DECREF(uniobj); + *fillcharloc = PyUnicode_READ_CHAR(obj, 0); return 1; } @@ -11114,59 +11055,43 @@ _PyUnicode_EQ(PyObject *aa, PyObject *bb) } int -PyUnicode_Contains(PyObject *container, PyObject *element) +PyUnicode_Contains(PyObject *str, PyObject *substr) { - PyObject *str, *sub; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; int result; - /* Coerce the two arguments */ - sub = PyUnicode_FromObject(element); - if (!sub) { + if (!PyUnicode_Check(substr)) { PyErr_Format(PyExc_TypeError, - "'in ' requires string as left operand, not %s", - element->ob_type->tp_name); + "'in ' requires string as left operand, not %.100s", + Py_TYPE(substr)->tp_name); return -1; } - - str = PyUnicode_FromObject(container); - if (!str) { - Py_DECREF(sub); + if (PyUnicode_READY(substr) == -1) + return -1; + if (ensure_unicode(str) < 0) return -1; - } kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(sub); - if (kind1 < kind2) { - Py_DECREF(sub); - Py_DECREF(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) return 0; - } len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(sub); - if (len1 < len2) { - Py_DECREF(sub); - Py_DECREF(str); + len2 = PyUnicode_GET_LENGTH(substr); + if (len1 < len2) return 0; - } buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(sub); + buf2 = PyUnicode_DATA(substr); if (len2 == 1) { Py_UCS4 ch = PyUnicode_READ(kind2, buf2, 0); result = findchar((const char *)buf1, kind1, len1, ch, 1) != -1; - Py_DECREF(sub); - Py_DECREF(str); return result; } if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sub, kind1); - if (!buf2) { - Py_DECREF(sub); - Py_DECREF(str); + buf2 = _PyUnicode_AsKind(substr, kind1); + if (!buf2) return -1; - } } switch (kind1) { @@ -11184,9 +11109,6 @@ PyUnicode_Contains(PyObject *container, PyObject *element) assert(0); } - Py_DECREF(str); - Py_DECREF(sub); - if (kind2 != kind1) PyMem_Free(buf2); @@ -11198,56 +11120,40 @@ PyUnicode_Contains(PyObject *container, PyObject *element) PyObject * PyUnicode_Concat(PyObject *left, PyObject *right) { - PyObject *u = NULL, *v = NULL, *w; + PyObject *result; Py_UCS4 maxchar, maxchar2; - Py_ssize_t u_len, v_len, new_len; + Py_ssize_t left_len, right_len, new_len; - /* Coerce the two arguments */ - u = PyUnicode_FromObject(left); - if (u == NULL) - goto onError; - v = PyUnicode_FromObject(right); - if (v == NULL) - goto onError; + if (ensure_unicode(left) < 0 || ensure_unicode(right) < 0) + return NULL; /* Shortcuts */ - if (v == unicode_empty) { - Py_DECREF(v); - return u; - } - if (u == unicode_empty) { - Py_DECREF(u); - return v; - } + if (left == unicode_empty) + return PyUnicode_FromObject(right); + if (right == unicode_empty) + return PyUnicode_FromObject(left); - u_len = PyUnicode_GET_LENGTH(u); - v_len = PyUnicode_GET_LENGTH(v); - if (u_len > PY_SSIZE_T_MAX - v_len) { + left_len = PyUnicode_GET_LENGTH(left); + right_len = PyUnicode_GET_LENGTH(right); + if (left_len > PY_SSIZE_T_MAX - right_len) { PyErr_SetString(PyExc_OverflowError, "strings are too large to concat"); - goto onError; + return NULL; } - new_len = u_len + v_len; + new_len = left_len + right_len; - maxchar = PyUnicode_MAX_CHAR_VALUE(u); - maxchar2 = PyUnicode_MAX_CHAR_VALUE(v); + maxchar = PyUnicode_MAX_CHAR_VALUE(left); + maxchar2 = PyUnicode_MAX_CHAR_VALUE(right); maxchar = Py_MAX(maxchar, maxchar2); /* Concat the two Unicode strings */ - w = PyUnicode_New(new_len, maxchar); - if (w == NULL) - goto onError; - _PyUnicode_FastCopyCharacters(w, 0, u, 0, u_len); - _PyUnicode_FastCopyCharacters(w, u_len, v, 0, v_len); - Py_DECREF(u); - Py_DECREF(v); - assert(_PyUnicode_CheckConsistency(w, 1)); - return w; - - onError: - Py_XDECREF(u); - Py_XDECREF(v); - return NULL; + result = PyUnicode_New(new_len, maxchar); + if (result == NULL) + return NULL; + _PyUnicode_FastCopyCharacters(result, 0, left, 0, left_len); + _PyUnicode_FastCopyCharacters(result, left_len, right, 0, right_len); + assert(_PyUnicode_CheckConsistency(result, 1)); + return result; } void @@ -11362,25 +11268,21 @@ unicode_count(PyObject *self, PyObject *args) kind1 = PyUnicode_KIND(self); kind2 = PyUnicode_KIND(substring); - if (kind1 < kind2) { - Py_DECREF(substring); + if (kind1 < kind2) return PyLong_FromLong(0); - } + len1 = PyUnicode_GET_LENGTH(self); len2 = PyUnicode_GET_LENGTH(substring); ADJUST_INDICES(start, end, len1); - if (end - start < len2) { - Py_DECREF(substring); + if (end - start < len2) return PyLong_FromLong(0); - } + buf1 = PyUnicode_DATA(self); buf2 = PyUnicode_DATA(substring); if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(substring, kind1); - if (!buf2) { - Py_DECREF(substring); + if (!buf2) return NULL; - } } switch (kind1) { case PyUnicode_1BYTE_KIND: @@ -11410,8 +11312,6 @@ unicode_count(PyObject *self, PyObject *args) if (kind2 != kind1) PyMem_Free(buf2); - Py_DECREF(substring); - return result; } @@ -11549,18 +11449,10 @@ unicode_find(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, 1); if (result == -2) return NULL; @@ -11637,18 +11529,10 @@ unicode_index(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - result = any_find_slice(1, self, substring, start, end); - - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, 1); if (result == -2) return NULL; @@ -12457,40 +12341,15 @@ unicode_repeat(PyObject *str, Py_ssize_t len) } PyObject * -PyUnicode_Replace(PyObject *obj, - PyObject *subobj, - PyObject *replobj, +PyUnicode_Replace(PyObject *str, + PyObject *substr, + PyObject *replstr, Py_ssize_t maxcount) { - PyObject *self; - PyObject *str1; - PyObject *str2; - PyObject *result; - - self = PyUnicode_FromObject(obj); - if (self == NULL) - return NULL; - str1 = PyUnicode_FromObject(subobj); - if (str1 == NULL) { - Py_DECREF(self); - return NULL; - } - str2 = PyUnicode_FromObject(replobj); - if (str2 == NULL) { - Py_DECREF(self); - Py_DECREF(str1); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0 || + ensure_unicode(replstr) < 0) return NULL; - } - if (PyUnicode_READY(self) == -1 || - PyUnicode_READY(str1) == -1 || - PyUnicode_READY(str2) == -1) - result = NULL; - else - result = replace(self, str1, str2, maxcount); - Py_DECREF(self); - Py_DECREF(str1); - Py_DECREF(str2); - return result; + return replace(str, substr, replstr, maxcount); } PyDoc_STRVAR(replace__doc__, @@ -12506,28 +12365,12 @@ unicode_replace(PyObject *self, PyObject *args) PyObject *str1; PyObject *str2; Py_ssize_t maxcount = -1; - PyObject *result; - if (!PyArg_ParseTuple(args, "OO|n:replace", &str1, &str2, &maxcount)) + if (!PyArg_ParseTuple(args, "UU|n:replace", &str1, &str2, &maxcount)) return NULL; if (PyUnicode_READY(self) == -1) return NULL; - str1 = PyUnicode_FromObject(str1); - if (str1 == NULL) - return NULL; - str2 = PyUnicode_FromObject(str2); - if (str2 == NULL) { - Py_DECREF(str1); - return NULL; - } - if (PyUnicode_READY(str1) == -1 || PyUnicode_READY(str2) == -1) - result = NULL; - else - result = replace(self, str1, str2, maxcount); - - Py_DECREF(str1); - Py_DECREF(str2); - return result; + return replace(self, str1, str2, maxcount); } static PyObject * @@ -12716,18 +12559,10 @@ unicode_rfind(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(-1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, -1); if (result == -2) return NULL; @@ -12753,18 +12588,10 @@ unicode_rindex(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(-1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, -1); if (result == -2) return NULL; @@ -12804,24 +12631,10 @@ unicode_rjust(PyObject *self, PyObject *args) PyObject * PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { - PyObject *result; - - s = PyUnicode_FromObject(s); - if (s == NULL) + if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) return NULL; - if (sep != NULL) { - sep = PyUnicode_FromObject(sep); - if (sep == NULL) { - Py_DECREF(s); - return NULL; - } - } - result = split(s, sep, maxsplit); - - Py_DECREF(s); - Py_XDECREF(sep); - return result; + return split(s, sep, maxsplit); } PyDoc_STRVAR(split__doc__, @@ -12846,35 +12659,26 @@ unicode_split(PyObject *self, PyObject *args, PyObject *kwds) if (substring == Py_None) return split(self, NULL, maxcount); - else if (PyUnicode_Check(substring)) + + if (PyUnicode_Check(substring)) return split(self, substring, maxcount); - else - return PyUnicode_Split(self, substring, maxcount); + + PyErr_Format(PyExc_TypeError, + "must be str or None, not %.100s", + Py_TYPE(substring)->tp_name); + return NULL; } PyObject * -PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) +PyUnicode_Partition(PyObject *str_obj, PyObject *sep_obj) { - PyObject* str_obj; - PyObject* sep_obj; PyObject* out; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str_in); - if (!str_obj) - return NULL; - sep_obj = PyUnicode_FromObject(sep_in); - if (!sep_obj) { - Py_DECREF(str_obj); + if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) return NULL; - } - if (PyUnicode_READY(sep_obj) == -1 || PyUnicode_READY(str_obj) == -1) { - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - return NULL; - } kind1 = PyUnicode_KIND(str_obj); kind2 = PyUnicode_KIND(sep_obj); @@ -12888,8 +12692,6 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) out = PyTuple_Pack(3, str_obj, unicode_empty, unicode_empty); Py_DECREF(unicode_empty); } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); return out; } buf1 = PyUnicode_DATA(str_obj); @@ -12897,7 +12699,7 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(sep_obj, kind1); if (!buf2) - goto onError; + return NULL; } switch (kind1) { @@ -12918,39 +12720,23 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) out = 0; } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); if (kind2 != kind1) PyMem_Free(buf2); return out; - onError: - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - if (kind2 != kind1 && buf2) - PyMem_Free(buf2); - return NULL; } PyObject * -PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) +PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) { - PyObject* str_obj; - PyObject* sep_obj; PyObject* out; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str_in); - if (!str_obj) + if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) return NULL; - sep_obj = PyUnicode_FromObject(sep_in); - if (!sep_obj) { - Py_DECREF(str_obj); - return NULL; - } kind1 = PyUnicode_KIND(str_obj); kind2 = PyUnicode_KIND(sep_obj); @@ -12964,8 +12750,6 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) out = PyTuple_Pack(3, unicode_empty, unicode_empty, str_obj); Py_DECREF(unicode_empty); } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); return out; } buf1 = PyUnicode_DATA(str_obj); @@ -12973,7 +12757,7 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(sep_obj, kind1); if (!buf2) - goto onError; + return NULL; } switch (kind1) { @@ -12994,18 +12778,10 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) out = 0; } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); if (kind2 != kind1) PyMem_Free(buf2); return out; - onError: - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - if (kind2 != kind1 && buf2) - PyMem_Free(buf2); - return NULL; } PyDoc_STRVAR(partition__doc__, @@ -13037,24 +12813,10 @@ unicode_rpartition(PyObject *self, PyObject *separator) PyObject * PyUnicode_RSplit(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { - PyObject *result; - - s = PyUnicode_FromObject(s); - if (s == NULL) + if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) return NULL; - if (sep != NULL) { - sep = PyUnicode_FromObject(sep); - if (sep == NULL) { - Py_DECREF(s); - return NULL; - } - } - result = rsplit(s, sep, maxsplit); - - Py_DECREF(s); - Py_XDECREF(sep); - return result; + return rsplit(s, sep, maxsplit); } PyDoc_STRVAR(rsplit__doc__, @@ -13079,10 +12841,14 @@ unicode_rsplit(PyObject *self, PyObject *args, PyObject *kwds) if (substring == Py_None) return rsplit(self, NULL, maxcount); - else if (PyUnicode_Check(substring)) + + if (PyUnicode_Check(substring)) return rsplit(self, substring, maxcount); - else - return PyUnicode_RSplit(self, substring, maxcount); + + PyErr_Format(PyExc_TypeError, + "must be str or None, not %.100s", + Py_TYPE(substring)->tp_name); + return NULL; } PyDoc_STRVAR(splitlines__doc__, @@ -13363,11 +13129,15 @@ unicode_startswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyUnicode_FromObject(PyTuple_GET_ITEM(subobj, i)); - if (substring == NULL) + substring = PyTuple_GET_ITEM(subobj, i); + if (!PyUnicode_Check(substring)) { + PyErr_Format(PyExc_TypeError, + "tuple for startswith must only contain str, " + "not %.100s", + Py_TYPE(substring)->tp_name); return NULL; + } result = tailmatch(self, substring, start, end, -1); - Py_DECREF(substring); if (result == -1) return NULL; if (result) { @@ -13377,15 +13147,13 @@ unicode_startswith(PyObject *self, /* nothing matched */ Py_RETURN_FALSE; } - substring = PyUnicode_FromObject(subobj); - if (substring == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "startswith first arg must be str or " - "a tuple of str, not %s", Py_TYPE(subobj)->tp_name); + if (!PyUnicode_Check(subobj)) { + PyErr_Format(PyExc_TypeError, + "startswith first arg must be str or " + "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, -1); - Py_DECREF(substring); + result = tailmatch(self, subobj, start, end, -1); if (result == -1) return NULL; return PyBool_FromLong(result); @@ -13415,12 +13183,15 @@ unicode_endswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyUnicode_FromObject( - PyTuple_GET_ITEM(subobj, i)); - if (substring == NULL) + substring = PyTuple_GET_ITEM(subobj, i); + if (!PyUnicode_Check(substring)) { + PyErr_Format(PyExc_TypeError, + "tuple for endswith must only contain str, " + "not %.100s", + Py_TYPE(substring)->tp_name); return NULL; + } result = tailmatch(self, substring, start, end, +1); - Py_DECREF(substring); if (result == -1) return NULL; if (result) { @@ -13429,15 +13200,13 @@ unicode_endswith(PyObject *self, } Py_RETURN_FALSE; } - substring = PyUnicode_FromObject(subobj); - if (substring == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "endswith first arg must be str or " - "a tuple of str, not %s", Py_TYPE(subobj)->tp_name); + if (!PyUnicode_Check(subobj)) { + PyErr_Format(PyExc_TypeError, + "endswith first arg must be str or " + "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, +1); - Py_DECREF(substring); + result = tailmatch(self, subobj, start, end, +1); if (result == -1) return NULL; return PyBool_FromLong(result); @@ -14907,13 +14676,10 @@ PyUnicode_Format(PyObject *format, PyObject *args) return NULL; } - ctx.fmtstr = PyUnicode_FromObject(format); - if (ctx.fmtstr == NULL) + if (ensure_unicode(format) < 0) return NULL; - if (PyUnicode_READY(ctx.fmtstr) == -1) { - Py_DECREF(ctx.fmtstr); - return NULL; - } + + ctx.fmtstr = format; ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); @@ -14973,11 +14739,9 @@ PyUnicode_Format(PyObject *format, PyObject *args) if (ctx.args_owned) { Py_DECREF(ctx.args); } - Py_DECREF(ctx.fmtstr); return _PyUnicodeWriter_Finish(&ctx.writer); onError: - Py_DECREF(ctx.fmtstr); _PyUnicodeWriter_Dealloc(&ctx.writer); if (ctx.args_owned) { Py_DECREF(ctx.args); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 31d9e0e..29fcffe 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1931,9 +1931,8 @@ builtin_input_impl(PyModuleDef *module, PyObject *prompt) Py_CLEAR(stringpo); if (po == NULL) goto _readline_errors; - promptstr = PyBytes_AsString(po); - if (promptstr == NULL) - goto _readline_errors; + assert(PyBytes_Check(po)); + promptstr = PyBytes_AS_STRING(po); } else { po = NULL; diff --git a/Python/getargs.c b/Python/getargs.c index 05ec27b..9858bd5 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1056,35 +1056,25 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return converterr("(AsCharBuffer failed)", arg, msgbuf, bufsize); } - else { - PyObject *u; - - /* Convert object to Unicode */ - u = PyUnicode_FromObject(arg); - if (u == NULL) - return converterr( - "string or unicode or text buffer", - arg, msgbuf, bufsize); - + else if (PyUnicode_Check(arg)) { /* Encode object; use default error handling */ - s = PyUnicode_AsEncodedString(u, + s = PyUnicode_AsEncodedString(arg, encoding, NULL); - Py_DECREF(u); if (s == NULL) return converterr("(encoding failed)", arg, msgbuf, bufsize); - if (!PyBytes_Check(s)) { - Py_DECREF(s); - return converterr( - "(encoder failed to return bytes)", - arg, msgbuf, bufsize); - } + assert(PyBytes_Check(s)); size = PyBytes_GET_SIZE(s); ptr = PyBytes_AS_STRING(s); if (ptr == NULL) ptr = ""; } + else { + return converterr( + recode_strings ? "str" : "str, bytes or bytearray", + arg, msgbuf, bufsize); + } /* Write output; output is guaranteed to be 0-terminated */ if (*format == '#') { -- cgit v0.12 From 23c5cbbddec66063f1cc44cc06045a39a119b3d6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Apr 2016 12:36:11 +0300 Subject: fs_unicode_converter is no longer used. --- Python/import.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Python/import.c b/Python/import.c index 0608924..6d71f2a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -38,14 +38,6 @@ module _imp #include "clinic/import.c.h" -/*[python input] -class fs_unicode_converter(CConverter): - type = 'PyObject *' - converter = 'PyUnicode_FSDecoder' - -[python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=9d6786230166006e]*/ - /* Initialize things */ void -- cgit v0.12 From 95702725ff7ce93c332757fec6ba00a59f5ded94 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 01:51:31 +1000 Subject: Add secrets module and tests. --- Lib/secrets.py | 149 +++++++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_secrets.py | 120 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 Lib/secrets.py create mode 100644 Lib/test/test_secrets.py diff --git a/Lib/secrets.py b/Lib/secrets.py new file mode 100644 index 0000000..ed018ad --- /dev/null +++ b/Lib/secrets.py @@ -0,0 +1,149 @@ +"""Generate cryptographically strong pseudo-random numbers suitable for +managing secrets such as account authentication, tokens, and similar. +See PEP 506 for more information. + +https://www.python.org/dev/peps/pep-0506/ + + +Random numbers +============== + +The ``secrets`` module provides the following pseudo-random functions, based +on SystemRandom, which in turn uses the most secure source of randomness your +operating system provides. + + + choice(sequence) + Choose a random element from a non-empty sequence. + + randbelow(n) + Return a random int in the range [0, n). + + randbits(k) + Generates an int with k random bits. + + SystemRandom + Class for generating random numbers using sources provided by + the operating system. See the ``random`` module for documentation. + + +Token functions +=============== + +The ``secrets`` module provides a number of functions for generating secure +tokens, suitable for applications such as password resets, hard-to-guess +URLs, and similar. All the ``token_*`` functions take an optional single +argument specifying the number of bytes of randomness to use. If that is +not given, or is ``None``, a reasonable default is used. That default is +subject to change at any time, including during maintenance releases. + + + token_bytes(nbytes=None) + Return a random byte-string containing ``nbytes`` number of bytes. + + >>> secrets.token_bytes(16) #doctest:+SKIP + b'\\xebr\\x17D*t\\xae\\xd4\\xe3S\\xb6\\xe2\\xebP1\\x8b' + + + token_hex(nbytes=None) + Return a random text-string, in hexadecimal. The string has ``nbytes`` + random bytes, each byte converted to two hex digits. + + >>> secrets.token_hex(16) #doctest:+SKIP + 'f9bf78b9a18ce6d46a0cd2b0b86df9da' + + token_urlsafe(nbytes=None) + Return a random URL-safe text-string, containing ``nbytes`` random + bytes. On average, each byte results in approximately 1.3 characters + in the final result. + + >>> secrets.token_urlsafe(16) #doctest:+SKIP + 'Drmhze6EPcv0fN_81Bj-nA' + + +(The examples above assume Python 3. In Python 2, byte-strings will display +using regular quotes ``''`` with no prefix, and text-strings will have a +``u`` prefix.) + + +Other functions +=============== + + compare_digest(a, b) + Return True if strings a and b are equal, otherwise False. + Performs the equality comparison in such a way as to reduce the + risk of timing attacks. + + See http://codahale.com/a-lesson-in-timing-attacks/ for a + discussion on how timing attacks against ``==`` can reveal + secrets from your application. + + +""" + +__all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom', + 'token_bytes', 'token_hex', 'token_urlsafe', + 'compare_digest', + ] + + +import base64 +import binascii +import os + +try: + from hmac import compare_digest +except ImportError: + # Python version is too old. Fall back to a pure-Python version. + + import operator + from functools import reduce + + def compare_digest(a, b): + """Return ``a == b`` using an approach resistant to timing analysis. + + a and b must both be of the same type: either both text strings, + or both byte strings. + + Note: If a and b are of different lengths, or if an error occurs, + a timing attack could theoretically reveal information about the + types and lengths of a and b, but not their values. + """ + # For a similar approach, see + # http://codahale.com/a-lesson-in-timing-attacks/ + for T in (bytes, str): + if isinstance(a, T) and isinstance(b, T): + break + else: # for...else + raise TypeError("arguments must be both strings or both bytes") + if len(a) != len(b): + return False + # Thanks to Raymond Hettinger for this one-liner. + return reduce(operator.and_, map(operator.eq, a, b), True) + + + +from random import SystemRandom + +_sysrand = SystemRandom() + +randbits = _sysrand.getrandbits +choice = _sysrand.choice + +def randbelow(exclusive_upper_bound): + return _sysrand._randbelow(exclusive_upper_bound) + +DEFAULT_ENTROPY = 32 # number of bytes to return by default + +def token_bytes(nbytes=None): + if nbytes is None: + nbytes = DEFAULT_ENTROPY + return os.urandom(nbytes) + +def token_hex(nbytes=None): + return binascii.hexlify(token_bytes(nbytes)).decode('ascii') + +def token_urlsafe(nbytes=None): + tok = token_bytes(nbytes) + return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii') + diff --git a/Lib/test/test_secrets.py b/Lib/test/test_secrets.py new file mode 100644 index 0000000..a3d1a8c --- /dev/null +++ b/Lib/test/test_secrets.py @@ -0,0 +1,120 @@ +"""Test the secrets module. + +As most of the functions in secrets are thin wrappers around functions +defined elsewhere, we don't need to test them exhaustively. +""" + + +import secrets +import unittest +import string + + +# === Unit tests === + +class Compare_Digest_Tests(unittest.TestCase): + """Test secrets.compare_digest function.""" + + def test_equal(self): + # Test compare_digest functionality with equal (byte/text) strings. + for s in ("a", "bcd", "xyz123"): + a = s*100 + b = s*100 + self.assertTrue(secrets.compare_digest(a, b)) + self.assertTrue(secrets.compare_digest(a.encode('utf-8'), b.encode('utf-8'))) + + def test_unequal(self): + # Test compare_digest functionality with unequal (byte/text) strings. + self.assertFalse(secrets.compare_digest("abc", "abcd")) + self.assertFalse(secrets.compare_digest(b"abc", b"abcd")) + for s in ("x", "mn", "a1b2c3"): + a = s*100 + "q" + b = s*100 + "k" + self.assertFalse(secrets.compare_digest(a, b)) + self.assertFalse(secrets.compare_digest(a.encode('utf-8'), b.encode('utf-8'))) + + def test_bad_types(self): + # Test that compare_digest raises with mixed types. + a = 'abcde' + b = a.encode('utf-8') + assert isinstance(a, str) + assert isinstance(b, bytes) + self.assertRaises(TypeError, secrets.compare_digest, a, b) + self.assertRaises(TypeError, secrets.compare_digest, b, a) + + def test_bool(self): + # Test that compare_digest returns a bool. + self.assertTrue(isinstance(secrets.compare_digest("abc", "abc"), bool)) + self.assertTrue(isinstance(secrets.compare_digest("abc", "xyz"), bool)) + + +class Random_Tests(unittest.TestCase): + """Test wrappers around SystemRandom methods.""" + + def test_randbits(self): + # Test randbits. + errmsg = "randbits(%d) returned %d" + for numbits in (3, 12, 30): + for i in range(6): + n = secrets.randbits(numbits) + self.assertTrue(0 <= n < 2**numbits, errmsg % (numbits, n)) + + def test_choice(self): + # Test choice. + items = [1, 2, 4, 8, 16, 32, 64] + for i in range(10): + self.assertTrue(secrets.choice(items) in items) + + def test_randbelow(self): + # Test randbelow. + errmsg = "randbelow(%d) returned %d" + for i in range(2, 10): + n = secrets.randbelow(i) + self.assertTrue(n in range(i), errmsg % (i, n)) + self.assertRaises(ValueError, secrets.randbelow, 0) + + +class Token_Tests(unittest.TestCase): + """Test token functions.""" + + def test_token_defaults(self): + # Test that token_* functions handle default size correctly. + for func in (secrets.token_bytes, secrets.token_hex, + secrets.token_urlsafe): + name = func.__name__ + try: + func() + except TypeError: + self.fail("%s cannot be called with no argument" % name) + try: + func(None) + except TypeError: + self.fail("%s cannot be called with None" % name) + size = secrets.DEFAULT_ENTROPY + self.assertEqual(len(secrets.token_bytes(None)), size) + self.assertEqual(len(secrets.token_hex(None)), 2*size) + + def test_token_bytes(self): + # Test token_bytes. + self.assertTrue(isinstance(secrets.token_bytes(11), bytes)) + for n in (1, 8, 17, 100): + self.assertEqual(len(secrets.token_bytes(n)), n) + + def test_token_hex(self): + # Test token_hex. + self.assertTrue(isinstance(secrets.token_hex(7), str)) + for n in (1, 12, 25, 90): + s = secrets.token_hex(n) + self.assertEqual(len(s), 2*n) + self.assertTrue(all(c in string.hexdigits for c in s)) + + def test_token_urlsafe(self): + # Test token_urlsafe. + self.assertTrue(isinstance(secrets.token_urlsafe(9), str)) + legal = string.ascii_letters + string.digits + '-_' + for n in (1, 11, 28, 76): + self.assertTrue(all(c in legal for c in secrets.token_urlsafe(n))) + + +if __name__ == '__main__': + unittest.main() -- cgit v0.12 From a873f6824886007ca1096b9d3a0a730c52c1c327 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 01:55:14 +1000 Subject: run Tools/reindent.py on secrets.py to satisfy the checkwhitespace hook --- Lib/secrets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/secrets.py b/Lib/secrets.py index ed018ad..e0f2656 100644 --- a/Lib/secrets.py +++ b/Lib/secrets.py @@ -146,4 +146,3 @@ def token_hex(nbytes=None): def token_urlsafe(nbytes=None): tok = token_bytes(nbytes) return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii') - -- cgit v0.12 From 08fbef040a02d684c1c8466b98f8d68d38c3de47 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 10:04:24 +1000 Subject: Improve tests with more modern assert* methods and subTests. --- Lib/test/test_secrets.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_secrets.py b/Lib/test/test_secrets.py index a3d1a8c..afcba84 100644 --- a/Lib/test/test_secrets.py +++ b/Lib/test/test_secrets.py @@ -44,8 +44,8 @@ class Compare_Digest_Tests(unittest.TestCase): def test_bool(self): # Test that compare_digest returns a bool. - self.assertTrue(isinstance(secrets.compare_digest("abc", "abc"), bool)) - self.assertTrue(isinstance(secrets.compare_digest("abc", "xyz"), bool)) + self.assertIsInstance(secrets.compare_digest("abc", "abc"), bool) + self.assertIsInstance(secrets.compare_digest("abc", "xyz"), bool) class Random_Tests(unittest.TestCase): @@ -67,10 +67,8 @@ class Random_Tests(unittest.TestCase): def test_randbelow(self): # Test randbelow. - errmsg = "randbelow(%d) returned %d" for i in range(2, 10): - n = secrets.randbelow(i) - self.assertTrue(n in range(i), errmsg % (i, n)) + self.assertIn(secrets.randbelow(i), range(i)) self.assertRaises(ValueError, secrets.randbelow, 0) @@ -81,39 +79,44 @@ class Token_Tests(unittest.TestCase): # Test that token_* functions handle default size correctly. for func in (secrets.token_bytes, secrets.token_hex, secrets.token_urlsafe): - name = func.__name__ - try: - func() - except TypeError: - self.fail("%s cannot be called with no argument" % name) - try: - func(None) - except TypeError: - self.fail("%s cannot be called with None" % name) + with self.subTest(func=func): + name = func.__name__ + try: + func() + except TypeError: + self.fail("%s cannot be called with no argument" % name) + try: + func(None) + except TypeError: + self.fail("%s cannot be called with None" % name) size = secrets.DEFAULT_ENTROPY self.assertEqual(len(secrets.token_bytes(None)), size) self.assertEqual(len(secrets.token_hex(None)), 2*size) def test_token_bytes(self): # Test token_bytes. - self.assertTrue(isinstance(secrets.token_bytes(11), bytes)) for n in (1, 8, 17, 100): - self.assertEqual(len(secrets.token_bytes(n)), n) + with self.subTest(n=n): + self.assertIsInstance(secrets.token_bytes(n), bytes) + self.assertEqual(len(secrets.token_bytes(n)), n) def test_token_hex(self): # Test token_hex. - self.assertTrue(isinstance(secrets.token_hex(7), str)) for n in (1, 12, 25, 90): - s = secrets.token_hex(n) - self.assertEqual(len(s), 2*n) - self.assertTrue(all(c in string.hexdigits for c in s)) + with self.subTest(n=n): + s = secrets.token_hex(n) + self.assertIsInstance(s, str)) + self.assertEqual(len(s), 2*n) + self.assertTrue(all(c in string.hexdigits for c in s)) def test_token_urlsafe(self): # Test token_urlsafe. - self.assertTrue(isinstance(secrets.token_urlsafe(9), str)) legal = string.ascii_letters + string.digits + '-_' for n in (1, 11, 28, 76): - self.assertTrue(all(c in legal for c in secrets.token_urlsafe(n))) + with self.subTest(n=n): + s = secrets.token_urlsafe(n) + self.assertIsInstance(s, str)) + self.assertTrue(all(c in legal for c in s)) if __name__ == '__main__': -- cgit v0.12 From 8ca020e1cc2590472ba83611727ae20d9f34bfe5 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 10:06:18 +1000 Subject: Fix missing parens. --- Lib/test/test_secrets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_secrets.py b/Lib/test/test_secrets.py index afcba84..4c65cf0 100644 --- a/Lib/test/test_secrets.py +++ b/Lib/test/test_secrets.py @@ -105,7 +105,7 @@ class Token_Tests(unittest.TestCase): for n in (1, 12, 25, 90): with self.subTest(n=n): s = secrets.token_hex(n) - self.assertIsInstance(s, str)) + self.assertIsInstance(s, str) self.assertEqual(len(s), 2*n) self.assertTrue(all(c in string.hexdigits for c in s)) @@ -115,7 +115,7 @@ class Token_Tests(unittest.TestCase): for n in (1, 11, 28, 76): with self.subTest(n=n): s = secrets.token_urlsafe(n) - self.assertIsInstance(s, str)) + self.assertIsInstance(s, str) self.assertTrue(all(c in legal for c in s)) -- cgit v0.12 From c9a59e6e4f221f492578a03546e3d7c96b9da1b8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 15 Apr 2016 14:11:10 +0300 Subject: Issue #26764: Fixed SystemError in bytes.__rmod__. --- Lib/test/test_bytes.py | 37 +++++++++++-------------------------- Objects/bytesobject.c | 10 ++++------ 2 files changed, 15 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index bd3410f..966e287 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -483,26 +483,33 @@ class BaseBytesTest: self.assertRaises(ValueError, b.rindex, w, 1, 3) def test_mod(self): - b = b'hello, %b!' + b = self.type2test(b'hello, %b!') orig = b b = b % b'world' self.assertEqual(b, b'hello, world!') self.assertEqual(orig, b'hello, %b!') self.assertFalse(b is orig) - b = b'%s / 100 = %d%%' + b = self.type2test(b'%s / 100 = %d%%') a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') + self.assertIs(type(a), bytes) def test_imod(self): - b = b'hello, %b!' + b = self.type2test(b'hello, %b!') orig = b b %= b'world' self.assertEqual(b, b'hello, world!') self.assertEqual(orig, b'hello, %b!') self.assertFalse(b is orig) - b = b'%s / 100 = %d%%' + b = self.type2test(b'%s / 100 = %d%%') b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') + self.assertIs(type(b), bytes) + + def test_rmod(self): + with self.assertRaises(TypeError): + object() % self.type2test(b'abc') + self.assertIs(self.type2test(b'abc').__rmod__('%r'), NotImplemented) def test_replace(self): b = self.type2test(b'mississippi') @@ -1064,28 +1071,6 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase): b[8:] = b self.assertEqual(b, bytearray(list(range(8)) + list(range(256)))) - def test_mod(self): - b = bytearray(b'hello, %b!') - orig = b - b = b % b'world' - self.assertEqual(b, b'hello, world!') - self.assertEqual(orig, bytearray(b'hello, %b!')) - self.assertFalse(b is orig) - b = bytearray(b'%s / 100 = %d%%') - a = b % (b'seventy-nine', 79) - self.assertEqual(a, bytearray(b'seventy-nine / 100 = 79%')) - - def test_imod(self): - b = bytearray(b'hello, %b!') - orig = b - b %= b'world' - self.assertEqual(b, b'hello, world!') - self.assertEqual(orig, bytearray(b'hello, %b!')) - self.assertFalse(b is orig) - b = bytearray(b'%s / 100 = %d%%') - b %= (b'seventy-nine', 79) - self.assertEqual(b, bytearray(b'seventy-nine / 100 = 79%')) - def test_iconcat(self): b = bytearray(b"abc") b1 = b diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index b935375..ec03233 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3282,15 +3282,13 @@ bytes_methods[] = { }; static PyObject * -bytes_mod(PyObject *self, PyObject *args) +bytes_mod(PyObject *self, PyObject *arg) { - if (self == NULL || !PyBytes_Check(self)) { - PyErr_BadInternalCall(); - return NULL; + if (!PyBytes_Check(self)) { + Py_RETURN_NOTIMPLEMENTED; } - return _PyBytes_FormatEx(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), - args, 0); + arg, 0); } static PyNumberMethods bytes_as_number = { -- cgit v0.12 From e914d413128e504f4bb4c46315c9afef104388c9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 15 Apr 2016 17:52:27 +0200 Subject: Issue #26766: Fix _PyBytesWriter_Finish() Return a bytearray object when bytearray is requested and when the small buffer is used. Fix also test_bytes: bytearray%args must return a bytearray type. --- Lib/test/test_bytes.py | 4 ++-- Objects/bytesobject.c | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 966e287..a10ad5e 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -492,7 +492,7 @@ class BaseBytesTest: b = self.type2test(b'%s / 100 = %d%%') a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') - self.assertIs(type(a), bytes) + self.assertIs(type(a), self.type2test) def test_imod(self): b = self.type2test(b'hello, %b!') @@ -504,7 +504,7 @@ class BaseBytesTest: b = self.type2test(b'%s / 100 = %d%%') b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') - self.assertIs(type(b), bytes) + self.assertIs(type(b), self.type2test) def test_rmod(self): with self.assertRaises(TypeError): diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ec03233..701ae9d 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -4150,7 +4150,12 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) result = PyBytes_FromStringAndSize(NULL, 0); } else if (writer->use_small_buffer) { - result = PyBytes_FromStringAndSize(writer->small_buffer, size); + if (writer->use_bytearray) { + result = PyByteArray_FromStringAndSize(writer->small_buffer, size); + } + else { + result = PyBytes_FromStringAndSize(writer->small_buffer, size); + } } else { result = writer->buffer; -- cgit v0.12 From 6dda1b14af1924a0d8f7bb891aa342c358213e8a Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Sat, 16 Apr 2016 04:33:55 +1000 Subject: Remove python fallback for compare_digest. See https://mail.python.org/pipermail/python-dev/2016-April/144198.html https://mail.python.org/pipermail/python-dev/2016-April/144203.html --- Lib/secrets.py | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/Lib/secrets.py b/Lib/secrets.py index e0f2656..e4e9714 100644 --- a/Lib/secrets.py +++ b/Lib/secrets.py @@ -91,38 +91,7 @@ import base64 import binascii import os -try: - from hmac import compare_digest -except ImportError: - # Python version is too old. Fall back to a pure-Python version. - - import operator - from functools import reduce - - def compare_digest(a, b): - """Return ``a == b`` using an approach resistant to timing analysis. - - a and b must both be of the same type: either both text strings, - or both byte strings. - - Note: If a and b are of different lengths, or if an error occurs, - a timing attack could theoretically reveal information about the - types and lengths of a and b, but not their values. - """ - # For a similar approach, see - # http://codahale.com/a-lesson-in-timing-attacks/ - for T in (bytes, str): - if isinstance(a, T) and isinstance(b, T): - break - else: # for...else - raise TypeError("arguments must be both strings or both bytes") - if len(a) != len(b): - return False - # Thanks to Raymond Hettinger for this one-liner. - return reduce(operator.and_, map(operator.eq, a, b), True) - - - +from hmac import compare_digest from random import SystemRandom _sysrand = SystemRandom() -- cgit v0.12 From 43de36d2c7a6138a47371cbe9411d3dd8026e5a4 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 16 Apr 2016 01:20:47 +0300 Subject: Issue #26766: Remove redundant bytearray_format() from bytearrayobject.c --- Objects/bytearrayobject.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 7748859..de7fb1e 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -279,19 +279,6 @@ PyByteArray_Concat(PyObject *a, PyObject *b) return (PyObject *)result; } -static PyObject * -bytearray_format(PyByteArrayObject *self, PyObject *args) -{ - if (self == NULL || !PyByteArray_Check(self)) { - PyErr_BadInternalCall(); - return NULL; - } - - return _PyBytes_FormatEx(PyByteArray_AS_STRING(self), - PyByteArray_GET_SIZE(self), - args, 1); -} - /* Functions stuffed into the type object */ static Py_ssize_t @@ -3014,7 +3001,7 @@ bytearray_mod(PyObject *v, PyObject *w) { if (!PyByteArray_Check(v)) Py_RETURN_NOTIMPLEMENTED; - return bytearray_format((PyByteArrayObject *)v, w); + return _PyBytes_FormatEx(PyByteArray_AS_STRING(v), PyByteArray_GET_SIZE(v), w, 1); } static PyNumberMethods bytearray_as_number = { -- cgit v0.12 From b2871faa874c5a7abd7d9f77dfeb6082253a40be Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Sun, 17 Apr 2016 01:42:33 +1000 Subject: Documentation for secrets.py --- Doc/library/crypto.rst | 1 + Doc/library/random.rst | 3 +- Doc/library/secrets.rst | 199 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 Doc/library/secrets.rst diff --git a/Doc/library/crypto.rst b/Doc/library/crypto.rst index 1eddfdc..ae45549 100644 --- a/Doc/library/crypto.rst +++ b/Doc/library/crypto.rst @@ -16,3 +16,4 @@ Here's an overview: hashlib.rst hmac.rst + secrets.rst diff --git a/Doc/library/random.rst b/Doc/library/random.rst index df502a0..e7b81ad 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -46,7 +46,8 @@ from sources provided by the operating system. .. warning:: The pseudo-random generators of this module should not be used for - security purposes. + security purposes. For security or cryptographic uses, see the + :mod:`secrets` module. Bookkeeping functions: diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst new file mode 100644 index 0000000..cc214af --- /dev/null +++ b/Doc/library/secrets.rst @@ -0,0 +1,199 @@ +:mod:`secrets` --- Generate secure random numbers for managing secrets +====================================================================== + +.. module:: secrets + :synopsis: Generate secure random numbers for managing secrets. + +.. moduleauthor:: Steven D'Aprano +.. sectionauthor:: Steven D'Aprano +.. versionadded:: 3.6 + +.. testsetup:: + + from secrets import * + __name__ = '' + +**Source code:** :source:`Lib/secrets.py` + +------------- + +The :mod:`secrets` module is used for generating cryptographically strong +random numbers suitable for managing data such as passwords, account +authentication, security tokens, and related secrets. + +In particularly, :mod:`secrets` should be used in preference to the +default pseudo-random number generator in the :mod:`random` module, which +is designed for modelling and simulation, not security or cryptography. + +.. seealso:: + + :pep:`506` + + +Random numbers +-------------- + +The :mod:`secrets` module provides access to the most secure source of +randomness that your operating system provides. + +.. class:: SystemRandom + + A class for generating random numbers using the highest-quality + sources provided by the operating system. See + :class:`random.SystemRandom` for additional details. + +.. function:: choice(sequence) + + Return a randomly-chosen element from a non-empty sequence. + +.. function:: randbelow(n) + + Return a random int in the range [0, *n*). + +.. function:: randbits(k) + + Return an int with *k* random bits. + + +Generating tokens +----------------- + +The :mod:`secrets` module provides functions for generating secure +tokens, suitable for applications such as password resets, +hard-to-guess URLs, and similar. + +.. function:: token_bytes([nbytes=None]) + + Return a random byte string containing *nbytes* number of bytes. + If *nbytes* is ``None`` or not supplied, a reasonable default is + used. + + .. doctest:: + + >>> token_bytes(16) #doctest:+SKIP + b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' + + +.. function:: token_hex([nbytes=None]) + + Return a random text string, in hexadecimal. The string has *nbytes* + random bytes, each byte converted to two hex digits. If *nbytes* is + ``None`` or not supplied, a reasonable default is used. + + .. doctest:: + + >>> token_hex(16) #doctest:+SKIP + 'f9bf78b9a18ce6d46a0cd2b0b86df9da' + +.. function:: token_urlsafe([nbytes=None]) + + Return a random URL-safe text string, containing *nbytes* random + bytes. The text is Base64 encoded, so on average, each byte results + in approximately 1.3 characters. If *nbytes* is ``None`` or not + supplied, a reasonable default is used. + + .. doctest:: + + >>> token_urlsafe(16) #doctest:+SKIP + 'Drmhze6EPcv0fN_81Bj-nA' + + +How many bytes should tokens use? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To be secure against +`brute-force attacks `_, +tokens need to have sufficient randomness. Unfortunately, what is +considered sufficient will necessarily increase as computers get more +powerful and able to make more guesses in a shorter period. As of 2015, +it is believed that 64 bytes (512 bits) of randomness is sufficient for +the typical use-case expected for the :mod:`secrets` module. + +For those who want to manage their own token length, you can explicitly +specify how much randomness is used for tokens by giving an :class:`int` +argument to the various ``token_*`` functions. That argument is taken +as the number of bytes of randomness to use. + +Otherwise, if no argument is provided, or if the argument is ``None``, +the ``token_*`` functions will use a reasonable default instead. + +.. note:: + + That default is subject to change at any time, including during + maintenance releases. + + +Other functions +--------------- + +.. function:: compare_digest(a, b) + + Return ``True`` if strings *a* and *b* are equal, otherwise ``False``, + in such a way as to redice the risk of + `timing attacks `_ . + See :func:`hmac.compare_digest` for additional details. + + +Recipes and best practices +-------------------------- + +This section shows recipes and best practices for using :mod:`secrets` +to manage a basic level of security. + +Generate an eight-character alphanumeric password: + +.. testcode:: + + import string + alphabet = string.ascii_letters + string.digits + password = ''.join(choice(alphabet) for i in range(8)) + + +.. note:: + + Applications should + `not store passwords in a recoverable format `_ , + whether plain text or encrypted. They should always be salted and + hashed using a cryptographically-strong one-way (irreversible) hash + function. + + +Generate a ten-character alphanumeric password with at least one +lowercase character, at least one uppercase character, and at least +three digits: + +.. testcode:: + + import string + alphabet = string.ascii_letters + string.digits + while True: + password = ''.join(choice(alphabet) for i in range(10)) + if (any(c.islower() for c in password) + and any(c.isupper() for c in password) + and sum(c.isdigit() for c in password) >= 3): + break + + +Generate an `XKCD-style passphrase `_ : + +.. testcode:: + + # On standard Linux systems, use a convenient dictionary file. + # Other platforms may need to provide their own word-list. + with open('/usr/share/dict/words') as f: + words = [word.strip() for word in f] + password = ' '.join(choice(words) for i in range(4)) + + +Generate a hard-to-guess temporary URL containing a security token +suitable for password recovery applications: + +.. testcode:: + + url = 'https://mydomain.com/reset=' + token_urlsafe() + + + +.. + # This modeline must appear within the last ten lines of the file. + kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; -- cgit v0.12 From 528619b6c3a9cdf987397eacb4fbc36dec7c6433 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Apr 2016 23:42:37 +0000 Subject: Issue #26782: Add STARTUPINFO to subprocess.__all__ on Windows --- Doc/whatsnew/3.6.rst | 2 +- Lib/subprocess.py | 3 ++- Lib/test/test_subprocess.py | 3 +-- Misc/NEWS | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index ef10ef2..adb8b73 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -500,7 +500,7 @@ Changes in the Python API attributes to match the documented APIs: :mod:`calendar`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, - :mod:`optparse`, :mod:`tarfile`, :mod:`threading` and + :mod:`optparse`, :mod:`subprocess`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 640519d..e980349 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -471,7 +471,8 @@ if _mswindows: __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", - "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) + "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTUPINFO"]) class Handle(int): closed = False diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index b5c86f2..7f70fd0 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2540,8 +2540,7 @@ class MiscTests(unittest.TestCase): def test__all__(self): """Ensure that __all__ is populated properly.""" - # STARTUPINFO added to __all__ in 3.6 - intentionally_excluded = {"list2cmdline", "STARTUPINFO", "Handle"} + intentionally_excluded = {"list2cmdline", "Handle"} exported = set(subprocess.__all__) possible_exports = set() import types diff --git a/Misc/NEWS b/Misc/NEWS index af2d654..c02cd3a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -245,6 +245,8 @@ Core and Builtins Library ------- +- Issue #26782: Add STARTUPINFO to subprocess.__all__ on Windows. + - Issue #26404: Add context manager to socketserver. Patch by Aviv Palivoda. - Issue #26735: Fix :func:`os.urandom` on Solaris 11.3 and newer when reading -- cgit v0.12 From 151f5d5971ad3c19e0c6635e4ff0bbbad4cd1a82 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Sun, 17 Apr 2016 13:05:10 +1000 Subject: Fix a few minor typos to secrets documentation. --- Doc/library/secrets.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index cc214af..9bf848f 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -88,7 +88,7 @@ hard-to-guess URLs, and similar. .. function:: token_urlsafe([nbytes=None]) Return a random URL-safe text string, containing *nbytes* random - bytes. The text is Base64 encoded, so on average, each byte results + bytes. The text is Base64 encoded, so on average each byte results in approximately 1.3 characters. If *nbytes* is ``None`` or not supplied, a reasonable default is used. @@ -106,7 +106,7 @@ To be secure against tokens need to have sufficient randomness. Unfortunately, what is considered sufficient will necessarily increase as computers get more powerful and able to make more guesses in a shorter period. As of 2015, -it is believed that 64 bytes (512 bits) of randomness is sufficient for +it is believed that 32 bytes (256 bits) of randomness is sufficient for the typical use-case expected for the :mod:`secrets` module. For those who want to manage their own token length, you can explicitly @@ -129,8 +129,8 @@ Other functions .. function:: compare_digest(a, b) Return ``True`` if strings *a* and *b* are equal, otherwise ``False``, - in such a way as to redice the risk of - `timing attacks `_ . + in such a way as to reduce the risk of + `timing attacks `_. See :func:`hmac.compare_digest` for additional details. @@ -151,11 +151,10 @@ Generate an eight-character alphanumeric password: .. note:: - Applications should - `not store passwords in a recoverable format `_ , - whether plain text or encrypted. They should always be salted and - hashed using a cryptographically-strong one-way (irreversible) hash - function. + Applications should not + `store passwords in a recoverable format `_, + whether plain text or encrypted. They should be salted and hashed + using a cryptographically-strong one-way (irreversible) hash function. Generate a ten-character alphanumeric password with at least one @@ -174,7 +173,7 @@ three digits: break -Generate an `XKCD-style passphrase `_ : +Generate an `XKCD-style passphrase `_: .. testcode:: -- cgit v0.12 From 4ad465424a6ab3f76b378d898e1bf06ec0976c26 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Sun, 17 Apr 2016 13:13:36 +1000 Subject: Shorten secrets module docstring, add function docstrings. --- Lib/secrets.py | 106 ++++++++++++++++----------------------------------------- 1 file changed, 30 insertions(+), 76 deletions(-) diff --git a/Lib/secrets.py b/Lib/secrets.py index e4e9714..27fa450 100644 --- a/Lib/secrets.py +++ b/Lib/secrets.py @@ -1,84 +1,9 @@ """Generate cryptographically strong pseudo-random numbers suitable for managing secrets such as account authentication, tokens, and similar. -See PEP 506 for more information. +See PEP 506 for more information. https://www.python.org/dev/peps/pep-0506/ - -Random numbers -============== - -The ``secrets`` module provides the following pseudo-random functions, based -on SystemRandom, which in turn uses the most secure source of randomness your -operating system provides. - - - choice(sequence) - Choose a random element from a non-empty sequence. - - randbelow(n) - Return a random int in the range [0, n). - - randbits(k) - Generates an int with k random bits. - - SystemRandom - Class for generating random numbers using sources provided by - the operating system. See the ``random`` module for documentation. - - -Token functions -=============== - -The ``secrets`` module provides a number of functions for generating secure -tokens, suitable for applications such as password resets, hard-to-guess -URLs, and similar. All the ``token_*`` functions take an optional single -argument specifying the number of bytes of randomness to use. If that is -not given, or is ``None``, a reasonable default is used. That default is -subject to change at any time, including during maintenance releases. - - - token_bytes(nbytes=None) - Return a random byte-string containing ``nbytes`` number of bytes. - - >>> secrets.token_bytes(16) #doctest:+SKIP - b'\\xebr\\x17D*t\\xae\\xd4\\xe3S\\xb6\\xe2\\xebP1\\x8b' - - - token_hex(nbytes=None) - Return a random text-string, in hexadecimal. The string has ``nbytes`` - random bytes, each byte converted to two hex digits. - - >>> secrets.token_hex(16) #doctest:+SKIP - 'f9bf78b9a18ce6d46a0cd2b0b86df9da' - - token_urlsafe(nbytes=None) - Return a random URL-safe text-string, containing ``nbytes`` random - bytes. On average, each byte results in approximately 1.3 characters - in the final result. - - >>> secrets.token_urlsafe(16) #doctest:+SKIP - 'Drmhze6EPcv0fN_81Bj-nA' - - -(The examples above assume Python 3. In Python 2, byte-strings will display -using regular quotes ``''`` with no prefix, and text-strings will have a -``u`` prefix.) - - -Other functions -=============== - - compare_digest(a, b) - Return True if strings a and b are equal, otherwise False. - Performs the equality comparison in such a way as to reduce the - risk of timing attacks. - - See http://codahale.com/a-lesson-in-timing-attacks/ for a - discussion on how timing attacks against ``==`` can reveal - secrets from your application. - - """ __all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom', @@ -100,18 +25,47 @@ randbits = _sysrand.getrandbits choice = _sysrand.choice def randbelow(exclusive_upper_bound): + """Return a random int in the range [0, n).""" return _sysrand._randbelow(exclusive_upper_bound) DEFAULT_ENTROPY = 32 # number of bytes to return by default def token_bytes(nbytes=None): + """Return a random byte string containing *nbytes* bytes. + + If *nbytes* is ``None`` or not supplied, a reasonable + default is used. + + >>> token_bytes(16) #doctest:+SKIP + b'\\xebr\\x17D*t\\xae\\xd4\\xe3S\\xb6\\xe2\\xebP1\\x8b' + + """ if nbytes is None: nbytes = DEFAULT_ENTROPY return os.urandom(nbytes) def token_hex(nbytes=None): + """Return a random text string, in hexadecimal. + + The string has *nbytes* random bytes, each byte converted to two + hex digits. If *nbytes* is ``None`` or not supplied, a reasonable + default is used. + + >>> token_hex(16) #doctest:+SKIP + 'f9bf78b9a18ce6d46a0cd2b0b86df9da' + + """ return binascii.hexlify(token_bytes(nbytes)).decode('ascii') def token_urlsafe(nbytes=None): + """Return a random URL-safe text string, in Base64 encoding. + + The string has *nbytes* random bytes. If *nbytes* is ``None`` + or not supplied, a reasonable default is used. + + >>> token_urlsafe(16) #doctest:+SKIP + 'Drmhze6EPcv0fN_81Bj-nA' + + """ tok = token_bytes(nbytes) return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii') -- cgit v0.12 From a858bbde03638e3145894029dbc40d3d777be24f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 17 Apr 2016 16:51:52 +0200 Subject: Avoid fcntl() if possible in set_inheritable() Issue #26770: set_inheritable() avoids calling fcntl() twice if the FD_CLOEXEC is already set/cleared. This change only impacts platforms using the fcntl() implementation of set_inheritable() (not Linux nor Windows). --- Python/fileutils.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index a710c99..4a0ef48 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -798,7 +798,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) int request; int err; #endif - int flags; + int flags, new_flags; int res; #endif @@ -884,10 +884,18 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) return -1; } - if (inheritable) - flags &= ~FD_CLOEXEC; - else - flags |= FD_CLOEXEC; + if (inheritable) { + new_flags = flags & ~FD_CLOEXEC; + } + else { + new_flags = flags | FD_CLOEXEC; + } + + if (new_flags == flags) { + /* FD_CLOEXEC flag already set/cleared: nothing to do */ + return 0; + } + res = fcntl(fd, F_SETFD, flags); if (res < 0) { if (raise) -- cgit v0.12 From 55c861f637f8d82ac74e5b86a9361ed68d8be3f9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 17 Apr 2016 20:31:51 +0300 Subject: Issue #26745: Removed redundant code in _PyObject_GenericSetAttrWithDict. Based on patch by Xiang Zhang. --- Objects/object.c | 51 +++++++++++++++++++++------------------------------ 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/Objects/object.c b/Objects/object.c index 0817311..cc1b2ff 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1040,8 +1040,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *dict) name->ob_type->tp_name); return NULL; } - else - Py_INCREF(name); + Py_INCREF(name); if (tp->tp_dict == NULL) { if (PyType_Ready(tp) < 0) @@ -1049,10 +1048,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *dict) } descr = _PyType_Lookup(tp, name); - Py_XINCREF(descr); f = NULL; if (descr != NULL) { + Py_INCREF(descr); f = descr->ob_type->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)obj->ob_type); @@ -1072,8 +1071,9 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *dict) if (tsize < 0) tsize = -tsize; size = _PyObject_VAR_SIZE(tp, tsize); + assert(size <= PY_SSIZE_T_MAX); - dictoffset += (long)size; + dictoffset += (Py_ssize_t)size; assert(dictoffset > 0); assert(dictoffset % SIZEOF_VOID_P == 0); } @@ -1141,12 +1141,11 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, Py_INCREF(name); descr = _PyType_Lookup(tp, name); - Py_XINCREF(descr); - f = NULL; if (descr != NULL) { + Py_INCREF(descr); f = descr->ob_type->tp_descr_set; - if (f != NULL && PyDescr_IsData(descr)) { + if (f != NULL) { res = f(descr, obj, value); goto done; } @@ -1154,40 +1153,32 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { dictptr = _PyObject_GetDictPtr(obj); - if (dictptr != NULL) { - res = _PyObjectDict_SetItem(Py_TYPE(obj), dictptr, name, value); - if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) - PyErr_SetObject(PyExc_AttributeError, name); + if (dictptr == NULL) { + if (descr == NULL) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + } + else { + PyErr_Format(PyExc_AttributeError, + "'%.50s' object attribute '%U' is read-only", + tp->tp_name, name); + } goto done; } + res = _PyObjectDict_SetItem(tp, dictptr, name, value); } - if (dict != NULL) { + else { Py_INCREF(dict); if (value == NULL) res = PyDict_DelItem(dict, name); else res = PyDict_SetItem(dict, name, value); Py_DECREF(dict); - if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) - PyErr_SetObject(PyExc_AttributeError, name); - goto done; } + if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) + PyErr_SetObject(PyExc_AttributeError, name); - if (f != NULL) { - res = f(descr, obj, value); - goto done; - } - - if (descr == NULL) { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); - goto done; - } - - PyErr_Format(PyExc_AttributeError, - "'%.50s' object attribute '%U' is read-only", - tp->tp_name, name); done: Py_XDECREF(descr); Py_DECREF(name); -- cgit v0.12 From a648339595d738fad40692a04e7ff9863d22dd3d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 17 Apr 2016 21:21:01 -0700 Subject: Fix spelling error --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 02709e3..0c95722 100644 --- a/README +++ b/README @@ -187,7 +187,7 @@ Proposals for enhancement ------------------------- If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or python-ideas mailing lists for inital feedback. A Python +comp.lang.python or python-ideas mailing lists for initial feedback. A Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. All current PEPs, as well as guidelines for submitting a new PEP, are listed at http://www.python.org/dev/peps/. -- cgit v0.12 From d2be07e1fdc7e18338efe5c6e1c95eb44f8dd869 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 18 Apr 2016 07:25:54 +0200 Subject: #25987: add versionadded to Reversible. --- Doc/library/collections.abc.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 608641b..bbd0eda 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -112,6 +112,8 @@ ABC Inherits from Abstract Methods Mixin ABC for classes that provide the :meth:`__reversed__` method. + .. versionadded:: 3.6 + .. class:: Generator ABC for generator classes that implement the protocol defined in -- cgit v0.12 From 0621e0ea86ab964110c612b5eca297a698fb5195 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Apr 2016 17:02:55 +0200 Subject: Don't define _PyMem_PymallocEnabled() if pymalloc is disabled Isse #26516. --- Objects/obmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 40c9fcd..6cf8ea9 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -286,13 +286,13 @@ static PyObjectArenaAllocator _PyObject_Arena = {NULL, #endif }; +#ifdef WITH_PYMALLOC static int _PyMem_DebugEnabled(void) { return (_PyObject.malloc == _PyMem_DebugMalloc); } -#ifdef WITH_PYMALLOC int _PyMem_PymallocEnabled(void) { -- cgit v0.12 From 9a8d0d5c7d1ad3bfee57efce7860cded9790eee7 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 19 Apr 2016 19:17:16 +0100 Subject: Mention types.SimpleNamespace in collections.namedtuple doc Issue #26805. --- Doc/library/collections.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 728cd48..37d9e00 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -792,6 +792,11 @@ they add the ability to access fields by name instead of position index. Named tuple instances do not have per-instance dictionaries, so they are lightweight and require no more memory than regular tuples. + For simple uses, where the only requirement is to be able to refer to a set + of values by name using attribute-style access, the + :class:`types.SimpleNamespace` type can be a suitable alternative to using + a namedtuple. + .. versionchanged:: 3.1 Added support for *rename*. -- cgit v0.12 From 79d6e8de9e07f31da726c28819e3506d38b22f1e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 19 Apr 2016 23:37:17 +0300 Subject: Issue #26802: Optimized calling a function with *args only positional arguments. Patch by Joe Jevnik. --- Python/ceval.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Python/ceval.c b/Python/ceval.c index d69b8d6..88dc113 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4890,6 +4890,21 @@ update_star_args(int nstack, int nstar, PyObject *stararg, { PyObject *callargs, *w; + if (!nstack) { + if (!stararg) { + /* There are no positional arguments on the stack and there is no + sequence to be unpacked. */ + return PyTuple_New(0); + } + if (PyTuple_CheckExact(stararg)) { + /* No arguments are passed on the stack and the sequence is not a + tuple subclass so we can just pass the stararg tuple directly + to the function. */ + Py_INCREF(stararg); + return stararg; + } + } + callargs = PyTuple_New(nstack + nstar); if (callargs == NULL) { return NULL; -- cgit v0.12 From 0b2e98d53dab5023d4df5a276e42ea3dab30982b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Apr 2016 22:54:37 +0200 Subject: Optimize func(*tuple) function call Issue #26802: Optimize function calls only using unpacking like "func(*tuple)" (no other positional argument, no keyword): avoid copying the tuple. Patch written by Joe Jevnik. --- Misc/NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index fba505f..d202362 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26802: Optimize function calls only using unpacking like + ``func(*tuple)`` (no other positional argument, no keyword): avoid copying + the tuple. Patch written by Joe Jevnik. + - Issue #26659: Make the builtin slice type support cycle collection. - Issue #26718: super.__init__ no longer leaks memory if called multiple times. -- cgit v0.12 From 614827c1497701e4fe45d5cd55552ebfaa5d924c Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 19 Apr 2016 04:05:59 +0000 Subject: Additional grammar fix --- Lib/test/test_traceback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 2f739ba..5e4b6a2 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -175,7 +175,7 @@ class SyntaxTracebackCases(unittest.TestCase): text, charset, 5) do_test(" \t\f\n# coding: {0}\n".format(charset), text, charset, 5) - # Issue #18960: coding spec should has no effect + # Issue #18960: coding spec should have no effect do_test("x=0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) def test_print_traceback_at_exit(self): -- cgit v0.12 From 61f68dbd17aa45472a5e47671d12d719495025a3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Apr 2016 09:58:12 +0200 Subject: Issue #21668: Fix author of the patch. Sorry, it was hard to retrieve the original author of the patch in this issue with a long history and many authors. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index b144861..1654541 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -999,7 +999,7 @@ Build ----- - Issue #21668: Link audioop, _datetime, _ctypes_test modules to libm, - except on Mac OS X. Patch written by Xavier de Gaye. + except on Mac OS X. Patch written by Chi Hsuan Yen. - Issue #25702: A --with-lto configure option has been added that will enable link time optimizations at build time during a make profile-opt. -- cgit v0.12 From a2bf3060d0c4fa951fa42f68c27742a17782f4d9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Apr 2016 10:02:04 +0200 Subject: Issue #21668: Add also Chi Hsuan Yen to Misc/ACKS --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index 615c473..13db2b7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1630,6 +1630,7 @@ Florent Xicluna Arnon Yaari Hirokazu Yamamoto Ka-Ping Yee +Chi Hsuan Yen Jason Yeo EungJun Yi Bob Yodlowski -- cgit v0.12 From 5439fc4901dbc7ab0fc0c8d2b2266e05d0019a92 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 21 Apr 2016 00:23:08 -0700 Subject: [minor] Doc fix in old python doc. --- Doc/whatsnew/2.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index e55eaac..6e192dc 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -367,7 +367,7 @@ dictionary:: This version works for simple things such as integers, but it has a side effect; the ``_cache`` dictionary holds a reference to the return values, so they'll -never be deallocated until the Python process exits and cleans up This isn't +never be deallocated until the Python process exits and cleans up. This isn't very noticeable for integers, but if :func:`f` returns an object, or a data structure that takes up a lot of memory, this can be a problem. -- cgit v0.12 From f5c4b99034fae12ac2b9498dd12b5b3f352b90c8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Apr 2016 16:26:23 +0200 Subject: PyMem_Malloc() now uses the fast pymalloc allocator Issue #26249: PyMem_Malloc() allocator family now uses the pymalloc allocator rather than system malloc(). Applications calling PyMem_Malloc() without holding the GIL can now crash: use PYTHONMALLOC=debug environment variable to validate the usage of memory allocators in your application. --- Doc/c-api/memory.rst | 60 +++++++++++++++++++++++++++++++-------------------- Doc/using/cmdline.rst | 11 +++++----- Doc/whatsnew/3.6.rst | 6 ++++++ Misc/NEWS | 7 ++++++ Objects/obmalloc.c | 6 +++--- 5 files changed, 58 insertions(+), 32 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index bd0bc12..3ff5452 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -165,15 +165,17 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -The default memory block allocator uses the following functions: -:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call -``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes. +By default, these functions use :ref:`pymalloc memory allocator `. .. warning:: The :term:`GIL ` must be held when using these functions. +.. versionchanged:: 3.6 + + The default allocator is now pymalloc instead of system :c:func:`malloc`. + .. c:function:: void* PyMem_Malloc(size_t n) Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the @@ -295,15 +297,32 @@ Customize Memory Allocators Enum used to identify an allocator domain. Domains: - * :c:data:`PYMEM_DOMAIN_RAW`: functions :c:func:`PyMem_RawMalloc`, - :c:func:`PyMem_RawRealloc`, :c:func:`PyMem_RawCalloc` and - :c:func:`PyMem_RawFree` - * :c:data:`PYMEM_DOMAIN_MEM`: functions :c:func:`PyMem_Malloc`, - :c:func:`PyMem_Realloc`, :c:func:`PyMem_Calloc` and :c:func:`PyMem_Free` - * :c:data:`PYMEM_DOMAIN_OBJ`: functions :c:func:`PyObject_Malloc`, - :c:func:`PyObject_Realloc`, :c:func:`PyObject_Calloc` and - :c:func:`PyObject_Free` + .. c:var:: PYMEM_DOMAIN_RAW + + Functions: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + + .. c:var:: PYMEM_DOMAIN_MEM + Functions: + + * :c:func:`PyMem_Malloc`, + * :c:func:`PyMem_Realloc` + * :c:func:`PyMem_Calloc` + * :c:func:`PyMem_Free` + + .. c:var:: PYMEM_DOMAIN_OBJ + + Functions: + + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:func:`PyObject_Free` .. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) @@ -328,18 +347,12 @@ Customize Memory Allocators .. c:function:: void PyMem_SetupDebugHooks(void) - Setup hooks to detect bugs in the following Python memory allocator - functions: - - - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`, - :c:func:`PyMem_RawCalloc`, :c:func:`PyMem_RawFree` - - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Calloc`, - :c:func:`PyMem_Free` - - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`, - :c:func:`PyObject_Calloc`, :c:func:`PyObject_Free` + Setup hooks to detect bugs in the Python memory allocator functions. Newly allocated memory is filled with the byte ``0xCB``, freed memory is - filled with the byte ``0xDB``. Additional checks: + filled with the byte ``0xDB``. + + Runtime checks: - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer allocated by :c:func:`PyMem_Malloc` @@ -377,8 +390,9 @@ to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. -*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_OBJ` domain -(ex: :c:func:`PyObject_Malloc`). +*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: +:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: +:c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index c1c700c..49fe3a0 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -628,12 +628,11 @@ conflict. Set the family of memory allocators used by Python: * ``malloc``: use the :c:func:`malloc` function of the C library - for all Python memory allocators (ex: :c:func:`PyMem_RawMalloc`, - :c:func:`PyMem_Malloc` and :c:func:`PyObject_Malloc`). - * ``pymalloc``: :c:func:`PyObject_Malloc`, :c:func:`PyObject_Calloc` and - :c:func:`PyObject_Realloc` use the :ref:`pymalloc allocator `. - Other Python memory allocators (ex: :c:func:`PyMem_RawMalloc` and - :c:func:`PyMem_Malloc`) use :c:func:`malloc`. + for all domains (:c:data:`PYMEM_DOMAIN_RAW`, :c:data:`PYMEM_DOMAIN_MEM`, + :c:data:`PYMEM_DOMAIN_OBJ`). + * ``pymalloc``: use the :ref:`pymalloc allocator ` for + :c:data:`PYMEM_DOMAIN_MEM` and :c:data:`PYMEM_DOMAIN_OBJ` domains and use + the :c:func:`malloc` function for the :c:data:`PYMEM_DOMAIN_RAW` domain. Install debug hooks: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 331112f..87854c8 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -531,5 +531,11 @@ Changes in the Python API Changes in the C API -------------------- +* :c:func:`PyMem_Malloc` allocator family now uses the :ref:`pymalloc allocator + ` rather than system :c:func:`malloc`. Applications calling + :c:func:`PyMem_Malloc` without holding the GIL can now crash. Set the + :envvar:`PYTHONMALLOC` environment variable to ``debug`` to validate the + usage of memory allocators in your application. See :issue:`26249`. + * :c:func:`Py_Exit` (and the main interpreter) now override the exit status with 120 if flushing buffered data failed. See :issue:`5319`. diff --git a/Misc/NEWS b/Misc/NEWS index 5d5f01b..3f4eb43 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,13 @@ Release date: tba Core and Builtins ----------------- +- Issue #26249: Memory functions of the :c:func:`PyMem_Malloc` domain + (:c:data:`PYMEM_DOMAIN_MEM`) now use the :ref:`pymalloc allocator ` + rather than system :c:func:`malloc`. Applications calling + :c:func:`PyMem_Malloc` without holding the GIL can now crash: use + ``PYTHONMALLOC=debug`` environment variable to validate the usage of memory + allocators in your application. + - Issue #26802: Optimize function calls only using unpacking like ``func(*tuple)`` (no other positional argument, no keyword): avoid copying the tuple. Patch written by Joe Jevnik. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 6cf8ea9..d305b1d 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -198,9 +198,9 @@ static PyMemAllocatorEx _PyMem_Raw = { static PyMemAllocatorEx _PyMem = { #ifdef Py_DEBUG - &_PyMem_Debug.mem, PYDBG_FUNCS + &_PyMem_Debug.obj, PYDBG_FUNCS #else - NULL, PYMEM_FUNCS + NULL, PYOBJ_FUNCS #endif }; @@ -256,7 +256,7 @@ _PyMem_SetupAllocators(const char *opt) PyMemAllocatorEx obj_alloc = {NULL, PYOBJ_FUNCS}; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &mem_alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &mem_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &obj_alloc); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &obj_alloc); if (strcmp(opt, "pymalloc_debug") == 0) -- cgit v0.12 From 15932593bae491ffad73f690c8a2bcf6f91a085b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Apr 2016 18:52:22 +0200 Subject: Issue #26249: Try test_capi on Windows --- Objects/obmalloc.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index d305b1d..3f95133 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -166,7 +166,7 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size) #else # define PYOBJ_FUNCS PYRAW_FUNCS #endif -#define PYMEM_FUNCS PYRAW_FUNCS +#define PYMEM_FUNCS PYOBJ_FUNCS typedef struct { /* We tag each block with an API ID in order to tag API violations */ @@ -198,9 +198,9 @@ static PyMemAllocatorEx _PyMem_Raw = { static PyMemAllocatorEx _PyMem = { #ifdef Py_DEBUG - &_PyMem_Debug.obj, PYDBG_FUNCS + &_PyMem_Debug.mem, PYDBG_FUNCS #else - NULL, PYOBJ_FUNCS + NULL, PYMEM_FUNCS #endif }; @@ -252,11 +252,12 @@ _PyMem_SetupAllocators(const char *opt) else if (strcmp(opt, "pymalloc") == 0 || strcmp(opt, "pymalloc_debug") == 0) { - PyMemAllocatorEx mem_alloc = {NULL, PYRAW_FUNCS}; + PyMemAllocatorEx raw_alloc = {NULL, PYRAW_FUNCS}; + PyMemAllocatorEx mem_alloc = {NULL, PYMEM_FUNCS}; PyMemAllocatorEx obj_alloc = {NULL, PYOBJ_FUNCS}; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &mem_alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &obj_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &raw_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &mem_alloc); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &obj_alloc); if (strcmp(opt, "pymalloc_debug") == 0) -- cgit v0.12 From ce18d8c2f4ea8bbdc51ab4512d73f1ffe5b5901a Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 24 Apr 2016 01:55:09 +0300 Subject: Issue #26089: Remove duplicate field 'license' from DistributionMetadata It was renamed to 'license' in 178d19cff163. Patch by Augustin Laville. --- Lib/distutils/dist.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py index ffb33ff6..62a2451 100644 --- a/Lib/distutils/dist.py +++ b/Lib/distutils/dist.py @@ -1018,8 +1018,7 @@ class DistributionMetadata: "maintainer", "maintainer_email", "url", "license", "description", "long_description", "keywords", "platforms", "fullname", "contact", - "contact_email", "license", "classifiers", - "download_url", + "contact_email", "classifiers", "download_url", # PEP 314 "provides", "requires", "obsoletes", ) -- cgit v0.12 From 1e8ee9b3808cd6c1a7a29c75115d1060a8ee877b Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 24 Apr 2016 07:31:42 +0300 Subject: Issue #23277: Remove unused sys and os imports Patch by Jon Dufresne. --- Lib/ctypes/test/test_objects.py | 2 +- Lib/ctypes/test/test_parameters.py | 2 +- Lib/ctypes/test/test_returnfuncptrs.py | 1 - Lib/ctypes/test/test_sizes.py | 1 - Lib/test/audiotests.py | 2 +- Lib/test/make_ssl_certs.py | 1 - Lib/test/test_aifc.py | 1 - Lib/test/test_binhex.py | 1 - Lib/test/test_csv.py | 1 - Lib/test/test_dbm.py | 1 - Lib/test/test_dbm_ndbm.py | 1 - Lib/test/test_devpoll.py | 1 - Lib/test/test_eintr.py | 1 - Lib/test/test_email/__init__.py | 1 - Lib/test/test_heapq.py | 1 - Lib/test/test_importlib/extension/test_case_sensitivity.py | 1 - Lib/test/test_importlib/extension/test_path_hook.py | 1 - Lib/test/test_importlib/frozen/test_loader.py | 2 -- Lib/test/test_importlib/import_/test_relative_imports.py | 1 - Lib/test/test_importlib/source/test_case_sensitivity.py | 1 - Lib/test/test_json/__init__.py | 1 - Lib/test/test_kqueue.py | 1 - Lib/test/test_msilib.py | 1 - Lib/test/test_multibytecodec.py | 2 +- Lib/test/test_nis.py | 1 - Lib/test/test_normalization.py | 1 - Lib/test/test_parser.py | 1 - Lib/test/test_pep3151.py | 1 - Lib/test/test_posixpath.py | 1 - Lib/test/test_pulldom.py | 1 - Lib/test/test_quopri.py | 2 +- Lib/test/test_sort.py | 1 - Lib/test/test_strptime.py | 1 - Lib/test/test_tools/test_md5sum.py | 1 - Lib/test/test_tools/test_pdeps.py | 1 - Lib/test/test_ttk_guionly.py | 1 - Lib/test/test_ttk_textonly.py | 1 - Lib/test/test_unpack_ex.py | 1 - Lib/test/test_urllibnet.py | 1 - Lib/test/test_weakset.py | 2 -- Lib/test/test_xml_etree_c.py | 2 +- Lib/test/test_xmlrpc_net.py | 1 - Lib/test/test_zipfile.py | 1 - Lib/tkinter/test/runtktests.py | 1 - Lib/unittest/test/testmock/support.py | 2 -- 45 files changed, 6 insertions(+), 48 deletions(-) diff --git a/Lib/ctypes/test/test_objects.py b/Lib/ctypes/test/test_objects.py index ef7b20b..19e3dc1 100644 --- a/Lib/ctypes/test/test_objects.py +++ b/Lib/ctypes/test/test_objects.py @@ -54,7 +54,7 @@ of 'x' ('_b_base_' is either None, or the root object owning the memory block): ''' -import unittest, doctest, sys +import unittest, doctest import ctypes.test.test_objects diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py index e56bccf..5c8d703 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/ctypes/test/test_parameters.py @@ -1,4 +1,4 @@ -import unittest, sys +import unittest from ctypes.test import need_symbol class SimpleTypesTestCase(unittest.TestCase): diff --git a/Lib/ctypes/test/test_returnfuncptrs.py b/Lib/ctypes/test/test_returnfuncptrs.py index 93eba6b..1974f40 100644 --- a/Lib/ctypes/test/test_returnfuncptrs.py +++ b/Lib/ctypes/test/test_returnfuncptrs.py @@ -1,6 +1,5 @@ import unittest from ctypes import * -import os import _ctypes_test diff --git a/Lib/ctypes/test/test_sizes.py b/Lib/ctypes/test/test_sizes.py index f9b5e97..4ceacbc 100644 --- a/Lib/ctypes/test/test_sizes.py +++ b/Lib/ctypes/test/test_sizes.py @@ -2,7 +2,6 @@ from ctypes import * -import sys import unittest diff --git a/Lib/test/audiotests.py b/Lib/test/audiotests.py index 0ae2242..e52c423 100644 --- a/Lib/test/audiotests.py +++ b/Lib/test/audiotests.py @@ -3,7 +3,7 @@ import unittest import array import io import pickle -import sys + class UnseekableIO(io.FileIO): def tell(self): diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py index 81d04f8..e4326d7 100644 --- a/Lib/test/make_ssl_certs.py +++ b/Lib/test/make_ssl_certs.py @@ -3,7 +3,6 @@ and friends.""" import os import shutil -import sys import tempfile from subprocess import * diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index ab51437..1bd1f89 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -2,7 +2,6 @@ from test.support import findfile, TESTFN, unlink import unittest from test import audiotests from audioop import byteswap -import os import io import sys import struct diff --git a/Lib/test/test_binhex.py b/Lib/test/test_binhex.py index 9d4c85a..21f4463 100644 --- a/Lib/test/test_binhex.py +++ b/Lib/test/test_binhex.py @@ -4,7 +4,6 @@ Based on an original test by Roger E. Masse. """ import binhex -import os import unittest from test import support diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 921fd4a..9095f72 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -4,7 +4,6 @@ import copy import io import sys -import os import unittest from io import StringIO from tempfile import TemporaryFile diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 623d992..f0a428d 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -1,6 +1,5 @@ """Test script for the dbm.open function based on testdumbdbm.py""" -import os import unittest import glob import test.support diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py index 2291561..13e31f2 100644 --- a/Lib/test/test_dbm_ndbm.py +++ b/Lib/test/test_dbm_ndbm.py @@ -1,7 +1,6 @@ from test import support support.import_module("dbm.ndbm") #skip if not supported import unittest -import os import random import dbm.ndbm from dbm.ndbm import error diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py index 955618a..d1933e2 100644 --- a/Lib/test/test_devpoll.py +++ b/Lib/test/test_devpoll.py @@ -5,7 +5,6 @@ import os import random import select -import sys import unittest from test.support import TESTFN, run_unittest, cpython_only diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 75452f2..1c9b84f 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -1,7 +1,6 @@ import os import signal import subprocess -import sys import unittest from test import support diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py index d2f7d31..1600159 100644 --- a/Lib/test/test_email/__init__.py +++ b/Lib/test/test_email/__init__.py @@ -1,5 +1,4 @@ import os -import sys import unittest import collections import email diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index b7e8259..2f8c648 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -1,6 +1,5 @@ """Unittests for heapq.""" -import sys import random import unittest diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 706c3e4..42ddb3d 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -1,5 +1,4 @@ from importlib import _bootstrap_external -import sys from test import support import unittest diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index 8f4b8bb..8829de3 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -3,7 +3,6 @@ from .. import util machinery = util.import_importlib('importlib.machinery') import collections -import sys import unittest diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 603c7d7..29ecff1 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -3,8 +3,6 @@ from .. import util machinery = util.import_importlib('importlib.machinery') - -import sys from test.support import captured_stdout import types import unittest diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 42edc78..8a95a32 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -1,6 +1,5 @@ """Test relative imports (PEP 328).""" from .. import util -import sys import unittest import warnings diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index c274b38..0d6c388 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -5,7 +5,6 @@ importlib = util.import_importlib('importlib') machinery = util.import_importlib('importlib.machinery') import os -import sys from test import support as test_support import unittest diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index 0807e6f..bac370d 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -1,5 +1,4 @@ import os -import sys import json import doctest import unittest diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index f822024..9f49886 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -5,7 +5,6 @@ import errno import os import select import socket -import sys import time import unittest diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index 8ef334f..f656f72 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -1,6 +1,5 @@ """ Test suite for the code in msilib """ import unittest -import os from test.support import import_module msilib = import_module('msilib') diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 8d7a213..01a1cd3 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -5,7 +5,7 @@ from test import support from test.support import TESTFN -import unittest, io, codecs, sys, os +import unittest, io, codecs, sys import _multibytecodec ALL_CJKENCODINGS = [ diff --git a/Lib/test/test_nis.py b/Lib/test/test_nis.py index 387a4e7..21074c6 100644 --- a/Lib/test/test_nis.py +++ b/Lib/test/test_nis.py @@ -1,6 +1,5 @@ from test import support import unittest -import sys # Skip test if nis module does not exist. nis = support.import_module('nis') diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index 30fa612..5b590e1 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -3,7 +3,6 @@ import unittest from http.client import HTTPException import sys -import os from unicodedata import normalize, unidata_version TESTDATAFILE = "NormalizationTest.txt" diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 3d301b4..7269206 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -1,6 +1,5 @@ import parser import unittest -import sys import operator import struct from test import support diff --git a/Lib/test/test_pep3151.py b/Lib/test/test_pep3151.py index 8d560cd..c57c3dc 100644 --- a/Lib/test/test_pep3151.py +++ b/Lib/test/test_pep3151.py @@ -2,7 +2,6 @@ import builtins import os import select import socket -import sys import unittest import errno from errno import EEXIST diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 9d20471..8b3ce70 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,7 +1,6 @@ import itertools import os import posixpath -import sys import unittest import warnings from posixpath import realpath, abspath, dirname, basename diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py index 1932c6b..3d89e3a 100644 --- a/Lib/test/test_pulldom.py +++ b/Lib/test/test_pulldom.py @@ -1,6 +1,5 @@ import io import unittest -import sys import xml.sax from xml.sax.xmlreader import AttributesImpl diff --git a/Lib/test/test_quopri.py b/Lib/test/test_quopri.py index 7cac013..715544c 100644 --- a/Lib/test/test_quopri.py +++ b/Lib/test/test_quopri.py @@ -1,6 +1,6 @@ import unittest -import sys, os, io, subprocess +import sys, io, subprocess import quopri diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index a5d0ebf..98ccab5 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -1,6 +1,5 @@ from test import support import random -import sys import unittest from functools import cmp_to_key diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 1cc8b42..8c8f97b 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -5,7 +5,6 @@ import time import locale import re import os -import sys from test import support from datetime import date as datetime_date diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py index 1305295..e6c83fb 100644 --- a/Lib/test/test_tools/test_md5sum.py +++ b/Lib/test/test_tools/test_md5sum.py @@ -1,7 +1,6 @@ """Tests for the md5sum script in the Tools directory.""" import os -import sys import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure diff --git a/Lib/test/test_tools/test_pdeps.py b/Lib/test/test_tools/test_pdeps.py index 091fa6a..6b5d5c8 100644 --- a/Lib/test/test_tools/test_pdeps.py +++ b/Lib/test/test_tools/test_pdeps.py @@ -1,7 +1,6 @@ """Tests for the pdeps script in the Tools directory.""" import os -import sys import unittest import tempfile from test import support diff --git a/Lib/test/test_ttk_guionly.py b/Lib/test/test_ttk_guionly.py index 490e723..462665d 100644 --- a/Lib/test/test_ttk_guionly.py +++ b/Lib/test/test_ttk_guionly.py @@ -1,4 +1,3 @@ -import os import unittest from test import support diff --git a/Lib/test/test_ttk_textonly.py b/Lib/test/test_ttk_textonly.py index 566fc9d..7540a431 100644 --- a/Lib/test/test_ttk_textonly.py +++ b/Lib/test/test_ttk_textonly.py @@ -1,4 +1,3 @@ -import os from test import support # Skip this test if _tkinter does not exist. diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index d27eef0..f426e5a 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -352,7 +352,6 @@ Some size constraints (all fail.) __test__ = {'doctests' : doctests} def test_main(verbose=False): - import sys from test import support from test import test_unpack_ex support.run_doctest(test_unpack_ex, verbose) diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index f6df54e..c318ef3 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -4,7 +4,6 @@ from test import support import contextlib import socket import urllib.request -import sys import os import email.message import time diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 9ce672b..3e1f4c8 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -3,9 +3,7 @@ from weakref import proxy, ref, WeakSet import operator import copy import string -import os from random import randrange, shuffle -import sys import warnings import collections from collections import UserString as ustr diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 96b446e..cd5fd6c 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -1,5 +1,5 @@ # xml.etree test for cElementTree -import sys, struct +import struct from test import support from test.support import import_fresh_module import types diff --git a/Lib/test/test_xmlrpc_net.py b/Lib/test/test_xmlrpc_net.py index b60b82f..1771db5 100644 --- a/Lib/test/test_xmlrpc_net.py +++ b/Lib/test/test_xmlrpc_net.py @@ -1,7 +1,6 @@ import collections.abc import errno import socket -import sys import unittest from test import support diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 8589342..61f561b 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1,7 +1,6 @@ import contextlib import io import os -import sys import importlib.util import posixpath import time diff --git a/Lib/tkinter/test/runtktests.py b/Lib/tkinter/test/runtktests.py index dbe5e88..1ca8bde 100644 --- a/Lib/tkinter/test/runtktests.py +++ b/Lib/tkinter/test/runtktests.py @@ -7,7 +7,6 @@ Extensions also should live in packages following the same rule as above. """ import os -import sys import unittest import importlib import test.support diff --git a/Lib/unittest/test/testmock/support.py b/Lib/unittest/test/testmock/support.py index f473879..205431a 100644 --- a/Lib/unittest/test/testmock/support.py +++ b/Lib/unittest/test/testmock/support.py @@ -1,5 +1,3 @@ -import sys - def is_instance(obj, klass): """Version of is_instance that doesn't access __class__""" return issubclass(type(obj), klass) -- cgit v0.12 From c7f44aa99aea074398b35d02afba65b49df36813 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 24 Apr 2016 13:25:01 +0300 Subject: Issue #23277: Remove more unused sys and os imports. --- Lib/distutils/tests/test_clean.py | 1 - Lib/distutils/tests/test_config.py | 1 - Lib/distutils/tests/test_install_data.py | 1 - Lib/distutils/tests/test_install_headers.py | 1 - Lib/distutils/tests/test_unixccompiler.py | 1 - Lib/lib2to3/tests/test_parser.py | 1 - Lib/lib2to3/tests/test_pytree.py | 1 - Lib/lib2to3/tests/test_util.py | 3 --- Lib/test/seq_tests.py | 1 - Lib/test/test_importlib/source/test_source_encoding.py | 1 - Lib/test/test_itertools.py | 1 - Lib/test/test_set.py | 1 - Lib/test/test_xmlrpc.py | 1 - 13 files changed, 15 deletions(-) diff --git a/Lib/distutils/tests/test_clean.py b/Lib/distutils/tests/test_clean.py index b64f300..df88ec1 100644 --- a/Lib/distutils/tests/test_clean.py +++ b/Lib/distutils/tests/test_clean.py @@ -1,5 +1,4 @@ """Tests for distutils.command.clean.""" -import sys import os import unittest import getpass diff --git a/Lib/distutils/tests/test_config.py b/Lib/distutils/tests/test_config.py index 4de825a..0929f8a 100644 --- a/Lib/distutils/tests/test_config.py +++ b/Lib/distutils/tests/test_config.py @@ -1,5 +1,4 @@ """Tests for distutils.pypirc.pypirc.""" -import sys import os import unittest import tempfile diff --git a/Lib/distutils/tests/test_install_data.py b/Lib/distutils/tests/test_install_data.py index 4d8c00a..d73624d 100644 --- a/Lib/distutils/tests/test_install_data.py +++ b/Lib/distutils/tests/test_install_data.py @@ -1,5 +1,4 @@ """Tests for distutils.command.install_data.""" -import sys import os import unittest import getpass diff --git a/Lib/distutils/tests/test_install_headers.py b/Lib/distutils/tests/test_install_headers.py index d953157..d9ed6b7 100644 --- a/Lib/distutils/tests/test_install_headers.py +++ b/Lib/distutils/tests/test_install_headers.py @@ -1,5 +1,4 @@ """Tests for distutils.command.install_headers.""" -import sys import os import unittest import getpass diff --git a/Lib/distutils/tests/test_unixccompiler.py b/Lib/distutils/tests/test_unixccompiler.py index 3d14e12..7c95be5 100644 --- a/Lib/distutils/tests/test_unixccompiler.py +++ b/Lib/distutils/tests/test_unixccompiler.py @@ -1,5 +1,4 @@ """Tests for distutils.unixccompiler.""" -import os import sys import unittest from test.support import EnvironmentVarGuard, run_unittest diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index b533c01..46c8c54 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -15,7 +15,6 @@ from test.support import verbose # Python imports import os -import sys import unittest import warnings import subprocess diff --git a/Lib/lib2to3/tests/test_pytree.py b/Lib/lib2to3/tests/test_pytree.py index 4d585a8..c93aa65 100644 --- a/Lib/lib2to3/tests/test_pytree.py +++ b/Lib/lib2to3/tests/test_pytree.py @@ -11,7 +11,6 @@ especially when debugging a test. from __future__ import with_statement -import sys import warnings # Testing imports diff --git a/Lib/lib2to3/tests/test_util.py b/Lib/lib2to3/tests/test_util.py index d2be82c..c6c6139 100644 --- a/Lib/lib2to3/tests/test_util.py +++ b/Lib/lib2to3/tests/test_util.py @@ -3,9 +3,6 @@ # Testing imports from . import support -# Python imports -import os.path - # Local imports from lib2to3.pytree import Node, Leaf from lib2to3 import fixer_util diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index 72f4845..1e7a6f6 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -318,7 +318,6 @@ class CommonTest(unittest.TestCase): self.assertEqual(id(s), id(s*1)) def test_bigrepeat(self): - import sys if sys.maxsize <= 2147483647: x = self.type2test([0]) x *= 2**16 diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index 1e0771b..980855f 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -5,7 +5,6 @@ machinery = util.import_importlib('importlib.machinery') import codecs import importlib.util import re -import sys import types # Because sys.path gets essentially blanked, need to have unicodedata already # imported for the parser to use. diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 5088b54..3bbfdf1 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -4,7 +4,6 @@ from itertools import * import weakref from decimal import Decimal from fractions import Fraction -import sys import operator import random import copy diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 15ae42c..e584252 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -6,7 +6,6 @@ import operator import copy import pickle from random import randrange, shuffle -import sys import warnings import collections import collections.abc diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 831a5a5..cfb5964 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -9,7 +9,6 @@ import xmlrpc.server import http.client import http, http.server import socket -import os import re import io import contextlib -- cgit v0.12 From 597d15afe45ea3a8613d336e68a0a98b8846e603 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 24 Apr 2016 13:45:58 +0300 Subject: Issue #23277: Remove unused support.run_unittest import. --- Lib/test/test_codecmaps_cn.py | 1 - Lib/test/test_codecmaps_hk.py | 1 - Lib/test/test_codecmaps_kr.py | 1 - Lib/test/test_codecmaps_tw.py | 1 - Lib/test/test_dictcomps.py | 2 -- Lib/test/test_email/test_asian_codecs.py | 1 - Lib/test/test_file.py | 2 +- Lib/test/test_fileinput.py | 2 +- Lib/test/test_htmlparser.py | 1 - Lib/test/test_importlib/test_namespace_pkgs.py | 1 - Lib/test/test_iterlen.py | 1 - Lib/test/test_macpath.py | 2 +- Lib/test/test_sched.py | 1 - Lib/test/test_tools/test_pdeps.py | 1 - Lib/test/test_userstring.py | 2 +- 15 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py index f1bd384..89e51c6 100644 --- a/Lib/test/test_codecmaps_cn.py +++ b/Lib/test/test_codecmaps_cn.py @@ -3,7 +3,6 @@ # Codec mapping tests for PRC encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py index 4c0c415..7a48d24 100644 --- a/Lib/test/test_codecmaps_hk.py +++ b/Lib/test/test_codecmaps_hk.py @@ -3,7 +3,6 @@ # Codec mapping tests for HongKong encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py index 6cb41c8..471cd74 100644 --- a/Lib/test/test_codecmaps_kr.py +++ b/Lib/test/test_codecmaps_kr.py @@ -3,7 +3,6 @@ # Codec mapping tests for ROK encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py index 2ea44b5..145a97d 100644 --- a/Lib/test/test_codecmaps_tw.py +++ b/Lib/test/test_codecmaps_tw.py @@ -3,7 +3,6 @@ # Codec mapping tests for ROC encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 3c8b95c..0873071 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -1,7 +1,5 @@ import unittest -from test import support - # For scope testing. g = "Global variable" diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index 089269f..42bb3e9 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -3,7 +3,6 @@ # email package unit tests for (optional) Asian codecs import unittest -from test.support import run_unittest from test.test_email.test_email import TestEmailBase from email.charset import Charset diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index 67c3d86..65be30f 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -7,7 +7,7 @@ from weakref import proxy import io import _pyio as pyio -from test.support import TESTFN, run_unittest +from test.support import TESTFN from collections import UserList class AutoFileTests: diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index bdf4252..4f67c25 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -22,7 +22,7 @@ except ImportError: from io import BytesIO, StringIO from fileinput import FileInput, hook_encoded -from test.support import verbose, TESTFN, run_unittest, check_warnings +from test.support import verbose, TESTFN, check_warnings from test.support import unlink as safe_unlink from test import support from unittest import mock diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 11420b2c..a7f53d3 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -3,7 +3,6 @@ import html.parser import pprint import unittest -from test import support class EventCollector(html.parser.HTMLParser): diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 6639612..cb49f19 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -7,7 +7,6 @@ import types import unittest from test.test_importlib import util -from test.support import run_unittest # needed tests: # diff --git a/Lib/test/test_iterlen.py b/Lib/test/test_iterlen.py index 152f5fc..41c9752 100644 --- a/Lib/test/test_iterlen.py +++ b/Lib/test/test_iterlen.py @@ -42,7 +42,6 @@ enumerate(iter('abc')). """ import unittest -from test import support from itertools import repeat from collections import deque from operator import length_hint diff --git a/Lib/test/test_macpath.py b/Lib/test/test_macpath.py index 80bec7a..0698ff5 100644 --- a/Lib/test/test_macpath.py +++ b/Lib/test/test_macpath.py @@ -1,5 +1,5 @@ import macpath -from test import support, test_genericpath +from test import test_genericpath import unittest diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py index fe8e785..f86f599 100644 --- a/Lib/test/test_sched.py +++ b/Lib/test/test_sched.py @@ -2,7 +2,6 @@ import queue import sched import time import unittest -from test import support try: import threading except ImportError: diff --git a/Lib/test/test_tools/test_pdeps.py b/Lib/test/test_tools/test_pdeps.py index 6b5d5c8..0097623 100644 --- a/Lib/test/test_tools/test_pdeps.py +++ b/Lib/test/test_tools/test_pdeps.py @@ -3,7 +3,6 @@ import os import unittest import tempfile -from test import support from test.test_tools import scriptsdir, skip_if_missing, import_tool diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 9bc8edd..35dce2c 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -3,7 +3,7 @@ import string import unittest -from test import support, string_tests +from test import string_tests from collections import UserString -- cgit v0.12 From e437a10d15ddfd21d406e591acccf12ff443194e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 24 Apr 2016 21:41:02 +0300 Subject: Issue #23277: Remove unused imports in tests. --- Lib/ctypes/test/test_parameters.py | 6 +++--- Lib/distutils/tests/test_bdist_rpm.py | 4 ---- Lib/distutils/tests/test_build_ext.py | 1 - Lib/distutils/tests/test_clean.py | 1 - Lib/distutils/tests/test_config.py | 1 - Lib/distutils/tests/test_cygwinccompiler.py | 3 +-- Lib/distutils/tests/test_dep_util.py | 1 - Lib/distutils/tests/test_file_util.py | 1 - Lib/distutils/tests/test_install_data.py | 1 - Lib/distutils/tests/test_install_headers.py | 1 - Lib/distutils/tests/test_spawn.py | 5 ++--- Lib/lib2to3/tests/test_all_fixers.py | 1 - Lib/lib2to3/tests/test_fixers.py | 3 +-- Lib/lib2to3/tests/test_parser.py | 2 +- Lib/lib2to3/tests/test_pytree.py | 2 -- Lib/lib2to3/tests/test_refactor.py | 4 ---- Lib/test/audiotests.py | 1 - Lib/test/test__locale.py | 1 - Lib/test/test__osx_support.py | 1 - Lib/test/test_array.py | 2 -- Lib/test/test_asynchat.py | 1 - Lib/test/test_bigmem.py | 1 - Lib/test/test_binop.py | 2 +- Lib/test/test_buffer.py | 1 - Lib/test/test_codeccallbacks.py | 1 - Lib/test/test_codecs.py | 1 - Lib/test/test_cprofile.py | 2 +- Lib/test/test_csv.py | 3 --- Lib/test/test_dbm_gnu.py | 2 +- Lib/test/test_dbm_ndbm.py | 1 - Lib/test/test_devpoll.py | 2 +- Lib/test/test_doctest.py | 4 +--- Lib/test/test_dynamic.py | 1 - Lib/test/test_email/test_email.py | 1 - Lib/test/test_email/test_headerregistry.py | 1 - Lib/test/test_float.py | 1 - Lib/test/test_gdb.py | 1 - Lib/test/test_gettext.py | 1 - Lib/test/test_idle.py | 1 - Lib/test/test_importlib/extension/test_path_hook.py | 1 - Lib/test/test_importlib/import_/test_packages.py | 1 - Lib/test/test_importlib/test_locks.py | 1 - Lib/test/test_importlib/test_namespace_pkgs.py | 3 --- Lib/test/test_mailbox.py | 5 ----- Lib/test/test_mailcap.py | 1 - Lib/test/test_math.py | 1 - Lib/test/test_multiprocessing_main_handling.py | 3 +-- Lib/test/test_pdb.py | 1 - Lib/test/test_pep352.py | 1 - Lib/test/test_pickletools.py | 1 - Lib/test/test_plistlib.py | 1 - Lib/test/test_poplib.py | 1 - Lib/test/test_posix.py | 1 - Lib/test/test_posixpath.py | 1 - Lib/test/test_regrtest.py | 2 -- Lib/test/test_robotparser.py | 2 -- Lib/test/test_socket.py | 1 - Lib/test/test_socketserver.py | 2 -- Lib/test/test_subprocess.py | 3 --- Lib/test/test_sunau.py | 1 - Lib/test/test_telnetlib.py | 1 - Lib/test/test_threading.py | 2 +- Lib/test/test_tools/test_gprof2html.py | 3 +-- Lib/test/test_tools/test_md5sum.py | 2 +- Lib/test/test_tools/test_pdeps.py | 2 +- Lib/test/test_urllib2.py | 2 +- Lib/test/test_userstring.py | 1 - Lib/test/test_uu.py | 1 - Lib/test/test_venv.py | 1 - Lib/test/test_wave.py | 1 - Lib/test/test_weakset.py | 7 +------ Lib/test/test_xml_etree_c.py | 2 +- Lib/test/test_xmlrpc.py | 1 - Lib/test/test_xmlrpc_net.py | 2 -- Lib/test/test_zipimport_support.py | 1 - Lib/tkinter/test/runtktests.py | 1 - Lib/tkinter/test/test_ttk/test_functions.py | 1 - Lib/unittest/test/testmock/testmagicmethods.py | 1 - Lib/unittest/test/testmock/testpatch.py | 4 ++-- 79 files changed, 23 insertions(+), 117 deletions(-) diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py index 5c8d703..363f586 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/ctypes/test/test_parameters.py @@ -49,7 +49,7 @@ class SimpleTypesTestCase(unittest.TestCase): # XXX Replace by c_char_p tests def test_cstrings(self): - from ctypes import c_char_p, byref + from ctypes import c_char_p # c_char_p.from_param on a Python String packs the string # into a cparam object @@ -68,7 +68,7 @@ class SimpleTypesTestCase(unittest.TestCase): @need_symbol('c_wchar_p') def test_cw_strings(self): - from ctypes import byref, c_wchar_p + from ctypes import c_wchar_p c_wchar_p.from_param("123") @@ -98,7 +98,7 @@ class SimpleTypesTestCase(unittest.TestCase): def test_byref_pointer(self): # The from_param class method of POINTER(typ) classes accepts what is # returned by byref(obj), it type(obj) == typ - from ctypes import c_short, c_uint, c_int, c_long, pointer, POINTER, byref + from ctypes import c_short, c_uint, c_int, c_long, POINTER, byref LPINT = POINTER(c_int) LPINT.from_param(byref(c_int(42))) diff --git a/Lib/distutils/tests/test_bdist_rpm.py b/Lib/distutils/tests/test_bdist_rpm.py index 25c14ab..c1a2a04 100644 --- a/Lib/distutils/tests/test_bdist_rpm.py +++ b/Lib/distutils/tests/test_bdist_rpm.py @@ -3,16 +3,12 @@ import unittest import sys import os -import tempfile -import shutil from test.support import run_unittest from distutils.core import Distribution from distutils.command.bdist_rpm import bdist_rpm from distutils.tests import support from distutils.spawn import find_executable -from distutils import spawn -from distutils.errors import DistutilsExecError SETUP_PY = """\ from distutils.core import setup diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index 366ffbe..3d84f8b 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -166,7 +166,6 @@ class BuildExtTestCase(TempdirManager, cmd = self.build_ext(dist) cmd.finalize_options() - from distutils import sysconfig py_include = sysconfig.get_python_inc() self.assertIn(py_include, cmd.include_dirs) diff --git a/Lib/distutils/tests/test_clean.py b/Lib/distutils/tests/test_clean.py index df88ec1..c605afd 100644 --- a/Lib/distutils/tests/test_clean.py +++ b/Lib/distutils/tests/test_clean.py @@ -1,7 +1,6 @@ """Tests for distutils.command.clean.""" import os import unittest -import getpass from distutils.command.clean import clean from distutils.tests import support diff --git a/Lib/distutils/tests/test_config.py b/Lib/distutils/tests/test_config.py index 0929f8a..d91dedd 100644 --- a/Lib/distutils/tests/test_config.py +++ b/Lib/distutils/tests/test_config.py @@ -1,7 +1,6 @@ """Tests for distutils.pypirc.pypirc.""" import os import unittest -import tempfile from distutils.core import PyPIRCCommand from distutils.core import Distribution diff --git a/Lib/distutils/tests/test_cygwinccompiler.py b/Lib/distutils/tests/test_cygwinccompiler.py index 8569216..9dc869d 100644 --- a/Lib/distutils/tests/test_cygwinccompiler.py +++ b/Lib/distutils/tests/test_cygwinccompiler.py @@ -3,11 +3,10 @@ import unittest import sys import os from io import BytesIO -import subprocess from test.support import run_unittest from distutils import cygwinccompiler -from distutils.cygwinccompiler import (CygwinCCompiler, check_config_h, +from distutils.cygwinccompiler import (check_config_h, CONFIG_H_OK, CONFIG_H_NOTOK, CONFIG_H_UNCERTAIN, get_versions, get_msvcr) diff --git a/Lib/distutils/tests/test_dep_util.py b/Lib/distutils/tests/test_dep_util.py index 3e1c366..c6fae39 100644 --- a/Lib/distutils/tests/test_dep_util.py +++ b/Lib/distutils/tests/test_dep_util.py @@ -1,7 +1,6 @@ """Tests for distutils.dep_util.""" import unittest import os -import time from distutils.dep_util import newer, newer_pairwise, newer_group from distutils.errors import DistutilsFileError diff --git a/Lib/distutils/tests/test_file_util.py b/Lib/distutils/tests/test_file_util.py index a6d04f0..03040af 100644 --- a/Lib/distutils/tests/test_file_util.py +++ b/Lib/distutils/tests/test_file_util.py @@ -1,7 +1,6 @@ """Tests for distutils.file_util.""" import unittest import os -import shutil import errno from unittest.mock import patch diff --git a/Lib/distutils/tests/test_install_data.py b/Lib/distutils/tests/test_install_data.py index d73624d..32ab296 100644 --- a/Lib/distutils/tests/test_install_data.py +++ b/Lib/distutils/tests/test_install_data.py @@ -1,7 +1,6 @@ """Tests for distutils.command.install_data.""" import os import unittest -import getpass from distutils.command.install_data import install_data from distutils.tests import support diff --git a/Lib/distutils/tests/test_install_headers.py b/Lib/distutils/tests/test_install_headers.py index d9ed6b7..2217b32 100644 --- a/Lib/distutils/tests/test_install_headers.py +++ b/Lib/distutils/tests/test_install_headers.py @@ -1,7 +1,6 @@ """Tests for distutils.command.install_headers.""" import os import unittest -import getpass from distutils.command.install_headers import install_headers from distutils.tests import support diff --git a/Lib/distutils/tests/test_spawn.py b/Lib/distutils/tests/test_spawn.py index 6c7eb20..f507ef7 100644 --- a/Lib/distutils/tests/test_spawn.py +++ b/Lib/distutils/tests/test_spawn.py @@ -1,11 +1,10 @@ """Tests for distutils.spawn.""" import unittest import os -import time -from test.support import captured_stdout, run_unittest +from test.support import run_unittest from distutils.spawn import _nt_quote_args -from distutils.spawn import spawn, find_executable +from distutils.spawn import spawn from distutils.errors import DistutilsExecError from distutils.tests import support diff --git a/Lib/lib2to3/tests/test_all_fixers.py b/Lib/lib2to3/tests/test_all_fixers.py index 15079fe..c0507cf 100644 --- a/Lib/lib2to3/tests/test_all_fixers.py +++ b/Lib/lib2to3/tests/test_all_fixers.py @@ -10,7 +10,6 @@ import unittest import test.support # Local imports -from lib2to3 import refactor from . import support diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index def9b0e..b97b73a 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -2,12 +2,11 @@ # Python imports import os -import unittest from itertools import chain from operator import itemgetter # Local imports -from lib2to3 import pygram, pytree, refactor, fixer_util +from lib2to3 import pygram, fixer_util from lib2to3.tests import support diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 46c8c54..36eb176 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -10,7 +10,7 @@ from __future__ import with_statement # Testing imports from . import support -from .support import driver, test_dir +from .support import driver from test.support import verbose # Python imports diff --git a/Lib/lib2to3/tests/test_pytree.py b/Lib/lib2to3/tests/test_pytree.py index c93aa65..a611d17 100644 --- a/Lib/lib2to3/tests/test_pytree.py +++ b/Lib/lib2to3/tests/test_pytree.py @@ -11,8 +11,6 @@ especially when debugging a test. from __future__ import with_statement -import warnings - # Testing imports from . import support diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index f30c1e8..ab5ce727 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -7,18 +7,14 @@ from __future__ import with_statement import sys import os import codecs -import operator import io import tempfile import shutil import unittest -import warnings from lib2to3 import refactor, pygram, fixer_base from lib2to3.pgen2 import token -from . import support - TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") FIXER_DIR = os.path.join(TEST_DATA_DIR, "fixers") diff --git a/Lib/test/audiotests.py b/Lib/test/audiotests.py index e52c423..d3e8e9e 100644 --- a/Lib/test/audiotests.py +++ b/Lib/test/audiotests.py @@ -1,5 +1,4 @@ from test.support import findfile, TESTFN, unlink -import unittest import array import io import pickle diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 58f2f04..ab4e247 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -4,7 +4,6 @@ try: except ImportError: nl_langinfo = None -import codecs import locale import sys import unittest diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index ac6325a..bcba8ca 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -4,7 +4,6 @@ Test suite for _osx_support: shared OS X support functions. import os import platform -import shutil import stat import sys import unittest diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index b4f2bf8..8fd54cc 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -7,8 +7,6 @@ from test import support import weakref import pickle import operator -import io -import math import struct import sys import warnings diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py index 3a33fc8..08090b8 100644 --- a/Lib/test/test_asynchat.py +++ b/Lib/test/test_asynchat.py @@ -12,7 +12,6 @@ import socket import sys import time import unittest -import warnings import unittest.mock try: import threading diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 0e54595..9488b30 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -14,7 +14,6 @@ from test.support import bigmemtest, _1G, _2G, _4G import unittest import operator import sys -import functools # These tests all use one of the bigmemtest decorators to indicate how much # memory they use and how much memory they need to be even meaningful. The diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index e9dbddc..fc8d30f 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -2,7 +2,7 @@ import unittest from test import support -from operator import eq, ne, lt, gt, le, ge +from operator import eq, le from abc import ABCMeta def gcd(a, b): diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 2eef9fc..b83f2f1 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -16,7 +16,6 @@ import unittest from test import support from itertools import permutations, product from random import randrange, sample, choice -from sysconfig import get_config_var import warnings import sys, array, io from decimal import Decimal diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index ee1e28a..c8cdacf 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -4,7 +4,6 @@ import sys import test.support import unicodedata import unittest -import warnings class PosReturn: # this can be used for configurable callbacks diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 45a1987..d875340 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -4,7 +4,6 @@ import io import locale import sys import unittest -import warnings import encodings from test import support diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index f18983f..53f8917 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -6,7 +6,7 @@ from test.support import run_unittest, TESTFN, unlink # rip off all interesting stuff from test_profile import cProfile from test.test_profile import ProfileTest, regenerate_expected_output -from test.profilee import testfunc + class CProfileTest(ProfileTest): profilerclass = cProfile.Profile diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 9095f72..e97c9f3 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -2,7 +2,6 @@ # csv package unit tests import copy -import io import sys import unittest from io import StringIO @@ -1078,7 +1077,6 @@ class TestUnicode(unittest.TestCase): "François Pinard"] def test_unicode_read(self): - import io with TemporaryFile("w+", newline='', encoding="utf-8") as fileobj: fileobj.write(",".join(self.names) + "\r\n") fileobj.seek(0) @@ -1087,7 +1085,6 @@ class TestUnicode(unittest.TestCase): def test_unicode_write(self): - import io with TemporaryFile("w+", newline='', encoding="utf-8") as fileobj: writer = csv.writer(fileobj) writer.writerow(self.names) diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index a7808f5..304b332 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -2,7 +2,7 @@ from test import support gdbm = support.import_module("dbm.gnu") #skip if not supported import unittest import os -from test.support import verbose, TESTFN, unlink +from test.support import TESTFN, unlink filename = TESTFN diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py index 13e31f2..49f4426 100644 --- a/Lib/test/test_dbm_ndbm.py +++ b/Lib/test/test_dbm_ndbm.py @@ -1,7 +1,6 @@ from test import support support.import_module("dbm.ndbm") #skip if not supported import unittest -import random import dbm.ndbm from dbm.ndbm import error diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py index d1933e2..c133c81 100644 --- a/Lib/test/test_devpoll.py +++ b/Lib/test/test_devpoll.py @@ -6,7 +6,7 @@ import os import random import select import unittest -from test.support import TESTFN, run_unittest, cpython_only +from test.support import run_unittest, cpython_only if not hasattr(select, 'devpoll') : raise unittest.SkipTest('test works only on Solaris OS family') diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index f7c4806..b9bd135 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -1876,7 +1876,6 @@ if not hasattr(sys, 'gettrace') or not sys.gettrace(): To demonstrate this, we'll create a fake standard input that captures our debugger input: - >>> import tempfile >>> real_stdin = sys.stdin >>> sys.stdin = _FakeInput([ ... 'print(x)', # print data defined by the example @@ -1917,7 +1916,7 @@ if not hasattr(sys, 'gettrace') or not sys.gettrace(): ... finally: ... sys.stdin = real_stdin --Return-- - > (3)calls_set_trace()->None + > (3)calls_set_trace()->None -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2804,7 +2803,6 @@ text files). ... _ = f.write(" 'abc def'\n") ... _ = f.write("\n") ... _ = f.write(' \"\"\"\n') - ... import shutil ... rc1, out1, err1 = script_helper.assert_python_failure( ... '-m', 'doctest', fn, fn2, TERM='') ... rc2, out2, err2 = script_helper.assert_python_ok( diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 5080ec9..3ae090f 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -1,7 +1,6 @@ # Test the most dynamic corner cases of Python's runtime semantics. import builtins -import contextlib import unittest from test.support import swap_item, swap_attr diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 894b800..af3fb85 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3421,7 +3421,6 @@ Do you like this message? class TestFeedParsers(TestEmailBase): def parse(self, chunks): - from email.feedparser import FeedParser feedparser = FeedParser() for chunk in chunks: feedparser.feed(chunk) diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 55ecdea..af836dc 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -1,7 +1,6 @@ import datetime import textwrap import unittest -import types from email import errors from email import policy from email.message import Message diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 48f7a70..2e187ac 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -1,6 +1,5 @@ import fractions -import math import operator import os import random diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index ccef422..730b628 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -5,7 +5,6 @@ import os import re -import pprint import subprocess import sys import sysconfig diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 3a94383..d345baa 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -1,6 +1,5 @@ import os import base64 -import shutil import gettext import unittest diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 141e89e..0b34e97 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,5 +1,4 @@ import unittest -from test import support from test.support import import_module # Skip test if _thread or _tkinter wasn't built or idlelib was deleted. diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index 8829de3..a4b5a64 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -2,7 +2,6 @@ from .. import util machinery = util.import_importlib('importlib.machinery') -import collections import unittest diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py index 3755b84..2439604 100644 --- a/Lib/test/test_importlib/import_/test_packages.py +++ b/Lib/test/test_importlib/import_/test_packages.py @@ -1,7 +1,6 @@ from .. import util import sys import unittest -import importlib from test import support diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index df0af12..b2aadff 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -3,7 +3,6 @@ from . import util as test_util init = test_util.import_importlib('importlib') import sys -import time import unittest import weakref diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index cb49f19..116eb75 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -1,9 +1,6 @@ import contextlib -import importlib.abc -import importlib.machinery import os import sys -import types import unittest from test.test_importlib import util diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 0991f74..1f30fa6 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -7,17 +7,12 @@ import email import email.message import re import io -import shutil import tempfile from test import support import unittest import textwrap import mailbox import glob -try: - import fcntl -except ImportError: - pass class TestBase: diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py index 22b2fcc..623fadb 100644 --- a/Lib/test/test_mailcap.py +++ b/Lib/test/test_mailcap.py @@ -1,6 +1,5 @@ import mailcap import os -import shutil import test.support import unittest diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 6c7b99d..d0bc790 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -6,7 +6,6 @@ from test import support import unittest import math import os -import platform import sys import struct import sysconfig diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py index 52273ea..2e15cd8 100644 --- a/Lib/test/test_multiprocessing_main_handling.py +++ b/Lib/test/test_multiprocessing_main_handling.py @@ -6,7 +6,6 @@ support.import_module('_multiprocessing') import importlib import importlib.machinery -import zipimport import unittest import sys import os @@ -15,7 +14,7 @@ import py_compile from test.support.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, - assert_python_ok, assert_python_failure, spawn_python, kill_python) + assert_python_ok) # Look up which start methods are available to test import multiprocessing diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 35044ad..2cddd57 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -558,7 +558,6 @@ def test_pdb_continue_in_bottomframe(): def pdb_invoke(method, arg): """Run pdb.method(arg).""" - import pdb getattr(pdb.Pdb(nosigint=True), method)(arg) diff --git a/Lib/test/test_pep352.py b/Lib/test/test_pep352.py index 7c98c46..27d514f 100644 --- a/Lib/test/test_pep352.py +++ b/Lib/test/test_pep352.py @@ -1,6 +1,5 @@ import unittest import builtins -import warnings import os from platform import system as platform_system diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 80221f0..86bebfa 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,4 +1,3 @@ -import struct import pickle import pickletools from test import support diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index f0e9e5a..12d92fa 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -7,7 +7,6 @@ import datetime import codecs import binascii import collections -import struct from test import support from io import BytesIO diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index bceeb93..7b9606d 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -8,7 +8,6 @@ import asyncore import asynchat import socket import os -import time import errno from unittest import TestCase, skipUnless diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 28cdd90..6a1c829 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -11,7 +11,6 @@ import time import os import platform import pwd -import shutil import stat import tempfile import unittest diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 8b3ce70..26faa48 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,4 +1,3 @@ -import itertools import os import posixpath import unittest diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 213853f..c11408d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -4,10 +4,8 @@ Tests of regrtest.py. Note: test_regrtest cannot be run twice in parallel. """ -import argparse import contextlib import faulthandler -import getopt import io import os.path import platform diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 90b3072..76f4f7c 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -2,8 +2,6 @@ import io import unittest import urllib.robotparser from collections import namedtuple -from urllib.error import URLError, HTTPError -from urllib.request import urlopen from test import support from http.server import BaseHTTPRequestHandler, HTTPServer try: diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 982a976..a7cc1e7 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -13,7 +13,6 @@ import queue import sys import os import array -import platform import contextlib from weakref import proxy import signal diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 554c106..01ac12b 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -7,8 +7,6 @@ import os import select import signal import socket -import select -import errno import tempfile import unittest import socketserver diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 7f70fd0..786b8cf 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -5,15 +5,12 @@ import subprocess import sys import signal import io -import locale import os import errno import tempfile import time -import re import selectors import sysconfig -import warnings import select import shutil import gc diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py index 0f4134e..bc1f46c 100644 --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,4 +1,3 @@ -from test.support import TESTFN import unittest from test import audiotests from audioop import byteswap diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 610377a..69a9ce2 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -1,7 +1,6 @@ import socket import selectors import telnetlib -import time import contextlib from test import support diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 71df14c..0e70016 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,7 +3,7 @@ Tests for the threading module. """ import test.support -from test.support import verbose, strip_python_stderr, import_module, cpython_only +from test.support import verbose, import_module, cpython_only from test.support.script_helper import assert_python_ok, assert_python_failure import random diff --git a/Lib/test/test_tools/test_gprof2html.py b/Lib/test/test_tools/test_gprof2html.py index 0c294ec..9489ed9 100644 --- a/Lib/test/test_tools/test_gprof2html.py +++ b/Lib/test/test_tools/test_gprof2html.py @@ -2,12 +2,11 @@ import os import sys -import importlib import unittest from unittest import mock import tempfile -from test.test_tools import scriptsdir, skip_if_missing, import_tool +from test.test_tools import skip_if_missing, import_tool skip_if_missing() diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py index e6c83fb..fb565b7 100644 --- a/Lib/test/test_tools/test_md5sum.py +++ b/Lib/test/test_tools/test_md5sum.py @@ -5,7 +5,7 @@ import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -from test.test_tools import scriptsdir, import_tool, skip_if_missing +from test.test_tools import scriptsdir, skip_if_missing skip_if_missing() diff --git a/Lib/test/test_tools/test_pdeps.py b/Lib/test/test_tools/test_pdeps.py index 0097623..27cbfe2 100644 --- a/Lib/test/test_tools/test_pdeps.py +++ b/Lib/test/test_tools/test_pdeps.py @@ -4,7 +4,7 @@ import os import unittest import tempfile -from test.test_tools import scriptsdir, skip_if_missing, import_tool +from test.test_tools import skip_if_missing, import_tool skip_if_missing() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 008c751..df8a891 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -462,7 +462,7 @@ class MockHTTPHandler(urllib.request.BaseHandler): self.requests = [] def http_open(self, req): - import email, http.client, copy + import email, copy self.requests.append(copy.deepcopy(req)) if self._count == 0: self._count = self._count + 1 diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 35dce2c..7152822 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -1,7 +1,6 @@ # UserString is a wrapper around the native builtin string type. # UserString instances should behave similar to builtin string objects. -import string import unittest from test import string_tests diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py index 25fffbf..ad2f2c5 100644 --- a/Lib/test/test_uu.py +++ b/Lib/test/test_uu.py @@ -8,7 +8,6 @@ from test import support import sys, os import uu -from io import BytesIO import io plaintext = b"The smooth-scaled python crept over the sleeping dog\n" diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 55bee23..a2842b0 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -15,7 +15,6 @@ import sys import tempfile from test.support import (captured_stdout, captured_stderr, can_symlink, EnvironmentVarGuard, rmtree) -import textwrap import unittest import venv diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index a67a8b0..8666f72 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,4 +1,3 @@ -from test.support import TESTFN import unittest from test import audiotests from test import support diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 3e1f4c8..691b95e 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -1,11 +1,6 @@ import unittest -from weakref import proxy, ref, WeakSet -import operator -import copy +from weakref import WeakSet import string -from random import randrange, shuffle -import warnings -import collections from collections import UserString as ustr import gc import contextlib diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index cd5fd6c..87f3f27 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -108,7 +108,7 @@ class SizeofTest(unittest.TestCase): struct.calcsize('8P')) def test_main(): - from test import test_xml_etree, test_xml_etree_c + from test import test_xml_etree # Run the tests specific to the C implementation support.run_unittest( diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index cfb5964..59f9c32 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -1132,7 +1132,6 @@ def captured_stdout(encoding='utf-8'): """A variation on support.captured_stdout() which gives a text stream having a `buffer` attribute. """ - import io orig_stdout = sys.stdout sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding=encoding) try: diff --git a/Lib/test/test_xmlrpc_net.py b/Lib/test/test_xmlrpc_net.py index 1771db5..ae0a28e 100644 --- a/Lib/test/test_xmlrpc_net.py +++ b/Lib/test/test_xmlrpc_net.py @@ -1,6 +1,4 @@ import collections.abc -import errno -import socket import unittest from test import support diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 5913622..84d526c 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -12,7 +12,6 @@ import zipimport import doctest import inspect import linecache -import pdb import unittest from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) diff --git a/Lib/tkinter/test/runtktests.py b/Lib/tkinter/test/runtktests.py index 1ca8bde..33dc54a 100644 --- a/Lib/tkinter/test/runtktests.py +++ b/Lib/tkinter/test/runtktests.py @@ -7,7 +7,6 @@ Extensions also should live in packages following the same rule as above. """ import os -import unittest import importlib import test.support diff --git a/Lib/tkinter/test/test_ttk/test_functions.py b/Lib/tkinter/test/test_ttk/test_functions.py index c68a650..a1b7cdf 100644 --- a/Lib/tkinter/test/test_ttk/test_functions.py +++ b/Lib/tkinter/test/test_ttk/test_functions.py @@ -1,6 +1,5 @@ # -*- encoding: utf-8 -*- import unittest -import tkinter from tkinter import ttk class MockTkApp: diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/unittest/test/testmock/testmagicmethods.py index bb9b956..24569b5 100644 --- a/Lib/unittest/test/testmock/testmagicmethods.py +++ b/Lib/unittest/test/testmock/testmagicmethods.py @@ -1,5 +1,4 @@ import unittest -import inspect import sys from unittest.mock import Mock, MagicMock, _magics diff --git a/Lib/unittest/test/testmock/testpatch.py b/Lib/unittest/test/testmock/testpatch.py index dfce369..2e0c08f 100644 --- a/Lib/unittest/test/testmock/testpatch.py +++ b/Lib/unittest/test/testmock/testpatch.py @@ -10,9 +10,9 @@ from unittest.test.testmock import support from unittest.test.testmock.support import SomeClass, is_instance from unittest.mock import ( - NonCallableMock, CallableMixin, patch, sentinel, + NonCallableMock, CallableMixin, sentinel, MagicMock, Mock, NonCallableMagicMock, patch, _patch, - DEFAULT, call, _get_target, _patch + DEFAULT, call, _get_target ) -- cgit v0.12 From 8153ac8f00efb5efb486805234887d49b22755ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 24 Apr 2016 22:33:26 +0200 Subject: Issue #26249: Mention PyMem_Malloc() change in What's New in Python 3.6 in the Optimizations section. --- Doc/whatsnew/3.6.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 87854c8..99223af 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -388,6 +388,13 @@ Optimizations * Optimize ``bytes.replace(b'', b'.')`` and ``bytearray.replace(b'', b'.')``: up to 80% faster. (Contributed by Josh Snider in :issue:`26574`). +* Allocator functions of the :c:func:`PyMem_Malloc` domain + (:c:data:`PYMEM_DOMAIN_MEM`) now use the :ref:`pymalloc memory allocator + ` instead of :c:func:`malloc` function of the C library. The + pymalloc allocator is optimized for objects smaller or equal to 512 bytes + with a short lifetime, and use :c:func:`malloc` for larger memory blocks. + (Contributed by Victor Stinner in :issue:`26249`). + Build and C API Changes ======================= -- cgit v0.12 From e4fbb0206d0794b58d096beafd9964340ea26640 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 24 Apr 2016 23:42:49 +0300 Subject: Remove unused support.run_unittest imports. It is not needed since tests use unittest.main(). --- Lib/test/test_charmapcodec.py | 2 +- Lib/test/test_codecencodings_cn.py | 1 - Lib/test/test_codecencodings_hk.py | 1 - Lib/test/test_codecencodings_iso2022.py | 1 - Lib/test/test_codecencodings_jp.py | 1 - Lib/test/test_codecencodings_kr.py | 1 - Lib/test/test_codecencodings_tw.py | 1 - Lib/test/test_codecmaps_jp.py | 1 - Lib/test/test_epoll.py | 1 - Lib/test/test_hmac.py | 1 - Lib/test/test_list.py | 2 +- Lib/test/test_pow.py | 2 +- Lib/test/test_range.py | 2 +- Lib/test/test_userdict.py | 2 +- Lib/test/test_userlist.py | 2 +- 15 files changed, 6 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_charmapcodec.py b/Lib/test/test_charmapcodec.py index 4064aef..0d4594d 100644 --- a/Lib/test/test_charmapcodec.py +++ b/Lib/test/test_charmapcodec.py @@ -9,7 +9,7 @@ Written by Marc-Andre Lemburg (mal@lemburg.com). """#" -import test.support, unittest +import unittest import codecs diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index d0e3a15..3bdf7d0 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -3,7 +3,6 @@ # Codec encoding tests for PRC encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py index bb9be11..c5e2f99 100644 --- a/Lib/test/test_codecencodings_hk.py +++ b/Lib/test/test_codecencodings_hk.py @@ -3,7 +3,6 @@ # Codec encoding tests for HongKong encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py index 8a3ca70..00ea1c3 100644 --- a/Lib/test/test_codecencodings_iso2022.py +++ b/Lib/test/test_codecencodings_iso2022.py @@ -1,6 +1,5 @@ # Codec encoding tests for ISO 2022 encodings. -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py index 44b63a0..94378d1 100644 --- a/Lib/test/test_codecencodings_jp.py +++ b/Lib/test/test_codecencodings_jp.py @@ -3,7 +3,6 @@ # Codec encoding tests for Japanese encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py index b6a74fb..863d16b 100644 --- a/Lib/test/test_codecencodings_kr.py +++ b/Lib/test/test_codecencodings_kr.py @@ -3,7 +3,6 @@ # Codec encoding tests for ROK encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py index 9174296..bb1d133 100644 --- a/Lib/test/test_codecencodings_tw.py +++ b/Lib/test/test_codecencodings_tw.py @@ -3,7 +3,6 @@ # Codec encoding tests for ROC encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py index 5773823..fdfec80 100644 --- a/Lib/test/test_codecmaps_jp.py +++ b/Lib/test/test_codecmaps_jp.py @@ -3,7 +3,6 @@ # Codec mapping tests for Japanese encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py index a7359e9..a7aff8a 100644 --- a/Lib/test/test_epoll.py +++ b/Lib/test/test_epoll.py @@ -28,7 +28,6 @@ import socket import time import unittest -from test import support if not hasattr(select, "epoll"): raise unittest.SkipTest("test works only on Linux 2.6") diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 98826b5..067e13f 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -3,7 +3,6 @@ import hmac import hashlib import unittest import warnings -from test import support def ignore_warning(func): diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 8f82ab5..aee62dc 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -1,5 +1,5 @@ import sys -from test import support, list_tests +from test import list_tests import pickle import unittest diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index 6feac40..ce99fe6 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -1,4 +1,4 @@ -import test.support, unittest +import unittest class PowTest(unittest.TestCase): diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 106c732..e03b570 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -1,6 +1,6 @@ # Python test set -- built-in functions -import test.support, unittest +import unittest import sys import pickle import itertools diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index f44456f..662c7f6 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,6 +1,6 @@ # Check every path through every method of UserDict -from test import support, mapping_tests +from test import mapping_tests import unittest import collections diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index f92e4d3..8de6c14 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -1,7 +1,7 @@ # Check every path through every method of UserList from collections import UserList -from test import support, list_tests +from test import list_tests import unittest class UserListTest(list_tests.CommonTest): -- cgit v0.12 From a6f26c1d3495670c02c2ceb1e38ada0d468579f7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 25 Apr 2016 00:05:30 +0300 Subject: Remove more unused imports in tests. --- Lib/ctypes/test/test_bitfields.py | 1 - Lib/ctypes/test/test_pep3118.py | 2 +- Lib/lib2to3/tests/support.py | 2 -- Lib/test/eintrdata/eintr_tester.py | 1 - Lib/test/test_eintr.py | 1 - Lib/test/test_imp.py | 1 - Lib/test/test_json/test_fail.py | 1 - Lib/test/test_subprocess.py | 1 - Lib/test/test_symbol.py | 1 - Lib/test/test_threading.py | 1 - Lib/test/test_tools/__init__.py | 1 - 11 files changed, 1 insertion(+), 12 deletions(-) diff --git a/Lib/ctypes/test/test_bitfields.py b/Lib/ctypes/test/test_bitfields.py index b39d82c..0eb09fb 100644 --- a/Lib/ctypes/test/test_bitfields.py +++ b/Lib/ctypes/test/test_bitfields.py @@ -3,7 +3,6 @@ from ctypes.test import need_symbol import unittest import os -import ctypes import _ctypes_test class BITS(Structure): diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py index 32f802c..d68397e 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/ctypes/test/test_pep3118.py @@ -1,6 +1,6 @@ import unittest from ctypes import * -import re, struct, sys +import re, sys if sys.byteorder == "little": THIS_ENDIAN = "<" diff --git a/Lib/lib2to3/tests/support.py b/Lib/lib2to3/tests/support.py index 6f2d214..7153bb6 100644 --- a/Lib/lib2to3/tests/support.py +++ b/Lib/lib2to3/tests/support.py @@ -3,10 +3,8 @@ # Python imports import unittest -import sys import os import os.path -import re from textwrap import dedent # Local imports diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 1cb86e5..ffcdb98 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -10,7 +10,6 @@ sub-second periodicity (contrarily to signal()). import contextlib import faulthandler -import io import os import select import signal diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 1c9b84f..25f86d3 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -1,6 +1,5 @@ import os import signal -import subprocess import unittest from test import support diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index efb0384..4ece365 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -6,7 +6,6 @@ import importlib import importlib.util import os import os.path -import shutil import sys from test import support import unittest diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 95ff5b8..7910521 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -1,5 +1,4 @@ from test.test_json import PyTest, CTest -import re # 2007-10-05 JSONDOCS = [ diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 786b8cf..e7519a2 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1,5 +1,4 @@ import unittest -from test.support import script_helper from test import support import subprocess import sys diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py index 2dcb9de..c1306f5 100644 --- a/Lib/test/test_symbol.py +++ b/Lib/test/test_symbol.py @@ -1,6 +1,5 @@ import unittest from test import support -import filecmp import os import sys import subprocess diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0e70016..d6317c7 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -7,7 +7,6 @@ from test.support import verbose, import_module, cpython_only from test.support.script_helper import assert_python_ok, assert_python_failure import random -import re import sys _thread = import_module('_thread') threading = import_module('threading') diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index 04c8726..4d0fca3 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -3,7 +3,6 @@ import os import unittest import importlib from test import support -from fnmatch import fnmatch basepath = os.path.dirname( # os.path.dirname( # Lib -- cgit v0.12 From ccd047ea4b92f09a84b67e69deb82ce42e510c4c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 25 Apr 2016 00:12:32 +0300 Subject: Removed unused imports. --- Lib/distutils/command/config.py | 2 +- Lib/distutils/command/register.py | 2 +- Lib/distutils/command/sdist.py | 1 - Lib/distutils/extension.py | 1 - Lib/distutils/text_file.py | 2 +- Lib/ftplib.py | 2 -- Lib/idlelib/help.py | 4 ++-- Lib/idlelib/macosxSupport.py | 1 - Lib/lib2to3/fixer_util.py | 2 -- Lib/lib2to3/fixes/fix_dict.py | 3 +-- Lib/lib2to3/fixes/fix_exec.py | 1 - Lib/lib2to3/fixes/fix_filter.py | 1 - Lib/lib2to3/fixes/fix_has_key.py | 1 - Lib/lib2to3/fixes/fix_metaclass.py | 2 +- Lib/lib2to3/fixes/fix_nonzero.py | 2 +- Lib/lib2to3/fixes/fix_print.py | 2 +- Lib/lib2to3/fixes/fix_types.py | 1 - Lib/lib2to3/fixes/fix_urllib.py | 1 - Lib/lib2to3/refactor.py | 1 - Lib/pyclbr.py | 3 +-- Lib/re.py | 1 - Lib/turtledemo/__main__.py | 1 - Lib/turtledemo/bytedesign.py | 1 - Lib/turtledemo/planet_and_moon.py | 1 - Lib/uuid.py | 1 - Tools/scripts/diff.py | 2 +- 26 files changed, 11 insertions(+), 31 deletions(-) diff --git a/Lib/distutils/command/config.py b/Lib/distutils/command/config.py index 847e858..b1fd09e 100644 --- a/Lib/distutils/command/config.py +++ b/Lib/distutils/command/config.py @@ -9,7 +9,7 @@ configure-like tasks: "try to compile this C code", or "figure out where this header file lives". """ -import sys, os, re +import os, re from distutils.core import Command from distutils.errors import DistutilsExecError diff --git a/Lib/distutils/command/register.py b/Lib/distutils/command/register.py index b49f86f..a3e0893 100644 --- a/Lib/distutils/command/register.py +++ b/Lib/distutils/command/register.py @@ -5,7 +5,7 @@ Implements the Distutils 'register' command (register with the repository). # created 2002/10/21, Richard Jones -import os, string, getpass +import getpass import io import urllib.parse, urllib.request from warnings import warn diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py index 7ea3d5f..35a06eb 100644 --- a/Lib/distutils/command/sdist.py +++ b/Lib/distutils/command/sdist.py @@ -3,7 +3,6 @@ Implements the Distutils 'sdist' command (create a source distribution).""" import os -import string import sys from types import * from glob import glob diff --git a/Lib/distutils/extension.py b/Lib/distutils/extension.py index 7efbb74..c507da3 100644 --- a/Lib/distutils/extension.py +++ b/Lib/distutils/extension.py @@ -4,7 +4,6 @@ Provides the Extension class, used to describe C/C++ extension modules in setup scripts.""" import os -import sys import warnings # This class is really only used by the "build_ext" command, so it might diff --git a/Lib/distutils/text_file.py b/Lib/distutils/text_file.py index 478336f..93abad3 100644 --- a/Lib/distutils/text_file.py +++ b/Lib/distutils/text_file.py @@ -4,7 +4,7 @@ provides the TextFile class, which gives an interface to text files that (optionally) takes care of stripping comments, ignoring blank lines, and joining lines with backslashes.""" -import sys, os, io +import sys, io class TextFile: diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 2afa19d..2ab1d56 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -36,10 +36,8 @@ python ftplib.py -d localhost -l -p -l # Modified by Giampaolo Rodola' to add TLS support. # -import os import sys import socket -import warnings from socket import _GLOBAL_DEFAULT_TIMEOUT __all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index d0c59c5..4cb8c76 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -25,8 +25,8 @@ copy_strip - Copy idle.html to help.html, rstripping each line. show_idlehelp - Create HelpWindow. Called in EditorWindow.help_dialog. """ from html.parser import HTMLParser -from os.path import abspath, dirname, isdir, isfile, join -from tkinter import Tk, Toplevel, Frame, Text, Scrollbar, Menu, Menubutton +from os.path import abspath, dirname, isfile, join +from tkinter import Toplevel, Frame, Text, Scrollbar, Menu, Menubutton from tkinter import font as tkfont from idlelib.configHandler import idleConf diff --git a/Lib/idlelib/macosxSupport.py b/Lib/idlelib/macosxSupport.py index b96bae1..268426f 100644 --- a/Lib/idlelib/macosxSupport.py +++ b/Lib/idlelib/macosxSupport.py @@ -3,7 +3,6 @@ A number of functions that enhance IDLE on Mac OSX. """ import sys import tkinter -from os import path import warnings def runningAsOSXApp(): diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py index 44502bf..babe6cb 100644 --- a/Lib/lib2to3/fixer_util.py +++ b/Lib/lib2to3/fixer_util.py @@ -1,8 +1,6 @@ """Utility functions, node construction macros, etc.""" # Author: Collin Winter -from itertools import islice - # Local imports from .pgen2 import token from .pytree import Leaf, Node diff --git a/Lib/lib2to3/fixes/fix_dict.py b/Lib/lib2to3/fixes/fix_dict.py index 963f952..d3655c9 100644 --- a/Lib/lib2to3/fixes/fix_dict.py +++ b/Lib/lib2to3/fixes/fix_dict.py @@ -30,9 +30,8 @@ as an argument to a function that introspects the argument). # Local imports from .. import pytree from .. import patcomp -from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Name, Call, LParen, RParen, ArgList, Dot +from ..fixer_util import Name, Call, Dot from .. import fixer_util diff --git a/Lib/lib2to3/fixes/fix_exec.py b/Lib/lib2to3/fixes/fix_exec.py index 2c9b72d..ab921ee 100644 --- a/Lib/lib2to3/fixes/fix_exec.py +++ b/Lib/lib2to3/fixes/fix_exec.py @@ -10,7 +10,6 @@ exec code in ns1, ns2 -> exec(code, ns1, ns2) """ # Local imports -from .. import pytree from .. import fixer_base from ..fixer_util import Comma, Name, Call diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py index 391889f..bb6718c 100644 --- a/Lib/lib2to3/fixes/fix_filter.py +++ b/Lib/lib2to3/fixes/fix_filter.py @@ -14,7 +14,6 @@ Python 2.6 figure it out. """ # Local imports -from ..pgen2 import token from .. import fixer_base from ..fixer_util import Name, Call, ListComp, in_special_context diff --git a/Lib/lib2to3/fixes/fix_has_key.py b/Lib/lib2to3/fixes/fix_has_key.py index 18c560f..439708c 100644 --- a/Lib/lib2to3/fixes/fix_has_key.py +++ b/Lib/lib2to3/fixes/fix_has_key.py @@ -31,7 +31,6 @@ CAVEATS: # Local imports from .. import pytree -from ..pgen2 import token from .. import fixer_base from ..fixer_util import Name, parenthesize diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py index f3f8708..eca3334 100644 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ b/Lib/lib2to3/fixes/fix_metaclass.py @@ -20,7 +20,7 @@ # Local imports from .. import fixer_base from ..pygram import token -from ..fixer_util import Name, syms, Node, Leaf +from ..fixer_util import syms, Node, Leaf def has_metaclass(parent): diff --git a/Lib/lib2to3/fixes/fix_nonzero.py b/Lib/lib2to3/fixes/fix_nonzero.py index ad91a29..c229596 100644 --- a/Lib/lib2to3/fixes/fix_nonzero.py +++ b/Lib/lib2to3/fixes/fix_nonzero.py @@ -3,7 +3,7 @@ # Local imports from .. import fixer_base -from ..fixer_util import Name, syms +from ..fixer_util import Name class FixNonzero(fixer_base.BaseFix): BM_compatible = True diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py index a1fe2f5..8780322 100644 --- a/Lib/lib2to3/fixes/fix_print.py +++ b/Lib/lib2to3/fixes/fix_print.py @@ -18,7 +18,7 @@ from .. import patcomp from .. import pytree from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Name, Call, Comma, String, is_tuple +from ..fixer_util import Name, Call, Comma, String parend_expr = patcomp.compile_pattern( diff --git a/Lib/lib2to3/fixes/fix_types.py b/Lib/lib2to3/fixes/fix_types.py index 00327a7..67bf51f 100644 --- a/Lib/lib2to3/fixes/fix_types.py +++ b/Lib/lib2to3/fixes/fix_types.py @@ -20,7 +20,6 @@ There should be another fixer that handles at least the following constants: """ # Local imports -from ..pgen2 import token from .. import fixer_base from ..fixer_util import Name diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py index 1481cd9..5a36049 100644 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ b/Lib/lib2to3/fixes/fix_urllib.py @@ -6,7 +6,6 @@ # Local imports from lib2to3.fixes.fix_imports import alternates, FixImports -from lib2to3 import fixer_base from lib2to3.fixer_util import (Name, Comma, FromImport, Newline, find_indentation, Node, syms) diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index 0728083..b60b9de 100644 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -26,7 +26,6 @@ from itertools import chain from .pgen2 import driver, tokenize, token from .fixer_util import find_root from . import pytree, pygram -from . import btm_utils as bu from . import btm_matcher as bm diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 4d40b87..d7dba97 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -40,12 +40,10 @@ Instances of this class have the following instance variables: """ import io -import os import sys import importlib.util import tokenize from token import NAME, DEDENT, OP -from operator import itemgetter __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] @@ -328,6 +326,7 @@ def _getname(g): def _main(): # Main program for testing. import os + from operator import itemgetter mod = sys.argv[1] if os.path.exists(mod): path = [os.path.dirname(mod)] diff --git a/Lib/re.py b/Lib/re.py index dde8901..661929e 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -119,7 +119,6 @@ This module also defines an exception 'error'. """ -import sys import sre_compile import sre_parse try: diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index 106d058..d9b3dd5 100644 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -95,7 +95,6 @@ from idlelib.textView import view_text from turtledemo import __doc__ as about_turtledemo import turtle -import time demo_dir = os.path.dirname(os.path.abspath(__file__)) darwin = sys.platform == 'darwin' diff --git a/Lib/turtledemo/bytedesign.py b/Lib/turtledemo/bytedesign.py index 64b1d7d..b3b095b 100755 --- a/Lib/turtledemo/bytedesign.py +++ b/Lib/turtledemo/bytedesign.py @@ -22,7 +22,6 @@ to 0, this animation runs in "line per line" mode as fast as possible. """ -import math from turtle import Turtle, mainloop from time import clock diff --git a/Lib/turtledemo/planet_and_moon.py b/Lib/turtledemo/planet_and_moon.py index 26abfdb..021ff99 100755 --- a/Lib/turtledemo/planet_and_moon.py +++ b/Lib/turtledemo/planet_and_moon.py @@ -18,7 +18,6 @@ mouse over the scrollbar of the canvas. """ from turtle import Shape, Turtle, mainloop, Vec2D as Vec -from time import sleep G = 8 diff --git a/Lib/uuid.py b/Lib/uuid.py index e96e7e0..200c800 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -487,7 +487,6 @@ try: # Assume that the uuid_generate functions are broken from 10.5 onward, # the test can be adjusted when a later version is fixed. if sys.platform == 'darwin': - import os if int(os.uname().release.split('.')[0]) >= 9: _uuid_generate_time = None diff --git a/Tools/scripts/diff.py b/Tools/scripts/diff.py index 9720a43..96199b8 100755 --- a/Tools/scripts/diff.py +++ b/Tools/scripts/diff.py @@ -8,7 +8,7 @@ """ -import sys, os, time, difflib, argparse +import sys, os, difflib, argparse from datetime import datetime, timezone def file_mtime(path): -- cgit v0.12 From 159e5359d95a007865fd5367c398aaa1fb00828c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 25 Apr 2016 13:49:11 +0300 Subject: Remove outdated TkVersion checks. Minimal supported Tcl/Tk version is 8.4, and this is checked in _tkinter.c. --- Lib/tkinter/__init__.py | 3 --- Lib/tkinter/commondialog.py | 5 ----- Lib/tkinter/dialog.py | 5 +---- Lib/tkinter/tix.py | 4 ---- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 5272b30..3f242ac 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1887,9 +1887,6 @@ class Tk(Misc, Wm): if tcl_version != _tkinter.TCL_VERSION: raise RuntimeError("tcl.h version (%s) doesn't match libtcl.a version (%s)" \ % (_tkinter.TCL_VERSION, tcl_version)) - if TkVersion < 4.0: - raise RuntimeError("Tk 4.0 or higher is required; found Tk %s" - % str(TkVersion)) # Create and register the tkerror and exit commands # We need to inline parts of _register here, _ register # would register differently-named commands. diff --git a/Lib/tkinter/commondialog.py b/Lib/tkinter/commondialog.py index d2688db..1e75cae 100644 --- a/Lib/tkinter/commondialog.py +++ b/Lib/tkinter/commondialog.py @@ -15,11 +15,6 @@ class Dialog: command = None def __init__(self, master=None, **options): - - # FIXME: should this be placed on the module level instead? - if TkVersion < 4.2: - raise TclError("this module requires Tk 4.2 or newer") - self.master = master self.options = options if not master and options.get('parent'): diff --git a/Lib/tkinter/dialog.py b/Lib/tkinter/dialog.py index be085ab..f61c5f7 100644 --- a/Lib/tkinter/dialog.py +++ b/Lib/tkinter/dialog.py @@ -3,10 +3,7 @@ from tkinter import * from tkinter import _cnfmerge -if TkVersion <= 3.6: - DIALOG_ICON = 'warning' -else: - DIALOG_ICON = 'questhead' +DIALOG_ICON = 'questhead' class Dialog(Widget): diff --git a/Lib/tkinter/tix.py b/Lib/tkinter/tix.py index adb629a..8a8ab91 100644 --- a/Lib/tkinter/tix.py +++ b/Lib/tkinter/tix.py @@ -29,10 +29,6 @@ from tkinter import * from tkinter import _cnfmerge, _default_root -# WARNING - TkVersion is a limited precision floating point number -if TkVersion < 3.999: - raise ImportError("This version of Tix.py requires Tk 4.0 or higher") - import _tkinter # If this fails your Python may not be configured for Tk # Some more constants (for consistency with Tkinter) -- cgit v0.12 From 1845d144bc58be7b245bb02da3245c5e01760621 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 25 Apr 2016 21:38:53 +0200 Subject: Issue #17905: Do not guard locale include with HAVE_LANGINFO_H. --- Python/sysmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0c68c54..b519912 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -20,6 +20,7 @@ Data members: #include "pythread.h" #include "osdefs.h" +#include #ifdef MS_WINDOWS #define WIN32_LEAN_AND_MEAN @@ -33,7 +34,6 @@ extern const char *PyWin_DLLVersionString; #endif #ifdef HAVE_LANGINFO_H -#include #include #endif -- cgit v0.12 From 8e1da5823b4dabbc5e60f8ee27411070a37804ef Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 25 Apr 2016 22:48:42 +0200 Subject: Issue #26846: Workaround for non-standard stdlib.h on Android. --- Modules/_decimal/libmpdec/basearith.c | 1 - Modules/_decimal/libmpdec/io.c | 1 - Modules/_decimal/libmpdec/memory.c | 2 +- Modules/_decimal/libmpdec/memory.h | 51 ----------------------------------- Modules/_decimal/libmpdec/mpalloc.h | 51 +++++++++++++++++++++++++++++++++++ Modules/_decimal/libmpdec/mpdecimal.c | 2 +- 6 files changed, 53 insertions(+), 55 deletions(-) delete mode 100644 Modules/_decimal/libmpdec/memory.h create mode 100644 Modules/_decimal/libmpdec/mpalloc.h diff --git a/Modules/_decimal/libmpdec/basearith.c b/Modules/_decimal/libmpdec/basearith.c index 35de6b8..dfe1523 100644 --- a/Modules/_decimal/libmpdec/basearith.c +++ b/Modules/_decimal/libmpdec/basearith.c @@ -32,7 +32,6 @@ #include #include #include "constants.h" -#include "memory.h" #include "typearith.h" #include "basearith.h" diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c index a45a429..3aadfb0 100644 --- a/Modules/_decimal/libmpdec/io.c +++ b/Modules/_decimal/libmpdec/io.c @@ -37,7 +37,6 @@ #include #include "bits.h" #include "constants.h" -#include "memory.h" #include "typearith.h" #include "io.h" diff --git a/Modules/_decimal/libmpdec/memory.c b/Modules/_decimal/libmpdec/memory.c index 61eb633..a854e09 100644 --- a/Modules/_decimal/libmpdec/memory.c +++ b/Modules/_decimal/libmpdec/memory.c @@ -30,7 +30,7 @@ #include #include #include "typearith.h" -#include "memory.h" +#include "mpalloc.h" #if defined(_MSC_VER) diff --git a/Modules/_decimal/libmpdec/memory.h b/Modules/_decimal/libmpdec/memory.h deleted file mode 100644 index 9c98d1a..0000000 --- a/Modules/_decimal/libmpdec/memory.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef MEMORY_H -#define MEMORY_H - - -#include "mpdecimal.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -int mpd_switch_to_dyn(mpd_t *result, mpd_ssize_t size, uint32_t *status); -int mpd_switch_to_dyn_zero(mpd_t *result, mpd_ssize_t size, uint32_t *status); -int mpd_realloc_dyn(mpd_t *result, mpd_ssize_t size, uint32_t *status); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif - - - diff --git a/Modules/_decimal/libmpdec/mpalloc.h b/Modules/_decimal/libmpdec/mpalloc.h new file mode 100644 index 0000000..efd7119 --- /dev/null +++ b/Modules/_decimal/libmpdec/mpalloc.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +#ifndef MPALLOC_H +#define MPALLOC_H + + +#include "mpdecimal.h" + + +/* Internal header file: all symbols have local scope in the DSO */ +MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) + + +int mpd_switch_to_dyn(mpd_t *result, mpd_ssize_t size, uint32_t *status); +int mpd_switch_to_dyn_zero(mpd_t *result, mpd_ssize_t size, uint32_t *status); +int mpd_realloc_dyn(mpd_t *result, mpd_ssize_t size, uint32_t *status); + + +MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ + + +#endif + + + diff --git a/Modules/_decimal/libmpdec/mpdecimal.c b/Modules/_decimal/libmpdec/mpdecimal.c index 593f9f5..328ab92 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.c +++ b/Modules/_decimal/libmpdec/mpdecimal.c @@ -36,7 +36,7 @@ #include "bits.h" #include "convolute.h" #include "crt.h" -#include "memory.h" +#include "mpalloc.h" #include "typearith.h" #include "umodarith.h" -- cgit v0.12 From 267b639a2655d283282813219730704a8d2743b7 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 26 Apr 2016 01:09:18 +0200 Subject: Issue #20306: The pw_gecos and pw_passwd fields are not required by POSIX. If they aren't present, set them to an empty string. --- Modules/pwdmodule.c | 8 ++++++++ configure | 27 +++++++++++++++++++++++++++ configure.ac | 4 ++++ pyconfig.h.in | 6 ++++++ 4 files changed, 45 insertions(+) diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index 281c30b..c503256 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -75,10 +75,18 @@ mkpwent(struct passwd *p) #define SETS(i,val) sets(v, i, val) SETS(setIndex++, p->pw_name); +#if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) SETS(setIndex++, p->pw_passwd); +#else + SETS(setIndex++, ""); +#endif PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromUid(p->pw_uid)); PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromGid(p->pw_gid)); +#if defined(HAVE_STRUCT_PASSWD_PW_GECOS) SETS(setIndex++, p->pw_gecos); +#else + SETS(setIndex++, ""); +#endif SETS(setIndex++, p->pw_dir); SETS(setIndex++, p->pw_shell); diff --git a/configure b/configure index ee8a8fc..a961f39 100755 --- a/configure +++ b/configure @@ -12918,6 +12918,33 @@ _ACEOF fi +ac_fn_c_check_member "$LINENO" "struct passwd" "pw_gecos" "ac_cv_member_struct_passwd_pw_gecos" " + #include + #include + +" +if test "x$ac_cv_member_struct_passwd_pw_gecos" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 +_ACEOF + + +fi +ac_fn_c_check_member "$LINENO" "struct passwd" "pw_passwd" "ac_cv_member_struct_passwd_pw_passwd" " + #include + #include + +" +if test "x$ac_cv_member_struct_passwd_pw_passwd" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 +_ACEOF + + +fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for time.h that defines altzone" >&5 $as_echo_n "checking for time.h that defines altzone... " >&6; } diff --git a/configure.ac b/configure.ac index 7a6b5be..bfd2b04 100644 --- a/configure.ac +++ b/configure.ac @@ -3762,6 +3762,10 @@ AC_CHECK_MEMBERS([struct stat.st_flags]) AC_CHECK_MEMBERS([struct stat.st_gen]) AC_CHECK_MEMBERS([struct stat.st_birthtime]) AC_CHECK_MEMBERS([struct stat.st_blocks]) +AC_CHECK_MEMBERS([struct passwd.pw_gecos, struct passwd.pw_passwd], [], [], [[ + #include + #include +]]) AC_MSG_CHECKING(for time.h that defines altzone) AC_CACHE_VAL(ac_cv_header_time_altzone,[ diff --git a/pyconfig.h.in b/pyconfig.h.in index d432a82..b51bd56 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -916,6 +916,12 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STROPTS_H +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#undef HAVE_STRUCT_PASSWD_PW_GECOS + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#undef HAVE_STRUCT_PASSWD_PW_PASSWD + /* Define to 1 if `st_birthtime' is a member of `struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BIRTHTIME -- cgit v0.12 From 144da4ec850583c51e1c9cf55d8aaf20cdc7af5d Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 26 Apr 2016 01:56:50 +0200 Subject: Issue #22747: Workaround for systems without langinfo.h. --- Python/pylifecycle.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 74bdb19..7187fe4 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -223,6 +223,8 @@ get_locale_encoding(void) return NULL; } return get_codec_name(codeset); +#elif defined(__ANDROID__) + return get_codec_name("UTF-8"); #else PyErr_SetNone(PyExc_NotImplementedError); return NULL; -- cgit v0.12 From ab425aa9d3353a15e2a673eb1af121dcde720a14 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 26 Apr 2016 00:10:00 -0700 Subject: Issue #16394: Note the tee() pure python equivalent is only a rough approximation. --- Doc/library/itertools.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 758e49b..8376f1a 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -588,7 +588,10 @@ loops that truncate the stream. .. function:: tee(iterable, n=2) - Return *n* independent iterators from a single iterable. Equivalent to:: + Return *n* independent iterators from a single iterable. + + The following Python code helps explain what *tee* does (although the actual + implementation is more complex and uses only a single underlying FIFO queue):: def tee(iterable, n=2): it = iter(iterable) -- cgit v0.12 From 45009778aa131b99bc95412390dae232d9f760a7 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 26 Apr 2016 11:43:21 +0200 Subject: Issue #20306: Android is the only system that returns NULL for the pw_passwd field. Rather than cluttering the tests, translate the arguably more correct "None" to an empty string. --- Modules/pwdmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index c503256..61be3b2 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -75,7 +75,7 @@ mkpwent(struct passwd *p) #define SETS(i,val) sets(v, i, val) SETS(setIndex++, p->pw_name); -#if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) +#if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) && !defined(__ANDROID__) SETS(setIndex++, p->pw_passwd); #else SETS(setIndex++, ""); -- cgit v0.12 From 71dc3d878a44c8fc6b9b1743750600d4902538f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Apr 2016 12:35:13 +0200 Subject: Issue #25349, #26249: Fix memleak in formatfloat() --- Objects/bytesobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 701ae9d..296c73b 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -438,6 +438,7 @@ formatfloat(PyObject *v, int flags, int prec, int type, if (str == NULL) return NULL; Py_MEMCPY(str, p, len); + PyMem_Free(p); str += len; return str; } -- cgit v0.12 From 432dfcf3bc2653a1163a2db9e6b5b38a3c9681e1 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 26 Apr 2016 16:20:17 +0200 Subject: Issue #26857: Workaround for missing symbol "gethostbyaddr_r" on Android. --- Modules/socketmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index ec35fb9..46eeed1 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -163,7 +163,7 @@ if_indextoname(index) -- return the corresponding interface name\n\ # include #endif -#ifndef WITH_THREAD +#if !defined(WITH_THREAD) || defined(__ANDROID__) # undef HAVE_GETHOSTBYNAME_R #endif -- cgit v0.12 From 8d013a8d36ea1a9041548527bd6d33609de3d3fd Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 26 Apr 2016 16:34:41 +0200 Subject: Issue #26846: Post commit cleanup. --- Modules/_decimal/_decimal.c | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 3c2ad85..e19bbf2 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -36,7 +36,6 @@ #include #include "docstrings.h" -#include "memory.h" #if !defined(MPD_VERSION_HEX) || MPD_VERSION_HEX < 0x02040100 diff --git a/setup.py b/setup.py index 4a0cb8b..2e9f9a4 100644 --- a/setup.py +++ b/setup.py @@ -2060,7 +2060,7 @@ class PyBuildExt(build_ext): '_decimal/libmpdec/fnt.h', '_decimal/libmpdec/fourstep.h', '_decimal/libmpdec/io.h', - '_decimal/libmpdec/memory.h', + '_decimal/libmpdec/mpalloc.h', '_decimal/libmpdec/mpdecimal.h', '_decimal/libmpdec/numbertheory.h', '_decimal/libmpdec/sixstep.h', -- cgit v0.12 From fa935c4727da6e34e42643fe54fe291e87f00ace Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 26 Apr 2016 16:48:48 +0200 Subject: Issue #26854: Android has a different include path for soundcard.h. --- Modules/ossaudiodev.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index d2fd5c8..2b7d71f 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -31,7 +31,11 @@ #endif #include +#ifdef __ANDROID__ +#include +#else #include +#endif #if defined(linux) -- cgit v0.12 From fb7c8ae4e7762d5b862cace2f68b48ff42e39cb0 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 26 Apr 2016 17:04:18 +0200 Subject: Issue #26863: HAVE_FACCESSAT should (currently) not be defined on Android. --- Modules/posixmodule.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a87bbbd..03ad07d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -32,6 +32,12 @@ #include "winreparse.h" #endif +/* On android API level 21, 'AT_EACCESS' is not declared although + * HAVE_FACCESSAT is defined. */ +#ifdef __ANDROID__ +#undef HAVE_FACCESSAT +#endif + #ifdef __cplusplus extern "C" { #endif @@ -5933,7 +5939,7 @@ os_openpty_impl(PyModuleDef *module) if (_Py_set_inheritable(master_fd, 0, NULL) < 0) goto posix_error; -#if !defined(__CYGWIN__) && !defined(HAVE_DEV_PTC) +#if !defined(__CYGWIN__) && !defined(__ANDROID__) && !defined(HAVE_DEV_PTC) ioctl(slave_fd, I_PUSH, "ptem"); /* push ptem */ ioctl(slave_fd, I_PUSH, "ldterm"); /* push ldterm */ #ifndef __hpux -- cgit v0.12 From b275210a3b0e04691ebd1c1d0374720be59911b9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 27 Apr 2016 23:13:46 +0300 Subject: Issue #25788: fileinput.hook_encoded() now supports an "errors" argument for passing to open. Original patch by Joseph Hackman. --- Doc/library/fileinput.rst | 10 +++++++--- Doc/whatsnew/3.6.rst | 7 +++++++ Lib/fileinput.py | 4 ++-- Lib/test/test_fileinput.py | 21 ++++++++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 3433682..8efe8e3 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -193,10 +193,14 @@ The two following opening hooks are provided by this module: Usage example: ``fi = fileinput.FileInput(openhook=fileinput.hook_compressed)`` -.. function:: hook_encoded(encoding) +.. function:: hook_encoded(encoding, errors=None) Returns a hook which opens each file with :func:`open`, using the given - *encoding* to read the file. + *encoding* and *errors* to read the file. Usage example: ``fi = - fileinput.FileInput(openhook=fileinput.hook_encoded("iso-8859-1"))`` + fileinput.FileInput(openhook=fileinput.hook_encoded("utf-8", + "surrogateescape"))`` + + .. versionchanged:: 3.6 + Added the optional *errors* parameter. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 99223af..be4c014 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -358,6 +358,13 @@ The :func:`~zlib.compress` function now accepts keyword arguments. (Contributed by Aviv Palivoda in :issue:`26243`.) +fileinput +--------- + +:func:`~fileinput.hook_encoded` now supports the *errors* argument. +(Contributed by Joseph Hackman in :issue:`25788`.) + + Optimizations ============= diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 1e19d24..721fe9c 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -400,9 +400,9 @@ def hook_compressed(filename, mode): return open(filename, mode) -def hook_encoded(encoding): +def hook_encoded(encoding, errors=None): def openhook(filename, mode): - return open(filename, mode, encoding=encoding) + return open(filename, mode, encoding=encoding, errors=errors) return openhook diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 4f67c25..565633f 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -945,7 +945,8 @@ class Test_hook_encoded(unittest.TestCase): def test(self): encoding = object() - result = fileinput.hook_encoded(encoding) + errors = object() + result = fileinput.hook_encoded(encoding, errors=errors) fake_open = InvocationRecorder() original_open = builtins.open @@ -963,8 +964,26 @@ class Test_hook_encoded(unittest.TestCase): self.assertIs(args[0], filename) self.assertIs(args[1], mode) self.assertIs(kwargs.pop('encoding'), encoding) + self.assertIs(kwargs.pop('errors'), errors) self.assertFalse(kwargs) + def test_errors(self): + with open(TESTFN, 'wb') as f: + f.write(b'\x80abc') + self.addCleanup(safe_unlink, TESTFN) + + def check(errors, expected_lines): + with FileInput(files=TESTFN, mode='r', + openhook=hook_encoded('utf-8', errors=errors)) as fi: + lines = list(fi) + self.assertEqual(lines, expected_lines) + + check('ignore', ['abc']) + with self.assertRaises(UnicodeDecodeError): + check('strict', ['abc']) + check('replace', ['\ufffdabc']) + check('backslashreplace', ['\\x80abc']) + def test_modes(self): with open(TESTFN, 'wb') as f: # UTF-7 is a convenient, seldom used encoding diff --git a/Misc/ACKS b/Misc/ACKS index dd3a567..ebc3fc6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -538,6 +538,7 @@ Michael Guravage Lars Gustäbel Thomas Güttler Jonas H. +Joseph Hackman Barry Haddow Philipp Hagemeister Paul ten Hagen diff --git a/Misc/NEWS b/Misc/NEWS index b6fb8f8..e68bbdf 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -256,6 +256,9 @@ Core and Builtins Library ------- +- Issue #25788: fileinput.hook_encoded() now supports an "errors" argument + for passing to open. Original patch by Joseph Hackman. + - Issue #26634: recursive_repr() now sets __qualname__ of wrapper. Patch by Xiang Zhang. -- cgit v0.12 From fa070298e982a39304cb5b6a326b534820d9cad0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 29 Apr 2016 11:31:52 +0300 Subject: Issue #26880: Removed redundant checks in set.__init__. --- Objects/setobject.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index ac71b2c..3d0f355 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1998,9 +1998,7 @@ set_init(PySetObject *self, PyObject *args, PyObject *kwds) { PyObject *iterable = NULL; - if (!PyAnySet_Check(self)) - return -1; - if (kwds != NULL && PySet_Check(self) && !_PyArg_NoKeywords("set()", kwds)) + if (kwds != NULL && !_PyArg_NoKeywords("set()", kwds)) return -1; if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; -- cgit v0.12 From 8421d714d030dbd0c368148aa818e15673db3b39 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 29 Apr 2016 01:37:05 -0700 Subject: Removed unused initialization and the uninteresting comment. --- Objects/setobject.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 3d0f355..6dd403f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1043,9 +1043,8 @@ PyDoc_STRVAR(update_doc, static PyObject * make_new_set(PyTypeObject *type, PyObject *iterable) { - PySetObject *so = NULL; + PySetObject *so; - /* create PySetObject structure */ so = (PySetObject *)type->tp_alloc(type, 0); if (so == NULL) return NULL; -- cgit v0.12 From 3a31cca4abdfdc4d623deb7e8c4be58b5ca936eb Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 29 Apr 2016 16:48:11 +0300 Subject: Issue #24902: Print server URL on http.server startup Initial patch by Felix Kaiser. --- Lib/http/server.py | 3 ++- Misc/NEWS | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index 1f6a62b..bd94eaa 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1177,7 +1177,8 @@ def test(HandlerClass=BaseHTTPRequestHandler, HandlerClass.protocol_version = protocol with ServerClass(server_address, HandlerClass) as httpd: sa = httpd.socket.getsockname() - print("Serving HTTP on", sa[0], "port", sa[1], "...") + serve_message = "Serving HTTP on {host} port {port} (http://{host}:{port}/) ..." + print(serve_message.format(host=sa[0], port=sa[1])) try: httpd.serve_forever() except KeyboardInterrupt: diff --git a/Misc/NEWS b/Misc/NEWS index e68bbdf..7b3bbd7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -256,6 +256,9 @@ Core and Builtins Library ------- +- Issue #24902: Print server URL on http.server startup. Initial patch by + Felix Kaiser. + - Issue #25788: fileinput.hook_encoded() now supports an "errors" argument for passing to open. Original patch by Joseph Hackman. -- cgit v0.12 From 0ac70c0e9052db0c8ca6da517019c9c8f3f5d30d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 29 Apr 2016 16:54:10 +0300 Subject: Fix typos. Reported by andportnoy on GitHub. --- Modules/_randommodule.c | 2 +- Objects/typeobject.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index a85d905..fd6b230 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -136,7 +136,7 @@ genrand_int32(RandomObject *self) * optimize the division away at compile-time. 67108864 is 2**26. In * effect, a contains 27 random bits shifted left 26, and b fills in the * lower 26 bits of the 53-bit numerator. - * The orginal code credited Isaku Wada for this algorithm, 2002/01/09. + * The original code credited Isaku Wada for this algorithm, 2002/01/09. */ static PyObject * random_random(RandomObject *self) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2086e9d..9e7b4e6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3944,7 +3944,7 @@ _PyObject_GetState(PyObject *obj, int required) } /* If we found some slot attributes, pack them in a tuple along - the orginal attribute dictionary. */ + the original attribute dictionary. */ if (PyDict_Size(slots) > 0) { PyObject *state2; -- cgit v0.12 From 0a5bd51dd30762e3e3b0f3b00ffe7899b9c58317 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 29 Apr 2016 19:50:02 +0300 Subject: Issue #13436: Add a test to make sure that ast.ImportFrom(level=None) works --- Lib/test/test_ast.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index a025c20..7b43be6 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -548,6 +548,17 @@ class ASTHelpers_Test(unittest.TestCase): compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) + def test_level_as_none(self): + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep')], + level=None, + lineno=0, col_offset=0)] + mod = ast.Module(body) + code = compile(mod, 'test', 'exec') + ns = {} + exec(code, ns) + self.assertIn('sleep', ns) + class ASTValidatorTests(unittest.TestCase): -- cgit v0.12 From 7a9579c0ce02ff0254c6aa02952dbcdcdbac6d33 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 2 May 2016 13:45:20 +0300 Subject: Got rid of redundand "self" parameter declarations. Argument Clinic is now able to infer all needed information. --- Modules/_bz2module.c | 3 +-- Modules/_lzmamodule.c | 14 ++++--------- Modules/_tkinter.c | 17 +++++---------- Objects/bytearrayobject.c | 47 +++++++++++++----------------------------- Objects/bytesobject.c | 46 ++++++++++++++++++----------------------- Objects/clinic/bytesobject.c.h | 22 ++++++++++---------- Python/bltinmodule.c | 5 ++--- 7 files changed, 57 insertions(+), 97 deletions(-) diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index e3e0eb1..f4077fa 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -592,7 +592,6 @@ error: /*[clinic input] _bz2.BZ2Decompressor.decompress - self: self(type="BZ2Decompressor *") data: Py_buffer max_length: Py_ssize_t=-1 @@ -615,7 +614,7 @@ the unused_data attribute. static PyObject * _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=23e41045deb240a3 input=9558b424c8b00516]*/ +/*[clinic end generated code: output=23e41045deb240a3 input=52e1ffc66a8ea624]*/ { PyObject *result = NULL; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f3bcd76..4126b0c 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -553,7 +553,6 @@ error: /*[clinic input] _lzma.LZMACompressor.compress - self: self(type="Compressor *") data: Py_buffer / @@ -567,7 +566,7 @@ flush() method to finish the compression process. static PyObject * _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) -/*[clinic end generated code: output=31f615136963e00f input=8b60cb13e0ce6420]*/ +/*[clinic end generated code: output=31f615136963e00f input=64019eac7f2cc8d0]*/ { PyObject *result = NULL; @@ -583,8 +582,6 @@ _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) /*[clinic input] _lzma.LZMACompressor.flush - self: self(type="Compressor *") - Finish the compression process. Returns the compressed data left in internal buffers. @@ -594,7 +591,7 @@ The compressor object may not be used after this method is called. static PyObject * _lzma_LZMACompressor_flush_impl(Compressor *self) -/*[clinic end generated code: output=fec21f3e22504f50 input=3060fb26f9b4042c]*/ +/*[clinic end generated code: output=fec21f3e22504f50 input=6b369303f67ad0a8]*/ { PyObject *result = NULL; @@ -698,7 +695,6 @@ Compressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) /*[-clinic input] _lzma.LZMACompressor.__init__ - self: self(type="Compressor *") format: int(c_default="FORMAT_XZ") = FORMAT_XZ The container format to use for the output. This can be FORMAT_XZ (default), FORMAT_ALONE, or FORMAT_RAW. @@ -1063,7 +1059,6 @@ error: /*[clinic input] _lzma.LZMADecompressor.decompress - self: self(type="Decompressor *") data: Py_buffer max_length: Py_ssize_t=-1 @@ -1086,7 +1081,7 @@ the unused_data attribute. static PyObject * _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ef4e20ec7122241d input=f2bb902cc1caf203]*/ +/*[clinic end generated code: output=ef4e20ec7122241d input=60c1f135820e309d]*/ { PyObject *result = NULL; @@ -1126,7 +1121,6 @@ Decompressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) /*[clinic input] _lzma.LZMADecompressor.__init__ - self: self(type="Decompressor *") format: int(c_default="FORMAT_AUTO") = FORMAT_AUTO Specifies the container format of the input stream. If this is FORMAT_AUTO (the default), the decompressor will automatically detect @@ -1152,7 +1146,7 @@ For one-shot decompression, use the decompress() function instead. static int _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, PyObject *memlimit, PyObject *filters) -/*[clinic end generated code: output=3e1821f8aa36564c input=458ca6132ef29801]*/ +/*[clinic end generated code: output=3e1821f8aa36564c input=81fe684a6c2f8a27]*/ { const uint32_t decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK; uint64_t memlimit_ = UINT64_MAX; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 768df81..a4f0042 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2485,7 +2485,6 @@ Tkapp_CommandProc(CommandEvent *ev, int flags) /*[clinic input] _tkinter.tkapp.createcommand - self: self(type="TkappObject *") name: str func: object / @@ -2495,7 +2494,7 @@ _tkinter.tkapp.createcommand static PyObject * _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, PyObject *func) -/*[clinic end generated code: output=2a1c79a4ee2af410 input=2bc2c046a0914234]*/ +/*[clinic end generated code: output=2a1c79a4ee2af410 input=255785cb70edc6a0]*/ { PythonCmd_ClientData *data; int err; @@ -2561,7 +2560,6 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, /*[clinic input] _tkinter.tkapp.deletecommand - self: self(type="TkappObject *") name: str / @@ -2569,7 +2567,7 @@ _tkinter.tkapp.deletecommand static PyObject * _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) -/*[clinic end generated code: output=a67e8cb5845e0d2d input=b6306468f10b219c]*/ +/*[clinic end generated code: output=a67e8cb5845e0d2d input=53e9952eae1f85f5]*/ { int err; @@ -2762,13 +2760,11 @@ typedef struct { /*[clinic input] _tkinter.tktimertoken.deletetimerhandler - self: self(type="TkttObject *") - [clinic start generated code]*/ static PyObject * _tkinter_tktimertoken_deletetimerhandler_impl(TkttObject *self) -/*[clinic end generated code: output=bd7fe17f328cfa55 input=25ba5dd594e52084]*/ +/*[clinic end generated code: output=bd7fe17f328cfa55 input=40bd070ff85f5cf3]*/ { TkttObject *v = self; PyObject *func = v->func; @@ -2894,7 +2890,6 @@ _tkinter_tkapp_createtimerhandler_impl(TkappObject *self, int milliseconds, /*[clinic input] _tkinter.tkapp.mainloop - self: self(type="TkappObject *") threshold: int = 0 / @@ -2902,7 +2897,7 @@ _tkinter.tkapp.mainloop static PyObject * _tkinter_tkapp_mainloop_impl(TkappObject *self, int threshold) -/*[clinic end generated code: output=0ba8eabbe57841b0 input=ad57c9c1dd2b9470]*/ +/*[clinic end generated code: output=0ba8eabbe57841b0 input=036bcdcf03d5eca0]*/ { #ifdef WITH_THREAD PyThreadState *tstate = PyThreadState_Get(); @@ -3072,13 +3067,11 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) /*[clinic input] _tkinter.tkapp.willdispatch - self: self(type="TkappObject *") - [clinic start generated code]*/ static PyObject * _tkinter_tkapp_willdispatch_impl(TkappObject *self) -/*[clinic end generated code: output=0e3f46d244642155 input=2630699767808970]*/ +/*[clinic end generated code: output=0e3f46d244642155 input=d88f5970843d6dab]*/ { self->dispatching = 1; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index d0260c9..fb3668e 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1243,14 +1243,12 @@ bytearray_count(PyByteArrayObject *self, PyObject *args) /*[clinic input] bytearray.clear - self: self(type="PyByteArrayObject *") - Remove all items from the bytearray. [clinic start generated code]*/ static PyObject * bytearray_clear_impl(PyByteArrayObject *self) -/*[clinic end generated code: output=85c2fe6aede0956c input=e524fd330abcdc18]*/ +/*[clinic end generated code: output=85c2fe6aede0956c input=ed6edae9de447ac4]*/ { if (PyByteArray_Resize((PyObject *)self, 0) < 0) return NULL; @@ -1260,14 +1258,12 @@ bytearray_clear_impl(PyByteArrayObject *self) /*[clinic input] bytearray.copy - self: self(type="PyByteArrayObject *") - Return a copy of B. [clinic start generated code]*/ static PyObject * bytearray_copy_impl(PyByteArrayObject *self) -/*[clinic end generated code: output=68cfbcfed484c132 input=6d5d2975aa0f33f3]*/ +/*[clinic end generated code: output=68cfbcfed484c132 input=6597b0c01bccaa9e]*/ { return PyByteArray_FromStringAndSize(PyByteArray_AS_STRING((PyObject *)self), PyByteArray_GET_SIZE(self)); @@ -1489,7 +1485,6 @@ bytearray_endswith(PyByteArrayObject *self, PyObject *args) /*[clinic input] bytearray.translate - self: self(type="PyByteArrayObject *") table: object Translation table, which must be a bytes object of length 256. [ @@ -1506,7 +1501,7 @@ The remaining characters are mapped through the given translation table. static PyObject * bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, int group_right_1, PyObject *deletechars) -/*[clinic end generated code: output=2bebc86a9a1ff083 input=b749ad85f4860824]*/ +/*[clinic end generated code: output=2bebc86a9a1ff083 input=846a01671bccc1c5]*/ { char *input, *output; const char *table_chars; @@ -2187,7 +2182,6 @@ bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, /*[clinic input] bytearray.partition - self: self(type="PyByteArrayObject *") sep: object / @@ -2203,7 +2197,7 @@ bytearray object and two empty bytearray objects. static PyObject * bytearray_partition(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=45d2525ddd35f957 input=7d7fe37b1696d506]*/ +/*[clinic end generated code: output=45d2525ddd35f957 input=86f89223892b70b5]*/ { PyObject *bytesep, *result; @@ -2225,7 +2219,6 @@ bytearray_partition(PyByteArrayObject *self, PyObject *sep) /*[clinic input] bytearray.rpartition - self: self(type="PyByteArrayObject *") sep: object / @@ -2241,7 +2234,7 @@ objects and the original bytearray object. static PyObject * bytearray_rpartition(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=440de3c9426115e8 input=9b8cd540c1b75853]*/ +/*[clinic end generated code: output=440de3c9426115e8 input=5f4094f2de87c8f3]*/ { PyObject *bytesep, *result; @@ -2299,14 +2292,12 @@ bytearray_rsplit_impl(PyByteArrayObject *self, PyObject *sep, /*[clinic input] bytearray.reverse - self: self(type="PyByteArrayObject *") - Reverse the order of the values in B in place. [clinic start generated code]*/ static PyObject * bytearray_reverse_impl(PyByteArrayObject *self) -/*[clinic end generated code: output=9f7616f29ab309d3 input=7933a499b8597bd1]*/ +/*[clinic end generated code: output=9f7616f29ab309d3 input=543356319fc78557]*/ { char swap, *head, *tail; Py_ssize_t i, j, n = Py_SIZE(self); @@ -2335,7 +2326,6 @@ class bytesvalue_converter(CConverter): /*[clinic input] bytearray.insert - self: self(type="PyByteArrayObject *") index: Py_ssize_t The index where the value is to be inserted. item: bytesvalue @@ -2347,7 +2337,7 @@ Insert a single item into the bytearray before the given index. static PyObject * bytearray_insert_impl(PyByteArrayObject *self, Py_ssize_t index, int item) -/*[clinic end generated code: output=76c775a70e7b07b7 input=833766836ba30e1e]*/ +/*[clinic end generated code: output=76c775a70e7b07b7 input=b2b5d07e9de6c070]*/ { Py_ssize_t n = Py_SIZE(self); char *buf; @@ -2377,7 +2367,6 @@ bytearray_insert_impl(PyByteArrayObject *self, Py_ssize_t index, int item) /*[clinic input] bytearray.append - self: self(type="PyByteArrayObject *") item: bytesvalue The item to be appended. / @@ -2387,7 +2376,7 @@ Append a single item to the end of the bytearray. static PyObject * bytearray_append_impl(PyByteArrayObject *self, int item) -/*[clinic end generated code: output=a154e19ed1886cb6 input=ae56ea87380407cc]*/ +/*[clinic end generated code: output=a154e19ed1886cb6 input=20d6bec3d1340593]*/ { Py_ssize_t n = Py_SIZE(self); @@ -2407,7 +2396,6 @@ bytearray_append_impl(PyByteArrayObject *self, int item) /*[clinic input] bytearray.extend - self: self(type="PyByteArrayObject *") iterable_of_ints: object The iterable of items to append. / @@ -2417,7 +2405,7 @@ Append all the items from the iterator or sequence to the end of the bytearray. static PyObject * bytearray_extend(PyByteArrayObject *self, PyObject *iterable_of_ints) -/*[clinic end generated code: output=98155dbe249170b1 input=ce83a5d75b70d850]*/ +/*[clinic end generated code: output=98155dbe249170b1 input=c617b3a93249ba28]*/ { PyObject *it, *item, *bytearray_obj; Py_ssize_t buf_size = 0, len = 0; @@ -2492,7 +2480,6 @@ bytearray_extend(PyByteArrayObject *self, PyObject *iterable_of_ints) /*[clinic input] bytearray.pop - self: self(type="PyByteArrayObject *") index: Py_ssize_t = -1 The index from where to remove the item. -1 (the default value) means remove the last item. @@ -2505,7 +2492,7 @@ If no index argument is given, will pop the last item. static PyObject * bytearray_pop_impl(PyByteArrayObject *self, Py_ssize_t index) -/*[clinic end generated code: output=e0ccd401f8021da8 input=0797e6c0ca9d5a85]*/ +/*[clinic end generated code: output=e0ccd401f8021da8 input=3591df2d06c0d237]*/ { int value; Py_ssize_t n = Py_SIZE(self); @@ -2537,7 +2524,6 @@ bytearray_pop_impl(PyByteArrayObject *self, Py_ssize_t index) /*[clinic input] bytearray.remove - self: self(type="PyByteArrayObject *") value: bytesvalue The value to remove. / @@ -2547,7 +2533,7 @@ Remove the first occurrence of a value in the bytearray. static PyObject * bytearray_remove_impl(PyByteArrayObject *self, int value) -/*[clinic end generated code: output=d659e37866709c13 input=47560b11fd856c24]*/ +/*[clinic end generated code: output=d659e37866709c13 input=121831240cd51ddf]*/ { Py_ssize_t where, n = Py_SIZE(self); char *buf = PyByteArray_AS_STRING(self); @@ -2858,14 +2844,12 @@ _common_reduce(PyByteArrayObject *self, int proto) /*[clinic input] bytearray.__reduce__ as bytearray_reduce - self: self(type="PyByteArrayObject *") - Return state information for pickling. [clinic start generated code]*/ static PyObject * bytearray_reduce_impl(PyByteArrayObject *self) -/*[clinic end generated code: output=52bf304086464cab input=fbb07de4d102a03a]*/ +/*[clinic end generated code: output=52bf304086464cab input=44b5737ada62dd3f]*/ { return _common_reduce(self, 2); } @@ -2873,7 +2857,6 @@ bytearray_reduce_impl(PyByteArrayObject *self) /*[clinic input] bytearray.__reduce_ex__ as bytearray_reduce_ex - self: self(type="PyByteArrayObject *") proto: int = 0 / @@ -2882,7 +2865,7 @@ Return state information for pickling. static PyObject * bytearray_reduce_ex_impl(PyByteArrayObject *self, int proto) -/*[clinic end generated code: output=52eac33377197520 input=0e091a42ca6dbd91]*/ +/*[clinic end generated code: output=52eac33377197520 input=f129bc1a1aa151ee]*/ { return _common_reduce(self, proto); } @@ -2890,14 +2873,12 @@ bytearray_reduce_ex_impl(PyByteArrayObject *self, int proto) /*[clinic input] bytearray.__sizeof__ as bytearray_sizeof - self: self(type="PyByteArrayObject *") - Returns the size of the bytearray object in memory, in bytes. [clinic start generated code]*/ static PyObject * bytearray_sizeof_impl(PyByteArrayObject *self) -/*[clinic end generated code: output=738abdd17951c427 input=6b23d305362b462b]*/ +/*[clinic end generated code: output=738abdd17951c427 input=e27320fd98a4bc5a]*/ { Py_ssize_t res; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 296c73b..4cf1dee 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -9,9 +9,9 @@ #include /*[clinic input] -class bytes "PyBytesObject*" "&PyBytes_Type" +class bytes "PyBytesObject *" "&PyBytes_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=1a1d9102afc1b00c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7a238f965d64892b]*/ #include "clinic/bytesobject.c.h" @@ -1752,8 +1752,8 @@ Return a list of the sections in the bytes, using sep as the delimiter. [clinic start generated code]*/ static PyObject * -bytes_split_impl(PyBytesObject*self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=8bde44dacb36ef2e input=8b809b39074abbfa]*/ +bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) +/*[clinic end generated code: output=52126b5844c1d8ef input=8b809b39074abbfa]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -1777,7 +1777,6 @@ bytes_split_impl(PyBytesObject*self, PyObject *sep, Py_ssize_t maxsplit) /*[clinic input] bytes.partition - self: self(type="PyBytesObject *") sep: Py_buffer / @@ -1793,7 +1792,7 @@ object and two empty bytes objects. static PyObject * bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=f532b392a17ff695 input=bc855dc63ca949de]*/ +/*[clinic end generated code: output=f532b392a17ff695 input=61cca95519406099]*/ { return stringlib_partition( (PyObject*) self, @@ -1805,7 +1804,6 @@ bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) /*[clinic input] bytes.rpartition - self: self(type="PyBytesObject *") sep: Py_buffer / @@ -1821,7 +1819,7 @@ objects and the original bytes object. static PyObject * bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=191b114cbb028e50 input=6588fff262a9170e]*/ +/*[clinic end generated code: output=191b114cbb028e50 input=67f689e63a62d478]*/ { return stringlib_rpartition( (PyObject*) self, @@ -1839,8 +1837,8 @@ Splitting is done starting at the end of the bytes and working to the front. [clinic start generated code]*/ static PyObject * -bytes_rsplit_impl(PyBytesObject*self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=0b6570b977911d88 input=0f86c9f28f7d7b7b]*/ +bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) +/*[clinic end generated code: output=ba698d9ea01e1c8f input=0f86c9f28f7d7b7b]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -1878,8 +1876,8 @@ Example: b'.'.join([b'ab', b'pq', b'rs']) -> b'ab.pq.rs'. [clinic start generated code]*/ static PyObject * -bytes_join(PyBytesObject*self, PyObject *iterable_of_bytes) -/*[clinic end generated code: output=634aff14764ff997 input=7fe377b95bd549d2]*/ +bytes_join(PyBytesObject *self, PyObject *iterable_of_bytes) +/*[clinic end generated code: output=a046f379f626f6f8 input=7fe377b95bd549d2]*/ { return stringlib_bytes_join((PyObject*)self, iterable_of_bytes); } @@ -2129,7 +2127,6 @@ do_argstrip(PyBytesObject *self, int striptype, PyObject *bytes) /*[clinic input] bytes.strip - self: self(type="PyBytesObject *") bytes: object = None / @@ -2140,7 +2137,7 @@ If the argument is omitted or None, strip leading and trailing ASCII whitespace. static PyObject * bytes_strip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=c7c228d3bd104a1b input=37daa5fad1395d95]*/ +/*[clinic end generated code: output=c7c228d3bd104a1b input=8a354640e4e0b3ef]*/ { return do_argstrip(self, BOTHSTRIP, bytes); } @@ -2148,7 +2145,6 @@ bytes_strip_impl(PyBytesObject *self, PyObject *bytes) /*[clinic input] bytes.lstrip - self: self(type="PyBytesObject *") bytes: object = None / @@ -2159,7 +2155,7 @@ If the argument is omitted or None, strip leading ASCII whitespace. static PyObject * bytes_lstrip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=28602e586f524e82 input=88811b09dfbc2988]*/ +/*[clinic end generated code: output=28602e586f524e82 input=9baff4398c3f6857]*/ { return do_argstrip(self, LEFTSTRIP, bytes); } @@ -2167,7 +2163,6 @@ bytes_lstrip_impl(PyBytesObject *self, PyObject *bytes) /*[clinic input] bytes.rstrip - self: self(type="PyBytesObject *") bytes: object = None / @@ -2178,7 +2173,7 @@ If the argument is omitted or None, strip trailing ASCII whitespace. static PyObject * bytes_rstrip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=547e3815c95447da input=8f93c9cd361f0140]*/ +/*[clinic end generated code: output=547e3815c95447da input=b78af445c727e32b]*/ { return do_argstrip(self, RIGHTSTRIP, bytes); } @@ -2235,7 +2230,6 @@ bytes_count(PyBytesObject *self, PyObject *args) /*[clinic input] bytes.translate - self: self(type="PyBytesObject *") table: object Translation table, which must be a bytes object of length 256. [ @@ -2252,7 +2246,7 @@ The remaining characters are mapped through the given translation table. static PyObject * bytes_translate_impl(PyBytesObject *self, PyObject *table, int group_right_1, PyObject *deletechars) -/*[clinic end generated code: output=233df850eb50bf8d input=d8fa5519d7cc4be7]*/ +/*[clinic end generated code: output=233df850eb50bf8d input=ca20edf39d780d49]*/ { char *input, *output; Py_buffer table_view = {NULL, NULL}; @@ -2909,9 +2903,9 @@ replaced. [clinic start generated code]*/ static PyObject * -bytes_replace_impl(PyBytesObject*self, Py_buffer *old, Py_buffer *new, +bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count) -/*[clinic end generated code: output=403dc9d7a83c5a1d input=b2fbbf0bf04de8e5]*/ +/*[clinic end generated code: output=994fa588b6b9c104 input=b2fbbf0bf04de8e5]*/ { return (PyObject *)replace((PyBytesObject *) self, (const char *)old->buf, old->len, @@ -3078,9 +3072,9 @@ Decode the bytes using the codec registered for encoding. [clinic start generated code]*/ static PyObject * -bytes_decode_impl(PyBytesObject*self, const char *encoding, +bytes_decode_impl(PyBytesObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=2d2016ff8e0bb176 input=958174769d2a40ca]*/ +/*[clinic end generated code: output=5649a53dde27b314 input=958174769d2a40ca]*/ { return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); } @@ -3098,8 +3092,8 @@ true. [clinic start generated code]*/ static PyObject * -bytes_splitlines_impl(PyBytesObject*self, int keepends) -/*[clinic end generated code: output=995c3598f7833cad input=7f4aac67144f9944]*/ +bytes_splitlines_impl(PyBytesObject *self, int keepends) +/*[clinic end generated code: output=3484149a5d880ffb input=7f4aac67144f9944]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 5a1a5e9..95ce817 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -20,10 +20,10 @@ PyDoc_STRVAR(bytes_split__doc__, {"split", (PyCFunction)bytes_split, METH_VARARGS|METH_KEYWORDS, bytes_split__doc__}, static PyObject * -bytes_split_impl(PyBytesObject*self, PyObject *sep, Py_ssize_t maxsplit); +bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit); static PyObject * -bytes_split(PyBytesObject*self, PyObject *args, PyObject *kwargs) +bytes_split(PyBytesObject *self, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; static char *_keywords[] = {"sep", "maxsplit", NULL}; @@ -133,10 +133,10 @@ PyDoc_STRVAR(bytes_rsplit__doc__, {"rsplit", (PyCFunction)bytes_rsplit, METH_VARARGS|METH_KEYWORDS, bytes_rsplit__doc__}, static PyObject * -bytes_rsplit_impl(PyBytesObject*self, PyObject *sep, Py_ssize_t maxsplit); +bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit); static PyObject * -bytes_rsplit(PyBytesObject*self, PyObject *args, PyObject *kwargs) +bytes_rsplit(PyBytesObject *self, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; static char *_keywords[] = {"sep", "maxsplit", NULL}; @@ -359,11 +359,11 @@ PyDoc_STRVAR(bytes_replace__doc__, {"replace", (PyCFunction)bytes_replace, METH_VARARGS, bytes_replace__doc__}, static PyObject * -bytes_replace_impl(PyBytesObject*self, Py_buffer *old, Py_buffer *new, +bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count); static PyObject * -bytes_replace(PyBytesObject*self, PyObject *args) +bytes_replace(PyBytesObject *self, PyObject *args) { PyObject *return_value = NULL; Py_buffer old = {NULL, NULL}; @@ -405,11 +405,11 @@ PyDoc_STRVAR(bytes_decode__doc__, {"decode", (PyCFunction)bytes_decode, METH_VARARGS|METH_KEYWORDS, bytes_decode__doc__}, static PyObject * -bytes_decode_impl(PyBytesObject*self, const char *encoding, +bytes_decode_impl(PyBytesObject *self, const char *encoding, const char *errors); static PyObject * -bytes_decode(PyBytesObject*self, PyObject *args, PyObject *kwargs) +bytes_decode(PyBytesObject *self, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; static char *_keywords[] = {"encoding", "errors", NULL}; @@ -438,10 +438,10 @@ PyDoc_STRVAR(bytes_splitlines__doc__, {"splitlines", (PyCFunction)bytes_splitlines, METH_VARARGS|METH_KEYWORDS, bytes_splitlines__doc__}, static PyObject * -bytes_splitlines_impl(PyBytesObject*self, int keepends); +bytes_splitlines_impl(PyBytesObject *self, int keepends); static PyObject * -bytes_splitlines(PyBytesObject*self, PyObject *args, PyObject *kwargs) +bytes_splitlines(PyBytesObject *self, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; static char *_keywords[] = {"keepends", NULL}; @@ -484,4 +484,4 @@ bytes_fromhex(PyTypeObject *type, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=bd0ce8f25d7e18f4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d0e9f5a1c0682910 input=a9049054013a1b77]*/ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 0637a2d..6e0d736 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1078,7 +1078,6 @@ builtin_hasattr_impl(PyModuleDef *module, PyObject *obj, PyObject *name) /*[clinic input] id as builtin_id - self: self(type="PyModuleDef *") obj as v: object / @@ -1089,8 +1088,8 @@ This is guaranteed to be unique among simultaneously existing objects. [clinic start generated code]*/ static PyObject * -builtin_id(PyModuleDef *self, PyObject *v) -/*[clinic end generated code: output=0aa640785f697f65 input=5a534136419631f4]*/ +builtin_id(PyModuleDef *module, PyObject *v) +/*[clinic end generated code: output=63635e497e09c2f7 input=57fb4a9aaff96384]*/ { return PyLong_FromVoidPtr(v); } -- cgit v0.12 From c2f7d878972db908f5a1bad7af94c692b6d8c564 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 4 May 2016 09:44:44 +0300 Subject: Issue #26932: Fixed support of RTLD_* constants defined as enum values, not via macros (in particular on Android). Patch by Chi Hsuan Yen. --- Misc/NEWS | 3 ++ Modules/_ctypes/_ctypes.c | 4 +-- Modules/_ctypes/callproc.c | 2 +- Modules/posixmodule.c | 14 ++++---- Python/pystate.c | 4 +-- configure | 79 ++++++++++++++++++++++++++++++++++++++++++++++ configure.ac | 2 ++ pyconfig.h.in | 28 ++++++++++++++++ 8 files changed, 124 insertions(+), 12 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 46cd9ca..bae8c7b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1045,6 +1045,9 @@ Tests Build ----- +- Issue #26932: Fixed support of RTLD_* constants defined as enum values, + not via macros (in particular on Android). Patch by Chi Hsuan Yen. + - Issue #22359: Disable the rules for running _freeze_importlib and pgen when cross-compiling. The output of these programs is normally saved with the source code anyway, and is still regenerated when doing a native build. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6b1e744..33a7f90 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5480,14 +5480,14 @@ PyInit__ctypes(void) #endif /* If RTLD_LOCAL is not defined (Windows!), set it to zero. */ -#ifndef RTLD_LOCAL +#if !HAVE_DECL_RTLD_LOCAL #define RTLD_LOCAL 0 #endif /* If RTLD_GLOBAL is not defined (cygwin), set it to the same value as RTLD_LOCAL. */ -#ifndef RTLD_GLOBAL +#if !HAVE_DECL_RTLD_GLOBAL #define RTLD_GLOBAL RTLD_LOCAL #endif diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 870a0d4..9ab0723 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1307,7 +1307,7 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) PyObject *name, *name2; char *name_str; void * handle; -#ifdef RTLD_LOCAL +#if HAVE_DECL_RTLD_LOCAL int mode = RTLD_NOW | RTLD_LOCAL; #else /* cygwin doesn't define RTLD_LOCAL */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 03ad07d..23b2a3c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12895,25 +12895,25 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, XATTR_SIZE_MAX)) return -1; #endif -#ifdef RTLD_LAZY +#if HAVE_DECL_RTLD_LAZY if (PyModule_AddIntMacro(m, RTLD_LAZY)) return -1; #endif -#ifdef RTLD_NOW +#if HAVE_DECL_RTLD_NOW if (PyModule_AddIntMacro(m, RTLD_NOW)) return -1; #endif -#ifdef RTLD_GLOBAL +#if HAVE_DECL_RTLD_GLOBAL if (PyModule_AddIntMacro(m, RTLD_GLOBAL)) return -1; #endif -#ifdef RTLD_LOCAL +#if HAVE_DECL_RTLD_LOCAL if (PyModule_AddIntMacro(m, RTLD_LOCAL)) return -1; #endif -#ifdef RTLD_NODELETE +#if HAVE_DECL_RTLD_NODELETE if (PyModule_AddIntMacro(m, RTLD_NODELETE)) return -1; #endif -#ifdef RTLD_NOLOAD +#if HAVE_DECL_RTLD_NOLOAD if (PyModule_AddIntMacro(m, RTLD_NOLOAD)) return -1; #endif -#ifdef RTLD_DEEPBIND +#if HAVE_DECL_RTLD_DEEPBIND if (PyModule_AddIntMacro(m, RTLD_DEEPBIND)) return -1; #endif diff --git a/Python/pystate.c b/Python/pystate.c index 0503f32..ba4dd4c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -25,7 +25,7 @@ to avoid the expense of doing their own locking). #ifdef HAVE_DLFCN_H #include #endif -#ifndef RTLD_LAZY +#if !HAVE_DECL_RTLD_LAZY #define RTLD_LAZY 1 #endif #endif @@ -91,7 +91,7 @@ PyInterpreterState_New(void) interp->fscodec_initialized = 0; interp->importlib = NULL; #ifdef HAVE_DLOPEN -#ifdef RTLD_NOW +#if HAVE_DECL_RTLD_NOW interp->dlopenflags = RTLD_NOW; #else interp->dlopenflags = RTLD_LAZY; diff --git a/configure b/configure index a961f39..a009afe 100755 --- a/configure +++ b/configure @@ -14255,6 +14255,85 @@ $as_echo "#define HAVE_BROKEN_SEM_GETVALUE 1" >>confdefs.h fi +ac_fn_c_check_decl "$LINENO" "RTLD_LAZY" "ac_cv_have_decl_RTLD_LAZY" "#include +" +if test "x$ac_cv_have_decl_RTLD_LAZY" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_RTLD_LAZY $ac_have_decl +_ACEOF +ac_fn_c_check_decl "$LINENO" "RTLD_NOW" "ac_cv_have_decl_RTLD_NOW" "#include +" +if test "x$ac_cv_have_decl_RTLD_NOW" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_RTLD_NOW $ac_have_decl +_ACEOF +ac_fn_c_check_decl "$LINENO" "RTLD_GLOBAL" "ac_cv_have_decl_RTLD_GLOBAL" "#include +" +if test "x$ac_cv_have_decl_RTLD_GLOBAL" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_RTLD_GLOBAL $ac_have_decl +_ACEOF +ac_fn_c_check_decl "$LINENO" "RTLD_LOCAL" "ac_cv_have_decl_RTLD_LOCAL" "#include +" +if test "x$ac_cv_have_decl_RTLD_LOCAL" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_RTLD_LOCAL $ac_have_decl +_ACEOF +ac_fn_c_check_decl "$LINENO" "RTLD_NODELETE" "ac_cv_have_decl_RTLD_NODELETE" "#include +" +if test "x$ac_cv_have_decl_RTLD_NODELETE" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_RTLD_NODELETE $ac_have_decl +_ACEOF +ac_fn_c_check_decl "$LINENO" "RTLD_NOLOAD" "ac_cv_have_decl_RTLD_NOLOAD" "#include +" +if test "x$ac_cv_have_decl_RTLD_NOLOAD" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_RTLD_NOLOAD $ac_have_decl +_ACEOF +ac_fn_c_check_decl "$LINENO" "RTLD_DEEPBIND" "ac_cv_have_decl_RTLD_DEEPBIND" "#include +" +if test "x$ac_cv_have_decl_RTLD_DEEPBIND" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_RTLD_DEEPBIND $ac_have_decl +_ACEOF + + # determine what size digit to use for Python's longs { $as_echo "$as_me:${as_lineno-$LINENO}: checking digit size for Python's longs" >&5 $as_echo_n "checking digit size for Python's longs... " >&6; } diff --git a/configure.ac b/configure.ac index bfd2b04..f947af9 100644 --- a/configure.ac +++ b/configure.ac @@ -4342,6 +4342,8 @@ then [define to 1 if your sem_getvalue is broken.]) fi +AC_CHECK_DECLS([RTLD_LAZY, RTLD_NOW, RTLD_GLOBAL, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_DEEPBIND], [], [], [[#include ]]) + # determine what size digit to use for Python's longs AC_MSG_CHECKING([digit size for Python's longs]) AC_ARG_ENABLE(big-digits, diff --git a/pyconfig.h.in b/pyconfig.h.in index b51bd56..0451855 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -167,6 +167,34 @@ */ #undef HAVE_DECL_ISNAN +/* Define to 1 if you have the declaration of `RTLD_DEEPBIND', and to 0 if you + don't. */ +#undef HAVE_DECL_RTLD_DEEPBIND + +/* Define to 1 if you have the declaration of `RTLD_GLOBAL', and to 0 if you + don't. */ +#undef HAVE_DECL_RTLD_GLOBAL + +/* Define to 1 if you have the declaration of `RTLD_LAZY', and to 0 if you + don't. */ +#undef HAVE_DECL_RTLD_LAZY + +/* Define to 1 if you have the declaration of `RTLD_LOCAL', and to 0 if you + don't. */ +#undef HAVE_DECL_RTLD_LOCAL + +/* Define to 1 if you have the declaration of `RTLD_NODELETE', and to 0 if you + don't. */ +#undef HAVE_DECL_RTLD_NODELETE + +/* Define to 1 if you have the declaration of `RTLD_NOLOAD', and to 0 if you + don't. */ +#undef HAVE_DECL_RTLD_NOLOAD + +/* Define to 1 if you have the declaration of `RTLD_NOW', and to 0 if you + don't. */ +#undef HAVE_DECL_RTLD_NOW + /* Define to 1 if you have the declaration of `tzname', and to 0 if you don't. */ #undef HAVE_DECL_TZNAME -- cgit v0.12 From 3b06e243527da5f4ca4dc2e3126dc9f5bbdc243c Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Thu, 5 May 2016 03:54:29 +1000 Subject: Issue 26002 and 25974 patches by Upendra Kumar and Stefan Krah speed up median by using bisect, and general speedup for Decimals using as_integer_ratio --- Lib/statistics.py | 68 ++++++++++++++++++++------------------------- Lib/test/test_statistics.py | 31 ++++++++++----------- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 518f546..af5d41e 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -105,6 +105,7 @@ import math from fractions import Fraction from decimal import Decimal from itertools import groupby +from bisect import bisect_left, bisect_right @@ -223,56 +224,26 @@ def _exact_ratio(x): # Optimise the common case of floats. We expect that the most often # used numeric type will be builtin floats, so try to make this as # fast as possible. - if type(x) is float: + if type(x) is float or type(x) is Decimal: return x.as_integer_ratio() try: # x may be an int, Fraction, or Integral ABC. return (x.numerator, x.denominator) except AttributeError: try: - # x may be a float subclass. + # x may be a float or Decimal subclass. return x.as_integer_ratio() except AttributeError: - try: - # x may be a Decimal. - return _decimal_to_ratio(x) - except AttributeError: - # Just give up? - pass + # Just give up? + pass except (OverflowError, ValueError): # float NAN or INF. - assert not math.isfinite(x) + assert not _isfinite(x) return (x, None) msg = "can't convert type '{}' to numerator/denominator" raise TypeError(msg.format(type(x).__name__)) -# FIXME This is faster than Fraction.from_decimal, but still too slow. -def _decimal_to_ratio(d): - """Convert Decimal d to exact integer ratio (numerator, denominator). - - >>> from decimal import Decimal - >>> _decimal_to_ratio(Decimal("2.6")) - (26, 10) - - """ - sign, digits, exp = d.as_tuple() - if exp in ('F', 'n', 'N'): # INF, NAN, sNAN - assert not d.is_finite() - return (d, None) - num = 0 - for digit in digits: - num = num*10 + digit - if exp < 0: - den = 10**-exp - else: - num *= 10**exp - den = 1 - if sign: - num = -num - return (num, den) - - def _convert(value, T): """Convert value to given numeric type T.""" if type(value) is T: @@ -305,6 +276,21 @@ def _counts(data): return table +def _find_lteq(a, x): + 'Locate the leftmost value exactly equal to x' + i = bisect_left(a, x) + if i != len(a) and a[i] == x: + return i + raise ValueError + + +def _find_rteq(a, l, x): + 'Locate the rightmost value exactly equal to x' + i = bisect_right(a, x, lo=l) + if i != (len(a)+1) and a[i-1] == x: + return i-1 + raise ValueError + # === Measures of central tendency (averages) === def mean(data): @@ -442,9 +428,15 @@ def median_grouped(data, interval=1): except TypeError: # Mixed type. For now we just coerce to float. L = float(x) - float(interval)/2 - cf = data.index(x) # Number of values below the median interval. - # FIXME The following line could be more efficient for big lists. - f = data.count(x) # Number of data points in the median interval. + + # Uses bisection search to search for x in data with log(n) time complexity + # Find the position of leftmost occurence of x in data + l1 = _find_lteq(data, x) + # Find the position of rightmost occurence of x in data[l1...len(data)] + # Assuming always l1 <= l2 + l2 = _find_rteq(data, l1, x) + cf = l1 + f = l2 - l1 + 1 return L + interval*(n/2 - cf)/f diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 9a54fe1..5275cb6 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -699,13 +699,12 @@ class ExactRatioTest(unittest.TestCase): num, den = statistics._exact_ratio(x) self.assertEqual(x, num/den) - @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal(self): D = Decimal _exact_ratio = statistics._exact_ratio - self.assertEqual(_exact_ratio(D("0.125")), (125, 1000)) - self.assertEqual(_exact_ratio(D("12.345")), (12345, 1000)) - self.assertEqual(_exact_ratio(D("-1.98")), (-198, 100)) + self.assertEqual(_exact_ratio(D("0.125")), (1, 8)) + self.assertEqual(_exact_ratio(D("12.345")), (2469, 200)) + self.assertEqual(_exact_ratio(D("-1.98")), (-99, 50)) def test_inf(self): INF = float("INF") @@ -731,7 +730,6 @@ class ExactRatioTest(unittest.TestCase): self.assertIs(ratio[1], None) self.assertEqual(type(ratio[0]), type(nan)) - @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_nan(self): NAN = Decimal("NAN") sNAN = Decimal("sNAN") @@ -745,18 +743,18 @@ class ExactRatioTest(unittest.TestCase): class DecimalToRatioTest(unittest.TestCase): - # Test _decimal_to_ratio private function. + # Test _exact_ratio private function. def test_infinity(self): # Test that INFs are handled correctly. inf = Decimal('INF') - self.assertEqual(statistics._decimal_to_ratio(inf), (inf, None)) - self.assertEqual(statistics._decimal_to_ratio(-inf), (-inf, None)) + self.assertEqual(statistics._exact_ratio(inf), (inf, None)) + self.assertEqual(statistics._exact_ratio(-inf), (-inf, None)) def test_nan(self): # Test that NANs are handled correctly. for nan in (Decimal('NAN'), Decimal('sNAN')): - num, den = statistics._decimal_to_ratio(nan) + num, den = statistics._exact_ratio(nan) # Because NANs always compare non-equal, we cannot use assertEqual. # Nor can we use an identity test, as we don't guarantee anything # about the object identity. @@ -769,30 +767,30 @@ class DecimalToRatioTest(unittest.TestCase): for d in numbers: # First test positive decimals. assert d > 0 - num, den = statistics._decimal_to_ratio(d) + num, den = statistics._exact_ratio(d) self.assertGreaterEqual(num, 0) self.assertGreater(den, 0) # Then test negative decimals. - num, den = statistics._decimal_to_ratio(-d) + num, den = statistics._exact_ratio(-d) self.assertLessEqual(num, 0) self.assertGreater(den, 0) def test_negative_exponent(self): # Test result when the exponent is negative. - t = statistics._decimal_to_ratio(Decimal("0.1234")) - self.assertEqual(t, (1234, 10000)) + t = statistics._exact_ratio(Decimal("0.1234")) + self.assertEqual(t, (617, 5000)) def test_positive_exponent(self): # Test results when the exponent is positive. - t = statistics._decimal_to_ratio(Decimal("1.234e7")) + t = statistics._exact_ratio(Decimal("1.234e7")) self.assertEqual(t, (12340000, 1)) def test_regression_20536(self): # Regression test for issue 20536. # See http://bugs.python.org/issue20536 - t = statistics._decimal_to_ratio(Decimal("1e2")) + t = statistics._exact_ratio(Decimal("1e2")) self.assertEqual(t, (100, 1)) - t = statistics._decimal_to_ratio(Decimal("1.47e5")) + t = statistics._exact_ratio(Decimal("1.47e5")) self.assertEqual(t, (147000, 1)) @@ -1260,7 +1258,6 @@ class SumSpecialValues(NumericTestCase): with decimal.localcontext(decimal.BasicContext): self.assertRaises(decimal.InvalidOperation, statistics._sum, data) - @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_snan_raises(self): # Adding sNAN should raise InvalidOperation. sNAN = Decimal('sNAN') -- cgit v0.12 From dd40fc3e57da1988bffa35e3aad33fa0a6dda813 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 4 May 2016 22:23:26 +0300 Subject: Issue #26765: Moved common code and docstrings for bytes and bytearray methods to bytes_methods.c. --- Include/bytes_methods.h | 21 ++ Objects/bytearrayobject.c | 352 +++----------------------------- Objects/bytes_methods.c | 424 +++++++++++++++++++++++++++++++++++++++ Objects/bytesobject.c | 362 +++------------------------------ Objects/stringlib/find.h | 73 ------- Objects/stringlib/transmogrify.h | 30 --- Objects/unicodeobject.c | 34 +++- 7 files changed, 519 insertions(+), 777 deletions(-) diff --git a/Include/bytes_methods.h b/Include/bytes_methods.h index dbc033c..7fa7540 100644 --- a/Include/bytes_methods.h +++ b/Include/bytes_methods.h @@ -21,6 +21,15 @@ extern void _Py_bytes_title(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len); +extern PyObject *_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args); +extern int _Py_bytes_contains(const char *str, Py_ssize_t len, PyObject *arg); +extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args); + /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); @@ -37,7 +46,19 @@ extern const char _Py_upper__doc__[]; extern const char _Py_title__doc__[]; extern const char _Py_capitalize__doc__[]; extern const char _Py_swapcase__doc__[]; +extern const char _Py_count__doc__[]; +extern const char _Py_find__doc__[]; +extern const char _Py_index__doc__[]; +extern const char _Py_rfind__doc__[]; +extern const char _Py_rindex__doc__[]; +extern const char _Py_startswith__doc__[]; +extern const char _Py_endswith__doc__[]; extern const char _Py_maketrans__doc__[]; +extern const char _Py_expandtabs__doc__[]; +extern const char _Py_ljust__doc__[]; +extern const char _Py_rjust__doc__[]; +extern const char _Py_center__doc__[]; +extern const char _Py_zfill__doc__[]; /* this is needed because some docs are shared from the .o, not static */ #define PyDoc_STRVAR_shared(name,str) const char name[] = PyDoc_STR(str) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index fb3668e..093cdbf 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1097,147 +1097,16 @@ bytearray_dealloc(PyByteArrayObject *self) #include "stringlib/transmogrify.h" -/* The following Py_LOCAL_INLINE and Py_LOCAL functions -were copied from the old char* style string object. */ - -/* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } - -Py_LOCAL_INLINE(Py_ssize_t) -bytearray_find_internal(PyByteArrayObject *self, PyObject *args, int dir) -{ - PyObject *subobj; - char byte; - Py_buffer subbuf; - const char *sub; - Py_ssize_t len, sub_len; - Py_ssize_t start=0, end=PY_SSIZE_T_MAX; - Py_ssize_t res; - - if (!stringlib_parse_args_finds_byte("find/rfind/index/rindex", - args, &subobj, &byte, &start, &end)) - return -2; - - if (subobj) { - if (PyObject_GetBuffer(subobj, &subbuf, PyBUF_SIMPLE) != 0) - return -2; - - sub = subbuf.buf; - sub_len = subbuf.len; - } - else { - sub = &byte; - sub_len = 1; - } - len = PyByteArray_GET_SIZE(self); - - ADJUST_INDICES(start, end, len); - if (end - start < sub_len) - res = -1; - else if (sub_len == 1) { - if (dir > 0) - res = stringlib_find_char( - PyByteArray_AS_STRING(self) + start, end - start, - *sub); - else - res = stringlib_rfind_char( - PyByteArray_AS_STRING(self) + start, end - start, - *sub); - if (res >= 0) - res += start; - } - else { - if (dir > 0) - res = stringlib_find_slice( - PyByteArray_AS_STRING(self), len, - sub, sub_len, start, end); - else - res = stringlib_rfind_slice( - PyByteArray_AS_STRING(self), len, - sub, sub_len, start, end); - } - - if (subobj) - PyBuffer_Release(&subbuf); - - return res; -} - -PyDoc_STRVAR(find__doc__, -"B.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - static PyObject * bytearray_find(PyByteArrayObject *self, PyObject *args) { - Py_ssize_t result = bytearray_find_internal(self, args, +1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); + return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); } -PyDoc_STRVAR(count__doc__, -"B.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of subsection sub in\n\ -bytes B[start:end]. Optional arguments start and end are interpreted\n\ -as in slice notation."); - static PyObject * bytearray_count(PyByteArrayObject *self, PyObject *args) { - PyObject *sub_obj; - const char *str = PyByteArray_AS_STRING(self), *sub; - Py_ssize_t sub_len; - char byte; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; - - Py_buffer vsub; - PyObject *count_obj; - - if (!stringlib_parse_args_finds_byte("count", args, &sub_obj, &byte, - &start, &end)) - return NULL; - - if (sub_obj) { - if (PyObject_GetBuffer(sub_obj, &vsub, PyBUF_SIMPLE) != 0) - return NULL; - - sub = vsub.buf; - sub_len = vsub.len; - } - else { - sub = &byte; - sub_len = 1; - } - - ADJUST_INDICES(start, end, PyByteArray_GET_SIZE(self)); - - count_obj = PyLong_FromSsize_t( - stringlib_count(str + start, end - start, sub, sub_len, PY_SSIZE_T_MAX) - ); - - if (sub_obj) - PyBuffer_Release(&vsub); - - return count_obj; + return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); } /*[clinic input] @@ -1269,216 +1138,40 @@ bytearray_copy_impl(PyByteArrayObject *self) PyByteArray_GET_SIZE(self)); } -PyDoc_STRVAR(index__doc__, -"B.index(sub[, start[, end]]) -> int\n\ -\n\ -Like B.find() but raise ValueError when the subsection is not found."); - static PyObject * bytearray_index(PyByteArrayObject *self, PyObject *args) { - Py_ssize_t result = bytearray_find_internal(self, args, +1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "subsection not found"); - return NULL; - } - return PyLong_FromSsize_t(result); + return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); } - -PyDoc_STRVAR(rfind__doc__, -"B.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - static PyObject * bytearray_rfind(PyByteArrayObject *self, PyObject *args) { - Py_ssize_t result = bytearray_find_internal(self, args, -1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); + return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); } - -PyDoc_STRVAR(rindex__doc__, -"B.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Like B.rfind() but raise ValueError when the subsection is not found."); - static PyObject * bytearray_rindex(PyByteArrayObject *self, PyObject *args) { - Py_ssize_t result = bytearray_find_internal(self, args, -1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "subsection not found"); - return NULL; - } - return PyLong_FromSsize_t(result); + return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); } - static int bytearray_contains(PyObject *self, PyObject *arg) { - Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) { - Py_buffer varg; - Py_ssize_t pos; - PyErr_Clear(); - if (PyObject_GetBuffer(arg, &varg, PyBUF_SIMPLE) != 0) - return -1; - pos = stringlib_find(PyByteArray_AS_STRING(self), Py_SIZE(self), - varg.buf, varg.len, 0); - PyBuffer_Release(&varg); - return pos >= 0; - } - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return -1; - } - - return memchr(PyByteArray_AS_STRING(self), (int) ival, Py_SIZE(self)) != NULL; + return _Py_bytes_contains(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), arg); } - -/* Matches the end (direction >= 0) or start (direction < 0) of self - * against substr, using the start and end arguments. Returns - * -1 on error, 0 if not found and 1 if found. - */ -Py_LOCAL(int) -_bytearray_tailmatch(PyByteArrayObject *self, PyObject *substr, Py_ssize_t start, - Py_ssize_t end, int direction) -{ - Py_ssize_t len = PyByteArray_GET_SIZE(self); - const char* str; - Py_buffer vsubstr; - int rv = 0; - - str = PyByteArray_AS_STRING(self); - - if (PyObject_GetBuffer(substr, &vsubstr, PyBUF_SIMPLE) != 0) - return -1; - - ADJUST_INDICES(start, end, len); - - if (direction < 0) { - /* startswith */ - if (start+vsubstr.len > len) { - goto done; - } - } else { - /* endswith */ - if (end-start < vsubstr.len || start > len) { - goto done; - } - - if (end-vsubstr.len > start) - start = end - vsubstr.len; - } - if (end-start >= vsubstr.len) - rv = ! memcmp(str+start, vsubstr.buf, vsubstr.len); - -done: - PyBuffer_Release(&vsubstr); - return rv; -} - - -PyDoc_STRVAR(startswith__doc__, -"B.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of bytes to try."); - static PyObject * bytearray_startswith(PyByteArrayObject *self, PyObject *args) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!stringlib_parse_args_finds("startswith", args, &subobj, &start, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _bytearray_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, -1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _bytearray_tailmatch(self, subobj, start, end, -1); - if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "startswith first arg must be bytes " - "or a tuple of bytes, not %s", Py_TYPE(subobj)->tp_name); - return NULL; - } - else - return PyBool_FromLong(result); + return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); } -PyDoc_STRVAR(endswith__doc__, -"B.endswith(suffix[, start[, end]]) -> bool\n\ -\n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of bytes to try."); - static PyObject * bytearray_endswith(PyByteArrayObject *self, PyObject *args) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!stringlib_parse_args_finds("endswith", args, &subobj, &start, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _bytearray_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, +1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _bytearray_tailmatch(self, subobj, start, end, +1); - if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "endswith first arg must be bytes or " - "a tuple of bytes, not %s", Py_TYPE(subobj)->tp_name); - return NULL; - } - else - return PyBool_FromLong(result); + return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); } @@ -1544,7 +1237,7 @@ bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, result = PyByteArray_FromStringAndSize((char *)NULL, inlen); if (result == NULL) goto done; - output_start = output = PyByteArray_AsString(result); + output_start = output = PyByteArray_AS_STRING(result); input = PyByteArray_AS_STRING(input_obj); if (vdel.len == 0 && table_chars != NULL) { @@ -2919,19 +2612,22 @@ bytearray_methods[] = { BYTEARRAY_APPEND_METHODDEF {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, _Py_capitalize__doc__}, - {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, + {"center", (PyCFunction)stringlib_center, METH_VARARGS, _Py_center__doc__}, BYTEARRAY_CLEAR_METHODDEF BYTEARRAY_COPY_METHODDEF - {"count", (PyCFunction)bytearray_count, METH_VARARGS, count__doc__}, + {"count", (PyCFunction)bytearray_count, METH_VARARGS, + _Py_count__doc__}, BYTEARRAY_DECODE_METHODDEF - {"endswith", (PyCFunction)bytearray_endswith, METH_VARARGS, endswith__doc__}, + {"endswith", (PyCFunction)bytearray_endswith, METH_VARARGS, + _Py_endswith__doc__}, {"expandtabs", (PyCFunction)stringlib_expandtabs, METH_VARARGS | METH_KEYWORDS, - expandtabs__doc__}, + _Py_expandtabs__doc__}, BYTEARRAY_EXTEND_METHODDEF - {"find", (PyCFunction)bytearray_find, METH_VARARGS, find__doc__}, + {"find", (PyCFunction)bytearray_find, METH_VARARGS, + _Py_find__doc__}, BYTEARRAY_FROMHEX_METHODDEF {"hex", (PyCFunction)bytearray_hex, METH_NOARGS, hex__doc__}, - {"index", (PyCFunction)bytearray_index, METH_VARARGS, index__doc__}, + {"index", (PyCFunction)bytearray_index, METH_VARARGS, _Py_index__doc__}, BYTEARRAY_INSERT_METHODDEF {"isalnum", (PyCFunction)stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, @@ -2948,7 +2644,7 @@ bytearray_methods[] = { {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS, _Py_isupper__doc__}, BYTEARRAY_JOIN_METHODDEF - {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, ljust__doc__}, + {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, _Py_ljust__doc__}, {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__}, BYTEARRAY_LSTRIP_METHODDEF BYTEARRAY_MAKETRANS_METHODDEF @@ -2957,23 +2653,23 @@ bytearray_methods[] = { BYTEARRAY_REMOVE_METHODDEF BYTEARRAY_REPLACE_METHODDEF BYTEARRAY_REVERSE_METHODDEF - {"rfind", (PyCFunction)bytearray_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction)bytearray_rindex, METH_VARARGS, rindex__doc__}, - {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, + {"rfind", (PyCFunction)bytearray_rfind, METH_VARARGS, _Py_rfind__doc__}, + {"rindex", (PyCFunction)bytearray_rindex, METH_VARARGS, _Py_rindex__doc__}, + {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, _Py_rjust__doc__}, BYTEARRAY_RPARTITION_METHODDEF BYTEARRAY_RSPLIT_METHODDEF BYTEARRAY_RSTRIP_METHODDEF BYTEARRAY_SPLIT_METHODDEF BYTEARRAY_SPLITLINES_METHODDEF {"startswith", (PyCFunction)bytearray_startswith, METH_VARARGS , - startswith__doc__}, + _Py_startswith__doc__}, BYTEARRAY_STRIP_METHODDEF {"swapcase", (PyCFunction)stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__}, BYTEARRAY_TRANSLATE_METHODDEF {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__}, - {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, zfill__doc__}, + {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, _Py_zfill__doc__}, {NULL} }; diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index d025351..fe666c6 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -387,3 +387,427 @@ _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to) return res; } + +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F +#define STRINGLIB_CHAR char +#define STRINGLIB_SIZEOF_CHAR 1 + +#include "stringlib/fastsearch.h" +#include "stringlib/count.h" +#include "stringlib/find.h" + +/* +Wraps stringlib_parse_args_finds() and additionally checks whether the +first argument is an integer in range(0, 256). + +If this is the case, writes the integer value to the byte parameter +and sets subobj to NULL. Otherwise, sets the first argument to subobj +and doesn't touch byte. The other parameters are similar to those of +stringlib_parse_args_finds(). +*/ + +Py_LOCAL_INLINE(int) +parse_args_finds_byte(const char *function_name, PyObject *args, + PyObject **subobj, char *byte, + Py_ssize_t *start, Py_ssize_t *end) +{ + PyObject *tmp_subobj; + Py_ssize_t ival; + PyObject *err; + + if(!stringlib_parse_args_finds(function_name, args, &tmp_subobj, + start, end)) + return 0; + + if (!PyNumber_Check(tmp_subobj)) { + *subobj = tmp_subobj; + return 1; + } + + ival = PyNumber_AsSsize_t(tmp_subobj, PyExc_OverflowError); + if (ival == -1) { + err = PyErr_Occurred(); + if (err && !PyErr_GivenExceptionMatches(err, PyExc_OverflowError)) { + PyErr_Clear(); + *subobj = tmp_subobj; + return 1; + } + } + + if (ival < 0 || ival > 255) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return 0; + } + + *subobj = NULL; + *byte = (char)ival; + return 1; +} + +/* helper macro to fixup start/end slice values */ +#define ADJUST_INDICES(start, end, len) \ + if (end > len) \ + end = len; \ + else if (end < 0) { \ + end += len; \ + if (end < 0) \ + end = 0; \ + } \ + if (start < 0) { \ + start += len; \ + if (start < 0) \ + start = 0; \ + } + +Py_LOCAL_INLINE(Py_ssize_t) +find_internal(const char *str, Py_ssize_t len, + const char *function_name, PyObject *args, int dir) +{ + PyObject *subobj; + char byte; + Py_buffer subbuf; + const char *sub; + Py_ssize_t sub_len; + Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; + Py_ssize_t res; + + if (!parse_args_finds_byte(function_name, args, + &subobj, &byte, &start, &end)) + return -2; + + if (subobj) { + if (PyObject_GetBuffer(subobj, &subbuf, PyBUF_SIMPLE) != 0) + return -2; + + sub = subbuf.buf; + sub_len = subbuf.len; + } + else { + sub = &byte; + sub_len = 1; + } + + ADJUST_INDICES(start, end, len); + if (end - start < sub_len) + res = -1; + else if (sub_len == 1) { + if (dir > 0) + res = stringlib_find_char( + str + start, end - start, + *sub); + else + res = stringlib_rfind_char( + str + start, end - start, + *sub); + if (res >= 0) + res += start; + } + else { + if (dir > 0) + res = stringlib_find_slice( + str, len, + sub, sub_len, start, end); + else + res = stringlib_rfind_slice( + str, len, + sub, sub_len, start, end); + } + + if (subobj) + PyBuffer_Release(&subbuf); + + return res; +} + +PyDoc_STRVAR_shared(_Py_find__doc__, +"B.find(sub[, start[, end]]) -> int\n\ +\n\ +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Return -1 on failure."); + +PyObject * +_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) +{ + Py_ssize_t result = find_internal(str, len, "find", args, +1); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); +} + +PyDoc_STRVAR_shared(_Py_index__doc__, +"B.index(sub[, start[, end]]) -> int\n\ +\n\ +Like B.find() but raise ValueError when the subsection is not found."); + +PyObject * +_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) +{ + Py_ssize_t result = find_internal(str, len, "index", args, +1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "subsection not found"); + return NULL; + } + return PyLong_FromSsize_t(result); +} + +PyDoc_STRVAR_shared(_Py_rfind__doc__, +"B.rfind(sub[, start[, end]]) -> int\n\ +\n\ +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Return -1 on failure."); + +PyObject * +_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) +{ + Py_ssize_t result = find_internal(str, len, "rfind", args, -1); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); +} + +PyDoc_STRVAR_shared(_Py_rindex__doc__, +"B.rindex(sub[, start[, end]]) -> int\n\ +\n\ +Like B.rfind() but raise ValueError when the subsection is not found."); + +PyObject * +_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) +{ + Py_ssize_t result = find_internal(str, len, "rindex", args, -1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "subsection not found"); + return NULL; + } + return PyLong_FromSsize_t(result); +} + +PyDoc_STRVAR_shared(_Py_count__doc__, +"B.count(sub[, start[, end]]) -> int\n\ +\n\ +Return the number of non-overlapping occurrences of subsection sub in\n\ +bytes B[start:end]. Optional arguments start and end are interpreted\n\ +as in slice notation."); + +PyObject * +_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args) +{ + PyObject *sub_obj; + const char *sub; + Py_ssize_t sub_len; + char byte; + Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; + + Py_buffer vsub; + PyObject *count_obj; + + if (!parse_args_finds_byte("count", args, + &sub_obj, &byte, &start, &end)) + return NULL; + + if (sub_obj) { + if (PyObject_GetBuffer(sub_obj, &vsub, PyBUF_SIMPLE) != 0) + return NULL; + + sub = vsub.buf; + sub_len = vsub.len; + } + else { + sub = &byte; + sub_len = 1; + } + + ADJUST_INDICES(start, end, len); + + count_obj = PyLong_FromSsize_t( + stringlib_count(str + start, end - start, sub, sub_len, PY_SSIZE_T_MAX) + ); + + if (sub_obj) + PyBuffer_Release(&vsub); + + return count_obj; +} + +int +_Py_bytes_contains(const char *str, Py_ssize_t len, PyObject *arg) +{ + Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) { + Py_buffer varg; + Py_ssize_t pos; + PyErr_Clear(); + if (PyObject_GetBuffer(arg, &varg, PyBUF_SIMPLE) != 0) + return -1; + pos = stringlib_find(str, len, + varg.buf, varg.len, 0); + PyBuffer_Release(&varg); + return pos >= 0; + } + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return -1; + } + + return memchr(str, (int) ival, len) != NULL; +} + + +/* Matches the end (direction >= 0) or start (direction < 0) of the buffer + * against substr, using the start and end arguments. Returns + * -1 on error, 0 if not found and 1 if found. + */ +Py_LOCAL(int) +tailmatch(const char *str, Py_ssize_t len, PyObject *substr, + Py_ssize_t start, Py_ssize_t end, int direction) +{ + Py_buffer sub_view = {NULL, NULL}; + const char *sub; + Py_ssize_t slen; + + if (PyBytes_Check(substr)) { + sub = PyBytes_AS_STRING(substr); + slen = PyBytes_GET_SIZE(substr); + } + else { + if (PyObject_GetBuffer(substr, &sub_view, PyBUF_SIMPLE) != 0) + return -1; + sub = sub_view.buf; + slen = sub_view.len; + } + + ADJUST_INDICES(start, end, len); + + if (direction < 0) { + /* startswith */ + if (start + slen > len) + goto notfound; + } else { + /* endswith */ + if (end - start < slen || start > len) + goto notfound; + + if (end - slen > start) + start = end - slen; + } + if (end - start < slen) + goto notfound; + if (memcmp(str + start, sub, slen) != 0) + goto notfound; + + PyBuffer_Release(&sub_view); + return 1; + +notfound: + PyBuffer_Release(&sub_view); + return 0; +} + +Py_LOCAL(PyObject *) +_Py_bytes_tailmatch(const char *str, Py_ssize_t len, + const char *function_name, PyObject *args, + int direction) +{ + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + PyObject *subobj; + int result; + + if (!stringlib_parse_args_finds(function_name, args, &subobj, &start, &end)) + return NULL; + if (PyTuple_Check(subobj)) { + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { + result = tailmatch(str, len, PyTuple_GET_ITEM(subobj, i), + start, end, direction); + if (result == -1) + return NULL; + else if (result) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + result = tailmatch(str, len, subobj, start, end, direction); + if (result == -1) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) + PyErr_Format(PyExc_TypeError, + "%s first arg must be bytes or a tuple of bytes, " + "not %s", + function_name, Py_TYPE(subobj)->tp_name); + return NULL; + } + else + return PyBool_FromLong(result); +} + +PyDoc_STRVAR_shared(_Py_startswith__doc__, +"B.startswith(prefix[, start[, end]]) -> bool\n\ +\n\ +Return True if B starts with the specified prefix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +prefix can also be a tuple of bytes to try."); + +PyObject * +_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args) +{ + return _Py_bytes_tailmatch(str, len, "startswith", args, -1); +} + +PyDoc_STRVAR_shared(_Py_endswith__doc__, +"B.endswith(suffix[, start[, end]]) -> bool\n\ +\n\ +Return True if B ends with the specified suffix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +suffix can also be a tuple of bytes to try."); + +PyObject * +_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args) +{ + return _Py_bytes_tailmatch(str, len, "endswith", args, +1); +} + +PyDoc_STRVAR_shared(_Py_expandtabs__doc__, +"B.expandtabs(tabsize=8) -> copy of B\n\ +\n\ +Return a copy of B where all tab characters are expanded using spaces.\n\ +If tabsize is not given, a tab size of 8 characters is assumed."); + +PyDoc_STRVAR_shared(_Py_ljust__doc__, +"B.ljust(width[, fillchar]) -> copy of B\n" +"\n" +"Return B left justified in a string of length width. Padding is\n" +"done using the specified fill character (default is a space)."); + +PyDoc_STRVAR_shared(_Py_rjust__doc__, +"B.rjust(width[, fillchar]) -> copy of B\n" +"\n" +"Return B right justified in a string of length width. Padding is\n" +"done using the specified fill character (default is a space)"); + +PyDoc_STRVAR_shared(_Py_center__doc__, +"B.center(width[, fillchar]) -> copy of B\n" +"\n" +"Return B centered in a string of length width. Padding is\n" +"done using the specified fill character (default is a space)."); + +PyDoc_STRVAR_shared(_Py_zfill__doc__, +"B.zfill(width) -> copy of B\n" +"\n" +"Pad a numeric string B with zeros on the left, to fill a field\n" +"of the specified width. B is never truncated."); + diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4cf1dee..2bf39e3 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -486,11 +486,11 @@ formatlong(PyObject *v, int flags, int prec, int type) static int byte_converter(PyObject *arg, char *p) { - if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) { + if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { *p = PyBytes_AS_STRING(arg)[0]; return 1; } - else if (PyByteArray_Check(arg) && PyByteArray_Size(arg) == 1) { + else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { *p = PyByteArray_AS_STRING(arg)[0]; return 1; } @@ -1488,24 +1488,7 @@ bytes_repeat(PyBytesObject *a, Py_ssize_t n) static int bytes_contains(PyObject *self, PyObject *arg) { - Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) { - Py_buffer varg; - Py_ssize_t pos; - PyErr_Clear(); - if (PyObject_GetBuffer(arg, &varg, PyBUF_SIMPLE) != 0) - return -1; - pos = stringlib_find(PyBytes_AS_STRING(self), Py_SIZE(self), - varg.buf, varg.len, 0); - PyBuffer_Release(&varg); - return pos >= 0; - } - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return -1; - } - - return memchr(PyBytes_AS_STRING(self), (int) ival, Py_SIZE(self)) != NULL; + return _Py_bytes_contains(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), arg); } static PyObject * @@ -1890,157 +1873,30 @@ _PyBytes_Join(PyObject *sep, PyObject *x) return bytes_join((PyBytesObject*)sep, x); } -/* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } - -Py_LOCAL_INLINE(Py_ssize_t) -bytes_find_internal(PyBytesObject *self, PyObject *args, int dir) -{ - PyObject *subobj; - char byte; - Py_buffer subbuf; - const char *sub; - Py_ssize_t len, sub_len; - Py_ssize_t start=0, end=PY_SSIZE_T_MAX; - Py_ssize_t res; - - if (!stringlib_parse_args_finds_byte("find/rfind/index/rindex", - args, &subobj, &byte, &start, &end)) - return -2; - - if (subobj) { - if (PyObject_GetBuffer(subobj, &subbuf, PyBUF_SIMPLE) != 0) - return -2; - - sub = subbuf.buf; - sub_len = subbuf.len; - } - else { - sub = &byte; - sub_len = 1; - } - len = PyBytes_GET_SIZE(self); - - ADJUST_INDICES(start, end, len); - if (end - start < sub_len) - res = -1; - else if (sub_len == 1) { - if (dir > 0) - res = stringlib_find_char( - PyBytes_AS_STRING(self) + start, end - start, - *sub); - else - res = stringlib_rfind_char( - PyBytes_AS_STRING(self) + start, end - start, - *sub); - if (res >= 0) - res += start; - } - else { - if (dir > 0) - res = stringlib_find_slice( - PyBytes_AS_STRING(self), len, - sub, sub_len, start, end); - else - res = stringlib_rfind_slice( - PyBytes_AS_STRING(self), len, - sub, sub_len, start, end); - } - - if (subobj) - PyBuffer_Release(&subbuf); - - return res; -} - - -PyDoc_STRVAR(find__doc__, -"B.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where substring sub is found,\n\ -such that sub is contained within B[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - static PyObject * bytes_find(PyBytesObject *self, PyObject *args) { - Py_ssize_t result = bytes_find_internal(self, args, +1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); + return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); } - -PyDoc_STRVAR(index__doc__, -"B.index(sub[, start[, end]]) -> int\n\ -\n\ -Like B.find() but raise ValueError when the substring is not found."); - static PyObject * bytes_index(PyBytesObject *self, PyObject *args) { - Py_ssize_t result = bytes_find_internal(self, args, +1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "substring not found"); - return NULL; - } - return PyLong_FromSsize_t(result); + return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); } -PyDoc_STRVAR(rfind__doc__, -"B.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where substring sub is found,\n\ -such that sub is contained within B[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - static PyObject * bytes_rfind(PyBytesObject *self, PyObject *args) { - Py_ssize_t result = bytes_find_internal(self, args, -1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); + return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); } -PyDoc_STRVAR(rindex__doc__, -"B.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Like B.rfind() but raise ValueError when the substring is not found."); - static PyObject * bytes_rindex(PyBytesObject *self, PyObject *args) { - Py_ssize_t result = bytes_find_internal(self, args, -1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "substring not found"); - return NULL; - } - return PyLong_FromSsize_t(result); + return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); } @@ -2179,51 +2035,10 @@ bytes_rstrip_impl(PyBytesObject *self, PyObject *bytes) } -PyDoc_STRVAR(count__doc__, -"B.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of substring sub in\n\ -string B[start:end]. Optional arguments start and end are interpreted\n\ -as in slice notation."); - static PyObject * bytes_count(PyBytesObject *self, PyObject *args) { - PyObject *sub_obj; - const char *str = PyBytes_AS_STRING(self), *sub; - Py_ssize_t sub_len; - char byte; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; - - Py_buffer vsub; - PyObject *count_obj; - - if (!stringlib_parse_args_finds_byte("count", args, &sub_obj, &byte, - &start, &end)) - return NULL; - - if (sub_obj) { - if (PyObject_GetBuffer(sub_obj, &vsub, PyBUF_SIMPLE) != 0) - return NULL; - - sub = vsub.buf; - sub_len = vsub.len; - } - else { - sub = &byte; - sub_len = 1; - } - - ADJUST_INDICES(start, end, PyBytes_GET_SIZE(self)); - - count_obj = PyLong_FromSsize_t( - stringlib_count(str + start, end - start, sub, sub_len, PY_SSIZE_T_MAX) - ); - - if (sub_obj) - PyBuffer_Release(&vsub); - - return count_obj; + return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); } @@ -2307,7 +2122,7 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, int group_right_1, PyBuffer_Release(&table_view); return NULL; } - output_start = output = PyBytes_AsString(result); + output_start = output = PyBytes_AS_STRING(result); input = PyBytes_AS_STRING(input_obj); if (dellen == 0 && table_chars != NULL) { @@ -2914,145 +2729,17 @@ bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, /** End DALKE **/ -/* Matches the end (direction >= 0) or start (direction < 0) of self - * against substr, using the start and end arguments. Returns - * -1 on error, 0 if not found and 1 if found. - */ -Py_LOCAL(int) -_bytes_tailmatch(PyBytesObject *self, PyObject *substr, Py_ssize_t start, - Py_ssize_t end, int direction) -{ - Py_ssize_t len = PyBytes_GET_SIZE(self); - Py_ssize_t slen; - Py_buffer sub_view = {NULL, NULL}; - const char* sub; - const char* str; - - if (PyBytes_Check(substr)) { - sub = PyBytes_AS_STRING(substr); - slen = PyBytes_GET_SIZE(substr); - } - else { - if (PyObject_GetBuffer(substr, &sub_view, PyBUF_SIMPLE) != 0) - return -1; - sub = sub_view.buf; - slen = sub_view.len; - } - str = PyBytes_AS_STRING(self); - - ADJUST_INDICES(start, end, len); - - if (direction < 0) { - /* startswith */ - if (start+slen > len) - goto notfound; - } else { - /* endswith */ - if (end-start < slen || start > len) - goto notfound; - - if (end-slen > start) - start = end - slen; - } - if (end-start < slen) - goto notfound; - if (memcmp(str+start, sub, slen) != 0) - goto notfound; - - PyBuffer_Release(&sub_view); - return 1; - -notfound: - PyBuffer_Release(&sub_view); - return 0; -} - - -PyDoc_STRVAR(startswith__doc__, -"B.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of bytes to try."); static PyObject * bytes_startswith(PyBytesObject *self, PyObject *args) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!stringlib_parse_args_finds("startswith", args, &subobj, &start, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _bytes_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, -1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _bytes_tailmatch(self, subobj, start, end, -1); - if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "startswith first arg must be bytes " - "or a tuple of bytes, not %s", Py_TYPE(subobj)->tp_name); - return NULL; - } - else - return PyBool_FromLong(result); + return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); } - -PyDoc_STRVAR(endswith__doc__, -"B.endswith(suffix[, start[, end]]) -> bool\n\ -\n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of bytes to try."); - static PyObject * bytes_endswith(PyBytesObject *self, PyObject *args) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!stringlib_parse_args_finds("endswith", args, &subobj, &start, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _bytes_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, +1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _bytes_tailmatch(self, subobj, start, end, +1); - if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "endswith first arg must be bytes or " - "a tuple of bytes, not %s", Py_TYPE(subobj)->tp_name); - return NULL; - } - else - return PyBool_FromLong(result); + return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); } @@ -3224,17 +2911,20 @@ bytes_methods[] = { {"__getnewargs__", (PyCFunction)bytes_getnewargs, METH_NOARGS}, {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, _Py_capitalize__doc__}, - {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, - {"count", (PyCFunction)bytes_count, METH_VARARGS, count__doc__}, + {"center", (PyCFunction)stringlib_center, METH_VARARGS, + _Py_center__doc__}, + {"count", (PyCFunction)bytes_count, METH_VARARGS, + _Py_count__doc__}, BYTES_DECODE_METHODDEF {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, - endswith__doc__}, + _Py_endswith__doc__}, {"expandtabs", (PyCFunction)stringlib_expandtabs, METH_VARARGS | METH_KEYWORDS, - expandtabs__doc__}, - {"find", (PyCFunction)bytes_find, METH_VARARGS, find__doc__}, + _Py_expandtabs__doc__}, + {"find", (PyCFunction)bytes_find, METH_VARARGS, + _Py_find__doc__}, BYTES_FROMHEX_METHODDEF {"hex", (PyCFunction)bytes_hex, METH_NOARGS, hex__doc__}, - {"index", (PyCFunction)bytes_index, METH_VARARGS, index__doc__}, + {"index", (PyCFunction)bytes_index, METH_VARARGS, _Py_index__doc__}, {"isalnum", (PyCFunction)stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, {"isalpha", (PyCFunction)stringlib_isalpha, METH_NOARGS, @@ -3250,29 +2940,29 @@ bytes_methods[] = { {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS, _Py_isupper__doc__}, BYTES_JOIN_METHODDEF - {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, ljust__doc__}, + {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, _Py_ljust__doc__}, {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__}, BYTES_LSTRIP_METHODDEF BYTES_MAKETRANS_METHODDEF BYTES_PARTITION_METHODDEF BYTES_REPLACE_METHODDEF - {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, rindex__doc__}, - {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, + {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, _Py_rfind__doc__}, + {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, _Py_rindex__doc__}, + {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, _Py_rjust__doc__}, BYTES_RPARTITION_METHODDEF BYTES_RSPLIT_METHODDEF BYTES_RSTRIP_METHODDEF BYTES_SPLIT_METHODDEF BYTES_SPLITLINES_METHODDEF {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS, - startswith__doc__}, + _Py_startswith__doc__}, BYTES_STRIP_METHODDEF {"swapcase", (PyCFunction)stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__}, BYTES_TRANSLATE_METHODDEF {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__}, - {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, zfill__doc__}, + {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, _Py_zfill__doc__}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index a7065fc..509b929 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -117,76 +117,3 @@ STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, } #undef FORMAT_BUFFER_SIZE - -#if STRINGLIB_IS_UNICODE - -/* -Wraps stringlib_parse_args_finds() and additionally ensures that the -first argument is a unicode object. -*/ - -Py_LOCAL_INLINE(int) -STRINGLIB(parse_args_finds_unicode)(const char * function_name, PyObject *args, - PyObject **substring, - Py_ssize_t *start, Py_ssize_t *end) -{ - if(STRINGLIB(parse_args_finds)(function_name, args, substring, - start, end)) { - if (ensure_unicode(*substring) < 0) - return 0; - return 1; - } - return 0; -} - -#else /* !STRINGLIB_IS_UNICODE */ - -/* -Wraps stringlib_parse_args_finds() and additionally checks whether the -first argument is an integer in range(0, 256). - -If this is the case, writes the integer value to the byte parameter -and sets subobj to NULL. Otherwise, sets the first argument to subobj -and doesn't touch byte. The other parameters are similar to those of -stringlib_parse_args_finds(). -*/ - -Py_LOCAL_INLINE(int) -STRINGLIB(parse_args_finds_byte)(const char *function_name, PyObject *args, - PyObject **subobj, char *byte, - Py_ssize_t *start, Py_ssize_t *end) -{ - PyObject *tmp_subobj; - Py_ssize_t ival; - PyObject *err; - - if(!STRINGLIB(parse_args_finds)(function_name, args, &tmp_subobj, - start, end)) - return 0; - - if (!PyNumber_Check(tmp_subobj)) { - *subobj = tmp_subobj; - return 1; - } - - ival = PyNumber_AsSsize_t(tmp_subobj, PyExc_OverflowError); - if (ival == -1) { - err = PyErr_Occurred(); - if (err && !PyErr_GivenExceptionMatches(err, PyExc_OverflowError)) { - PyErr_Clear(); - *subobj = tmp_subobj; - return 1; - } - } - - if (ival < 0 || ival > 255) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return 0; - } - - *subobj = NULL; - *byte = (char)ival; - return 1; -} - -#endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/stringlib/transmogrify.h b/Objects/stringlib/transmogrify.h index b559b53..298c5a6 100644 --- a/Objects/stringlib/transmogrify.h +++ b/Objects/stringlib/transmogrify.h @@ -4,12 +4,6 @@ /* the more complicated methods. parts of these should be pulled out into the shared code in bytes_methods.c to cut down on duplicate code bloat. */ -PyDoc_STRVAR(expandtabs__doc__, -"B.expandtabs(tabsize=8) -> copy of B\n\ -\n\ -Return a copy of B where all tab characters are expanded using spaces.\n\ -If tabsize is not given, a tab size of 8 characters is assumed."); - static PyObject* stringlib_expandtabs(PyObject *self, PyObject *args, PyObject *kwds) { @@ -120,12 +114,6 @@ pad(PyObject *self, Py_ssize_t left, Py_ssize_t right, char fill) return u; } -PyDoc_STRVAR(ljust__doc__, -"B.ljust(width[, fillchar]) -> copy of B\n" -"\n" -"Return B left justified in a string of length width. Padding is\n" -"done using the specified fill character (default is a space)."); - static PyObject * stringlib_ljust(PyObject *self, PyObject *args) { @@ -150,12 +138,6 @@ stringlib_ljust(PyObject *self, PyObject *args) } -PyDoc_STRVAR(rjust__doc__, -"B.rjust(width[, fillchar]) -> copy of B\n" -"\n" -"Return B right justified in a string of length width. Padding is\n" -"done using the specified fill character (default is a space)"); - static PyObject * stringlib_rjust(PyObject *self, PyObject *args) { @@ -180,12 +162,6 @@ stringlib_rjust(PyObject *self, PyObject *args) } -PyDoc_STRVAR(center__doc__, -"B.center(width[, fillchar]) -> copy of B\n" -"\n" -"Return B centered in a string of length width. Padding is\n" -"done using the specified fill character (default is a space)."); - static PyObject * stringlib_center(PyObject *self, PyObject *args) { @@ -213,12 +189,6 @@ stringlib_center(PyObject *self, PyObject *args) return pad(self, left, marg - left, fillchar); } -PyDoc_STRVAR(zfill__doc__, -"B.zfill(width) -> copy of B\n" -"\n" -"Pad a numeric string B with zeros on the left, to fill a field\n" -"of the specified width. B is never truncated."); - static PyObject * stringlib_zfill(PyObject *self, PyObject *args) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f1de3d5..1e7cba6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -11244,6 +11244,25 @@ PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) Py_XDECREF(right); } +/* +Wraps stringlib_parse_args_finds() and additionally ensures that the +first argument is a unicode object. +*/ + +Py_LOCAL_INLINE(int) +parse_args_finds_unicode(const char * function_name, PyObject *args, + PyObject **substring, + Py_ssize_t *start, Py_ssize_t *end) +{ + if(stringlib_parse_args_finds(function_name, args, substring, + start, end)) { + if (ensure_unicode(*substring) < 0) + return 0; + return 1; + } + return 0; +} + PyDoc_STRVAR(count__doc__, "S.count(sub[, start[, end]]) -> int\n\ \n\ @@ -11262,8 +11281,7 @@ unicode_count(PyObject *self, PyObject *args) void *buf1, *buf2; Py_ssize_t len1, len2, iresult; - if (!stringlib_parse_args_finds_unicode("count", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("count", args, &substring, &start, &end)) return NULL; kind1 = PyUnicode_KIND(self); @@ -11445,8 +11463,7 @@ unicode_find(PyObject *self, PyObject *args) Py_ssize_t end = 0; Py_ssize_t result; - if (!stringlib_parse_args_finds_unicode("find", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("find", args, &substring, &start, &end)) return NULL; if (PyUnicode_READY(self) == -1) @@ -11525,8 +11542,7 @@ unicode_index(PyObject *self, PyObject *args) Py_ssize_t start = 0; Py_ssize_t end = 0; - if (!stringlib_parse_args_finds_unicode("index", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("index", args, &substring, &start, &end)) return NULL; if (PyUnicode_READY(self) == -1) @@ -12555,8 +12571,7 @@ unicode_rfind(PyObject *self, PyObject *args) Py_ssize_t end = 0; Py_ssize_t result; - if (!stringlib_parse_args_finds_unicode("rfind", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end)) return NULL; if (PyUnicode_READY(self) == -1) @@ -12584,8 +12599,7 @@ unicode_rindex(PyObject *self, PyObject *args) Py_ssize_t end = 0; Py_ssize_t result; - if (!stringlib_parse_args_finds_unicode("rindex", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end)) return NULL; if (PyUnicode_READY(self) == -1) -- cgit v0.12 From fb81d3cbe72e406a47962cee0a832d7b67ad11a3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 May 2016 09:26:07 +0300 Subject: Issue #26765: Moved common code for the replace() method of bytes and bytearray to a template file. --- Objects/bytearrayobject.c | 503 +--------------------------------- Objects/bytesobject.c | 508 +--------------------------------- Objects/stringlib/transmogrify.h | 578 +++++++++++++++++++++++++++++++++++---- 3 files changed, 527 insertions(+), 1062 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 093cdbf..9457768 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1307,503 +1307,6 @@ bytearray_maketrans_impl(Py_buffer *frm, Py_buffer *to) } -/* find and count characters and substrings */ - -#define findchar(target, target_len, c) \ - ((char *)memchr((const void *)(target), c, target_len)) - - -/* Bytes ops must return a string, create a copy */ -Py_LOCAL(PyByteArrayObject *) -return_self(PyByteArrayObject *self) -{ - /* always return a new bytearray */ - return (PyByteArrayObject *)PyByteArray_FromStringAndSize( - PyByteArray_AS_STRING(self), - PyByteArray_GET_SIZE(self)); -} - -Py_LOCAL_INLINE(Py_ssize_t) -countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount) -{ - Py_ssize_t count=0; - const char *start=target; - const char *end=target+target_len; - - while ( (start=findchar(start, end-start, c)) != NULL ) { - count++; - if (count >= maxcount) - break; - start += 1; - } - return count; -} - - -/* Algorithms for different cases of string replacement */ - -/* len(self)>=1, from="", len(to)>=1, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_interleave(PyByteArrayObject *self, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - Py_ssize_t self_len, result_len; - Py_ssize_t count, i; - PyByteArrayObject *result; - - self_len = PyByteArray_GET_SIZE(self); - - /* 1 at the end plus 1 after every character; - count = min(maxcount, self_len + 1) */ - if (maxcount <= self_len) - count = maxcount; - else - /* Can't overflow: self_len + 1 <= maxcount <= PY_SSIZE_T_MAX. */ - count = self_len + 1; - - /* Check for overflow */ - /* result_len = count * to_len + self_len; */ - assert(count > 0); - if (to_len > (PY_SSIZE_T_MAX - self_len) / count) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - result_len = count * to_len + self_len; - - if (! (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) ) - return NULL; - - self_s = PyByteArray_AS_STRING(self); - result_s = PyByteArray_AS_STRING(result); - - if (to_len > 1) { - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; - - for (i = 0; i < count; i++) { - *result_s++ = *self_s++; - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - } - } - else { - result_s[0] = to_s[0]; - result_s += to_len; - count -= 1; - for (i = 0; i < count; i++) { - *result_s++ = *self_s++; - result_s[0] = to_s[0]; - result_s += to_len; - } - } - - /* Copy the rest of the original string */ - Py_MEMCPY(result_s, self_s, self_len-i); - - return result; -} - -/* Special case for deleting a single character */ -/* len(self)>=1, len(from)==1, to="", maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_delete_single_character(PyByteArrayObject *self, - char from_c, Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count; - PyByteArrayObject *result; - - self_len = PyByteArray_GET_SIZE(self); - self_s = PyByteArray_AS_STRING(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - return return_self(self); - } - - result_len = self_len - count; /* from_len == 1 */ - assert(result_len>=0); - - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - start = next+1; - } - Py_MEMCPY(result_s, start, end-start); - - return result; -} - -/* len(self)>=1, len(from)>=2, to="", maxcount>=1 */ - -Py_LOCAL(PyByteArrayObject *) -replace_delete_substring(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset; - PyByteArrayObject *result; - - self_len = PyByteArray_GET_SIZE(self); - self_s = PyByteArray_AS_STRING(self); - - count = stringlib_count(self_s, self_len, - from_s, from_len, - maxcount); - - if (count == 0) { - /* no matches */ - return return_self(self); - } - - result_len = self_len - (count * from_len); - assert (result_len>=0); - - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL ) - return NULL; - - result_s = PyByteArray_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = stringlib_find(start, end-start, - from_s, from_len, - 0); - if (offset == -1) - break; - next = start + offset; - - Py_MEMCPY(result_s, start, next-start); - - result_s += (next-start); - start = next+from_len; - } - Py_MEMCPY(result_s, start, end-start); - return result; -} - -/* len(self)>=1, len(from)==len(to)==1, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_single_character_in_place(PyByteArrayObject *self, - char from_c, char to_c, - Py_ssize_t maxcount) -{ - char *self_s, *result_s, *start, *end, *next; - Py_ssize_t self_len; - PyByteArrayObject *result; - - /* The result string will be the same size */ - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); - - next = findchar(self_s, self_len, from_c); - - if (next == NULL) { - /* No matches; return the original bytes */ - return return_self(self); - } - - /* Need to make a new bytes */ - result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); - - /* change everything in-place, starting with this one */ - start = result_s + (next-self_s); - *start = to_c; - start++; - end = result_s + self_len; - - while (--maxcount > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - *next = to_c; - start = next+1; - } - - return result; -} - -/* len(self)>=1, len(from)==len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_substring_in_place(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *result_s, *start, *end; - char *self_s; - Py_ssize_t self_len, offset; - PyByteArrayObject *result; - - /* The result bytes will be the same size */ - - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); - - offset = stringlib_find(self_s, self_len, - from_s, from_len, - 0); - if (offset == -1) { - /* No matches; return the original bytes */ - return return_self(self); - } - - /* Need to make a new bytes */ - result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); - - /* change everything in-place, starting with this one */ - start = result_s + offset; - Py_MEMCPY(start, to_s, from_len); - start += from_len; - end = result_s + self_len; - - while ( --maxcount > 0) { - offset = stringlib_find(start, end-start, - from_s, from_len, - 0); - if (offset==-1) - break; - Py_MEMCPY(start+offset, to_s, from_len); - start += offset+from_len; - } - - return result; -} - -/* len(self)>=1, len(from)==1, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_single_character(PyByteArrayObject *self, - char from_c, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count; - PyByteArrayObject *result; - - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } - - /* use the difference between current and new, hence the "-1" */ - /* result_len = self_len + count * (to_len-1) */ - assert(count > 0); - if (to_len - 1 > (PY_SSIZE_T_MAX - self_len) / count) { - PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); - return NULL; - } - result_len = self_len + count * (to_len - 1); - - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += 1; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+1; - } - } - /* Copy the remainder of the remaining bytes */ - Py_MEMCPY(result_s, start, end-start); - - return result; -} - -/* len(self)>=1, len(from)>=2, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_substring(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset; - PyByteArrayObject *result; - - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); - - count = stringlib_count(self_s, self_len, - from_s, from_len, - maxcount); - - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } - - /* Check for overflow */ - /* result_len = self_len + count * (to_len-from_len) */ - assert(count > 0); - if (to_len - from_len > (PY_SSIZE_T_MAX - self_len) / count) { - PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); - return NULL; - } - result_len = self_len + count * (to_len - from_len); - - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = stringlib_find(start, end-start, - from_s, from_len, - 0); - if (offset == -1) - break; - next = start+offset; - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += from_len; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+from_len; - } - } - /* Copy the remainder of the remaining bytes */ - Py_MEMCPY(result_s, start, end-start); - - return result; -} - - -Py_LOCAL(PyByteArrayObject *) -replace(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - if (maxcount < 0) { - maxcount = PY_SSIZE_T_MAX; - } else if (maxcount == 0 || PyByteArray_GET_SIZE(self) == 0) { - /* nothing to do; return the original bytes */ - return return_self(self); - } - - if (maxcount == 0 || - (from_len == 0 && to_len == 0)) { - /* nothing to do; return the original bytes */ - return return_self(self); - } - - /* Handle zero-length special cases */ - - if (from_len == 0) { - /* insert the 'to' bytes everywhere. */ - /* >>> "Python".replace("", ".") */ - /* '.P.y.t.h.o.n.' */ - return replace_interleave(self, to_s, to_len, maxcount); - } - - /* Except for "".replace("", "A") == "A" there is no way beyond this */ - /* point for an empty self bytes to generate a non-empty bytes */ - /* Special case so the remaining code always gets a non-empty bytes */ - if (PyByteArray_GET_SIZE(self) == 0) { - return return_self(self); - } - - if (to_len == 0) { - /* delete all occurrences of 'from' bytes */ - if (from_len == 1) { - return replace_delete_single_character( - self, from_s[0], maxcount); - } else { - return replace_delete_substring(self, from_s, from_len, maxcount); - } - } - - /* Handle special case where both bytes have the same length */ - - if (from_len == to_len) { - if (from_len == 1) { - return replace_single_character_in_place( - self, - from_s[0], - to_s[0], - maxcount); - } else { - return replace_substring_in_place( - self, from_s, from_len, to_s, to_len, maxcount); - } - } - - /* Otherwise use the more generic algorithms */ - if (from_len == 1) { - return replace_single_character(self, from_s[0], - to_s, to_len, maxcount); - } else { - /* len('from')>=2, len('to')>=1 */ - return replace_substring(self, from_s, from_len, to_s, to_len, maxcount); - } -} - - /*[clinic input] bytearray.replace @@ -1825,9 +1328,9 @@ bytearray_replace_impl(PyByteArrayObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count) /*[clinic end generated code: output=d39884c4dc59412a input=aa379d988637c7fb]*/ { - return (PyObject *)replace((PyByteArrayObject *) self, - old->buf, old->len, - new->buf, new->len, count); + return stringlib_replace((PyObject *)self, + (const char *)old->buf, old->len, + (const char *)new->buf, new->len, count); } /*[clinic input] diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2bf39e3..aeddf53 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2198,508 +2198,6 @@ bytes_maketrans_impl(Py_buffer *frm, Py_buffer *to) return _Py_bytes_maketrans(frm, to); } -/* find and count characters and substrings */ - -#define findchar(target, target_len, c) \ - ((char *)memchr((const void *)(target), c, target_len)) - -/* String ops must return a string. */ -/* If the object is subclass of string, create a copy */ -Py_LOCAL(PyBytesObject *) -return_self(PyBytesObject *self) -{ - if (PyBytes_CheckExact(self)) { - Py_INCREF(self); - return self; - } - return (PyBytesObject *)PyBytes_FromStringAndSize( - PyBytes_AS_STRING(self), - PyBytes_GET_SIZE(self)); -} - -Py_LOCAL_INLINE(Py_ssize_t) -countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount) -{ - Py_ssize_t count=0; - const char *start=target; - const char *end=target+target_len; - - while ( (start=findchar(start, end-start, c)) != NULL ) { - count++; - if (count >= maxcount) - break; - start += 1; - } - return count; -} - - -/* Algorithms for different cases of string replacement */ - -/* len(self)>=1, from="", len(to)>=1, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_interleave(PyBytesObject *self, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - Py_ssize_t self_len, result_len; - Py_ssize_t count, i; - PyBytesObject *result; - - self_len = PyBytes_GET_SIZE(self); - - /* 1 at the end plus 1 after every character; - count = min(maxcount, self_len + 1) */ - if (maxcount <= self_len) - count = maxcount; - else - /* Can't overflow: self_len + 1 <= maxcount <= PY_SSIZE_T_MAX. */ - count = self_len + 1; - - /* Check for overflow */ - /* result_len = count * to_len + self_len; */ - assert(count > 0); - if (to_len > (PY_SSIZE_T_MAX - self_len) / count) { - PyErr_SetString(PyExc_OverflowError, - "replacement bytes are too long"); - return NULL; - } - result_len = count * to_len + self_len; - - if (! (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) ) - return NULL; - - self_s = PyBytes_AS_STRING(self); - result_s = PyBytes_AS_STRING(result); - - if (to_len > 1) { - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; - - for (i = 0; i < count; i++) { - *result_s++ = *self_s++; - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - } - } - else { - result_s[0] = to_s[0]; - result_s += to_len; - count -= 1; - for (i = 0; i < count; i++) { - *result_s++ = *self_s++; - result_s[0] = to_s[0]; - result_s += to_len; - } - } - - /* Copy the rest of the original string */ - Py_MEMCPY(result_s, self_s, self_len-i); - - return result; -} - -/* Special case for deleting a single character */ -/* len(self)>=1, len(from)==1, to="", maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_delete_single_character(PyBytesObject *self, - char from_c, Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count; - PyBytesObject *result; - - self_len = PyBytes_GET_SIZE(self); - self_s = PyBytes_AS_STRING(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - return return_self(self); - } - - result_len = self_len - count; /* from_len == 1 */ - assert(result_len>=0); - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - start = next+1; - } - Py_MEMCPY(result_s, start, end-start); - - return result; -} - -/* len(self)>=1, len(from)>=2, to="", maxcount>=1 */ - -Py_LOCAL(PyBytesObject *) -replace_delete_substring(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - Py_ssize_t maxcount) { - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset; - PyBytesObject *result; - - self_len = PyBytes_GET_SIZE(self); - self_s = PyBytes_AS_STRING(self); - - count = stringlib_count(self_s, self_len, - from_s, from_len, - maxcount); - - if (count == 0) { - /* no matches */ - return return_self(self); - } - - result_len = self_len - (count * from_len); - assert (result_len>=0); - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL ) - return NULL; - - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = stringlib_find(start, end-start, - from_s, from_len, - 0); - if (offset == -1) - break; - next = start + offset; - - Py_MEMCPY(result_s, start, next-start); - - result_s += (next-start); - start = next+from_len; - } - Py_MEMCPY(result_s, start, end-start); - return result; -} - -/* len(self)>=1, len(from)==len(to)==1, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_single_character_in_place(PyBytesObject *self, - char from_c, char to_c, - Py_ssize_t maxcount) -{ - char *self_s, *result_s, *start, *end, *next; - Py_ssize_t self_len; - PyBytesObject *result; - - /* The result string will be the same size */ - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - next = findchar(self_s, self_len, from_c); - - if (next == NULL) { - /* No matches; return the original string */ - return return_self(self); - } - - /* Need to make a new string */ - result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); - - /* change everything in-place, starting with this one */ - start = result_s + (next-self_s); - *start = to_c; - start++; - end = result_s + self_len; - - while (--maxcount > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - *next = to_c; - start = next+1; - } - - return result; -} - -/* len(self)>=1, len(from)==len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_substring_in_place(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *result_s, *start, *end; - char *self_s; - Py_ssize_t self_len, offset; - PyBytesObject *result; - - /* The result string will be the same size */ - - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - offset = stringlib_find(self_s, self_len, - from_s, from_len, - 0); - if (offset == -1) { - /* No matches; return the original string */ - return return_self(self); - } - - /* Need to make a new string */ - result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); - - /* change everything in-place, starting with this one */ - start = result_s + offset; - Py_MEMCPY(start, to_s, from_len); - start += from_len; - end = result_s + self_len; - - while ( --maxcount > 0) { - offset = stringlib_find(start, end-start, - from_s, from_len, - 0); - if (offset==-1) - break; - Py_MEMCPY(start+offset, to_s, from_len); - start += offset+from_len; - } - - return result; -} - -/* len(self)>=1, len(from)==1, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_single_character(PyBytesObject *self, - char from_c, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count; - PyBytesObject *result; - - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } - - /* use the difference between current and new, hence the "-1" */ - /* result_len = self_len + count * (to_len-1) */ - assert(count > 0); - if (to_len - 1 > (PY_SSIZE_T_MAX - self_len) / count) { - PyErr_SetString(PyExc_OverflowError, - "replacement bytes are too long"); - return NULL; - } - result_len = self_len + count * (to_len - 1); - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += 1; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+1; - } - } - /* Copy the remainder of the remaining string */ - Py_MEMCPY(result_s, start, end-start); - - return result; -} - -/* len(self)>=1, len(from)>=2, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_substring(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) { - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset; - PyBytesObject *result; - - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - count = stringlib_count(self_s, self_len, - from_s, from_len, - maxcount); - - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } - - /* Check for overflow */ - /* result_len = self_len + count * (to_len-from_len) */ - assert(count > 0); - if (to_len - from_len > (PY_SSIZE_T_MAX - self_len) / count) { - PyErr_SetString(PyExc_OverflowError, - "replacement bytes are too long"); - return NULL; - } - result_len = self_len + count * (to_len-from_len); - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = stringlib_find(start, end-start, - from_s, from_len, - 0); - if (offset == -1) - break; - next = start+offset; - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += from_len; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+from_len; - } - } - /* Copy the remainder of the remaining string */ - Py_MEMCPY(result_s, start, end-start); - - return result; -} - - -Py_LOCAL(PyBytesObject *) -replace(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - if (maxcount < 0) { - maxcount = PY_SSIZE_T_MAX; - } else if (maxcount == 0 || PyBytes_GET_SIZE(self) == 0) { - /* nothing to do; return the original string */ - return return_self(self); - } - - if (maxcount == 0 || - (from_len == 0 && to_len == 0)) { - /* nothing to do; return the original string */ - return return_self(self); - } - - /* Handle zero-length special cases */ - - if (from_len == 0) { - /* insert the 'to' string everywhere. */ - /* >>> "Python".replace("", ".") */ - /* '.P.y.t.h.o.n.' */ - return replace_interleave(self, to_s, to_len, maxcount); - } - - /* Except for "".replace("", "A") == "A" there is no way beyond this */ - /* point for an empty self string to generate a non-empty string */ - /* Special case so the remaining code always gets a non-empty string */ - if (PyBytes_GET_SIZE(self) == 0) { - return return_self(self); - } - - if (to_len == 0) { - /* delete all occurrences of 'from' string */ - if (from_len == 1) { - return replace_delete_single_character( - self, from_s[0], maxcount); - } else { - return replace_delete_substring(self, from_s, - from_len, maxcount); - } - } - - /* Handle special case where both strings have the same length */ - - if (from_len == to_len) { - if (from_len == 1) { - return replace_single_character_in_place( - self, - from_s[0], - to_s[0], - maxcount); - } else { - return replace_substring_in_place( - self, from_s, from_len, to_s, to_len, - maxcount); - } - } - - /* Otherwise use the more generic algorithms */ - if (from_len == 1) { - return replace_single_character(self, from_s[0], - to_s, to_len, maxcount); - } else { - /* len('from')>=2, len('to')>=1 */ - return replace_substring(self, from_s, from_len, to_s, to_len, - maxcount); - } -} - /*[clinic input] bytes.replace @@ -2722,9 +2220,9 @@ bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count) /*[clinic end generated code: output=994fa588b6b9c104 input=b2fbbf0bf04de8e5]*/ { - return (PyObject *)replace((PyBytesObject *) self, - (const char *)old->buf, old->len, - (const char *)new->buf, new->len, count); + return stringlib_replace((PyObject *)self, + (const char *)old->buf, old->len, + (const char *)new->buf, new->len, count); } /** End DALKE **/ diff --git a/Objects/stringlib/transmogrify.h b/Objects/stringlib/transmogrify.h index 298c5a6..8b147de 100644 --- a/Objects/stringlib/transmogrify.h +++ b/Objects/stringlib/transmogrify.h @@ -4,6 +4,18 @@ /* the more complicated methods. parts of these should be pulled out into the shared code in bytes_methods.c to cut down on duplicate code bloat. */ +Py_LOCAL_INLINE(PyObject *) +return_self(PyObject *self) +{ +#if !STRINGLIB_MUTABLE + if (STRINGLIB_CHECK_EXACT(self)) { + Py_INCREF(self); + return self; + } +#endif + return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + static PyObject* stringlib_expandtabs(PyObject *self, PyObject *args, PyObject *kwds) { @@ -87,28 +99,20 @@ pad(PyObject *self, Py_ssize_t left, Py_ssize_t right, char fill) if (right < 0) right = 0; - if (left == 0 && right == 0 && STRINGLIB_CHECK_EXACT(self)) { -#if STRINGLIB_MUTABLE - /* We're defined as returning a copy; If the object is mutable - * that means we must make an identical copy. */ - return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); -#else - Py_INCREF(self); - return (PyObject *)self; -#endif /* STRINGLIB_MUTABLE */ + if (left == 0 && right == 0) { + return return_self(self); } - u = STRINGLIB_NEW(NULL, - left + STRINGLIB_LEN(self) + right); + u = STRINGLIB_NEW(NULL, left + STRINGLIB_LEN(self) + right); if (u) { if (left) memset(STRINGLIB_STR(u), fill, left); Py_MEMCPY(STRINGLIB_STR(u) + left, - STRINGLIB_STR(self), - STRINGLIB_LEN(self)); + STRINGLIB_STR(self), + STRINGLIB_LEN(self)); if (right) memset(STRINGLIB_STR(u) + left + STRINGLIB_LEN(self), - fill, right); + fill, right); } return u; @@ -123,15 +127,8 @@ stringlib_ljust(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "n|c:ljust", &width, &fillchar)) return NULL; - if (STRINGLIB_LEN(self) >= width && STRINGLIB_CHECK_EXACT(self)) { -#if STRINGLIB_MUTABLE - /* We're defined as returning a copy; If the object is mutable - * that means we must make an identical copy. */ - return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); -#else - Py_INCREF(self); - return (PyObject*) self; -#endif + if (STRINGLIB_LEN(self) >= width) { + return return_self(self); } return pad(self, 0, width - STRINGLIB_LEN(self), fillchar); @@ -147,15 +144,8 @@ stringlib_rjust(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "n|c:rjust", &width, &fillchar)) return NULL; - if (STRINGLIB_LEN(self) >= width && STRINGLIB_CHECK_EXACT(self)) { -#if STRINGLIB_MUTABLE - /* We're defined as returning a copy; If the object is mutable - * that means we must make an identical copy. */ - return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); -#else - Py_INCREF(self); - return (PyObject*) self; -#endif + if (STRINGLIB_LEN(self) >= width) { + return return_self(self); } return pad(self, width - STRINGLIB_LEN(self), 0, fillchar); @@ -172,15 +162,8 @@ stringlib_center(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "n|c:center", &width, &fillchar)) return NULL; - if (STRINGLIB_LEN(self) >= width && STRINGLIB_CHECK_EXACT(self)) { -#if STRINGLIB_MUTABLE - /* We're defined as returning a copy; If the object is mutable - * that means we must make an identical copy. */ - return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); -#else - Py_INCREF(self); - return (PyObject*) self; -#endif + if (STRINGLIB_LEN(self) >= width) { + return return_self(self); } marg = width - STRINGLIB_LEN(self); @@ -201,21 +184,7 @@ stringlib_zfill(PyObject *self, PyObject *args) return NULL; if (STRINGLIB_LEN(self) >= width) { - if (STRINGLIB_CHECK_EXACT(self)) { -#if STRINGLIB_MUTABLE - /* We're defined as returning a copy; If the object is mutable - * that means we must make an identical copy. */ - return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); -#else - Py_INCREF(self); - return (PyObject*) self; -#endif - } - else - return STRINGLIB_NEW( - STRINGLIB_STR(self), - STRINGLIB_LEN(self) - ); + return return_self(self); } fill = width - STRINGLIB_LEN(self); @@ -232,5 +201,500 @@ stringlib_zfill(PyObject *self, PyObject *args) p[fill] = '0'; } - return (PyObject*) s; + return s; +} + + +/* find and count characters and substrings */ + +#define findchar(target, target_len, c) \ + ((char *)memchr((const void *)(target), c, target_len)) + + +Py_LOCAL_INLINE(Py_ssize_t) +countchar(const char *target, Py_ssize_t target_len, char c, + Py_ssize_t maxcount) +{ + Py_ssize_t count = 0; + const char *start = target; + const char *end = target + target_len; + + while ((start = findchar(start, end - start, c)) != NULL) { + count++; + if (count >= maxcount) + break; + start += 1; + } + return count; +} + + +/* Algorithms for different cases of string replacement */ + +/* len(self)>=1, from="", len(to)>=1, maxcount>=1 */ +Py_LOCAL(PyObject *) +stringlib_replace_interleave(PyObject *self, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + const char *self_s; + char *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count, i; + PyObject *result; + + self_len = STRINGLIB_LEN(self); + + /* 1 at the end plus 1 after every character; + count = min(maxcount, self_len + 1) */ + if (maxcount <= self_len) { + count = maxcount; + } + else { + /* Can't overflow: self_len + 1 <= maxcount <= PY_SSIZE_T_MAX. */ + count = self_len + 1; + } + + /* Check for overflow */ + /* result_len = count * to_len + self_len; */ + assert(count > 0); + if (to_len > (PY_SSIZE_T_MAX - self_len) / count) { + PyErr_SetString(PyExc_OverflowError, + "replace bytes are too long"); + return NULL; + } + result_len = count * to_len + self_len; + result = STRINGLIB_NEW(NULL, result_len); + if (result == NULL) { + return NULL; + } + + self_s = STRINGLIB_STR(self); + result_s = STRINGLIB_STR(result); + + if (to_len > 1) { + /* Lay the first one down (guaranteed this will occur) */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + count -= 1; + + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } + } + else { + result_s[0] = to_s[0]; + result_s += to_len; + count -= 1; + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + result_s[0] = to_s[0]; + result_s += to_len; + } + } + + /* Copy the rest of the original string */ + Py_MEMCPY(result_s, self_s, self_len - i); + + return result; +} + +/* Special case for deleting a single character */ +/* len(self)>=1, len(from)==1, to="", maxcount>=1 */ +Py_LOCAL(PyObject *) +stringlib_replace_delete_single_character(PyObject *self, + char from_c, Py_ssize_t maxcount) +{ + const char *self_s, *start, *next, *end; + char *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count; + PyObject *result; + + self_len = STRINGLIB_LEN(self); + self_s = STRINGLIB_STR(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + return return_self(self); + } + + result_len = self_len - count; /* from_len == 1 */ + assert(result_len>=0); + + result = STRINGLIB_NEW(NULL, result_len); + if (result == NULL) { + return NULL; + } + result_s = STRINGLIB_STR(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end - start, from_c); + if (next == NULL) + break; + Py_MEMCPY(result_s, start, next - start); + result_s += (next - start); + start = next + 1; + } + Py_MEMCPY(result_s, start, end - start); + + return result; +} + +/* len(self)>=1, len(from)>=2, to="", maxcount>=1 */ + +Py_LOCAL(PyObject *) +stringlib_replace_delete_substring(PyObject *self, + const char *from_s, Py_ssize_t from_len, + Py_ssize_t maxcount) +{ + const char *self_s, *start, *next, *end; + char *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset; + PyObject *result; + + self_len = STRINGLIB_LEN(self); + self_s = STRINGLIB_STR(self); + + count = stringlib_count(self_s, self_len, + from_s, from_len, + maxcount); + + if (count == 0) { + /* no matches */ + return return_self(self); + } + + result_len = self_len - (count * from_len); + assert (result_len>=0); + + result = STRINGLIB_NEW(NULL, result_len); + if (result == NULL) { + return NULL; + } + result_s = STRINGLIB_STR(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = stringlib_find(start, end - start, + from_s, from_len, + 0); + if (offset == -1) + break; + next = start + offset; + + Py_MEMCPY(result_s, start, next - start); + + result_s += (next - start); + start = next + from_len; + } + Py_MEMCPY(result_s, start, end - start); + return result; } + +/* len(self)>=1, len(from)==len(to)==1, maxcount>=1 */ +Py_LOCAL(PyObject *) +stringlib_replace_single_character_in_place(PyObject *self, + char from_c, char to_c, + Py_ssize_t maxcount) +{ + const char *self_s, *end; + char *result_s, *start, *next; + Py_ssize_t self_len; + PyObject *result; + + /* The result string will be the same size */ + self_s = STRINGLIB_STR(self); + self_len = STRINGLIB_LEN(self); + + next = findchar(self_s, self_len, from_c); + + if (next == NULL) { + /* No matches; return the original bytes */ + return return_self(self); + } + + /* Need to make a new bytes */ + result = STRINGLIB_NEW(NULL, self_len); + if (result == NULL) { + return NULL; + } + result_s = STRINGLIB_STR(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + (next - self_s); + *start = to_c; + start++; + end = result_s + self_len; + + while (--maxcount > 0) { + next = findchar(start, end - start, from_c); + if (next == NULL) + break; + *next = to_c; + start = next + 1; + } + + return result; +} + +/* len(self)>=1, len(from)==len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyObject *) +stringlib_replace_substring_in_place(PyObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + const char *self_s, *end; + char *result_s, *start; + Py_ssize_t self_len, offset; + PyObject *result; + + /* The result bytes will be the same size */ + + self_s = STRINGLIB_STR(self); + self_len = STRINGLIB_LEN(self); + + offset = stringlib_find(self_s, self_len, + from_s, from_len, + 0); + if (offset == -1) { + /* No matches; return the original bytes */ + return return_self(self); + } + + /* Need to make a new bytes */ + result = STRINGLIB_NEW(NULL, self_len); + if (result == NULL) { + return NULL; + } + result_s = STRINGLIB_STR(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + offset; + Py_MEMCPY(start, to_s, from_len); + start += from_len; + end = result_s + self_len; + + while ( --maxcount > 0) { + offset = stringlib_find(start, end - start, + from_s, from_len, + 0); + if (offset == -1) + break; + Py_MEMCPY(start + offset, to_s, from_len); + start += offset + from_len; + } + + return result; +} + +/* len(self)>=1, len(from)==1, len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyObject *) +stringlib_replace_single_character(PyObject *self, + char from_c, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + const char *self_s, *start, *next, *end; + char *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count; + PyObject *result; + + self_s = STRINGLIB_STR(self); + self_len = STRINGLIB_LEN(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } + + /* use the difference between current and new, hence the "-1" */ + /* result_len = self_len + count * (to_len-1) */ + assert(count > 0); + if (to_len - 1 > (PY_SSIZE_T_MAX - self_len) / count) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + result_len = self_len + count * (to_len - 1); + + result = STRINGLIB_NEW(NULL, result_len); + if (result == NULL) { + return NULL; + } + result_s = STRINGLIB_STR(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end - start, from_c); + if (next == NULL) + break; + + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += 1; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next - start); + result_s += (next - start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next + 1; + } + } + /* Copy the remainder of the remaining bytes */ + Py_MEMCPY(result_s, start, end - start); + + return result; +} + +/* len(self)>=1, len(from)>=2, len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyObject *) +stringlib_replace_substring(PyObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + const char *self_s, *start, *next, *end; + char *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset; + PyObject *result; + + self_s = STRINGLIB_STR(self); + self_len = STRINGLIB_LEN(self); + + count = stringlib_count(self_s, self_len, + from_s, from_len, + maxcount); + + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } + + /* Check for overflow */ + /* result_len = self_len + count * (to_len-from_len) */ + assert(count > 0); + if (to_len - from_len > (PY_SSIZE_T_MAX - self_len) / count) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + result_len = self_len + count * (to_len - from_len); + + result = STRINGLIB_NEW(NULL, result_len); + if (result == NULL) { + return NULL; + } + result_s = STRINGLIB_STR(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = stringlib_find(start, end - start, + from_s, from_len, + 0); + if (offset == -1) + break; + next = start + offset; + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += from_len; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next - start); + result_s += (next - start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next + from_len; + } + } + /* Copy the remainder of the remaining bytes */ + Py_MEMCPY(result_s, start, end - start); + + return result; +} + + +Py_LOCAL(PyObject *) +stringlib_replace(PyObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + if (maxcount < 0) { + maxcount = PY_SSIZE_T_MAX; + } else if (maxcount == 0 || STRINGLIB_LEN(self) == 0) { + /* nothing to do; return the original bytes */ + return return_self(self); + } + + /* Handle zero-length special cases */ + if (from_len == 0) { + if (to_len == 0) { + /* nothing to do; return the original bytes */ + return return_self(self); + } + /* insert the 'to' bytes everywhere. */ + /* >>> b"Python".replace(b"", b".") */ + /* b'.P.y.t.h.o.n.' */ + return stringlib_replace_interleave(self, to_s, to_len, maxcount); + } + + /* Except for b"".replace(b"", b"A") == b"A" there is no way beyond this */ + /* point for an empty self bytes to generate a non-empty bytes */ + /* Special case so the remaining code always gets a non-empty bytes */ + if (STRINGLIB_LEN(self) == 0) { + return return_self(self); + } + + if (to_len == 0) { + /* delete all occurrences of 'from' bytes */ + if (from_len == 1) { + return stringlib_replace_delete_single_character( + self, from_s[0], maxcount); + } else { + return stringlib_replace_delete_substring( + self, from_s, from_len, maxcount); + } + } + + /* Handle special case where both bytes have the same length */ + + if (from_len == to_len) { + if (from_len == 1) { + return stringlib_replace_single_character_in_place( + self, from_s[0], to_s[0], maxcount); + } else { + return stringlib_replace_substring_in_place( + self, from_s, from_len, to_s, to_len, maxcount); + } + } + + /* Otherwise use the more generic algorithms */ + if (from_len == 1) { + return stringlib_replace_single_character( + self, from_s[0], to_s, to_len, maxcount); + } else { + /* len('from')>=2, len('to')>=1 */ + return stringlib_replace_substring( + self, from_s, from_len, to_s, to_len, maxcount); + } +} + +#undef findchar -- cgit v0.12 From d7062de95d035121abbab526c3b59904361aa256 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 May 2016 10:55:45 +0300 Subject: Issue #26918: Skipped some tests in test_pipes on Android. Patch by Xavier de Gaye. --- Lib/test/test_pipes.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_pipes.py b/Lib/test/test_pipes.py index 6a7b45f..ad01d08 100644 --- a/Lib/test/test_pipes.py +++ b/Lib/test/test_pipes.py @@ -2,6 +2,7 @@ import pipes import os import string import unittest +import shutil from test.support import TESTFN, run_unittest, unlink, reap_children if os.name != 'posix': @@ -18,6 +19,8 @@ class SimplePipeTests(unittest.TestCase): unlink(f) def testSimplePipe1(self): + if shutil.which('tr') is None: + self.skipTest('tr is not available') t = pipes.Template() t.append(s_command, pipes.STDIN_STDOUT) f = t.open(TESTFN, 'w') @@ -27,6 +30,8 @@ class SimplePipeTests(unittest.TestCase): self.assertEqual(f.read(), 'HELLO WORLD #1') def testSimplePipe2(self): + if shutil.which('tr') is None: + self.skipTest('tr is not available') with open(TESTFN, 'w') as f: f.write('hello world #2') t = pipes.Template() @@ -36,6 +41,8 @@ class SimplePipeTests(unittest.TestCase): self.assertEqual(f.read(), 'HELLO WORLD #2') def testSimplePipe3(self): + if shutil.which('tr') is None: + self.skipTest('tr is not available') with open(TESTFN, 'w') as f: f.write('hello world #2') t = pipes.Template() -- cgit v0.12 From 584e8aedc3d66721efcdcbd1a43d4c5b7476427b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 5 May 2016 11:14:06 +0300 Subject: Issue 26915: Add identity checks to the collections ABC __contains__ methods. --- Lib/_collections_abc.py | 7 ++++--- Lib/test/test_collections.py | 22 +++++++++++++++++++++- Misc/NEWS | 5 +++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index d337584..3158373 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -689,7 +689,7 @@ class ItemsView(MappingView, Set): except KeyError: return False else: - return v == value + return v is value or v == value def __iter__(self): for key in self._mapping: @@ -704,7 +704,8 @@ class ValuesView(MappingView): def __contains__(self, value): for key in self._mapping: - if value == self._mapping[key]: + v = self._mapping[key] + if v is value or v == value: return True return False @@ -839,7 +840,7 @@ class Sequence(Sized, Reversible, Container): def __contains__(self, value): for v in self: - if v == value: + if v is value or v == value: return True return False diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 4202462..3824a87 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -23,7 +23,7 @@ from collections.abc import Awaitable, Coroutine, AsyncIterator, AsyncIterable from collections.abc import Hashable, Iterable, Iterator, Generator, Reversible from collections.abc import Sized, Container, Callable from collections.abc import Set, MutableSet -from collections.abc import Mapping, MutableMapping, KeysView, ItemsView +from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence from collections.abc import ByteString @@ -1074,6 +1074,26 @@ class TestCollectionABCs(ABCTestCase): self.assertFalse(ncs > cs) self.assertTrue(ncs >= cs) + def test_issue26915(self): + # Container membership test should check identity first + class CustomEqualObject: + def __eq__(self, other): + return False + class CustomSequence(list): + def __contains__(self, value): + return Sequence.__contains__(self, value) + + nan = float('nan') + obj = CustomEqualObject() + containers = [ + CustomSequence([nan, obj]), + ItemsView({1: nan, 2: obj}), + ValuesView({1: nan, 2: obj}) + ] + for container in containers: + for elem in container: + self.assertIn(elem, container) + def assertSameSet(self, s1, s2): # coerce both to a real set then check equality self.assertSetEqual(set(s1), set(s2)) diff --git a/Misc/NEWS b/Misc/NEWS index 4833aa3..875c113 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -268,6 +268,11 @@ Library - Issue #26873: xmlrpc now raises ResponseError on unsupported type tags instead of silently return incorrect result. +- Issue #26915: The __contains__ methods in the collections ABCs now check + for identity before checking equality. This better matches the behavior + of the concrete classes, allows sensible handling of NaNs, and makes it + easier to reason about container invariants. + - Issue #26711: Fixed the comparison of plistlib.Data with other types. - Issue #24114: Fix an uninitialized variable in `ctypes.util`. -- cgit v0.12 From 24182a3aaa6054dc00658cb96e6c606cc459a946 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 May 2016 16:21:35 +0300 Subject: Restored parameter name "self" since gdb needs exact specific parameter names. --- Python/bltinmodule.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 6e0d736..0637a2d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1078,6 +1078,7 @@ builtin_hasattr_impl(PyModuleDef *module, PyObject *obj, PyObject *name) /*[clinic input] id as builtin_id + self: self(type="PyModuleDef *") obj as v: object / @@ -1088,8 +1089,8 @@ This is guaranteed to be unique among simultaneously existing objects. [clinic start generated code]*/ static PyObject * -builtin_id(PyModuleDef *module, PyObject *v) -/*[clinic end generated code: output=63635e497e09c2f7 input=57fb4a9aaff96384]*/ +builtin_id(PyModuleDef *self, PyObject *v) +/*[clinic end generated code: output=0aa640785f697f65 input=5a534136419631f4]*/ { return PyLong_FromVoidPtr(v); } -- cgit v0.12 From deab18dfd0e16a75686d99f243f38826385bc8d1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 May 2016 16:45:18 +0300 Subject: Issue #26708: Use the "const" qualifier for immutable strings. This can help to avoid unintentional modification. --- Modules/posixmodule.c | 76 ++++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 23b2a3c..9247843 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -810,8 +810,8 @@ typedef struct { const char *argument_name; int nullable; int allow_fd; - wchar_t *wide; - char *narrow; + const wchar_t *wide; + const char *narrow; int fd; Py_ssize_t length; PyObject *object; @@ -834,7 +834,7 @@ path_converter(PyObject *o, void *p) path_t *path = (path_t *)p; PyObject *bytes; Py_ssize_t length; - char *narrow; + const char *narrow; #define FORMAT_EXCEPTION(exc, fmt) \ PyErr_Format(exc, "%s%s" fmt, \ @@ -862,7 +862,7 @@ path_converter(PyObject *o, void *p) if (PyUnicode_Check(o)) { #ifdef MS_WINDOWS - wchar_t *wide; + const wchar_t *wide; wide = PyUnicode_AsUnicodeAndSize(o, &length); if (!wide) { @@ -1164,7 +1164,7 @@ convertenviron(void) for (e = _wenviron; *e != NULL; e++) { PyObject *k; PyObject *v; - wchar_t *p = wcschr(*e, L'='); + const wchar_t *p = wcschr(*e, L'='); if (p == NULL) continue; k = PyUnicode_FromWideChar(*e, (Py_ssize_t)(p-*e)); @@ -1192,7 +1192,7 @@ convertenviron(void) for (e = environ; *e != NULL; e++) { PyObject *k; PyObject *v; - char *p = strchr(*e, '='); + const char *p = strchr(*e, '='); if (p == NULL) continue; k = PyBytes_FromStringAndSize(*e, (int)(p-*e)); @@ -3483,7 +3483,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (!path->narrow) { WIN32_FIND_DATAW wFileData; - wchar_t *po_wchars; + const wchar_t *po_wchars; if (!path->wide) { /* Default arg: "." */ po_wchars = L"."; @@ -3649,7 +3649,7 @@ _posix_listdir(path_t *path, PyObject *list) else #endif { - char *name; + const char *name; if (path->narrow) { name = path->narrow; /* only return bytes if they specified a bytes object */ @@ -3831,7 +3831,7 @@ os__getfinalpathname_impl(PyModuleDef *module, PyObject *path) wchar_t *target_path; int result_length; PyObject *result; - wchar_t *path_wchar; + const wchar_t *path_wchar; path_wchar = PyUnicode_AsUnicode(path); if (path_wchar == NULL) @@ -3920,7 +3920,8 @@ os__getvolumepathname_impl(PyModuleDef *module, PyObject *path) /*[clinic end generated code: output=79a0ba729f956dbe input=7eacadc40acbda6b]*/ { PyObject *result; - wchar_t *path_wchar, *mountpath=NULL; + const wchar_t *path_wchar; + wchar_t *mountpath=NULL; size_t buflen; BOOL ret; @@ -4118,7 +4119,7 @@ os_setpriority_impl(PyModuleDef *module, int which, int who, int priority) static PyObject * internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is_replace) { - char *function_name = is_replace ? "replace" : "rename"; + const char *function_name = is_replace ? "replace" : "rename"; int dir_fd_specified; #ifdef MS_WINDOWS @@ -4298,7 +4299,7 @@ os_system_impl(PyModuleDef *module, PyObject *command) /*[clinic end generated code: output=800f775e10b7be55 input=86a58554ba6094af]*/ { long result; - char *bytes = PyBytes_AsString(command); + const char *bytes = PyBytes_AsString(command); Py_BEGIN_ALLOW_THREADS result = system(bytes); Py_END_ALLOW_THREADS @@ -4937,7 +4938,8 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) Py_ssize_t i, pos, envc; PyObject *keys=NULL, *vals=NULL; PyObject *key, *val, *key2, *val2; - char *p, *k, *v; + char *p; + const char *k, *v; size_t len; i = PyMapping_Size(env); @@ -5052,7 +5054,7 @@ static PyObject * os_execv_impl(PyModuleDef *module, PyObject *path, PyObject *argv) /*[clinic end generated code: output=9221f08143146fff input=96041559925e5229]*/ { - char *path_char; + const char *path_char; char **argvlist; Py_ssize_t argc; @@ -5173,7 +5175,7 @@ static PyObject * os_spawnv_impl(PyModuleDef *module, int mode, PyObject *path, PyObject *argv) /*[clinic end generated code: output=140a7945484c8cc5 input=042c91dfc1e6debc]*/ { - char *path_char; + const char *path_char; char **argvlist; int i; Py_ssize_t argc; @@ -5251,7 +5253,7 @@ os_spawnve_impl(PyModuleDef *module, int mode, PyObject *path, PyObject *argv, PyObject *env) /*[clinic end generated code: output=e7f5f0703610531f input=02362fd937963f8f]*/ { - char *path_char; + const char *path_char; char **argvlist; char **envlist; PyObject *res = NULL; @@ -6264,7 +6266,7 @@ static PyObject * posix_initgroups(PyObject *self, PyObject *args) { PyObject *oname; - char *username; + const char *username; int res; #ifdef __APPLE__ int gid; @@ -7138,16 +7140,16 @@ exit: static PyObject * win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) { - wchar_t *path; + const wchar_t *path; DWORD n_bytes_returned; DWORD io_result; PyObject *po, *result; - int dir_fd; + int dir_fd; HANDLE reparse_point_handle; char target_buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; REPARSE_DATA_BUFFER *rdb = (REPARSE_DATA_BUFFER *)target_buffer; - wchar_t *print_name; + const wchar_t *print_name; static char *keywords[] = {"path", "dir_fd", NULL}; @@ -7215,8 +7217,8 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) #if defined(MS_WINDOWS) /* Grab CreateSymbolicLinkW dynamically from kernel32 */ -static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPWSTR, LPWSTR, DWORD) = NULL; -static DWORD (CALLBACK *Py_CreateSymbolicLinkA)(LPSTR, LPSTR, DWORD) = NULL; +static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; +static DWORD (CALLBACK *Py_CreateSymbolicLinkA)(LPCSTR, LPCSTR, DWORD) = NULL; static int check_CreateSymbolicLink(void) @@ -7321,7 +7323,7 @@ _joinA(char *dest_path, const char *root, const char *rest) /* Return True if the path at src relative to dest is a directory */ static int -_check_dirW(WCHAR *src, WCHAR *dest) +_check_dirW(LPCWSTR src, LPCWSTR dest) { WIN32_FILE_ATTRIBUTE_DATA src_info; WCHAR dest_parent[MAX_PATH]; @@ -7340,7 +7342,7 @@ _check_dirW(WCHAR *src, WCHAR *dest) /* Return True if the path at src relative to dest is a directory */ static int -_check_dirA(const char *src, char *dest) +_check_dirA(LPCSTR src, LPCSTR dest) { WIN32_FILE_ATTRIBUTE_DATA src_info; char dest_parent[MAX_PATH]; @@ -9030,7 +9032,7 @@ static PyObject * os_putenv_impl(PyModuleDef *module, PyObject *name, PyObject *value) /*[clinic end generated code: output=a2438cf95e5a0c1c input=ba586581c2e6105f]*/ { - wchar_t *env; + const wchar_t *env; PyObject *unicode = PyUnicode_FromFormat("%U=%U", name, value); if (unicode == NULL) { @@ -9076,8 +9078,8 @@ os_putenv_impl(PyModuleDef *module, PyObject *name, PyObject *value) { PyObject *bytes = NULL; char *env; - char *name_string = PyBytes_AsString(name); - char *value_string = PyBytes_AsString(value); + const char *name_string = PyBytes_AsString(name); + const char *value_string = PyBytes_AsString(value); bytes = PyBytes_FromFormat("%s=%s", name_string, value_string); if (bytes == NULL) { @@ -10469,7 +10471,7 @@ cmp_constdefs(const void *v1, const void *v2) static int setup_confname_table(struct constdef *table, size_t tablesize, - char *tablename, PyObject *module) + const char *tablename, PyObject *module) { PyObject *d = NULL; size_t i; @@ -10596,9 +10598,9 @@ static PyObject * win32_startfile(PyObject *self, PyObject *args) { PyObject *ofilepath; - char *filepath; - char *operation = NULL; - wchar_t *wpath, *woperation; + const char *filepath; + const char *operation = NULL; + const wchar_t *wpath, *woperation; HINSTANCE rc; PyObject *unipath, *uoperation = NULL; @@ -11003,7 +11005,7 @@ os_listxattr_impl(PyModuleDef *module, path_t *path, int follow_symlinks) name = path->narrow ? path->narrow : "."; for (i = 0; ; i++) { - char *start, *trace, *end; + const char *start, *trace, *end; ssize_t length; static const Py_ssize_t buffer_sizes[] = { 256, XATTR_LIST_MAX, 0 }; Py_ssize_t buffer_size = buffer_sizes[i]; @@ -11482,7 +11484,7 @@ DirEntry_fetch_stat(DirEntry *self, int follow_symlinks) struct _Py_stat_struct st; #ifdef MS_WINDOWS - wchar_t *path; + const wchar_t *path; path = PyUnicode_AsUnicode(self->path); if (!path) @@ -11499,7 +11501,7 @@ DirEntry_fetch_stat(DirEntry *self, int follow_symlinks) } #else /* POSIX */ PyObject *bytes; - char *path; + const char *path; if (!PyUnicode_FSConverter(self->path, &bytes)) return NULL; @@ -11683,7 +11685,7 @@ DirEntry_inode(DirEntry *self) { #ifdef MS_WINDOWS if (!self->got_file_index) { - wchar_t *path; + const wchar_t *path; struct _Py_stat_struct stat; path = PyUnicode_AsUnicode(self->path); @@ -11777,7 +11779,7 @@ static PyTypeObject DirEntryType = { #ifdef MS_WINDOWS static wchar_t * -join_path_filenameW(wchar_t *path_wide, wchar_t* filename) +join_path_filenameW(const wchar_t *path_wide, const wchar_t *filename) { Py_ssize_t path_len; Py_ssize_t size; @@ -12208,7 +12210,7 @@ posix_scandir(PyObject *self, PyObject *args, PyObject *kwargs) #ifdef MS_WINDOWS wchar_t *path_strW; #else - char *path; + const char *path; #endif iterator = PyObject_New(ScandirIterator, &ScandirIteratorType); -- cgit v0.12 From 0ce9cd985b7c8a13de538c071e44856f5c7001d2 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 7 May 2016 20:39:20 +0300 Subject: Issue #26924: Do not define _multiprocessing.sem_unlink under Android Android declares sem_unlink but doesn't implement it. --- Modules/_multiprocessing/multiprocessing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 4ae638e..f6938f9 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -128,7 +128,7 @@ static PyMethodDef module_methods[] = { {"recv", multiprocessing_recv, METH_VARARGS, ""}, {"send", multiprocessing_send, METH_VARARGS, ""}, #endif -#ifndef POSIX_SEMAPHORES_NOT_ENABLED +#if defined(HAVE_SEM_UNLINK) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) {"sem_unlink", _PyMp_sem_unlink, METH_VARARGS, ""}, #endif {NULL} -- cgit v0.12 From 1a269d09f65c69d9deecf943472335becd63ff06 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 7 May 2016 21:13:50 +0300 Subject: Issue #26924: Fix Windows buildbots sem_unlink is defined as #define SEM_UNLINK(name) 0 under Windows. --- Modules/_multiprocessing/multiprocessing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index f6938f9..d92a8bf 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -128,7 +128,7 @@ static PyMethodDef module_methods[] = { {"recv", multiprocessing_recv, METH_VARARGS, ""}, {"send", multiprocessing_send, METH_VARARGS, ""}, #endif -#if defined(HAVE_SEM_UNLINK) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) {"sem_unlink", _PyMp_sem_unlink, METH_VARARGS, ""}, #endif {NULL} -- cgit v0.12 From cc22984d9e5e5357212db2eb1841f93e75fe75f5 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Sun, 8 May 2016 22:14:38 +1000 Subject: Issue 26977, remove unneeded line in pvariance (duplicate call to _ss). --- Lib/statistics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index af5d41e..fc0c3d2 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -593,7 +593,6 @@ def pvariance(data, mu=None): n = len(data) if n < 1: raise StatisticsError('pvariance requires at least one data point') - ss = _ss(data, mu) T, ss = _ss(data, mu) return _convert(ss/n, T) -- cgit v0.12 From ce41287e996351735ec1564bc1d701dd9057bdcf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 May 2016 23:36:44 +0300 Subject: Issue #18531: Single var-keyword argument of dict subtype was passed unscathed to the C-defined function. Now it is converted to exact dict. --- Lib/test/test_getargs2.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_testcapimodule.c | 22 +++++++++++++++++++ Python/ceval.c | 2 +- 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index be777af..6e2fa79 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -70,6 +70,12 @@ class BadInt3(int): def __int__(self): return True +class TupleSubclass(tuple): + pass + +class DictSubclass(dict): + pass + class Unsigned_TestCase(unittest.TestCase): def test_b(self): @@ -321,6 +327,33 @@ class Boolean_TestCase(unittest.TestCase): class Tuple_TestCase(unittest.TestCase): + def test_args(self): + from _testcapi import get_args + + ret = get_args(1, 2) + self.assertEqual(ret, (1, 2)) + self.assertIs(type(ret), tuple) + + ret = get_args(1, *(2, 3)) + self.assertEqual(ret, (1, 2, 3)) + self.assertIs(type(ret), tuple) + + ret = get_args(*[1, 2]) + self.assertEqual(ret, (1, 2)) + self.assertIs(type(ret), tuple) + + ret = get_args(*TupleSubclass([1, 2])) + self.assertEqual(ret, (1, 2)) + self.assertIs(type(ret), tuple) + + ret = get_args() + self.assertIn(ret, ((), None)) + self.assertIn(type(ret), (tuple, type(None))) + + ret = get_args(*()) + self.assertIn(ret, ((), None)) + self.assertIn(type(ret), (tuple, type(None))) + def test_tuple(self): from _testcapi import getargs_tuple @@ -336,6 +369,29 @@ class Tuple_TestCase(unittest.TestCase): self.assertRaises(TypeError, getargs_tuple, 1, seq()) class Keywords_TestCase(unittest.TestCase): + def test_kwargs(self): + from _testcapi import get_kwargs + + ret = get_kwargs(a=1, b=2) + self.assertEqual(ret, {'a': 1, 'b': 2}) + self.assertIs(type(ret), dict) + + ret = get_kwargs(a=1, **{'b': 2, 'c': 3}) + self.assertEqual(ret, {'a': 1, 'b': 2, 'c': 3}) + self.assertIs(type(ret), dict) + + ret = get_kwargs(**DictSubclass({'a': 1, 'b': 2})) + self.assertEqual(ret, {'a': 1, 'b': 2}) + self.assertIs(type(ret), dict) + + ret = get_kwargs() + self.assertIn(ret, ({}, None)) + self.assertIn(type(ret), (dict, type(None))) + + ret = get_kwargs(**{}) + self.assertIn(ret, ({}, None)) + self.assertIn(type(ret), (dict, type(None))) + def test_positional_args(self): # using all positional args self.assertEqual( diff --git a/Misc/NEWS b/Misc/NEWS index 727f84e..63cdce2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #18531: Single var-keyword argument of dict subtype was passed + unscathed to the C-defined function. Now it is converted to exact dict. + - Issue #26811: gc.get_objects() no longer contains a broken tuple with NULL pointer. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 8c79485..5083be6 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -872,6 +872,26 @@ test_L_code(PyObject *self) #endif /* ifdef HAVE_LONG_LONG */ +static PyObject * +get_args(PyObject *self, PyObject *args) +{ + if (args == NULL) { + args = Py_None; + } + Py_INCREF(args); + return args; +} + +static PyObject * +get_kwargs(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (kwargs == NULL) { + kwargs = Py_None; + } + Py_INCREF(kwargs); + return kwargs; +} + /* Test tuple argument processing */ static PyObject * getargs_tuple(PyObject *self, PyObject *args) @@ -3784,6 +3804,8 @@ static PyMethodDef TestMethods[] = { {"test_pep3118_obsolete_write_locks", (PyCFunction)test_pep3118_obsolete_write_locks, METH_NOARGS}, #endif {"getbuffer_with_null_view", getbuffer_with_null_view, METH_O}, + {"get_args", get_args, METH_VARARGS}, + {"get_kwargs", (PyCFunction)get_kwargs, METH_VARARGS|METH_KEYWORDS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, {"getargs_keywords", (PyCFunction)getargs_keywords, METH_VARARGS|METH_KEYWORDS}, diff --git a/Python/ceval.c b/Python/ceval.c index 88dc113..b461030 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4993,7 +4993,7 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) if (flags & CALL_FLAG_KW) { kwdict = EXT_POP(*pp_stack); - if (!PyDict_Check(kwdict)) { + if (!PyDict_CheckExact(kwdict)) { PyObject *d; d = PyDict_New(); if (d == NULL) -- cgit v0.12 From d65018b17c6c794a0c91e0e53fb691d0f666cb5d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 10 May 2016 15:29:05 -0600 Subject: Fixes #19711: Add tests for reloading namespace packages. --- Lib/test/test_importlib/test_namespace_pkgs.py | 34 +++++++++++++++++++++++++- Misc/NEWS | 2 ++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 116eb75..e37d8a1 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -1,4 +1,5 @@ import contextlib +import importlib import os import sys import unittest @@ -67,6 +68,7 @@ class NamespacePackageTest(unittest.TestCase): # TODO: will we ever want to pass exc_info to __exit__? self.ctx.__exit__(None, None, None) + class SingleNamespacePackage(NamespacePackageTest): paths = ['portion1'] @@ -83,7 +85,7 @@ class SingleNamespacePackage(NamespacePackageTest): self.assertEqual(repr(foo), "") -class DynamicPatheNamespacePackage(NamespacePackageTest): +class DynamicPathNamespacePackage(NamespacePackageTest): paths = ['portion1'] def test_dynamic_path(self): @@ -285,5 +287,35 @@ class ModuleAndNamespacePackageInSameDir(NamespacePackageTest): self.assertEqual(a_test.attr, 'in module') +class ReloadTests(NamespacePackageTest): + paths = ['portion1'] + + def test_simple_package(self): + import foo.one + foo = importlib.reload(foo) + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_cant_import_other(self): + import foo + with self.assertRaises(ImportError): + import foo.two + foo = importlib.reload(foo) + with self.assertRaises(ImportError): + import foo.two + + def test_dynamic_path(self): + import foo.one + with self.assertRaises(ImportError): + import foo.two + + # Now modify sys.path and reload. + sys.path.append(os.path.join(self.root, 'portion2')) + foo = importlib.reload(foo) + + # And make sure foo.two is now importable + import foo.two + self.assertEqual(foo.two.attr, 'portion2 foo two') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 55963b0..07d36ec 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -52,6 +52,8 @@ Core and Builtins - Issue #26581: If coding cookie is specified multiple times on a line in Python source code file, only the first one is taken to account. +- Issue #19711: Add tests for reloading namespace packages. + - Issue #26563: Debug hooks on Python memory allocators now raise a fatal error if functions of the :c:func:`PyMem_Malloc` family are called without holding the GIL. -- cgit v0.12 From 441581a79ff9666380bea83ad26895f0d09b6695 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Wed, 11 May 2016 13:01:42 +1000 Subject: Update NEWS. --- Misc/NEWS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index cc06868..0ce1956 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -275,6 +275,14 @@ Library - Issue #26977: Removed unnecessary, and ignored, call to sum of squares helper in statistics.pvariance. +- Issue #26002: Use bisect in statistics.median instead of a linear search. + Patch by Upendra Kuma. + +- Issue #25974: Make use of new Decimal.as_integer_ratio() method in statistics + module. Patch by Stefan Krah. + +- Issue #26996: Add secrets module as described in PEP 506. + - Issue #26881: The modulefinder module now supports extended opcode arguments. - Issue #23815: Fixed crashes related to directly created instances of types in -- cgit v0.12 From 25885d1dc588ce82f4b05899705fb54cb8704a35 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 12 May 2016 10:21:14 +0300 Subject: Issue #27005: Optimized the float.fromhex() class method for exact float. --- Misc/NEWS | 3 +++ Objects/floatobject.c | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 87fa371..c5ad0f7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #27005: Optimized the float.fromhex() class method for exact float. + It is now 2 times faster. + - Issue #18531: Single var-keyword argument of dict subtype was passed unscathed to the C-defined function. Now it is converted to exact dict. diff --git a/Objects/floatobject.c b/Objects/floatobject.c index eb60659..f640dd3 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1195,7 +1195,7 @@ Return a hexadecimal representation of a floating-point number.\n\ static PyObject * float_fromhex(PyObject *cls, PyObject *arg) { - PyObject *result_as_float, *result; + PyObject *result; double x; long exp, top_exp, lsb, key_digit; char *s, *coeff_start, *s_store, *coeff_end, *exp_start, *s_end; @@ -1410,11 +1410,10 @@ float_fromhex(PyObject *cls, PyObject *arg) s++; if (s != s_end) goto parse_error; - result_as_float = Py_BuildValue("(d)", negate ? -x : x); - if (result_as_float == NULL) - return NULL; - result = PyObject_CallObject(cls, result_as_float); - Py_DECREF(result_as_float); + result = PyFloat_FromDouble(negate ? -x : x); + if (cls != (PyObject *)&PyFloat_Type && result != NULL) { + Py_SETREF(result, PyObject_CallFunctionObjArgs(cls, result)); + } return result; overflow_error: -- cgit v0.12 From 5787ef621a76dfe225308f0001d60e5e46e9a55f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 12 May 2016 10:32:30 +0300 Subject: Issue #27005: Fixed the call of PyObject_CallFunctionObjArgs(). --- Objects/floatobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index f640dd3..5b2742a 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1412,7 +1412,7 @@ float_fromhex(PyObject *cls, PyObject *arg) goto parse_error; result = PyFloat_FromDouble(negate ? -x : x); if (cls != (PyObject *)&PyFloat_Type && result != NULL) { - Py_SETREF(result, PyObject_CallFunctionObjArgs(cls, result)); + Py_SETREF(result, PyObject_CallFunctionObjArgs(cls, result, NULL)); } return result; -- cgit v0.12 From 228ab1ff6b5b02d8325a678b8afc140a153d4cf5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 10 May 2016 16:21:03 -0600 Subject: Issue #21099: Switch applicable importlib tests to use PEP 451 API. --- .../extension/test_case_sensitivity.py | 2 - Lib/test/test_importlib/extension/test_finder.py | 1 - Lib/test/test_importlib/import_/test_path.py | 68 +++++++++++++++++----- Lib/test/test_importlib/source/test_file_loader.py | 9 +-- Lib/test/test_importlib/source/test_path_hook.py | 11 +++- Lib/test/test_importlib/test_abc.py | 22 +++++-- Lib/test/test_importlib/test_api.py | 31 ++++++++-- Misc/NEWS | 2 + 8 files changed, 108 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 42ddb3d..b74e900 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -7,8 +7,6 @@ from .. import util machinery = util.import_importlib('importlib.machinery') -# XXX find_spec tests - @unittest.skipIf(util.EXTENSIONS.filename is None, '_testcapi not available') @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest: diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 71bf67f..c9b4a37 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -6,7 +6,6 @@ machinery = util.import_importlib('importlib.machinery') import unittest import warnings -# XXX find_spec tests class FinderTests(abc.FinderTests): diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index b32a876..7aa26b0 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -16,11 +16,14 @@ class FinderTests: """Tests for PathFinder.""" + find = None + check_found = None + def test_failure(self): # Test None returned upon not finding a suitable loader. module = '' with util.import_state(): - self.assertIsNone(self.machinery.PathFinder.find_module(module)) + self.assertIsNone(self.find(module)) def test_sys_path(self): # Test that sys.path is used when 'path' is None. @@ -30,8 +33,8 @@ class FinderTests: importer = util.mock_spec(module) with util.import_state(path_importer_cache={path: importer}, path=[path]): - loader = self.machinery.PathFinder.find_module(module) - self.assertIs(loader, importer) + found = self.find(module) + self.check_found(found, importer) def test_path(self): # Test that 'path' is used when set. @@ -40,8 +43,8 @@ class FinderTests: path = '' importer = util.mock_spec(module) with util.import_state(path_importer_cache={path: importer}): - loader = self.machinery.PathFinder.find_module(module, [path]) - self.assertIs(loader, importer) + found = self.find(module, [path]) + self.check_found(found, importer) def test_empty_list(self): # An empty list should not count as asking for sys.path. @@ -50,7 +53,7 @@ class FinderTests: importer = util.mock_spec(module) with util.import_state(path_importer_cache={path: importer}, path=[path]): - self.assertIsNone(self.machinery.PathFinder.find_module('module', [])) + self.assertIsNone(self.find('module', [])) def test_path_hooks(self): # Test that sys.path_hooks is used. @@ -60,8 +63,8 @@ class FinderTests: importer = util.mock_spec(module) hook = util.mock_path_hook(path, importer=importer) with util.import_state(path_hooks=[hook]): - loader = self.machinery.PathFinder.find_module(module, [path]) - self.assertIs(loader, importer) + found = self.find(module, [path]) + self.check_found(found, importer) self.assertIn(path, sys.path_importer_cache) self.assertIs(sys.path_importer_cache[path], importer) @@ -73,7 +76,7 @@ class FinderTests: path=[path_entry]): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - self.assertIsNone(self.machinery.PathFinder.find_module('os')) + self.assertIsNone(self.find('os')) self.assertIsNone(sys.path_importer_cache[path_entry]) self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, ImportWarning)) @@ -85,8 +88,8 @@ class FinderTests: importer = util.mock_spec(module) hook = util.mock_path_hook(os.getcwd(), importer=importer) with util.import_state(path=[path], path_hooks=[hook]): - loader = self.machinery.PathFinder.find_module(module) - self.assertIs(loader, importer) + found = self.find(module) + self.check_found(found, importer) self.assertIn(os.getcwd(), sys.path_importer_cache) def test_None_on_sys_path(self): @@ -182,16 +185,33 @@ class FinderTests: self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) +class FindModuleTests(FinderTests): + def find(self, *args, **kwargs): + return self.machinery.PathFinder.find_module(*args, **kwargs) + def check_found(self, found, importer): + self.assertIs(found, importer) + + +(Frozen_FindModuleTests, + Source_FindModuleTests +) = util.test_both(FindModuleTests, importlib=importlib, machinery=machinery) -(Frozen_FinderTests, - Source_FinderTests - ) = util.test_both(FinderTests, importlib=importlib, machinery=machinery) +class FindSpecTests(FinderTests): + def find(self, *args, **kwargs): + return self.machinery.PathFinder.find_spec(*args, **kwargs) + def check_found(self, found, importer): + self.assertIs(found.loader, importer) + + +(Frozen_FindSpecTests, + Source_FindSpecTests + ) = util.test_both(FindSpecTests, importlib=importlib, machinery=machinery) class PathEntryFinderTests: - def test_finder_with_failing_find_module(self): + def test_finder_with_failing_find_spec(self): # PathEntryFinder with find_module() defined should work. # Issue #20763. class Finder: @@ -209,6 +229,24 @@ class PathEntryFinderTests: path_hooks=[Finder]): self.machinery.PathFinder.find_spec('importlib') + def test_finder_with_failing_find_module(self): + # PathEntryFinder with find_module() defined should work. + # Issue #20763. + class Finder: + path_location = 'test_finder_with_find_module' + def __init__(self, path): + if path != self.path_location: + raise ImportError + + @staticmethod + def find_module(fullname): + return None + + + with util.import_state(path=[Finder.path_location]+sys.path[:], + path_hooks=[Finder]): + self.machinery.PathFinder.find_module('importlib') + (Frozen_PEFTests, Source_PEFTests diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 73f4c62..a151149 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -217,7 +217,7 @@ class SimpleTest(abc.LoaderTests): # PEP 302 with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) - mod = loader.load_module('_temp') # XXX + mod = loader.load_module('_temp') # Sanity checks. self.assertEqual(mod.__cached__, compiled) self.assertEqual(mod.x, 5) @@ -245,12 +245,7 @@ class SimpleTest(abc.LoaderTests): class BadBytecodeTest: def import_(self, file, module_name): - loader = self.loader(module_name, file) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - # XXX Change to use exec_module(). - module = loader.load_module(module_name) - self.assertIn(module_name, sys.modules) + raise NotImplementedError def manipulate_bytecode(self, name, mapping, manipulator, *, del_source=False): diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index e6a2415..795d436 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -16,10 +16,19 @@ class PathHookTest: def test_success(self): with util.create_modules('dummy') as mapping: self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_module')) + 'find_spec')) + + def test_success_legacy(self): + with util.create_modules('dummy') as mapping: + self.assertTrue(hasattr(self.path_hook()(mapping['.root']), + 'find_module')) def test_empty_string(self): # The empty string represents the cwd. + self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) + + def test_empty_string_legacy(self): + # The empty string represents the cwd. self.assertTrue(hasattr(self.path_hook()(''), 'find_module')) diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index d4bf915..c862480 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -207,6 +207,10 @@ class LoaderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(Loader) + def test_create_module(self): + spec = 'a spec' + self.assertIsNone(self.ins.create_module(spec)) + def test_load_module(self): with self.assertRaises(ImportError): self.ins.load_module('something') @@ -519,6 +523,12 @@ class InspectLoaderLoadModuleTests: support.unload(self.module_name) self.addCleanup(support.unload, self.module_name) + def load(self, loader): + spec = self.util.spec_from_loader(self.module_name, loader) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return self.init._bootstrap._load_unlocked(spec) + def mock_get_code(self): return mock.patch.object(self.InspectLoaderSubclass, 'get_code') @@ -528,9 +538,7 @@ class InspectLoaderLoadModuleTests: mocked_get_code.side_effect = ImportError with self.assertRaises(ImportError): loader = self.InspectLoaderSubclass() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module(self.module_name) + self.load(loader) def test_get_code_None(self): # If get_code() returns None, raise ImportError. @@ -538,7 +546,7 @@ class InspectLoaderLoadModuleTests: mocked_get_code.return_value = None with self.assertRaises(ImportError): loader = self.InspectLoaderSubclass() - loader.load_module(self.module_name) + self.load(loader) def test_module_returned(self): # The loaded module should be returned. @@ -546,14 +554,16 @@ class InspectLoaderLoadModuleTests: with self.mock_get_code() as mocked_get_code: mocked_get_code.return_value = code loader = self.InspectLoaderSubclass() - module = loader.load_module(self.module_name) + module = self.load(loader) self.assertEqual(module, sys.modules[self.module_name]) (Frozen_ILLoadModuleTests, Source_ILLoadModuleTests ) = test_util.test_both(InspectLoaderLoadModuleTests, - InspectLoaderSubclass=SPLIT_IL) + InspectLoaderSubclass=SPLIT_IL, + init=init, + util=util) ##### ExecutionLoader concrete methods ######################################### diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 6bc3c56..b0a94aa 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -99,9 +99,7 @@ class ImportModuleTests: class FindLoaderTests: - class FakeMetaFinder: - @staticmethod - def find_module(name, path=None): return name, path + FakeMetaFinder = None def test_sys_modules(self): # If a module with __loader__ is in sys.modules, then return it. @@ -171,9 +169,30 @@ class FindLoaderTests: self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule')) -(Frozen_FindLoaderTests, - Source_FindLoaderTests - ) = test_util.test_both(FindLoaderTests, init=init) +class FindLoaderPEP451Tests(FindLoaderTests): + + class FakeMetaFinder: + @staticmethod + def find_spec(name, path=None, target=None): + return machinery['Source'].ModuleSpec(name, (name, path)) + + +(Frozen_FindLoaderPEP451Tests, + Source_FindLoaderPEP451Tests + ) = test_util.test_both(FindLoaderPEP451Tests, init=init) + + +class FindLoaderPEP302Tests(FindLoaderTests): + + class FakeMetaFinder: + @staticmethod + def find_module(name, path=None): + return name, path + + +(Frozen_FindLoaderPEP302Tests, + Source_FindLoaderPEP302Tests + ) = test_util.test_both(FindLoaderPEP302Tests, init=init) class ReloadTests: diff --git a/Misc/NEWS b/Misc/NEWS index 15d278b..8d469e2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -59,6 +59,8 @@ Core and Builtins - Issue #19711: Add tests for reloading namespace packages. +- Issue #21099: Switch applicable importlib tests to use PEP 451 API. + - Issue #26563: Debug hooks on Python memory allocators now raise a fatal error if functions of the :c:func:`PyMem_Malloc` family are called without holding the GIL. -- cgit v0.12 From 18ee29d0b870caddc0806916ca2c823254f1a1f9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 13 May 2016 13:52:49 +0300 Subject: Issue #26039: zipfile.ZipFile.open() can now be used to write data into a ZIP file, as well as for extracting data. Patch by Thomas Kluyver. --- Doc/library/zipfile.rst | 42 ++++--- Doc/whatsnew/3.6.rst | 4 + Lib/test/test_zipfile.py | 83 +++++++++++++- Lib/zipfile.py | 290 ++++++++++++++++++++++++++++++----------------- Misc/NEWS | 3 + 5 files changed, 295 insertions(+), 127 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 19b354d..cf8d547 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -207,15 +207,15 @@ ZipFile Objects .. index:: single: universal newlines; zipfile.ZipFile.open method -.. method:: ZipFile.open(name, mode='r', pwd=None) +.. method:: ZipFile.open(name, mode='r', pwd=None, force_zip64=False) - Extract a member from the archive as a file-like object (ZipExtFile). *name* - is the name of the file in the archive, or a :class:`ZipInfo` object. The + Access a member of the archive as a file-like object. *name* + is the name of the file in the archive, or a :class:`ZipInfo` object. The *mode* parameter, if included, must be one of the following: ``'r'`` (the - default), ``'U'``, or ``'rU'``. Choosing ``'U'`` or ``'rU'`` will enable - :term:`universal newlines` support in the read-only object. *pwd* is the - password used for encrypted files. Calling :meth:`.open` on a closed - ZipFile will raise a :exc:`RuntimeError`. + default), ``'U'``, ``'rU'`` or ``'w'``. Choosing ``'U'`` or ``'rU'`` will + enable :term:`universal newlines` support in the read-only object. *pwd* is + the password used to decrypt encrypted ZIP files. Calling :meth:`.open` on + a closed ZipFile will raise a :exc:`RuntimeError`. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -224,17 +224,23 @@ ZipFile Objects with myzip.open('eggs.txt') as myfile: print(myfile.read()) - .. note:: - - The file-like object is read-only and provides the following methods: - :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, - :meth:`~io.IOBase.readlines`, :meth:`__iter__`, - :meth:`~iterator.__next__`. + With *mode* ``'r'``, ``'U'`` or ``'rU'``, the file-like object + (``ZipExtFile``) is read-only and provides the following methods: + :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, + :meth:`~io.IOBase.readlines`, :meth:`__iter__`, + :meth:`~iterator.__next__`. These objects can operate independently of + the ZipFile. - .. note:: + With ``mode='w'``, a writable file handle is returned, which supports the + :meth:`~io.BufferedIOBase.write` method. While a writable file handle is open, + attempting to read or write other files in the ZIP file will raise a + :exc:`RuntimeError`. - Objects returned by :meth:`.open` can operate independently of the - ZipFile. + When writing a file, if the file size is not known in advance but may exceed + 2 GiB, pass ``force_zip64=True`` to ensure that the header format is + capable of supporting large files. If the file size is known in advance, + construct a :class:`ZipInfo` object with :attr:`~ZipInfo.file_size` set, and + use that as the *name* parameter. .. note:: @@ -246,6 +252,10 @@ ZipFile Objects The ``'U'`` or ``'rU'`` mode. Use :class:`io.TextIOWrapper` for reading compressed text files in :term:`universal newlines` mode. + .. versionchanged:: 3.6 + :meth:`open` can now be used to write files into the archive with the + ``mode='w'`` option. + .. method:: ZipFile.extract(member, path=None, pwd=None) Extract a member from the archive to the current working directory; *member* diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index be4c014..384e35a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -350,6 +350,10 @@ A new :meth:`ZipInfo.is_dir() ` method can be used to check if the :class:`~zipfile.ZipInfo` instance represents a directory. (Contributed by Thomas Kluyver in :issue:`26039`.) +The :meth:`ZipFile.open() ` method can now be used to +write data into a ZIP file, as well as for extracting data. +(Contributed by Thomas Kluyver in :issue:`26039`.) + zlib ---- diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 61f561b..b339955 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -61,6 +61,9 @@ class AbstractTestsWithSourceFile: zipfp.write(TESTFN, "another.name") zipfp.write(TESTFN, TESTFN) zipfp.writestr("strfile", self.data) + with zipfp.open('written-open-w', mode='w') as f: + for line in self.line_gen: + f.write(line) def zip_test(self, f, compression): self.make_test_archive(f, compression) @@ -76,7 +79,7 @@ class AbstractTestsWithSourceFile: zipfp.printdir(file=fp) directory = fp.getvalue() lines = directory.splitlines() - self.assertEqual(len(lines), 4) # Number of files + header + self.assertEqual(len(lines), 5) # Number of files + header self.assertIn('File Name', lines[0]) self.assertIn('Modified', lines[0]) @@ -90,23 +93,25 @@ class AbstractTestsWithSourceFile: # Check the namelist names = zipfp.namelist() - self.assertEqual(len(names), 3) + self.assertEqual(len(names), 4) self.assertIn(TESTFN, names) self.assertIn("another.name", names) self.assertIn("strfile", names) + self.assertIn("written-open-w", names) # Check infolist infos = zipfp.infolist() names = [i.filename for i in infos] - self.assertEqual(len(names), 3) + self.assertEqual(len(names), 4) self.assertIn(TESTFN, names) self.assertIn("another.name", names) self.assertIn("strfile", names) + self.assertIn("written-open-w", names) for i in infos: self.assertEqual(i.file_size, len(self.data)) # check getinfo - for nm in (TESTFN, "another.name", "strfile"): + for nm in (TESTFN, "another.name", "strfile", "written-open-w"): info = zipfp.getinfo(nm) self.assertEqual(info.filename, nm) self.assertEqual(info.file_size, len(self.data)) @@ -372,14 +377,18 @@ class StoredTestsWithSourceFile(AbstractTestsWithSourceFile, test_low_compression = None def zip_test_writestr_permissions(self, f, compression): - # Make sure that writestr creates files with mode 0600, - # when it is passed a name rather than a ZipInfo instance. + # Make sure that writestr and open(... mode='w') create files with + # mode 0600, when they are passed a name rather than a ZipInfo + # instance. self.make_test_archive(f, compression) with zipfile.ZipFile(f, "r") as zipfp: zinfo = zipfp.getinfo('strfile') self.assertEqual(zinfo.external_attr, 0o600 << 16) + zinfo2 = zipfp.getinfo('written-open-w') + self.assertEqual(zinfo2.external_attr, 0o600 << 16) + def test_writestr_permissions(self): for f in get_files(self): self.zip_test_writestr_permissions(f, zipfile.ZIP_STORED) @@ -451,6 +460,10 @@ class StoredTestsWithSourceFile(AbstractTestsWithSourceFile, with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: self.assertRaises(RuntimeError, zipfp.write, TESTFN) + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with self.assertRaises(RuntimeError): + zipfp.open(TESTFN, mode='w') + def test_add_file_before_1980(self): # Set atime and mtime to 1970-01-01 os.utime(TESTFN, (0, 0)) @@ -1428,6 +1441,35 @@ class OtherTests(unittest.TestCase): # testzip returns the name of the first corrupt file, or None self.assertIsNone(zipf.testzip()) + def test_open_conflicting_handles(self): + # It's only possible to open one writable file handle at a time + msg1 = b"It's fun to charter an accountant!" + msg2 = b"And sail the wide accountant sea" + msg3 = b"To find, explore the funds offshore" + with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_STORED) as zipf: + with zipf.open('foo', mode='w') as w2: + w2.write(msg1) + with zipf.open('bar', mode='w') as w1: + with self.assertRaises(RuntimeError): + zipf.open('handle', mode='w') + with self.assertRaises(RuntimeError): + zipf.open('foo', mode='r') + with self.assertRaises(RuntimeError): + zipf.writestr('str', 'abcde') + with self.assertRaises(RuntimeError): + zipf.write(__file__, 'file') + with self.assertRaises(RuntimeError): + zipf.close() + w1.write(msg2) + with zipf.open('baz', mode='w') as w2: + w2.write(msg3) + + with zipfile.ZipFile(TESTFN2, 'r') as zipf: + self.assertEqual(zipf.read('foo'), msg1) + self.assertEqual(zipf.read('bar'), msg2) + self.assertEqual(zipf.read('baz'), msg3) + self.assertEqual(zipf.namelist(), ['foo', 'bar', 'baz']) + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) @@ -1761,6 +1803,22 @@ class UnseekableTests(unittest.TestCase): with zipf.open('twos') as zopen: self.assertEqual(zopen.read(), b'222') + def test_open_write(self): + for wrapper in (lambda f: f), Tellable, Unseekable: + with self.subTest(wrapper=wrapper): + f = io.BytesIO() + f.write(b'abc') + bf = io.BufferedWriter(f) + with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipf: + with zipf.open('ones', 'w') as zopen: + zopen.write(b'111') + with zipf.open('twos', 'w') as zopen: + zopen.write(b'222') + self.assertEqual(f.getvalue()[:5], b'abcPK') + with zipfile.ZipFile(f) as zipf: + self.assertEqual(zipf.read('ones'), b'111') + self.assertEqual(zipf.read('twos'), b'222') + @requires_zlib class TestsWithMultipleOpens(unittest.TestCase): @@ -1870,6 +1928,19 @@ class TestsWithMultipleOpens(unittest.TestCase): with open(os.devnull) as f: self.assertLess(f.fileno(), 100) + def test_write_while_reading(self): + with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf: + zipf.writestr('ones', self.data1) + with zipfile.ZipFile(TESTFN2, 'a', zipfile.ZIP_DEFLATED) as zipf: + with zipf.open('ones', 'r') as r1: + data1 = r1.read(500) + with zipf.open('twos', 'w') as w1: + w1.write(self.data2) + data1 += r1.read() + self.assertEqual(data1, self.data1) + with zipfile.ZipFile(TESTFN2) as zipf: + self.assertEqual(zipf.read('twos'), self.data2) + def tearDown(self): unlink(TESTFN2) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index e0598d2..03dead5 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -686,14 +686,19 @@ def _get_decompressor(compress_type): class _SharedFile: - def __init__(self, file, pos, close, lock): + def __init__(self, file, pos, close, lock, writing): self._file = file self._pos = pos self._close = close self._lock = lock + self._writing = writing def read(self, n=-1): with self._lock: + if self._writing(): + raise RuntimeError("Can't read from the ZIP file while there " + "is an open writing handle on it. " + "Close the writing handle before trying to read.") self._file.seek(self._pos) data = self._file.read(n) self._pos = self._file.tell() @@ -993,6 +998,76 @@ class ZipExtFile(io.BufferedIOBase): super().close() +class _ZipWriteFile(io.BufferedIOBase): + def __init__(self, zf, zinfo, zip64): + self._zinfo = zinfo + self._zip64 = zip64 + self._zipfile = zf + self._compressor = _get_compressor(zinfo.compress_type) + self._file_size = 0 + self._compress_size = 0 + self._crc = 0 + + @property + def _fileobj(self): + return self._zipfile.fp + + def writable(self): + return True + + def write(self, data): + nbytes = len(data) + self._file_size += nbytes + self._crc = crc32(data, self._crc) + if self._compressor: + data = self._compressor.compress(data) + self._compress_size += len(data) + self._fileobj.write(data) + return nbytes + + def close(self): + super().close() + # Flush any data from the compressor, and update header info + if self._compressor: + buf = self._compressor.flush() + self._compress_size += len(buf) + self._fileobj.write(buf) + self._zinfo.compress_size = self._compress_size + else: + self._zinfo.compress_size = self._file_size + self._zinfo.CRC = self._crc + self._zinfo.file_size = self._file_size + + # Write updated header info + if self._zinfo.flag_bits & 0x08: + # Write CRC and file sizes after the file data + fmt = ' ZIP64_LIMIT: + raise RuntimeError('File size unexpectedly exceeded ZIP64 ' + 'limit') + if self._compress_size > ZIP64_LIMIT: + raise RuntimeError('Compressed size unexpectedly exceeded ' + 'ZIP64 limit') + # Seek backwards and write file header (which will now include + # correct CRC and file sizes) + + # Preserve current position in file + self._zipfile.start_dir = self._fileobj.tell() + self._fileobj.seek(self._zinfo.header_offset) + self._fileobj.write(self._zinfo.FileHeader(self._zip64)) + self._fileobj.seek(self._zipfile.start_dir) + + self._zipfile._writing = False + + # Successfully written: Add file to our caches + self._zipfile.filelist.append(self._zinfo) + self._zipfile.NameToInfo[self._zinfo.filename] = self._zinfo + class ZipFile: """ Class with methods to open, read, write, close, list zip files. @@ -1055,6 +1130,7 @@ class ZipFile: self._fileRefCnt = 1 self._lock = threading.RLock() self._seekable = True + self._writing = False try: if mode == 'r': @@ -1267,30 +1343,59 @@ class ZipFile: with self.open(name, "r", pwd) as fp: return fp.read() - def open(self, name, mode="r", pwd=None): - """Return file-like object for 'name'.""" - if mode not in ("r", "U", "rU"): - raise RuntimeError('open() requires mode "r", "U", or "rU"') + def open(self, name, mode="r", pwd=None, force_zip64=False): + """Return file-like object for 'name'. + + name is a string for the file name within the ZIP file, or a ZipInfo + object. + + mode should be 'r' to read a file already in the ZIP file, or 'w' to + write to a file newly added to the archive. + + pwd is the password to decrypt files (only used for reading). + + When writing, if the file size is not known in advance but may exceed + 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large + files. If the size is known in advance, it is best to pass a ZipInfo + instance for name, with zinfo.file_size set. + """ + if mode not in {"r", "w", "U", "rU"}: + raise RuntimeError('open() requires mode "r", "w", "U", or "rU"') if 'U' in mode: import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) if pwd and not isinstance(pwd, bytes): raise TypeError("pwd: expected bytes, got %s" % type(pwd)) + if pwd and (mode == "w"): + raise ValueError("pwd is only supported for reading files") if not self.fp: raise RuntimeError( - "Attempt to read ZIP archive that was already closed") + "Attempt to use ZIP archive that was already closed") # Make sure we have an info object if isinstance(name, ZipInfo): # 'name' is already an info object zinfo = name + elif mode == 'w': + zinfo = ZipInfo(name) + zinfo.compress_type = self.compression else: # Get info object for name zinfo = self.getinfo(name) + if mode == 'w': + return self._open_to_write(zinfo, force_zip64=force_zip64) + + if self._writing: + raise RuntimeError("Can't read from the ZIP file while there " + "is an open writing handle on it. " + "Close the writing handle before trying to read.") + + # Open for reading: self._fileRefCnt += 1 - zef_file = _SharedFile(self.fp, zinfo.header_offset, self._fpclose, self._lock) + zef_file = _SharedFile(self.fp, zinfo.header_offset, + self._fpclose, self._lock, lambda: self._writing) try: # Skip the file header: fheader = zef_file.read(sizeFileHeader) @@ -1355,6 +1460,49 @@ class ZipFile: zef_file.close() raise + def _open_to_write(self, zinfo, force_zip64=False): + if force_zip64 and not self._allowZip64: + raise ValueError( + "force_zip64 is True, but allowZip64 was False when opening " + "the ZIP file." + ) + if self._writing: + raise RuntimeError("Can't write to the ZIP file while there is " + "another write handle open on it. " + "Close the first handle before opening another.") + + # Sizes and CRC are overwritten with correct data after processing the file + if not hasattr(zinfo, 'file_size'): + zinfo.file_size = 0 + zinfo.compress_size = 0 + zinfo.CRC = 0 + + zinfo.flag_bits = 0x00 + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= 0x02 + if not self._seekable: + zinfo.flag_bits |= 0x08 + + if not zinfo.external_attr: + zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- + + # Compressed size can be larger than uncompressed size + zip64 = self._allowZip64 and \ + (force_zip64 or zinfo.file_size * 1.05 > ZIP64_LIMIT) + + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() + + self._writecheck(zinfo) + self._didModify = True + + self.fp.write(zinfo.FileHeader(zip64)) + + self._writing = True + return _ZipWriteFile(self, zinfo, zip64) + def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately @@ -1464,6 +1612,10 @@ class ZipFile: if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise RuntimeError( + "Can't write to ZIP archive while an open writing handle exists" + ) zinfo = ZipInfo.from_file(filename, arcname) @@ -1476,75 +1628,25 @@ class ZipFile: else: zinfo.compress_type = self.compression - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header bytes - if zinfo.compress_type == ZIP_LZMA: + if zinfo.is_dir(): + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() # Start of header bytes + if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 + zinfo.flag_bits |= 0x02 - self._writecheck(zinfo) - self._didModify = True + self._writecheck(zinfo) + self._didModify = True - if zinfo.is_dir(): self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) self.start_dir = self.fp.tell() - return - - cmpr = _get_compressor(zinfo.compress_type) - if not self._seekable: - zinfo.flag_bits |= 0x08 - with open(filename, "rb") as fp: - # Must overwrite CRC and sizes with correct data later - zinfo.CRC = CRC = 0 - zinfo.compress_size = compress_size = 0 - # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - zinfo.file_size * 1.05 > ZIP64_LIMIT - self.fp.write(zinfo.FileHeader(zip64)) - file_size = 0 - while 1: - buf = fp.read(1024 * 8) - if not buf: - break - file_size = file_size + len(buf) - CRC = crc32(buf, CRC) - if cmpr: - buf = cmpr.compress(buf) - compress_size = compress_size + len(buf) - self.fp.write(buf) - if cmpr: - buf = cmpr.flush() - compress_size = compress_size + len(buf) - self.fp.write(buf) - zinfo.compress_size = compress_size - else: - zinfo.compress_size = file_size - zinfo.CRC = CRC - zinfo.file_size = file_size - if zinfo.flag_bits & 0x08: - # Write CRC and file sizes after the file data - fmt = ' ZIP64_LIMIT: - raise RuntimeError('File size has increased during compressing') - if compress_size > ZIP64_LIMIT: - raise RuntimeError('Compressed size larger than uncompressed size') - # Seek backwards and write file header (which will now include - # correct CRC and file sizes) - self.start_dir = self.fp.tell() # Preserve current position in file - self.fp.seek(zinfo.header_offset) - self.fp.write(zinfo.FileHeader(zip64)) - self.fp.seek(self.start_dir) - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo + else: + with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: + shutil.copyfileobj(src, dest, 1024*8) def writestr(self, zinfo_or_arcname, data, compress_type=None): """Write a file into the archive. The contents is 'data', which @@ -1569,45 +1671,18 @@ class ZipFile: if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise RuntimeError( + "Can't write to ZIP archive while an open writing handle exists." + ) + + if compress_type is not None: + zinfo.compress_type = compress_type zinfo.file_size = len(data) # Uncompressed size with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header data - if compress_type is not None: - zinfo.compress_type = compress_type - zinfo.header_offset = self.fp.tell() # Start of header data - if compress_type is not None: - zinfo.compress_type = compress_type - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - - self._writecheck(zinfo) - self._didModify = True - zinfo.CRC = crc32(data) # CRC-32 checksum - co = _get_compressor(zinfo.compress_type) - if co: - data = co.compress(data) + co.flush() - zinfo.compress_size = len(data) # Compressed size - else: - zinfo.compress_size = zinfo.file_size - zip64 = zinfo.file_size > ZIP64_LIMIT or \ - zinfo.compress_size > ZIP64_LIMIT - if zip64 and not self._allowZip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - self.fp.write(zinfo.FileHeader(zip64)) - self.fp.write(data) - if zinfo.flag_bits & 0x08: - # Write CRC and file sizes after the file data - fmt = ' Date: Sat, 14 May 2016 06:17:43 +0000 Subject: Remove old Python 2 compatibility from ctypes test --- Lib/ctypes/test/test_structures.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 84d456c..e00bb04 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -326,11 +326,8 @@ class StructureTestCase(unittest.TestCase): cls, msg = self.get_except(Person, b"Someone", (b"a", b"b", b"c")) self.assertEqual(cls, RuntimeError) - if issubclass(Exception, object): - self.assertEqual(msg, - "(Phone) : too many initializers") - else: - self.assertEqual(msg, "(Phone) TypeError: too many initializers") + self.assertEqual(msg, + "(Phone) : too many initializers") def test_huge_field_name(self): # issue12881: segfault with large structure field names -- cgit v0.12 From f0dbf7a6abc821c77e912deae81e3a27cf334c54 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 15 May 2016 01:26:25 +0000 Subject: Issue #26870: Add readline.set_auto_history(), originally by Tyler Crompton --- Doc/library/readline.rst | 14 ++++++++++++++ Doc/whatsnew/3.6.rst | 8 ++++++++ Lib/test/test_readline.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ Modules/readline.c | 21 +++++++++++++++++++- 6 files changed, 96 insertions(+), 1 deletion(-) diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 42e0ad5..f777920 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -156,6 +156,20 @@ The following functions operate on a global history list: This calls :c:func:`add_history` in the underlying library. +.. function:: set_auto_history(enabled) + + Enable or disable automatic calls to :c:func:`add_history` when reading + input via readline. The *enabled* argument should be a Boolean value + that when true, enables auto history, and that when False, disables + auto history. + + .. versionadded:: 3.6 + + .. impl-detail:: + Auto history is enabled by default, and changes to this do not persist + across multiple sessions. + + Startup hooks ------------- diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 384e35a..bad0f9e 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -239,6 +239,14 @@ Protocol version 4 already supports this case. (Contributed by Serhiy Storchaka in :issue:`24164`.) +readline +-------- + +Added :func:`~readline.set_auto_history` to enable or disable +automatic addition of input to the history list. (Contributed by +Tyler Crompton in :issue:`26870`.) + + rlcompleter ----------- diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 35330ab..84fd119 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -1,7 +1,11 @@ """ Very minimal unittests for parts of the readline module. """ +from errno import EIO import os +import selectors +import subprocess +import sys import tempfile import unittest from test.support import import_module, unlink @@ -96,6 +100,51 @@ class TestReadline(unittest.TestCase): TERM='xterm-256color') self.assertEqual(stdout, b'') + auto_history_script = """\ +import readline +readline.set_auto_history({}) +input() +print("History length:", readline.get_current_history_length()) +""" + + def test_auto_history_enabled(self): + output = run_pty(self.auto_history_script.format(True)) + self.assertIn(b"History length: 1\r\n", output) + + def test_auto_history_disabled(self): + output = run_pty(self.auto_history_script.format(False)) + self.assertIn(b"History length: 0\r\n", output) + + +def run_pty(script, input=b"dummy input\r"): + pty = import_module('pty') + output = bytearray() + [master, slave] = pty.openpty() + args = (sys.executable, '-c', script) + proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave) + os.close(slave) + with proc, selectors.DefaultSelector() as sel: + sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) + os.set_blocking(master, False) + while True: + for [_, events] in sel.select(): + if events & selectors.EVENT_READ: + try: + chunk = os.read(master, 0x10000) + except OSError as err: + # Linux raises EIO when the slave is closed + if err.errno != EIO: + raise + chunk = b"" + if not chunk: + os.close(master) + return output + output.extend(chunk) + if events & selectors.EVENT_WRITE: + input = input[os.write(master, input):] + if not input: + sel.modify(master, selectors.EVENT_READ) + if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index ebc3fc6..65d7040 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -307,6 +307,7 @@ Ryan Coyner Christopher A. Craig Jeremy Craven Laura Creighton +Tyler Crompton Simon Cross Felipe Cruz Drew Csillag diff --git a/Misc/NEWS b/Misc/NEWS index bede515..80996bd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -277,6 +277,10 @@ Core and Builtins Library ------- +- Issue #26870: Added readline.set_auto_history(), which can stop entries + being automatically added to the history list. Based on patch by Tyler + Crompton. + - Issue #26039: zipfile.ZipFile.open() can now be used to write data into a ZIP file, as well as for extracting data. Patch by Thomas Kluyver. diff --git a/Modules/readline.c b/Modules/readline.c index a323b69..de1cc17 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -575,6 +575,24 @@ PyDoc_STRVAR(doc_add_history, "add_history(string) -> None\n\ add an item to the history buffer"); +static int should_auto_add_history = 1; + +/* Enable or disable automatic history */ + +static PyObject * +py_set_auto_history(PyObject *self, PyObject *args) +{ + if (!PyArg_ParseTuple(args, "p:set_auto_history", + &should_auto_add_history)) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(doc_set_auto_history, +"set_auto_history(enabled) -> None\n\ +Enables or disables automatic history."); + /* Get the tab-completion word-delimiters that readline uses */ @@ -791,6 +809,7 @@ static struct PyMethodDef readline_methods[] = {"set_completer_delims", set_completer_delims, METH_VARARGS, doc_set_completer_delims}, + {"set_auto_history", py_set_auto_history, METH_VARARGS, doc_set_auto_history}, {"add_history", py_add_history, METH_VARARGS, doc_add_history}, {"remove_history_item", py_remove_history, METH_VARARGS, doc_remove_history}, {"replace_history_item", py_replace_history, METH_VARARGS, doc_replace_history}, @@ -1266,7 +1285,7 @@ call_readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) /* we have a valid line */ n = strlen(p); - if (n > 0) { + if (should_auto_add_history && n > 0) { const char *line; int length = _py_get_history_length(); if (length > 0) -- cgit v0.12 From 3712686956b2bf9e3c98f394a1a27f99fafdbaa8 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 15 May 2016 03:05:36 +0000 Subject: Issue #26870: Close pty master in case of exception --- Lib/test/test_readline.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 84fd119..372b055 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -1,6 +1,7 @@ """ Very minimal unittests for parts of the readline module. """ +from contextlib import ExitStack from errno import EIO import os import selectors @@ -123,7 +124,10 @@ def run_pty(script, input=b"dummy input\r"): args = (sys.executable, '-c', script) proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave) os.close(slave) - with proc, selectors.DefaultSelector() as sel: + with ExitStack() as cleanup: + cleanup.enter_context(proc) + cleanup.callback(os.close, master) + sel = cleanup.enter_context(selectors.DefaultSelector()) sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) os.set_blocking(master, False) while True: @@ -137,7 +141,6 @@ def run_pty(script, input=b"dummy input\r"): raise chunk = b"" if not chunk: - os.close(master) return output output.extend(chunk) if events & selectors.EVENT_WRITE: -- cgit v0.12 From 3e2a0715d73011682956c5a397b35efc2377d6d4 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 15 May 2016 03:59:59 +0000 Subject: Issue #26870: Temporary debugging for OS X Snow Leopard lockup --- Lib/test/test_readline.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 372b055..cfb4af1 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -9,7 +9,7 @@ import subprocess import sys import tempfile import unittest -from test.support import import_module, unlink +from test.support import import_module, unlink, get_original_stdout from test.support.script_helper import assert_python_ok # Skip tests if there is no readline module @@ -131,20 +131,25 @@ def run_pty(script, input=b"dummy input\r"): sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) os.set_blocking(master, False) while True: + get_original_stdout().write(f"test_readline: select()\n") for [_, events] in sel.select(): if events & selectors.EVENT_READ: try: + get_original_stdout().write(f"test_readline: read()\n") chunk = os.read(master, 0x10000) except OSError as err: # Linux raises EIO when the slave is closed if err.errno != EIO: raise chunk = b"" + get_original_stdout().write(f"test_readline: read {chunk!r}\n") if not chunk: return output output.extend(chunk) if events & selectors.EVENT_WRITE: + get_original_stdout().write(f"test_readline: write()\n") input = input[os.write(master, input):] + get_original_stdout().write(f"test_readline: remaining input = {input!r}\n") if not input: sel.modify(master, selectors.EVENT_READ) -- cgit v0.12 From f47fc5553be1945d4d3ca1386c523fcf2ed28cef Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 15 May 2016 12:27:16 +0300 Subject: Issue #26039: Document ZipInfo.is_dir() and make force_zip64 keyword-only. Patch by Thomas Kluyver. --- Doc/library/zipfile.rst | 12 ++++++++++-- Lib/zipfile.py | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index cf8d547..f5e161e 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -207,7 +207,7 @@ ZipFile Objects .. index:: single: universal newlines; zipfile.ZipFile.open method -.. method:: ZipFile.open(name, mode='r', pwd=None, force_zip64=False) +.. method:: ZipFile.open(name, mode='r', pwd=None, *, force_zip64=False) Access a member of the archive as a file-like object. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The @@ -490,7 +490,15 @@ file: .. versionadded:: 3.6 -Instances have the following attributes: +Instances have the following methods and attributes: + +.. method:: ZipInfo.is_dir() + + Return True if this archive member is a directory. + + This uses the entry's name: directories should always end with ``/``. + + .. versionadded:: 3.6 .. attribute:: ZipInfo.filename diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 03dead5..27a4c71 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -502,6 +502,7 @@ class ZipInfo (object): return zinfo def is_dir(self): + """Return True if this archive member is a directory.""" return self.filename[-1] == '/' @@ -1343,7 +1344,7 @@ class ZipFile: with self.open(name, "r", pwd) as fp: return fp.read() - def open(self, name, mode="r", pwd=None, force_zip64=False): + def open(self, name, mode="r", pwd=None, *, force_zip64=False): """Return file-like object for 'name'. name is a string for the file name within the ZIP file, or a ZipInfo -- cgit v0.12 From 2e1d8683c17f05cb1667e48f205856d8ad02a02a Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 15 May 2016 13:21:25 +0000 Subject: Issue #26870: Avoid using kqueue() with pseudo-terminals Also force terminate the child process in case it hangs for any reason. --- Lib/test/test_readline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index cfb4af1..f8e627a 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -9,7 +9,7 @@ import subprocess import sys import tempfile import unittest -from test.support import import_module, unlink, get_original_stdout +from test.support import import_module, unlink from test.support.script_helper import assert_python_ok # Skip tests if there is no readline module @@ -126,30 +126,30 @@ def run_pty(script, input=b"dummy input\r"): os.close(slave) with ExitStack() as cleanup: cleanup.enter_context(proc) + cleanup.callback(proc.terminate) cleanup.callback(os.close, master) - sel = cleanup.enter_context(selectors.DefaultSelector()) + # Avoid using DefaultSelector, because it may choose a kqueue() + # implementation, which does not work with pseudo-terminals on OS X + # < 10.9 (Issue 20365) and Open BSD (Issue 20667). + sel = getattr(selectors, "PollSelector", selectors.DefaultSelector)() + cleanup.enter_context(sel) sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) os.set_blocking(master, False) while True: - get_original_stdout().write(f"test_readline: select()\n") for [_, events] in sel.select(): if events & selectors.EVENT_READ: try: - get_original_stdout().write(f"test_readline: read()\n") chunk = os.read(master, 0x10000) except OSError as err: # Linux raises EIO when the slave is closed if err.errno != EIO: raise chunk = b"" - get_original_stdout().write(f"test_readline: read {chunk!r}\n") if not chunk: return output output.extend(chunk) if events & selectors.EVENT_WRITE: - get_original_stdout().write(f"test_readline: write()\n") input = input[os.write(master, input):] - get_original_stdout().write(f"test_readline: remaining input = {input!r}\n") if not input: sel.modify(master, selectors.EVENT_READ) -- cgit v0.12 From 79f561d126d09d6d7ea1457a2a6ef267d93e6448 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 15 May 2016 15:04:58 +0000 Subject: Issue #26870: Poll() also fails on OS X; try select() Also work around separate Open BSD bug with kill() of a zombie. --- Lib/test/test_readline.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index f8e627a..c1864ef 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -126,13 +126,20 @@ def run_pty(script, input=b"dummy input\r"): os.close(slave) with ExitStack() as cleanup: cleanup.enter_context(proc) - cleanup.callback(proc.terminate) + def terminate(proc): + try: + proc.terminate() + except ProcessLookupError: + # Workaround for Open/Net BSD bug (Issue 16762) + pass + cleanup.callback(terminate, proc) cleanup.callback(os.close, master) - # Avoid using DefaultSelector, because it may choose a kqueue() - # implementation, which does not work with pseudo-terminals on OS X - # < 10.9 (Issue 20365) and Open BSD (Issue 20667). - sel = getattr(selectors, "PollSelector", selectors.DefaultSelector)() - cleanup.enter_context(sel) + # Avoid using DefaultSelector and PollSelector. Kqueue() does not + # work with pseudo-terminals on OS X < 10.9 (Issue 20365) and Open + # BSD (Issue 20667). Poll() does not work with OS X 10.6 or 10.4 + # either (Issue 20472). Hopefully the file descriptor is low enough + # to use with select(). + sel = cleanup.enter_context(selectors.SelectSelector()) sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) os.set_blocking(master, False) while True: -- cgit v0.12 From 98019e1cf6dec54ca6c56ceb150869c1430bfda3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 16 May 2016 09:10:43 +0300 Subject: Issue #27034: Removed deprecated class asynchat.fifo. --- Doc/library/asynchat.rst | 16 ++++++++-------- Lib/asynchat.py | 29 ----------------------------- Lib/test/test_asynchat.py | 31 ------------------------------- Misc/NEWS | 2 ++ 4 files changed, 10 insertions(+), 68 deletions(-) diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst index 56ad4f8..1ff3dd8 100644 --- a/Doc/library/asynchat.rst +++ b/Doc/library/asynchat.rst @@ -57,13 +57,13 @@ connection requests. The asynchronous output buffer size (default ``4096``). Unlike :class:`asyncore.dispatcher`, :class:`async_chat` allows you to - define a first-in-first-out queue (fifo) of *producers*. A producer need + define a :abbr:`FIFO (first-in, first-out)` queue of *producers*. A producer need have only one method, :meth:`more`, which should return data to be transmitted on the channel. The producer indicates exhaustion (*i.e.* that it contains no more data) by having its :meth:`more` method return the empty bytes object. At this point - the :class:`async_chat` object removes the producer from the fifo and starts - using the next producer, if any. When the producer fifo is empty the + the :class:`async_chat` object removes the producer from the queue and starts + using the next producer, if any. When the producer queue is empty the :meth:`handle_write` method does nothing. You use the channel object's :meth:`set_terminator` method to describe how to recognize the end of, or an important breakpoint in, an incoming transmission from the remote @@ -77,8 +77,8 @@ connection requests. .. method:: async_chat.close_when_done() - Pushes a ``None`` on to the producer fifo. When this producer is popped off - the fifo it causes the channel to be closed. + Pushes a ``None`` on to the producer queue. When this producer is popped off + the queue it causes the channel to be closed. .. method:: async_chat.collect_incoming_data(data) @@ -91,7 +91,7 @@ connection requests. .. method:: async_chat.discard_buffers() In emergencies this method will discard any data held in the input and/or - output buffers and the producer fifo. + output buffers and the producer queue. .. method:: async_chat.found_terminator() @@ -109,7 +109,7 @@ connection requests. .. method:: async_chat.push(data) - Pushes data on to the channel's fifo to ensure its transmission. + Pushes data on to the channel's queue to ensure its transmission. This is all you need to do to have the channel write the data out to the network, although it is possible to use your own producers in more complex schemes to implement encryption and chunking, for example. @@ -117,7 +117,7 @@ connection requests. .. method:: async_chat.push_with_producer(producer) - Takes a producer object and adds it to the producer fifo associated with + Takes a producer object and adds it to the producer queue associated with the channel. When all currently-pushed producers have been exhausted the channel will consume this producer's data by calling its :meth:`more` method and send the data to the remote endpoint. diff --git a/Lib/asynchat.py b/Lib/asynchat.py index f728d1b..fc1146a 100644 --- a/Lib/asynchat.py +++ b/Lib/asynchat.py @@ -285,35 +285,6 @@ class simple_producer: return result -class fifo: - def __init__(self, list=None): - import warnings - warnings.warn('fifo class will be removed in Python 3.6', - DeprecationWarning, stacklevel=2) - if not list: - self.list = deque() - else: - self.list = deque(list) - - def __len__(self): - return len(self.list) - - def is_empty(self): - return not self.list - - def first(self): - return self.list[0] - - def push(self, data): - self.list.append(data) - - def pop(self): - if self.list: - return (1, self.list.popleft()) - else: - return (0, None) - - # Given 'haystack', see if any prefix of 'needle' is at its end. This # assumes an exact match has already been checked. Return the number of # characters matched. diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py index 08090b8..0eba76d 100644 --- a/Lib/test/test_asynchat.py +++ b/Lib/test/test_asynchat.py @@ -296,37 +296,6 @@ class TestHelperFunctions(unittest.TestCase): self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0) -class TestFifo(unittest.TestCase): - def test_basic(self): - with self.assertWarns(DeprecationWarning) as cm: - f = asynchat.fifo() - self.assertEqual(str(cm.warning), - "fifo class will be removed in Python 3.6") - f.push(7) - f.push(b'a') - self.assertEqual(len(f), 2) - self.assertEqual(f.first(), 7) - self.assertEqual(f.pop(), (1, 7)) - self.assertEqual(len(f), 1) - self.assertEqual(f.first(), b'a') - self.assertEqual(f.is_empty(), False) - self.assertEqual(f.pop(), (1, b'a')) - self.assertEqual(len(f), 0) - self.assertEqual(f.is_empty(), True) - self.assertEqual(f.pop(), (0, None)) - - def test_given_list(self): - with self.assertWarns(DeprecationWarning) as cm: - f = asynchat.fifo([b'x', 17, 3]) - self.assertEqual(str(cm.warning), - "fifo class will be removed in Python 3.6") - self.assertEqual(len(f), 3) - self.assertEqual(f.pop(), (1, b'x')) - self.assertEqual(f.pop(), (1, 17)) - self.assertEqual(f.pop(), (1, 3)) - self.assertEqual(f.pop(), (0, None)) - - class TestNotConnected(unittest.TestCase): def test_disallow_negative_terminator(self): # Issue #11259 diff --git a/Misc/NEWS b/Misc/NEWS index 5745cc3..8c9a1dd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -277,6 +277,8 @@ Core and Builtins Library ------- +- Issue #27034: Removed deprecated class asynchat.fifo. + - Issue #26870: Added readline.set_auto_history(), which can stop entries being automatically added to the history list. Based on patch by Tyler Crompton. -- cgit v0.12 From 4ecfa455ae47bd955857348238c8bf8476819a1e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 16 May 2016 09:31:54 +0300 Subject: Expand abbreviations FIFO and LIFO. --- Doc/library/asyncio-eventloop.rst | 5 +++-- Doc/library/collections.rst | 5 +++-- Doc/library/itertools.rst | 3 ++- Doc/library/multiprocessing.rst | 5 +++-- Doc/library/queue.rst | 11 +++++++---- Doc/library/unittest.rst | 6 +++--- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index f68b19d..a25f9d1 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -101,8 +101,9 @@ keywords to your callback, use :func:`functools.partial`. For example, called after :meth:`call_soon` returns, when control returns to the event loop. - This operates as a FIFO queue, callbacks are called in the order in - which they are registered. Each callback will be called exactly once. + This operates as a :abbr:`FIFO (first-in, first-out)` queue, callbacks + are called in the order in which they are registered. Each callback + will be called exactly once. Any positional arguments after the callback will be passed to the callback when it is called. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index b67db53..e41f4cc 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -987,8 +987,9 @@ the items are returned in the order their keys were first added. .. method:: popitem(last=True) The :meth:`popitem` method for ordered dictionaries returns and removes a - (key, value) pair. The pairs are returned in LIFO order if *last* is true - or FIFO order if false. + (key, value) pair. The pairs are returned in + :abbr:`LIFO (last-in, first-out)` order if *last* is true + or :abbr:`FIFO (first-in, first-out)` order if false. .. method:: move_to_end(key, last=True) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 8376f1a..d80bef3 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -591,7 +591,8 @@ loops that truncate the stream. Return *n* independent iterators from a single iterable. The following Python code helps explain what *tee* does (although the actual - implementation is more complex and uses only a single underlying FIFO queue):: + implementation is more complex and uses only a single underlying + :abbr:`FIFO (first-in, first-out)` queue):: def tee(iterable, n=2): it = iter(iterable) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 1b1d386..b2b02b0 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -644,8 +644,9 @@ primitives like locks. For passing messages one can use :func:`Pipe` (for a connection between two processes) or a queue (which allows multiple producers and consumers). -The :class:`Queue`, :class:`SimpleQueue` and :class:`JoinableQueue` types are multi-producer, -multi-consumer FIFO queues modelled on the :class:`queue.Queue` class in the +The :class:`Queue`, :class:`SimpleQueue` and :class:`JoinableQueue` types +are multi-producer, multi-consumer :abbr:`FIFO (first-in, first-out)` +queues modelled on the :class:`queue.Queue` class in the standard library. They differ in that :class:`Queue` lacks the :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join` methods introduced into Python 2.5's :class:`queue.Queue` class. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 1cb0935..e3eaa3f 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -16,8 +16,9 @@ availability of thread support in Python; see the :mod:`threading` module. The module implements three types of queue, which differ only in the order in -which the entries are retrieved. In a FIFO queue, the first tasks added are -the first retrieved. In a LIFO queue, the most recently added entry is +which the entries are retrieved. In a :abbr:`FIFO (first-in, first-out)` +queue, the first tasks added are the first retrieved. In a +:abbr:`LIFO (last-in, first-out)` queue, the most recently added entry is the first retrieved (operating like a stack). With a priority queue, the entries are kept sorted (using the :mod:`heapq` module) and the lowest valued entry is retrieved first. @@ -27,14 +28,16 @@ The :mod:`queue` module defines the following classes and exceptions: .. class:: Queue(maxsize=0) - Constructor for a FIFO queue. *maxsize* is an integer that sets the upperbound + Constructor for a :abbr:`FIFO (first-in, first-out)` queue. *maxsize* is + an integer that sets the upperbound limit on the number of items that can be placed in the queue. Insertion will block once this size has been reached, until queue items are consumed. If *maxsize* is less than or equal to zero, the queue size is infinite. .. class:: LifoQueue(maxsize=0) - Constructor for a LIFO queue. *maxsize* is an integer that sets the upperbound + Constructor for a :abbr:`LIFO (last-in, first-out)` queue. *maxsize* is + an integer that sets the upperbound limit on the number of items that can be placed in the queue. Insertion will block once this size has been reached, until queue items are consumed. If *maxsize* is less than or equal to zero, the queue size is infinite. diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 8482f20..55a6aaf 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1388,9 +1388,9 @@ Test cases Add a function to be called after :meth:`tearDown` to cleanup resources used during the test. Functions will be called in reverse order to the - order they are added (LIFO). They are called with any arguments and - keyword arguments passed into :meth:`addCleanup` when they are - added. + order they are added (:abbr:`LIFO (last-in, first-out)`). They + are called with any arguments and keyword arguments passed into + :meth:`addCleanup` when they are added. If :meth:`setUp` fails, meaning that :meth:`tearDown` is not called, then any cleanup functions added will still be called. -- cgit v0.12 From cbcc2fd641de178e139b59f0c08bb738e9d4835e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 16 May 2016 09:36:31 +0300 Subject: Issue #27033: The default value of the decode_data parameter for smtpd.SMTPChannel and smtpd.SMTPServer constructors is changed to False. --- Doc/library/smtpd.rst | 50 +++++++++++++++++++++++++++----------------------- Lib/smtpd.py | 45 +++++++++++++++------------------------------ Lib/test/test_smtpd.py | 41 ++++++++++++----------------------------- Misc/NEWS | 3 +++ 4 files changed, 57 insertions(+), 82 deletions(-) diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index 977f9a8..2466e38 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -29,7 +29,7 @@ SMTPServer Objects .. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=True) + map=None, enable_SMTPUTF8=False, decode_data=False) Create a new :class:`SMTPServer` object, which binds to local address *localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. It @@ -45,20 +45,19 @@ SMTPServer Objects global socket map is used. *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. If set to - ``True``, *decode_data* must be ``False`` (otherwise an error is raised). + in :RFC:`6531`) should be enabled. The default is ``False``. When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL`` command and when present is passed to :meth:`process_message` in the - ``kwargs['mail_options']`` list. + ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* + cannot be set to ``True`` at the same time. *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. The default is ``True`` for backward - compatibility reasons, but will change to ``False`` in Python 3.6; specify - the keyword value explicitly to avoid the :exc:`DeprecationWarning`. When - *decode_data* is set to ``False`` the server advertises the ``8BITMIME`` + should be decoded using UTF-8. The default is ``False``. When + *decode_data* is not set to ``True`` the server advertises the ``8BITMIME`` extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to the ``MAIL`` command, and when present passes it to :meth:`process_message` - in the ``kwargs['mail_options']`` list. + in the ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* + cannot be set to ``True`` at the same time. .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs) @@ -71,13 +70,12 @@ SMTPServer Objects format). If the *decode_data* constructor keyword is set to ``True``, the *data* - argument will be a unicode string. If it is set to ``False``, it + parameter will be a unicode string. If it is set to ``False``, it will be a bytes object. *kwargs* is a dictionary containing additional information. It is empty - unless at least one of ``decode_data=False`` or ``enable_SMTPUTF8=True`` - was given as an init parameter, in which case it contains the following - keys: + if ``decode_data=True`` was given as an init argument, otherwise + it contains the following keys: *mail_options*: a list of all received parameters to the ``MAIL`` @@ -108,10 +106,13 @@ SMTPServer Objects *localaddr* and *remoteaddr* may now contain IPv6 addresses. .. versionadded:: 3.5 - the *decode_data* and *enable_SMTPUTF8* constructor arguments, and the - *kwargs* argument to :meth:`process_message` when one or more of these is + The *decode_data* and *enable_SMTPUTF8* constructor parameters, and the + *kwargs* parameter to :meth:`process_message` when one or more of these is specified. + .. versionchanged:: 3.6 + *decode_data* is now ``False`` by default. + DebuggingServer Objects ----------------------- @@ -150,7 +151,7 @@ SMTPChannel Objects ------------------- .. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=True) + map=None, enable_SMTPUTF8=False, decode_data=False) Create a new :class:`SMTPChannel` object which manages the communication between the server and a single SMTP client. @@ -162,22 +163,25 @@ SMTPChannel Objects limit. *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. A - :exc:`ValueError` is raised if both *enable_SMTPUTF8* and *decode_data* are - set to ``True`` at the same time. + in :RFC:`6531`) should be enabled. The default is ``False``. + *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same + time. A dictionary can be specified in *map* to avoid using a global socket map. *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. The default is ``True`` for backward - compatibility reasons, but will change to ``False`` in Python 3.6. Specify - the keyword value explicitly to avoid the :exc:`DeprecationWarning`. + should be decoded using UTF-8. The default is ``False``. + *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same + time. To use a custom SMTPChannel implementation you need to override the :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`. .. versionchanged:: 3.5 - the *decode_data* and *enable_SMTPUTF8* arguments were added. + The *decode_data* and *enable_SMTPUTF8* parameters were added. + + .. versionchanged:: 3.6 + *decode_data* is now ``False`` by default. The :class:`SMTPChannel` has the following instance variables: diff --git a/Lib/smtpd.py b/Lib/smtpd.py index 732066e..d63ae95 100755 --- a/Lib/smtpd.py +++ b/Lib/smtpd.py @@ -128,24 +128,17 @@ class SMTPChannel(asynchat.async_chat): return self.command_size_limit def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, - map=None, enable_SMTPUTF8=False, decode_data=None): + map=None, enable_SMTPUTF8=False, decode_data=False): asynchat.async_chat.__init__(self, conn, map=map) self.smtp_server = server self.conn = conn self.addr = addr self.data_size_limit = data_size_limit - self.enable_SMTPUTF8 = enable_SMTPUTF8 - if enable_SMTPUTF8: - if decode_data: - raise ValueError("decode_data and enable_SMTPUTF8 cannot" - " be set to True at the same time") - decode_data = False - if decode_data is None: - warn("The decode_data default of True will change to False in 3.6;" - " specify an explicit value for this keyword", - DeprecationWarning, 2) - decode_data = True - self._decode_data = decode_data + self.enable_SMTPUTF8 = enable_SMTPUTF8 = bool(enable_SMTPUTF8) + self._decode_data = decode_data = bool(decode_data) + if enable_SMTPUTF8 and decode_data: + raise ValueError("decode_data and enable_SMTPUTF8 cannot" + " be set to True at the same time") if decode_data: self._emptystring = '' self._linesep = '\r\n' @@ -635,23 +628,15 @@ class SMTPServer(asyncore.dispatcher): def __init__(self, localaddr, remoteaddr, data_size_limit=DATA_SIZE_DEFAULT, map=None, - enable_SMTPUTF8=False, decode_data=None): + enable_SMTPUTF8=False, decode_data=False): self._localaddr = localaddr self._remoteaddr = remoteaddr self.data_size_limit = data_size_limit - self.enable_SMTPUTF8 = enable_SMTPUTF8 - if enable_SMTPUTF8: - if decode_data: - raise ValueError("The decode_data and enable_SMTPUTF8" - " parameters cannot be set to True at the" - " same time.") - decode_data = False - if decode_data is None: - warn("The decode_data default of True will change to False in 3.6;" - " specify an explicit value for this keyword", - DeprecationWarning, 2) - decode_data = True - self._decode_data = decode_data + self.enable_SMTPUTF8 = enable_SMTPUTF8 = bool(enable_SMTPUTF8) + self._decode_data = decode_data = bool(decode_data) + if enable_SMTPUTF8 and decode_data: + raise ValueError("decode_data and enable_SMTPUTF8 cannot" + " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: gai_results = socket.getaddrinfo(*localaddr, @@ -698,9 +683,9 @@ class SMTPServer(asyncore.dispatcher): containing a `.' followed by other text has had the leading dot removed. - kwargs is a dictionary containing additional information. It is empty - unless decode_data=False or enable_SMTPUTF8=True was given as init - parameter, in which case ut will contain the following keys: + kwargs is a dictionary containing additional information. It is + empty if decode_data=True was given as init parameter, otherwise + it will contain the following keys: 'mail_options': list of parameters to the mail command. All elements are uppercase strings. Example: ['BODY=8BITMIME', 'SMTPUTF8']. diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py index 88dbfdf..d92c9b3 100644 --- a/Lib/test/test_smtpd.py +++ b/Lib/test/test_smtpd.py @@ -53,10 +53,6 @@ class SMTPDServerTest(unittest.TestCase): write_line(b'DATA') self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') - def test_decode_data_default_warns(self): - with self.assertWarns(DeprecationWarning): - smtpd.SMTPServer((support.HOST, 0), ('b', 0)) - def test_decode_data_and_enable_SMTPUTF8_raises(self): self.assertRaises( ValueError, @@ -108,10 +104,9 @@ class DebuggingServerTest(unittest.TestCase): """)) def test_process_message_with_decode_data_false(self): - server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), - decode_data=False) + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) with support.captured_stdout() as s: self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') stdout = s.getvalue() @@ -175,13 +170,11 @@ class TestFamilyDetection(unittest.TestCase): @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") def test_socket_uses_IPv6(self): - server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOST, 0), - decode_data=False) + server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOST, 0)) self.assertEqual(server.socket.family, socket.AF_INET6) def test_socket_uses_IPv4(self): - server = smtpd.SMTPServer((support.HOST, 0), (support.HOSTv6, 0), - decode_data=False) + server = smtpd.SMTPServer((support.HOST, 0), (support.HOSTv6, 0)) self.assertEqual(server.socket.family, socket.AF_INET) @@ -204,18 +197,18 @@ class TestRcptOptionParsing(unittest.TestCase): channel.handle_read() def test_params_rejected(self): - server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) self.write_line(channel, b'EHLO example') self.write_line(channel, b'MAIL from: size=20') self.write_line(channel, b'RCPT to: foo=bar') self.assertEqual(channel.socket.last, self.error_response) def test_nothing_accepted(self): - server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) self.write_line(channel, b'EHLO example') self.write_line(channel, b'MAIL from: size=20') self.write_line(channel, b'RCPT to: ') @@ -257,9 +250,9 @@ class TestMailOptionParsing(unittest.TestCase): self.assertEqual(channel.socket.last, b'250 OK\r\n') def test_with_decode_data_false(self): - server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) self.write_line(channel, b'EHLO example') for line in [ b'MAIL from: size=20 SMTPUTF8', @@ -765,13 +758,6 @@ class SMTPDChannelTest(unittest.TestCase): with support.check_warnings(('', DeprecationWarning)): self.channel._SMTPChannel__addr = 'spam' - def test_decode_data_default_warning(self): - with self.assertWarns(DeprecationWarning): - server = DummyServer((support.HOST, 0), ('b', 0)) - conn, addr = self.server.accept() - with self.assertWarns(DeprecationWarning): - smtpd.SMTPChannel(server, conn, addr) - @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") class SMTPDChannelIPv6Test(SMTPDChannelTest): def setUp(self): @@ -845,12 +831,9 @@ class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): smtpd.socket = asyncore.socket = mock_socket self.old_debugstream = smtpd.DEBUGSTREAM self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((support.HOST, 0), ('b', 0), - decode_data=False) + self.server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = self.server.accept() - # Set decode_data to False - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=False) + self.channel = smtpd.SMTPChannel(self.server, conn, addr) def tearDown(self): asyncore.close_all() diff --git a/Misc/NEWS b/Misc/NEWS index 8c9a1dd..fe75bf9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -277,6 +277,9 @@ Core and Builtins Library ------- +- Issue #27033: The default value of the decode_data parameter for + smtpd.SMTPChannel and smtpd.SMTPServer constructors is changed to False. + - Issue #27034: Removed deprecated class asynchat.fifo. - Issue #26870: Added readline.set_auto_history(), which can stop entries -- cgit v0.12 From bcde10aa7e47da68afb68fe75c65b9339dd89f86 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 16 May 2016 09:42:29 +0300 Subject: Issue #26765: Ensure that bytes- and unicode-specific stringlib files are used with correct type. --- Objects/stringlib/codecs.h | 6 +++--- Objects/stringlib/ctype.h | 5 +++-- Objects/stringlib/find_max_char.h | 5 +++-- Objects/stringlib/join.h | 2 +- Objects/stringlib/localeutil.h | 4 ++-- Objects/stringlib/transmogrify.h | 5 +++-- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 2beb604..2846d7e 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -1,6 +1,8 @@ /* stringlib: codec implementations */ -#if STRINGLIB_IS_UNICODE +#if !STRINGLIB_IS_UNICODE +# error "codecs.h is specific to Unicode" +#endif /* Mask to quickly check whether a C 'long' contains a non-ASCII, UTF8-encoded char. */ @@ -823,5 +825,3 @@ STRINGLIB(utf32_encode)(const STRINGLIB_CHAR *in, #undef SWAB4 #endif - -#endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/stringlib/ctype.h b/Objects/stringlib/ctype.h index 739cf3d..f054625 100644 --- a/Objects/stringlib/ctype.h +++ b/Objects/stringlib/ctype.h @@ -1,5 +1,6 @@ -/* NOTE: this API is -ONLY- for use with single byte character strings. */ -/* Do not use it with Unicode. */ +#if STRINGLIB_IS_UNICODE +# error "ctype.h only compatible with byte-wise strings" +#endif #include "bytes_methods.h" diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h index eb3fe88..8ccbc30 100644 --- a/Objects/stringlib/find_max_char.h +++ b/Objects/stringlib/find_max_char.h @@ -1,6 +1,8 @@ /* Finding the optimal width of unicode characters in a buffer */ -#if STRINGLIB_IS_UNICODE +#if !STRINGLIB_IS_UNICODE +# error "find_max_char.h is specific to Unicode" +#endif /* Mask to quickly check whether a C 'long' contains a non-ASCII, UTF8-encoded char. */ @@ -129,5 +131,4 @@ STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) #undef MAX_CHAR_UCS4 #endif /* STRINGLIB_SIZEOF_CHAR == 1 */ -#endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/stringlib/join.h b/Objects/stringlib/join.h index cbf81be..90f966d 100644 --- a/Objects/stringlib/join.h +++ b/Objects/stringlib/join.h @@ -1,6 +1,6 @@ /* stringlib: bytes joining implementation */ -#if STRINGLIB_SIZEOF_CHAR != 1 +#if STRINGLIB_IS_UNICODE #error join.h only compatible with byte-wise strings #endif diff --git a/Objects/stringlib/localeutil.h b/Objects/stringlib/localeutil.h index 6e2f073..df501ed 100644 --- a/Objects/stringlib/localeutil.h +++ b/Objects/stringlib/localeutil.h @@ -2,8 +2,8 @@ #include -#ifndef STRINGLIB_IS_UNICODE -# error "localeutil is specific to Unicode" +#if !STRINGLIB_IS_UNICODE +# error "localeutil.h is specific to Unicode" #endif typedef struct { diff --git a/Objects/stringlib/transmogrify.h b/Objects/stringlib/transmogrify.h index 8b147de..625507d 100644 --- a/Objects/stringlib/transmogrify.h +++ b/Objects/stringlib/transmogrify.h @@ -1,5 +1,6 @@ -/* NOTE: this API is -ONLY- for use with single byte character strings. */ -/* Do not use it with Unicode. */ +#if STRINGLIB_IS_UNICODE +# error "transmogrify.h only compatible with byte-wise strings" +#endif /* the more complicated methods. parts of these should be pulled out into the shared code in bytes_methods.c to cut down on duplicate code bloat. */ -- cgit v0.12 From cae752aec8d5b17b8ce03d72c827f77dc4f98294 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 16 May 2016 13:44:52 -0400 Subject: Update pydoc topics for 3.6.0a1 --- Lib/pydoc_data/topics.py | 12763 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 12685 insertions(+), 78 deletions(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 38857d2..a0b071b 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,79 +1,12686 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Sat May 23 17:38:41 2015 -topics = {'assert': u'\nThe "assert" statement\n**********************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, "assert expression", is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, "assert expression1, expression2", is equivalent to\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that "__debug__" and "AssertionError" refer\nto the built-in variables with those names. In the current\nimplementation, the built-in variable "__debug__" is "True" under\nnormal circumstances, "False" when optimization is requested (command\nline option -O). The current code generator emits no code for an\nassert statement when optimization is requested at compile time. Note\nthat it is unnecessary to include the source code for the expression\nthat failed in the error message; it will be displayed as part of the\nstack trace.\n\nAssignments to "__debug__" are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', - 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n | "*" target\n\n(See section *Primaries* for the syntax definitions for\n*attributeref*, *subscription*, and *slicing*.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list, optionally enclosed in\nparentheses or square brackets, is recursively defined as follows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The\n object must be an iterable with the same number of items as there\n are targets in the target list, and the items are assigned, from\n left to right, to the corresponding targets.\n\n * If the target list contains one target prefixed with an\n asterisk, called a "starred" target: The object must be a sequence\n with at least as many items as there are targets in the target\n list, minus one. The first items of the sequence are assigned,\n from left to right, to the targets before the starred target. The\n final items of the sequence are assigned to the targets after the\n starred target. A list of the remaining items in the sequence is\n then assigned to the starred target (the list can be empty).\n\n * Else: The object must be a sequence with the same number of\n items as there are targets in the target list, and the items are\n assigned, from left to right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a "global" or "nonlocal" statement\n in the current code block: the name is bound to the object in the\n current local namespace.\n\n * Otherwise: the name is bound to the object in the global\n namespace or the outer namespace determined by "nonlocal",\n respectively.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in\n square brackets: The object must be an iterable with the same number\n of items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, "TypeError" is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily "AttributeError").\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n "a.x" can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target "a.x" is always\n set as an instance attribute, creating it if necessary. Thus, the\n two occurrences of "a.x" do not necessarily refer to the same\n attribute: if the RHS expression refers to a class attribute, the\n LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with "property()".\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield an integer. If it is negative, the sequence\'s\n length is added to it. The resulting value must be a nonnegative\n integer less than the sequence\'s length, and the sequence is asked\n to assign the assigned object to its item with that index. If the\n index is out of range, "IndexError" is raised (assignment to a\n subscripted sequence cannot add new items to a list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n For user-defined objects, the "__setitem__()" method is called with\n appropriate arguments.\n\n* If the target is a slicing: The primary expression in the\n reference is evaluated. It should yield a mutable sequence object\n (such as a list). The assigned object should be a sequence object\n of the same type. Next, the lower and upper bound expressions are\n evaluated, insofar they are present; defaults are zero and the\n sequence\'s length. The bounds should evaluate to integers. If\n either bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the target\n sequence allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nAlthough the definition of assignment implies that overlaps between\nthe left-hand side and the right-hand side are \'simultanenous\' (for\nexample "a, b = b, a" swaps two variables), overlaps *within* the\ncollection of assigned-to variables occur left-to-right, sometimes\nresulting in confusion. For instance, the following program prints\n"[0, 2]":\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2 # i is updated, then x[i] is updated\n print(x)\n\nSee also: **PEP 3132** - Extended Iterable Unpacking\n\n The specification for the "*target" feature.\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions of the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like "x += 1" can be rewritten as\n"x = x + 1" to achieve a similar, but not exactly equal effect. In the\naugmented version, "x" is only evaluated once. Also, when possible,\nthe actual operation is performed *in-place*, meaning that rather than\ncreating a new object and assigning that to the target, the old object\nis modified instead.\n\nUnlike normal assignments, augmented assignments evaluate the left-\nhand side *before* evaluating the right-hand side. For example, "a[i]\n+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs\nthe addition, and lastly, it writes the result back to "a[i]".\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', - 'atom-identifiers': u'\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a "NameError" exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name, with leading underscores removed and a single underscore\ninserted, in front of the name. For example, the identifier "__spam"\noccurring in a class named "Ham" will be transformed to "_Ham__spam".\nThis transformation is independent of the syntactical context in which\nthe identifier is used. If the transformed name is extremely long\n(longer than 255 characters), implementation defined truncation may\nhappen. If the class name consists only of underscores, no\ntransformation is done.\n', - 'atom-literals': u"\nLiterals\n********\n\nPython supports string and bytes literals and various numeric\nliterals:\n\n literal ::= stringliteral | bytesliteral\n | integer | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\nbytes, integer, floating point number, complex number) with the given\nvalue. The value may be approximated in the case of floating point\nand imaginary (complex) literals. See section *Literals* for details.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value. Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n", - 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of "x.name") for\nclass instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for "self"). "name" is the attribute name. This\n method should return the (computed) attribute value or raise an\n "AttributeError" exception.\n\n Note that if the attribute is found through the normal mechanism,\n "__getattr__()" is not called. (This is an intentional asymmetry\n between "__getattr__()" and "__setattr__()".) This is done both for\n efficiency reasons and because otherwise "__getattr__()" would have\n no way to access other attributes of the instance. Note that at\n least for instance variables, you can fake total control by not\n inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n "__getattribute__()" method below for a way to actually get total\n control over attribute access.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines "__getattr__()",\n the latter will not be called unless "__getattribute__()" either\n calls it explicitly or raises an "AttributeError". This method\n should return the (computed) attribute value or raise an\n "AttributeError" exception. In order to avoid infinite recursion in\n this method, its implementation should always call the base class\n method with the same name to access any attributes it needs, for\n example, "object.__getattribute__(self, name)".\n\n Note: This method may still be bypassed when looking up special\n methods as the result of implicit invocation via language syntax\n or built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If "__setattr__()" wants to assign to an instance attribute, it\n should call the base class method with the same name, for example,\n "object.__setattr__(self, name, value)".\n\nobject.__delattr__(self, name)\n\n Like "__setattr__()" but for attribute deletion instead of\n assignment. This should only be implemented if "del obj.name" is\n meaningful for the object.\n\nobject.__dir__(self)\n\n Called when "dir()" is called on the object. A sequence must be\n returned. "dir()" converts the returned sequence to a list and\n sorts it.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' "__dict__".\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or "None" when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an "AttributeError"\n exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\nThe attribute "__objclass__" is interpreted by the "inspect" module as\nspecifying the class where this object was defined (setting this\nappropriately can assist in runtime introspection of dynamic class\nattributes). For callables, it may indicate that an instance of the\ngiven type (or a subclass) is expected or required as the first\npositional argument (for example, CPython sets this attribute for\nunbound methods that are implemented in C).\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: "__get__()", "__set__()", and\n"__delete__()". If any of those methods are defined for an object, it\nis said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, "a.x" has a\nlookup chain starting with "a.__dict__[\'x\']", then\n"type(a).__dict__[\'x\']", and continuing through the base classes of\n"type(a)" excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, "a.x". How\nthe arguments are assembled depends on "a":\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: "x.__get__(a)".\n\nInstance Binding\n If binding to an object instance, "a.x" is transformed into the\n call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n\nClass Binding\n If binding to a class, "A.x" is transformed into the call:\n "A.__dict__[\'x\'].__get__(None, A)".\n\nSuper Binding\n If "a" is an instance of "super", then the binding "super(B,\n obj).m()" searches "obj.__class__.__mro__" for the base class "A"\n immediately preceding "B" and then invokes the descriptor with the\n call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of "__get__()", "__set__()" and "__delete__()". If it\ndoes not define "__get__()", then accessing the attribute will return\nthe descriptor object itself unless there is a value in the object\'s\ninstance dictionary. If the descriptor defines "__set__()" and/or\n"__delete__()", it is a data descriptor; if it defines neither, it is\na non-data descriptor. Normally, data descriptors define both\n"__get__()" and "__set__()", while non-data descriptors have just the\n"__get__()" method. Data descriptors with "__set__()" and "__get__()"\ndefined always override a redefinition in an instance dictionary. In\ncontrast, non-data descriptors can be overridden by instances.\n\nPython methods (including "staticmethod()" and "classmethod()") are\nimplemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe "property()" function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of classes have a dictionary for attribute\nstorage. This wastes space for objects having very few instance\nvariables. The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable. Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. *__slots__*\n reserves space for the declared variables and prevents the\n automatic creation of *__dict__* and *__weakref__* for each\n instance.\n\n\nNotes on using *__slots__*\n--------------------------\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises "AttributeError". If\n dynamic assignment of new variables is desired, then add\n "\'__dict__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* Without a *__weakref__* variable for each instance, classes\n defining *__slots__* do not support weak references to its\n instances. If weak reference support is needed, then add\n "\'__weakref__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the\n instance variable defined by the base class slot is inaccessible\n (except by retrieving its descriptor directly from the base class).\n This renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as "int", "bytes" and "tuple".\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings\n may also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n', - 'attribute-references': u'\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, which most objects do. This object is then\nasked to produce the attribute whose name is the identifier. This\nproduction can be customized by overriding the "__getattr__()" method.\nIf this attribute is not available, the exception "AttributeError" is\nraised. Otherwise, the type and value of the object produced is\ndetermined by the object. Multiple evaluations of the same attribute\nreference may yield different objects.\n', - 'augassign': u'\nAugmented assignment statements\n*******************************\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions of the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like "x += 1" can be rewritten as\n"x = x + 1" to achieve a similar, but not exactly equal effect. In the\naugmented version, "x" is only evaluated once. Also, when possible,\nthe actual operation is performed *in-place*, meaning that rather than\ncreating a new object and assigning that to the target, the old object\nis modified instead.\n\nUnlike normal assignments, augmented assignments evaluate the left-\nhand side *before* evaluating the right-hand side. For example, "a[i]\n+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs\nthe addition, and lastly, it writes the result back to "a[i]".\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', - 'binary': u'\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels. Note that some of these operations also apply to certain non-\nnumeric types. Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr |\n m_expr "//" u_expr| m_expr "/" u_expr |\n m_expr "%" u_expr\n a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe "*" (multiplication) operator yields the product of its arguments.\nThe arguments must either both be numbers, or one argument must be an\ninteger and the other must be a sequence. In the former case, the\nnumbers are converted to a common type and then multiplied together.\nIn the latter case, sequence repetition is performed; a negative\nrepetition factor yields an empty sequence.\n\nThe "@" (at) operator is intended to be used for matrix\nmultiplication. No builtin Python types implement this operator.\n\nNew in version 3.5.\n\nThe "/" (division) and "//" (floor division) operators yield the\nquotient of their arguments. The numeric arguments are first\nconverted to a common type. Division of integers yields a float, while\nfloor division of integers results in an integer; the result is that\nof mathematical division with the \'floor\' function applied to the\nresult. Division by zero raises the "ZeroDivisionError" exception.\n\nThe "%" (modulo) operator yields the remainder from the division of\nthe first argument by the second. The numeric arguments are first\nconverted to a common type. A zero right argument raises the\n"ZeroDivisionError" exception. The arguments may be floating point\nnumbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals "4*0.7 +\n0.34".) The modulo operator always yields a result with the same sign\nas its second operand (or zero); the absolute value of the result is\nstrictly smaller than the absolute value of the second operand [1].\n\nThe floor division and modulo operators are connected by the following\nidentity: "x == (x//y)*y + (x%y)". Floor division and modulo are also\nconnected with the built-in function "divmod()": "divmod(x, y) ==\n(x//y, x%y)". [2].\n\nIn addition to performing the modulo operation on numbers, the "%"\noperator is also overloaded by string objects to perform old-style\nstring formatting (also known as interpolation). The syntax for\nstring formatting is described in the Python Library Reference,\nsection *printf-style String Formatting*.\n\nThe floor division operator, the modulo operator, and the "divmod()"\nfunction are not defined for complex numbers. Instead, convert to a\nfloating point number using the "abs()" function if appropriate.\n\nThe "+" (addition) operator yields the sum of its arguments. The\narguments must either both be numbers or both be sequences of the same\ntype. In the former case, the numbers are converted to a common type\nand then added together. In the latter case, the sequences are\nconcatenated.\n\nThe "-" (subtraction) operator yields the difference of its arguments.\nThe numeric arguments are first converted to a common type.\n', - 'bitwise': u'\nBinary bitwise operations\n*************************\n\nEach of the three bitwise operations has a different priority level:\n\n and_expr ::= shift_expr | and_expr "&" shift_expr\n xor_expr ::= and_expr | xor_expr "^" and_expr\n or_expr ::= xor_expr | or_expr "|" xor_expr\n\nThe "&" operator yields the bitwise AND of its arguments, which must\nbe integers.\n\nThe "^" operator yields the bitwise XOR (exclusive OR) of its\narguments, which must be integers.\n\nThe "|" operator yields the bitwise (inclusive) OR of its arguments,\nwhich must be integers.\n', - 'bltin-code-objects': u'\nCode Objects\n************\n\nCode objects are used by the implementation to represent "pseudo-\ncompiled" executable Python code such as a function body. They differ\nfrom function objects because they don\'t contain a reference to their\nglobal execution environment. Code objects are returned by the built-\nin "compile()" function and can be extracted from function objects\nthrough their "__code__" attribute. See also the "code" module.\n\nA code object can be executed or evaluated by passing it (instead of a\nsource string) to the "exec()" or "eval()" built-in functions.\n\nSee *The standard type hierarchy* for more information.\n', - 'bltin-ellipsis-object': u'\nThe Ellipsis Object\n*******************\n\nThis object is commonly used by slicing (see *Slicings*). It supports\nno special operations. There is exactly one ellipsis object, named\n"Ellipsis" (a built-in name). "type(Ellipsis)()" produces the\n"Ellipsis" singleton.\n\nIt is written as "Ellipsis" or "...".\n', - 'bltin-null-object': u'\nThe Null Object\n***************\n\nThis object is returned by functions that don\'t explicitly return a\nvalue. It supports no special operations. There is exactly one null\nobject, named "None" (a built-in name). "type(None)()" produces the\nsame singleton.\n\nIt is written as "None".\n', - 'bltin-type-objects': u'\nType Objects\n************\n\nType objects represent the various object types. An object\'s type is\naccessed by the built-in function "type()". There are no special\noperations on types. The standard module "types" defines names for\nall standard built-in types.\n\nTypes are written like this: "".\n', - 'booleans': u'\nBoolean operations\n******************\n\n or_test ::= and_test | or_test "or" and_test\n and_test ::= not_test | and_test "and" not_test\n not_test ::= comparison | "not" not_test\n\nIn the context of Boolean operations, and also when expressions are\nused by control flow statements, the following values are interpreted\nas false: "False", "None", numeric zero of all types, and empty\nstrings and containers (including strings, tuples, lists,\ndictionaries, sets and frozensets). All other values are interpreted\nas true. User-defined objects can customize their truth value by\nproviding a "__bool__()" method.\n\nThe operator "not" yields "True" if its argument is false, "False"\notherwise.\n\nThe expression "x and y" first evaluates *x*; if *x* is false, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\nThe expression "x or y" first evaluates *x*; if *x* is true, its value\nis returned; otherwise, *y* is evaluated and the resulting value is\nreturned.\n\n(Note that neither "and" nor "or" restrict the value and type they\nreturn to "False" and "True", but rather return the last evaluated\nargument. This is sometimes useful, e.g., if "s" is a string that\nshould be replaced by a default value if it is empty, the expression\n"s or \'foo\'" yields the desired value. Because "not" has to create a\nnew value, it returns a boolean value regardless of the type of its\nargument (for example, "not \'foo\'" produces "False" rather than "\'\'".)\n', - 'break': u'\nThe "break" statement\n*********************\n\n break_stmt ::= "break"\n\n"break" may only occur syntactically nested in a "for" or "while"\nloop, but not nested in a function or class definition within that\nloop.\n\nIt terminates the nearest enclosing loop, skipping the optional "else"\nclause if the loop has one.\n\nIf a "for" loop is terminated by "break", the loop control target\nkeeps its current value.\n\nWhen "break" passes control out of a "try" statement with a "finally"\nclause, that "finally" clause is executed before really leaving the\nloop.\n', - 'callable-types': u'\nEmulating callable objects\n**************************\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, "x(arg1, arg2, ...)" is a shorthand for\n "x.__call__(arg1, arg2, ...)".\n', - 'calls': u'\nCalls\n*****\n\nA call calls a callable object (e.g., a *function*) with a possibly\nempty series of *arguments*:\n\n call ::= primary "(" [argument_list [","] | comprehension] ")"\n argument_list ::= positional_arguments ["," keyword_arguments]\n ["," "*" expression] ["," keyword_arguments]\n ["," "**" expression]\n | keyword_arguments ["," "*" expression]\n ["," keyword_arguments] ["," "**" expression]\n | "*" expression ["," keyword_arguments] ["," "**" expression]\n | "**" expression\n positional_arguments ::= expression ("," expression)*\n keyword_arguments ::= keyword_item ("," keyword_item)*\n keyword_item ::= identifier "=" expression\n\nAn optional trailing comma may be present after the positional and\nkeyword arguments but does not affect the semantics.\n\nThe primary must evaluate to a callable object (user-defined\nfunctions, built-in functions, methods of built-in objects, class\nobjects, methods of class instances, and all objects having a\n"__call__()" method are callable). All argument expressions are\nevaluated before the call is attempted. Please refer to section\n*Function definitions* for the syntax of formal *parameter* lists.\n\nIf keyword arguments are present, they are first converted to\npositional arguments, as follows. First, a list of unfilled slots is\ncreated for the formal parameters. If there are N positional\narguments, they are placed in the first N slots. Next, for each\nkeyword argument, the identifier is used to determine the\ncorresponding slot (if the identifier is the same as the first formal\nparameter name, the first slot is used, and so on). If the slot is\nalready filled, a "TypeError" exception is raised. Otherwise, the\nvalue of the argument is placed in the slot, filling it (even if the\nexpression is "None", it fills the slot). When all arguments have\nbeen processed, the slots that are still unfilled are filled with the\ncorresponding default value from the function definition. (Default\nvalues are calculated, once, when the function is defined; thus, a\nmutable object such as a list or dictionary used as default value will\nbe shared by all calls that don\'t specify an argument value for the\ncorresponding slot; this should usually be avoided.) If there are any\nunfilled slots for which no default value is specified, a "TypeError"\nexception is raised. Otherwise, the list of filled slots is used as\nthe argument list for the call.\n\n**CPython implementation detail:** An implementation may provide\nbuilt-in functions whose positional parameters do not have names, even\nif they are \'named\' for the purpose of documentation, and which\ntherefore cannot be supplied by keyword. In CPython, this is the case\nfor functions implemented in C that use "PyArg_ParseTuple()" to parse\ntheir arguments.\n\nIf there are more positional arguments than there are formal parameter\nslots, a "TypeError" exception is raised, unless a formal parameter\nusing the syntax "*identifier" is present; in this case, that formal\nparameter receives a tuple containing the excess positional arguments\n(or an empty tuple if there were no excess positional arguments).\n\nIf any keyword argument does not correspond to a formal parameter\nname, a "TypeError" exception is raised, unless a formal parameter\nusing the syntax "**identifier" is present; in this case, that formal\nparameter receives a dictionary containing the excess keyword\narguments (using the keywords as keys and the argument values as\ncorresponding values), or a (new) empty dictionary if there were no\nexcess keyword arguments.\n\nIf the syntax "*expression" appears in the function call, "expression"\nmust evaluate to an iterable. Elements from this iterable are treated\nas if they were additional positional arguments; if there are\npositional arguments *x1*, ..., *xN*, and "expression" evaluates to a\nsequence *y1*, ..., *yM*, this is equivalent to a call with M+N\npositional arguments *x1*, ..., *xN*, *y1*, ..., *yM*.\n\nA consequence of this is that although the "*expression" syntax may\nappear *after* some keyword arguments, it is processed *before* the\nkeyword arguments (and the "**expression" argument, if any -- see\nbelow). So:\n\n >>> def f(a, b):\n ... print(a, b)\n ...\n >>> f(b=1, *(2,))\n 2 1\n >>> f(a=1, *(2,))\n Traceback (most recent call last):\n File "", line 1, in ?\n TypeError: f() got multiple values for keyword argument \'a\'\n >>> f(1, *(2,))\n 1 2\n\nIt is unusual for both keyword arguments and the "*expression" syntax\nto be used in the same call, so in practice this confusion does not\narise.\n\nIf the syntax "**expression" appears in the function call,\n"expression" must evaluate to a mapping, the contents of which are\ntreated as additional keyword arguments. In the case of a keyword\nappearing in both "expression" and as an explicit keyword argument, a\n"TypeError" exception is raised.\n\nFormal parameters using the syntax "*identifier" or "**identifier"\ncannot be used as positional argument slots or as keyword argument\nnames.\n\nA call always returns some value, possibly "None", unless it raises an\nexception. How this value is computed depends on the type of the\ncallable object.\n\nIf it is---\n\na user-defined function:\n The code block for the function is executed, passing it the\n argument list. The first thing the code block will do is bind the\n formal parameters to the arguments; this is described in section\n *Function definitions*. When the code block executes a "return"\n statement, this specifies the return value of the function call.\n\na built-in function or method:\n The result is up to the interpreter; see *Built-in Functions* for\n the descriptions of built-in functions and methods.\n\na class object:\n A new instance of that class is returned.\n\na class instance method:\n The corresponding user-defined function is called, with an argument\n list that is one longer than the argument list of the call: the\n instance becomes the first argument.\n\na class instance:\n The class must define a "__call__()" method; the effect is then the\n same as if that method was called.\n', - 'class': u'\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [parameter_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class "object"; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with "self.name = value". Both class and\ninstance attributes are accessible through the notation ""self.name"",\nand an instance attribute hides a class attribute with the same name\nwhen accessed in this way. Class attributes can be used as defaults\nfor instance attributes, but using mutable values there can lead to\nunexpected results. *Descriptors* can be used to create instance\nvariables with different implementation details.\n\nSee also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n Class Decorators\n', - 'comparisons': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like "a < b < c" have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: "True" or "False".\n\nComparisons can be chained arbitrarily, e.g., "x < y <= z" is\nequivalent to "x < y and y <= z", except that "y" is evaluated only\nonce (but in both cases "z" is not evaluated at all when "x < y" is\nfound to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then "a op1 b op2 c ... y\nopN z" is equivalent to "a op1 b and b op2 c and ... y opN z", except\nthat each expression is evaluated at most once.\n\nNote that "a op1 b op2 c" doesn\'t imply any kind of comparison between\n*a* and *c*, so that, e.g., "x < y > z" is perfectly legal (though\nperhaps not pretty).\n\nThe operators "<", ">", "==", ">=", "<=", and "!=" compare the values\nof two objects. The objects need not have the same type. If both are\nnumbers, they are converted to a common type. Otherwise, the "==" and\n"!=" operators *always* consider objects of different types to be\nunequal, while the "<", ">", ">=" and "<=" operators raise a\n"TypeError" when comparing objects of different types that do not\nimplement these operators for the given pair of types. You can\ncontrol comparison behavior of objects of non-built-in types by\ndefining rich comparison methods like "__gt__()", described in section\n*Basic customization*.\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* The values "float(\'NaN\')" and "Decimal(\'NaN\')" are special. They\n are identical to themselves, "x is x" but are not equal to\n themselves, "x != x". Additionally, comparing any value to a\n not-a-number value will return "False". For example, both "3 <\n float(\'NaN\')" and "float(\'NaN\') < 3" will return "False".\n\n* Bytes objects are compared lexicographically using the numeric\n values of their elements.\n\n* Strings are compared lexicographically using the numeric\n equivalents (the result of the built-in function "ord()") of their\n characters. [3] String and bytes object can\'t be compared!\n\n* Tuples and lists are compared lexicographically using comparison\n of corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, "[1,2,x] <= [1,2,y]" has the same\n value as "x <= y". If the corresponding element does not exist, the\n shorter sequence is ordered first (for example, "[1,2] < [1,2,3]").\n\n* Mappings (dictionaries) compare equal if and only if they have the\n same "(key, value)" pairs. Order comparisons "(\'<\', \'<=\', \'>=\',\n \'>\')" raise "TypeError".\n\n* Sets and frozensets define comparison operators to mean subset and\n superset tests. Those relations do not define total orderings (the\n two sets "{1,2}" and {2,3} are not equal, nor subsets of one\n another, nor supersets of one another). Accordingly, sets are not\n appropriate arguments for functions which depend on total ordering.\n For example, "min()", "max()", and "sorted()" produce undefined\n results given a list of sets as inputs.\n\n* Most other objects of built-in types compare unequal unless they\n are the same object; the choice whether one object is considered\n smaller or larger than another one is made arbitrarily but\n consistently within one execution of a program.\n\nComparison of objects of differing types depends on whether either of\nthe types provide explicit support for the comparison. Most numeric\ntypes can be compared with one another. When cross-type comparison is\nnot supported, the comparison method returns "NotImplemented".\n\nThe operators "in" and "not in" test for membership. "x in s"\nevaluates to true if *x* is a member of *s*, and false otherwise. "x\nnot in s" returns the negation of "x in s". All built-in sequences\nand set types support this as well as dictionary, for which "in" tests\nwhether the dictionary has a given key. For container types such as\nlist, tuple, set, frozenset, dict, or collections.deque, the\nexpression "x in y" is equivalent to "any(x is e or x == e for e in\ny)".\n\nFor the string and bytes types, "x in y" is true if and only if *x* is\na substring of *y*. An equivalent test is "y.find(x) != -1". Empty\nstrings are always considered to be a substring of any other string,\nso """ in "abc"" will return "True".\n\nFor user-defined classes which define the "__contains__()" method, "x\nin y" is true if and only if "y.__contains__(x)" is true.\n\nFor user-defined classes which do not define "__contains__()" but do\ndefine "__iter__()", "x in y" is true if some value "z" with "x == z"\nis produced while iterating over "y". If an exception is raised\nduring the iteration, it is as if "in" raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n"__getitem__()", "x in y" is true if and only if there is a non-\nnegative integer index *i* such that "x == y[i]", and all lower\ninteger indices do not raise "IndexError" exception. (If any other\nexception is raised, it is as if "in" raised that exception).\n\nThe operator "not in" is defined to have the inverse true value of\n"in".\n\nThe operators "is" and "is not" test for object identity: "x is y" is\ntrue if and only if *x* and *y* are the same object. "x is not y"\nyields the inverse truth value. [4]\n', - 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe "if", "while" and "for" statements implement traditional control\nflow constructs. "try" specifies exception handlers and/or cleanup\ncode for a group of statements, while the "with" statement allows the\nexecution of initialization and finalization code around a block of\ncode. Function and class definitions are also syntactically compound\nstatements.\n\nA compound statement consists of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of a suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which "if" clause a following "else" clause would belong:\n\n if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n"print()" calls are executed:\n\n if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | async_with_stmt\n | async_for_stmt\n | async_funcdef\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a "NEWLINE" possibly followed by a\n"DEDENT". Also note that optional continuation clauses always begin\nwith a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling "else"\' problem is solved in Python by\nrequiring nested "if" statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe "if" statement\n==================\n\nThe "if" statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n\n\nThe "while" statement\n=====================\n\nThe "while" statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the "else" clause, if present, is executed\nand the loop terminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and goes back\nto testing the expression.\n\n\nThe "for" statement\n===================\n\nThe "for" statement is used to iterate over the elements of a sequence\n(such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n"expression_list". The suite is then executed once for each item\nprovided by the iterator, in the order returned by the iterator. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted. When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a "StopIteration" exception),\nthe suite in the "else" clause, if present, is executed, and the loop\nterminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and continues\nwith the next item, or with the "else" clause if there is no next\nitem.\n\nThe for-loop makes assignments to the variables(s) in the target list.\nThis overwrites all previous assignments to those variables including\nthose made in the suite of the for-loop:\n\n for i in range(10):\n print(i)\n i = 5 # this will not affect the for-loop\n # because i will be overwritten with the next\n # index in the range\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, they will not have been assigned to at\nall by the loop. Hint: the built-in function "range()" returns an\niterator of integers suitable to emulate the effect of Pascal\'s "for i\n:= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n\nNote: There is a subtlety when the sequence is being modified by the\n loop (this can only occur for mutable sequences, i.e. lists). An\n internal counter is used to keep track of which item is used next,\n and this is incremented on each iteration. When this counter has\n reached the length of the sequence the loop terminates. This means\n that if the suite deletes the current (or a previous) item from the\n sequence, the next item will be skipped (since it gets the index of\n the current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe "try" statement\n===================\n\nThe "try" statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" identifier]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe "except" clause(s) specify one or more exception handlers. When no\nexception occurs in the "try" clause, no exception handler is\nexecuted. When an exception occurs in the "try" suite, a search for an\nexception handler is started. This search inspects the except clauses\nin turn until one is found that matches the exception. An expression-\nless except clause, if present, must be last; it matches any\nexception. For an except clause with an expression, that expression\nis evaluated, and the clause matches the exception if the resulting\nobject is "compatible" with the exception. An object is compatible\nwith an exception if it is the class or a base class of the exception\nobject or a tuple containing an item compatible with the exception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire "try" statement raised\nthe exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the "as" keyword in that except clause, if\npresent, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using "as target", it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the "sys" module and can be accessed via\n"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of the\nexception class, the exception instance and a traceback object (see\nsection *The standard type hierarchy*) identifying the point in the\nprogram where the exception occurred. "sys.exc_info()" values are\nrestored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional "else" clause is executed if and when control flows off\nthe end of the "try" clause. [2] Exceptions in the "else" clause are\nnot handled by the preceding "except" clauses.\n\nIf "finally" is present, it specifies a \'cleanup\' handler. The "try"\nclause is executed, including any "except" and "else" clauses. If an\nexception occurs in any of the clauses and is not handled, the\nexception is temporarily saved. The "finally" clause is executed. If\nthere is a saved exception it is re-raised at the end of the "finally"\nclause. If the "finally" clause raises another exception, the saved\nexception is set as the context of the new exception. If the "finally"\nclause executes a "return" or "break" statement, the saved exception\nis discarded:\n\n >>> def f():\n ... try:\n ... 1/0\n ... finally:\n ... return 42\n ...\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the "finally" clause.\n\nWhen a "return", "break" or "continue" statement is executed in the\n"try" suite of a "try"..."finally" statement, the "finally" clause is\nalso executed \'on the way out.\' A "continue" statement is illegal in\nthe "finally" clause. (The reason is a problem with the current\nimplementation --- this restriction may be lifted in the future).\n\nThe return value of a function is determined by the last "return"\nstatement executed. Since the "finally" clause always executes, a\n"return" statement executed in the "finally" clause will always be the\nlast one executed:\n\n >>> def foo():\n ... try:\n ... return \'try\'\n ... finally:\n ... return \'finally\'\n ...\n >>> foo()\n \'finally\'\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the "raise" statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe "with" statement\n====================\n\nThe "with" statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common "try"..."except"..."finally"\nusage patterns to be encapsulated for convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the "with" statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the "with_item")\n is evaluated to obtain a context manager.\n\n2. The context manager\'s "__exit__()" is loaded for later use.\n\n3. The context manager\'s "__enter__()" method is invoked.\n\n4. If a target was included in the "with" statement, the return\n value from "__enter__()" is assigned to it.\n\n Note: The "with" statement guarantees that if the "__enter__()"\n method returns without an error, then "__exit__()" will always be\n called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s "__exit__()" method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to "__exit__()". Otherwise, three\n "None" arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the "__exit__()" method was false, the exception is reraised.\n If the return value was true, the exception is suppressed, and\n execution continues with the statement following the "with"\n statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from "__exit__()" is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple "with" statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n | "*" [parameter] ("," defparameter)* ["," "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated from left to right when the\nfunction definition is executed.** This means that the expression is\nevaluated once, when the function is defined, and that the same "pre-\ncomputed" value is used for each call. This is especially important\nto understand when a default parameter is a mutable object, such as a\nlist or a dictionary: if the function modifies the object (e.g. by\nappending an item to a list), the default value is in effect modified.\nThis is generally not what was intended. A way around this is to use\n"None" as the default, and explicitly test for it in the body of the\nfunction, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple. If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name. Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier". Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist. These annotations can be any valid Python expression and are\nevaluated when the function definition is executed. Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction. The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda\nexpressions, described in section *Lambdas*. Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression. The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around. Free variables used\nin the nested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n The original specification for function annotations.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [parameter_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class "object"; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with "self.name = value". Both class and\ninstance attributes are accessible through the notation ""self.name"",\nand an instance attribute hides a class attribute with the same name\nwhen accessed in this way. Class attributes can be used as defaults\nfor instance attributes, but using mutable values there can lead to\nunexpected results. *Descriptors* can be used to create instance\nvariables with different implementation details.\n\nSee also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n Class Decorators\n\n\nCoroutines\n==========\n\n\nCoroutine function definition\n-----------------------------\n\n async_funcdef ::= "async" funcdef\n\nExecution of Python coroutines can be suspended and resumed at many\npoints (see *coroutine*.) "await" expressions, "async for" and "async\nwith" can only be used in their bodies.\n\nFunctions defined with "async def" syntax are always coroutine\nfunctions, even if they do not contain "await" or "async" keywords.\n\nIt is a "SyntaxError" to use "yield" expressions in coroutines.\n\nNew in version 3.5.\n\n\nThe "async for" statement\n-------------------------\n\n async_for_stmt ::= "async" for_stmt\n\nAn *asynchronous iterable* is able to call asynchronous code in its\n*iter* implementation, and *asynchronous iterator* can call\nasynchronous code in its *next* method.\n\nThe "async for" statement allows convenient iteration over\nasynchronous iterators.\n\nThe following code:\n\n async for TARGET in ITER:\n BLOCK\n else:\n BLOCK2\n\nIs semantically equivalent to:\n\n iter = (ITER)\n iter = await type(iter).__aiter__(iter)\n running = True\n while running:\n try:\n TARGET = await type(iter).__anext__(iter)\n except StopAsyncIteration:\n running = False\n else:\n BLOCK\n else:\n BLOCK2\n\nSee also "__aiter__()" and "__anext__()" for details.\n\nNew in version 3.5.\n\n\nThe "async with" statement\n--------------------------\n\n async_with_stmt ::= "async" with_stmt\n\nAn *asynchronous context manager* is a *context manager* that is able\nto suspend execution in its *enter* and *exit* methods.\n\nThe following code:\n\n async with EXPR as VAR:\n BLOCK\n\nIs semantically equivalent to:\n\n mgr = (EXPR)\n aexit = type(mgr).__aexit__\n aenter = type(mgr).__aenter__(mgr)\n exc = True\n\n VAR = await aenter\n try:\n BLOCK\n except:\n if not await aexit(mgr, *sys.exc_info()):\n raise\n else:\n await aexit(mgr, None, None, None)\n\nSee also "__aenter__()" and "__aexit__()" for details.\n\nNew in version 3.5.\n\nSee also: **PEP 492** - Coroutines with async and await syntax\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless\n there is a "finally" clause which happens to raise another\n exception. That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of\n an exception or the execution of a "return", "continue", or\n "break" statement.\n\n[3] A string literal appearing as the first statement in the\n function body is transformed into the function\'s "__doc__"\n attribute and therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s "__doc__" item and\n therefore the class\'s *docstring*.\n', - 'context-managers': u'\nWith Statement Context Managers\n*******************************\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a "with" statement. The context manager\nhandles the entry into, and the exit from, the desired runtime context\nfor the execution of the block of code. Context managers are normally\ninvoked using the "with" statement (described in section *The with\nstatement*), but can also be used by directly invoking their methods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The "with"\n statement will bind this method\'s return value to the target(s)\n specified in the "as" clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be "None".\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that "__exit__()" methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n', - 'continue': u'\nThe "continue" statement\n************************\n\n continue_stmt ::= "continue"\n\n"continue" may only occur syntactically nested in a "for" or "while"\nloop, but not nested in a function or class definition or "finally"\nclause within that loop. It continues with the next cycle of the\nnearest enclosing loop.\n\nWhen "continue" passes control out of a "try" statement with a\n"finally" clause, that "finally" clause is executed before really\nstarting the next loop cycle.\n', - 'conversions': u'\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," this means\nthat the operator implementation for built-in types works as follows:\n\n* If either argument is a complex number, the other is converted to\n complex;\n\n* otherwise, if either argument is a floating point number, the\n other is converted to floating point;\n\n* otherwise, both must be integers and no conversion is necessary.\n\nSome additional rules apply for certain operators (e.g., a string as a\nleft argument to the \'%\' operator). Extensions must define their own\nconversion behavior.\n', - 'customization': u'\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. "__new__()" is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of "__new__()" should be the new object instance (usually an\n instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s "__new__()" method using\n "super(currentclass, cls).__new__(cls[, ...])" with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If "__new__()" returns an instance of *cls*, then the new\n instance\'s "__init__()" method will be invoked like\n "__init__(self[, ...])", where *self* is the new instance and the\n remaining arguments are the same as were passed to "__new__()".\n\n If "__new__()" does not return an instance of *cls*, then the new\n instance\'s "__init__()" method will not be invoked.\n\n "__new__()" is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called after the instance has been created (by "__new__()"), but\n before it is returned to the caller. The arguments are those\n passed to the class constructor expression. If a base class has an\n "__init__()" method, the derived class\'s "__init__()" method, if\n any, must explicitly call it to ensure proper initialization of the\n base class part of the instance; for example:\n "BaseClass.__init__(self, [args...])".\n\n Because "__new__()" and "__init__()" work together in constructing\n objects ("__new__()" to create it, and "__init__()" to customise\n it), no non-"None" value may be returned by "__init__()"; doing so\n will cause a "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a "__del__()" method, the\n derived class\'s "__del__()" method, if any, must explicitly call it\n to ensure proper deletion of the base class part of the instance.\n Note that it is possible (though not recommended!) for the\n "__del__()" method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n "__del__()" methods are called for objects that still exist when\n the interpreter exits.\n\n Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n decrements the reference count for "x" by one, and the latter is\n only called when "x"\'s reference count reaches zero. Some common\n situations that may prevent the reference count of an object from\n going to zero include: circular references between objects (e.g.,\n a doubly-linked list or a tree data structure with parent and\n child pointers); a reference to the object on the stack frame of\n a function that caught an exception (the traceback stored in\n "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n to the object on the stack frame that raised an unhandled\n exception in interactive mode (the traceback stored in\n "sys.last_traceback" keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the second can be resolved by freeing the reference to the\n traceback object when it is no longer useful, and the third can\n be resolved by storing "None" in "sys.last_traceback". Circular\n references which are garbage are detected and cleaned up when the\n cyclic garbage collector is enabled (it\'s on by default). Refer\n to the documentation for the "gc" module for more information\n about this topic.\n\n Warning: Due to the precarious circumstances under which\n "__del__()" methods are invoked, exceptions that occur during\n their execution are ignored, and a warning is printed to\n "sys.stderr" instead. Also, when "__del__()" is invoked in\n response to a module being deleted (e.g., when execution of the\n program is done), other globals referenced by the "__del__()"\n method may already have been deleted or in the process of being\n torn down (e.g. the import machinery shutting down). For this\n reason, "__del__()" methods should do the absolute minimum needed\n to maintain external invariants. Starting with version 1.5,\n Python guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n Called by the "repr()" built-in function to compute the "official"\n string representation of an object. If at all possible, this\n should look like a valid Python expression that could be used to\n recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n "<...some useful description...>" should be returned. The return\n value must be a string object. If a class defines "__repr__()" but\n not "__str__()", then "__repr__()" is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by "str(object)" and the built-in functions "format()" and\n "print()" to compute the "informal" or nicely printable string\n representation of an object. The return value must be a *string*\n object.\n\n This method differs from "object.__repr__()" in that there is no\n expectation that "__str__()" return a valid Python expression: a\n more convenient or concise representation can be used.\n\n The default implementation defined by the built-in type "object"\n calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n Called by "bytes()" to compute a byte-string representation of an\n object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n Called by the "format()" built-in function (and by extension, the\n "str.format()" method of class "str") to produce a "formatted"\n string representation of an object. The "format_spec" argument is a\n string that contains a description of the formatting options\n desired. The interpretation of the "format_spec" argument is up to\n the type implementing "__format__()", however most classes will\n either delegate formatting to one of the built-in types, or use a\n similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\n Changed in version 3.4: The __format__ method of "object" itself\n raises a "TypeError" if passed any non-empty string.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: "xy" calls\n "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n A rich comparison method may return the singleton "NotImplemented"\n if it does not implement the operation for a given pair of\n arguments. By convention, "False" and "True" are returned for a\n successful comparison. However, these methods can return any value,\n so if the comparison operator is used in a Boolean context (e.g.,\n in the condition of an "if" statement), Python will call "bool()"\n on the value to determine if the result is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of "x==y" does not imply that "x!=y" is false.\n Accordingly, when defining "__eq__()", one should also define\n "__ne__()" so that the operators will behave as expected. See the\n paragraph on "__hash__()" for some important notes on creating\n *hashable* objects which support custom comparison operations and\n are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n and "__eq__()" and "__ne__()" are their own reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see "functools.total_ordering()".\n\nobject.__hash__(self)\n\n Called by built-in function "hash()" and for operations on members\n of hashed collections including "set", "frozenset", and "dict".\n "__hash__()" should return an integer. The only required property\n is that objects which compare equal have the same hash value; it is\n advised to somehow mix together (e.g. using exclusive or) the hash\n values for the components of the object that also play a part in\n comparison of objects.\n\n Note: "hash()" truncates the value returned from an object\'s\n custom "__hash__()" method to the size of a "Py_ssize_t". This\n is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n builds. If an object\'s "__hash__()" must interoperate on builds\n of different bit sizes, be sure to check the width on all\n supported builds. An easy way to do this is with "python -c\n "import sys; print(sys.hash_info.width)""\n\n If a class does not define an "__eq__()" method it should not\n define a "__hash__()" operation either; if it defines "__eq__()"\n but not "__hash__()", its instances will not be usable as items in\n hashable collections. If a class defines mutable objects and\n implements an "__eq__()" method, it should not implement\n "__hash__()", since the implementation of hashable collections\n requires that a key\'s hash value is immutable (if the object\'s hash\n value changes, it will be in the wrong hash bucket).\n\n User-defined classes have "__eq__()" and "__hash__()" methods by\n default; with them, all objects compare unequal (except with\n themselves) and "x.__hash__()" returns an appropriate value such\n that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n A class that overrides "__eq__()" and does not define "__hash__()"\n will have its "__hash__()" implicitly set to "None". When the\n "__hash__()" method of a class is "None", instances of the class\n will raise an appropriate "TypeError" when a program attempts to\n retrieve their hash value, and will also be correctly identified as\n unhashable when checking "isinstance(obj, collections.Hashable").\n\n If a class that overrides "__eq__()" needs to retain the\n implementation of "__hash__()" from a parent class, the interpreter\n must be told this explicitly by setting "__hash__ =\n .__hash__".\n\n If a class that does not override "__eq__()" wishes to suppress\n hash support, it should include "__hash__ = None" in the class\n definition. A class which defines its own "__hash__()" that\n explicitly raises a "TypeError" would be incorrectly identified as\n hashable by an "isinstance(obj, collections.Hashable)" call.\n\n Note: By default, the "__hash__()" values of str, bytes and\n datetime objects are "salted" with an unpredictable random value.\n Although they remain constant within an individual Python\n process, they are not predictable between repeated invocations of\n Python.This is intended to provide protection against a denial-\n of-service caused by carefully-chosen inputs that exploit the\n worst case performance of a dict insertion, O(n^2) complexity.\n See http://www.ocert.org/advisories/ocert-2011-003.html for\n details.Changing hash values affects the iteration order of\n dicts, sets and other mappings. Python has never made guarantees\n about this ordering (and it typically varies between 32-bit and\n 64-bit builds).See also "PYTHONHASHSEED".\n\n Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n "bool()"; should return "False" or "True". When this method is not\n defined, "__len__()" is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither "__len__()" nor "__bool__()", all its instances are\n considered true.\n', - 'debugger': u'\n"pdb" --- The Python Debugger\n*****************************\n\n**Source code:** Lib/pdb.py\n\n======================================================================\n\nThe module "pdb" defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible -- it is actually defined as the class\n"Pdb". This is currently undocumented but easily understood by reading\nthe source. The extension interface uses the modules "bdb" and "cmd".\n\nThe debugger\'s prompt is "(Pdb)". Typical usage to run a program under\ncontrol of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > (0)?()\n (Pdb) continue\n > (1)?()\n (Pdb) continue\n NameError: \'spam\'\n > (1)?()\n (Pdb)\n\nChanged in version 3.3: Tab-completion via the "readline" module is\navailable for commands and command arguments, e.g. the current global\nand local names are offered as arguments of the "p" command.\n\n"pdb.py" can also be invoked as a script to debug other scripts. For\nexample:\n\n python3 -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 3.2: "pdb.py" now accepts a "-c" option that executes\ncommands as if given in a ".pdbrc" file, see *Debugger Commands*.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the "continue" command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print(spam)\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print(spam)\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement, globals=None, locals=None)\n\n Execute the *statement* (given as a string or a code object) under\n debugger control. The debugger prompt appears before any code is\n executed; you can set breakpoints and type "continue", or you can\n step through the statement using "step" or "next" (all these\n commands are explained below). The optional *globals* and *locals*\n arguments specify the environment in which the code is executed; by\n default the dictionary of the module "__main__" is used. (See the\n explanation of the built-in "exec()" or "eval()" functions.)\n\npdb.runeval(expression, globals=None, locals=None)\n\n Evaluate the *expression* (given as a string or a code object)\n under debugger control. When "runeval()" returns, it returns the\n value of the expression. Otherwise this function is similar to\n "run()".\n\npdb.runcall(function, *args, **kwds)\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When "runcall()" returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem(traceback=None)\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n "sys.last_traceback".\n\nThe "run*" functions and "set_trace()" are aliases for instantiating\nthe "Pdb" class and calling the method of the same name. If you want\nto access further features, you have to do this yourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None, nosigint=False)\n\n "Pdb" is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying "cmd.Cmd" class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n By default, Pdb sets a handler for the SIGINT signal (which is sent\n when the user presses Ctrl-C on the console) when you give a\n "continue" command. This allows you to break into the debugger\n again by pressing Ctrl-C. If you want Pdb not to touch the SIGINT\n handler, set *nosigint* tot true.\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 3.1: The *skip* argument.\n\n New in version 3.2: The *nosigint* argument. Previously, a SIGINT\n handler was never set by Pdb.\n\n run(statement, globals=None, locals=None)\n runeval(expression, globals=None, locals=None)\n runcall(function, *args, **kwds)\n set_trace()\n\n See the documentation for the functions explained above.\n\n\nDebugger Commands\n=================\n\nThe commands recognized by the debugger are listed below. Most\ncommands can be abbreviated to one or two letters as indicated; e.g.\n"h(elp)" means that either "h" or "help" can be used to enter the help\ncommand (but not "he" or "hel", nor "H" or "Help" or "HELP").\nArguments to commands must be separated by whitespace (spaces or\ntabs). Optional arguments are enclosed in square brackets ("[]") in\nthe command syntax; the square brackets must not be typed.\nAlternatives in the command syntax are separated by a vertical bar\n("|").\n\nEntering a blank line repeats the last command entered. Exception: if\nthe last command was a "list" command, the next 11 lines are listed.\n\nCommands that the debugger doesn\'t recognize are assumed to be Python\nstatements and are executed in the context of the program being\ndebugged. Python statements can also be prefixed with an exclamation\npoint ("!"). This is a powerful way to inspect the program being\ndebugged; it is even possible to change a variable or call a function.\nWhen an exception occurs in such a statement, the exception name is\nprinted but the debugger\'s state is not changed.\n\nThe debugger supports *aliases*. Aliases can have parameters which\nallows one a certain level of adaptability to the context under\nexamination.\n\nMultiple commands may be entered on a single line, separated by ";;".\n(A single ";" is not used as it is the separator for multiple commands\nin a line that is passed to the Python parser.) No intelligence is\napplied to separating the commands; the input is split at the first\n";;" pair, even if it is in the middle of a quoted string.\n\nIf a file ".pdbrc" exists in the user\'s home directory or in the\ncurrent directory, it is read in and executed as if it had been typed\nat the debugger prompt. This is particularly useful for aliases. If\nboth files exist, the one in the home directory is read first and\naliases defined there can be overridden by the local file.\n\nChanged in version 3.2: ".pdbrc" can now contain commands that\ncontinue debugging, such as "continue" or "next". Previously, these\ncommands had no effect.\n\nh(elp) [command]\n\n Without argument, print the list of available commands. With a\n *command* as argument, print help about that command. "help pdb"\n displays the full documentation (the docstring of the "pdb"\n module). Since the *command* argument must be an identifier, "help\n exec" must be entered to get help on the "!" command.\n\nw(here)\n\n Print a stack trace, with the most recent frame at the bottom. An\n arrow indicates the current frame, which determines the context of\n most commands.\n\nd(own) [count]\n\n Move the current frame *count* (default one) levels down in the\n stack trace (to a newer frame).\n\nu(p) [count]\n\n Move the current frame *count* (default one) levels up in the stack\n trace (to an older frame).\n\nb(reak) [([filename:]lineno | function) [, condition]]\n\n With a *lineno* argument, set a break there in the current file.\n With a *function* argument, set a break at the first executable\n statement within that function. The line number may be prefixed\n with a filename and a colon, to specify a breakpoint in another\n file (probably one that hasn\'t been loaded yet). The file is\n searched on "sys.path". Note that each breakpoint is assigned a\n number to which all the other breakpoint commands refer.\n\n If a second argument is present, it is an expression which must\n evaluate to true before the breakpoint is honored.\n\n Without argument, list all breaks, including for each breakpoint,\n the number of times that breakpoint has been hit, the current\n ignore count, and the associated condition if any.\n\ntbreak [([filename:]lineno | function) [, condition]]\n\n Temporary breakpoint, which is removed automatically when it is\n first hit. The arguments are the same as for "break".\n\ncl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n\n With a *filename:lineno* argument, clear all the breakpoints at\n this line. With a space separated list of breakpoint numbers, clear\n those breakpoints. Without argument, clear all breaks (but first\n ask confirmation).\n\ndisable [bpnumber [bpnumber ...]]\n\n Disable the breakpoints given as a space separated list of\n breakpoint numbers. Disabling a breakpoint means it cannot cause\n the program to stop execution, but unlike clearing a breakpoint, it\n remains in the list of breakpoints and can be (re-)enabled.\n\nenable [bpnumber [bpnumber ...]]\n\n Enable the breakpoints specified.\n\nignore bpnumber [count]\n\n Set the ignore count for the given breakpoint number. If count is\n omitted, the ignore count is set to 0. A breakpoint becomes active\n when the ignore count is zero. When non-zero, the count is\n decremented each time the breakpoint is reached and the breakpoint\n is not disabled and any associated condition evaluates to true.\n\ncondition bpnumber [condition]\n\n Set a new *condition* for the breakpoint, an expression which must\n evaluate to true before the breakpoint is honored. If *condition*\n is absent, any existing condition is removed; i.e., the breakpoint\n is made unconditional.\n\ncommands [bpnumber]\n\n Specify a list of commands for breakpoint number *bpnumber*. The\n commands themselves appear on the following lines. Type a line\n containing just "end" to terminate the commands. An example:\n\n (Pdb) commands 1\n (com) p some_variable\n (com) end\n (Pdb)\n\n To remove all commands from a breakpoint, type commands and follow\n it immediately with "end"; that is, give no commands.\n\n With no *bpnumber* argument, commands refers to the last breakpoint\n set.\n\n You can use breakpoint commands to start your program up again.\n Simply use the continue command, or step, or any other command that\n resumes execution.\n\n Specifying any command resuming execution (currently continue,\n step, next, return, jump, quit and their abbreviations) terminates\n the command list (as if that command was immediately followed by\n end). This is because any time you resume execution (even with a\n simple next or step), you may encounter another breakpoint--which\n could have its own command list, leading to ambiguities about which\n list to execute.\n\n If you use the \'silent\' command in the command list, the usual\n message about stopping at a breakpoint is not printed. This may be\n desirable for breakpoints that are to print a specific message and\n then continue. If none of the other commands print anything, you\n see no sign that the breakpoint was reached.\n\ns(tep)\n\n Execute the current line, stop at the first possible occasion\n (either in a function that is called or on the next line in the\n current function).\n\nn(ext)\n\n Continue execution until the next line in the current function is\n reached or it returns. (The difference between "next" and "step"\n is that "step" stops inside a called function, while "next"\n executes called functions at (nearly) full speed, only stopping at\n the next line in the current function.)\n\nunt(il) [lineno]\n\n Without argument, continue execution until the line with a number\n greater than the current one is reached.\n\n With a line number, continue execution until a line with a number\n greater or equal to that is reached. In both cases, also stop when\n the current frame returns.\n\n Changed in version 3.2: Allow giving an explicit line number.\n\nr(eturn)\n\n Continue execution until the current function returns.\n\nc(ont(inue))\n\n Continue execution, only stop when a breakpoint is encountered.\n\nj(ump) lineno\n\n Set the next line that will be executed. Only available in the\n bottom-most frame. This lets you jump back and execute code again,\n or jump forward to skip code that you don\'t want to run.\n\n It should be noted that not all jumps are allowed -- for instance\n it is not possible to jump into the middle of a "for" loop or out\n of a "finally" clause.\n\nl(ist) [first[, last]]\n\n List source code for the current file. Without arguments, list 11\n lines around the current line or continue the previous listing.\n With "." as argument, list 11 lines around the current line. With\n one argument, list 11 lines around at that line. With two\n arguments, list the given range; if the second argument is less\n than the first, it is interpreted as a count.\n\n The current line in the current frame is indicated by "->". If an\n exception is being debugged, the line where the exception was\n originally raised or propagated is indicated by ">>", if it differs\n from the current line.\n\n New in version 3.2: The ">>" marker.\n\nll | longlist\n\n List all source code for the current function or frame.\n Interesting lines are marked as for "list".\n\n New in version 3.2.\n\na(rgs)\n\n Print the argument list of the current function.\n\np expression\n\n Evaluate the *expression* in the current context and print its\n value.\n\n Note: "print()" can also be used, but is not a debugger command\n --- this executes the Python "print()" function.\n\npp expression\n\n Like the "p" command, except the value of the expression is pretty-\n printed using the "pprint" module.\n\nwhatis expression\n\n Print the type of the *expression*.\n\nsource expression\n\n Try to get source code for the given object and display it.\n\n New in version 3.2.\n\ndisplay [expression]\n\n Display the value of the expression if it changed, each time\n execution stops in the current frame.\n\n Without expression, list all display expressions for the current\n frame.\n\n New in version 3.2.\n\nundisplay [expression]\n\n Do not display the expression any more in the current frame.\n Without expression, clear all display expressions for the current\n frame.\n\n New in version 3.2.\n\ninteract\n\n Start an interative interpreter (using the "code" module) whose\n global namespace contains all the (global and local) names found in\n the current scope.\n\n New in version 3.2.\n\nalias [name [command]]\n\n Create an alias called *name* that executes *command*. The command\n must *not* be enclosed in quotes. Replaceable parameters can be\n indicated by "%1", "%2", and so on, while "%*" is replaced by all\n the parameters. If no command is given, the current alias for\n *name* is shown. If no arguments are given, all aliases are listed.\n\n Aliases may be nested and can contain anything that can be legally\n typed at the pdb prompt. Note that internal pdb commands *can* be\n overridden by aliases. Such a command is then hidden until the\n alias is removed. Aliasing is recursively applied to the first\n word of the command line; all other words in the line are left\n alone.\n\n As an example, here are two useful aliases (especially when placed\n in the ".pdbrc" file):\n\n # Print instance variables (usage "pi classInst")\n alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])\n # Print instance variables in self\n alias ps pi self\n\nunalias name\n\n Delete the specified alias.\n\n! statement\n\n Execute the (one-line) *statement* in the context of the current\n stack frame. The exclamation point can be omitted unless the first\n word of the statement resembles a debugger command. To set a\n global variable, you can prefix the assignment command with a\n "global" statement on the same line, e.g.:\n\n (Pdb) global list_options; list_options = [\'-l\']\n (Pdb)\n\nrun [args ...]\nrestart [args ...]\n\n Restart the debugged Python program. If an argument is supplied,\n it is split with "shlex" and the result is used as the new\n "sys.argv". History, breakpoints, actions and debugger options are\n preserved. "restart" is an alias for "run".\n\nq(uit)\n\n Quit from the debugger. The program being executed is aborted.\n\n-[ Footnotes ]-\n\n[1] Whether a frame is considered to originate in a certain module\n is determined by the "__name__" in the frame globals.\n', - 'del': u'\nThe "del" statement\n*******************\n\n del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather than spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a "global"\nstatement in the same code block. If the name is unbound, a\n"NameError" exception will be raised.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n\nChanged in version 3.2: Previously it was illegal to delete a name\nfrom the local namespace if it occurs as a free variable in a nested\nblock.\n', - 'dict': u'\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n dict_display ::= "{" [key_datum_list | dict_comprehension] "}"\n key_datum_list ::= key_datum ("," key_datum)* [","]\n key_datum ::= expression ":" expression\n dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum. This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*. (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.) Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': u'\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names. Names may be resolved in the local\nand global namespaces of the caller. Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace. [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace. If only one namespace is\nspecified, it is used for both.\n', - 'else': u'\nThe "if" statement\n******************\n\nThe "if" statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n', - 'exceptions': u'\nExceptions\n**********\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the "raise" statement. Exception\nhandlers are specified with the "try" ... "except" statement. The\n"finally" clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n"SystemExit".\n\nExceptions are identified by class instances. The "except" clause is\nselected depending on the class of the instance: it must reference the\nclass of the instance or a base class thereof. The instance can be\nreceived by the handler and can carry additional information about the\nexceptional condition.\n\nNote: Exception messages are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the "try" statement in section *The try\nstatement* and "raise" statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by\n these operations is not available at the time the module is\n compiled.\n', - 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\nas a command line argument to the interpreter) is a code block. A\nscript command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The string argument passed\nto the built-in functions "eval()" and "exec()" is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes comprehensions and generator\nexpressions since they are implemented using a function scope. This\nmeans that the following will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block,\nunless declared as "nonlocal". If a name is bound at the module\nlevel, it is a global variable. (The variables of the module code\nblock are local and global.) If a variable is used in a code block\nbut not defined there, it is a *free variable*.\n\nWhen a name is not found at all, a "NameError" exception is raised.\nIf the name refers to a local variable that has not been bound, an\n"UnboundLocalError" exception is raised. "UnboundLocalError" is a\nsubclass of "NameError".\n\nThe following constructs bind names: formal parameters to functions,\n"import" statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, "for" loop header, or after\n"as" in a "with" statement or "except" clause. The "import" statement\nof the form "from ... import *" binds all names defined in the\nimported module, except those beginning with an underscore. This form\nmay only be used at the module level.\n\nA target occurring in a "del" statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the "global" statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module "builtins". The global namespace is searched first. If\nthe name is not found there, the builtins namespace is searched. The\n"global" statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name "__builtins__" in its global\nnamespace; this should be a dictionary or a module (in the latter case\nthe module\'s dictionary is used). By default, when in the "__main__"\nmodule, "__builtins__" is the built-in module "builtins"; when in any\nother module, "__builtins__" is an alias for the dictionary of the\n"builtins" module itself. "__builtins__" can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n"__builtins__"; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should "import"\nthe "builtins" module and modify its attributes appropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n"__main__".\n\nThe "global" statement has the same scope as a name binding operation\nin the same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names. Names may be resolved in the local\nand global namespaces of the caller. Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace. [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace. If only one namespace is\nspecified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the "raise" statement. Exception\nhandlers are specified with the "try" ... "except" statement. The\n"finally" clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n"SystemExit".\n\nExceptions are identified by class instances. The "except" clause is\nselected depending on the class of the instance: it must reference the\nclass of the instance or a base class thereof. The instance can be\nreceived by the handler and can carry additional information about the\nexceptional condition.\n\nNote: Exception messages are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the "try" statement in section *The try\nstatement* and "raise" statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by\n these operations is not available at the time the module is\n compiled.\n', - 'exprlists': u'\nExpression lists\n****************\n\n expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple. The\nlength of the tuple is the number of expressions in the list. The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases. A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: "()".)\n', - 'floating': u'\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts are always interpreted using\nradix 10. For example, "077e010" is legal, and denotes the same number\nas "77e10". The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like "-1"\nis actually an expression composed of the unary operator "-" and the\nliteral "1".\n', - 'for': u'\nThe "for" statement\n*******************\n\nThe "for" statement is used to iterate over the elements of a sequence\n(such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n"expression_list". The suite is then executed once for each item\nprovided by the iterator, in the order returned by the iterator. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted. When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a "StopIteration" exception),\nthe suite in the "else" clause, if present, is executed, and the loop\nterminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and continues\nwith the next item, or with the "else" clause if there is no next\nitem.\n\nThe for-loop makes assignments to the variables(s) in the target list.\nThis overwrites all previous assignments to those variables including\nthose made in the suite of the for-loop:\n\n for i in range(10):\n print(i)\n i = 5 # this will not affect the for-loop\n # because i will be overwritten with the next\n # index in the range\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, they will not have been assigned to at\nall by the loop. Hint: the built-in function "range()" returns an\niterator of integers suitable to emulate the effect of Pascal\'s "for i\n:= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n\nNote: There is a subtlety when the sequence is being modified by the\n loop (this can only occur for mutable sequences, i.e. lists). An\n internal counter is used to keep track of which item is used next,\n and this is incremented on each iteration. When this counter has\n reached the length of the sequence the loop terminates. This means\n that if the suite deletes the current (or a previous) item from the\n sequence, the next item will be skipped (since it gets the index of\n the current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n', - 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe "str.format()" method and the "Formatter" class share the same\nsyntax for format strings (although in the case of "Formatter",\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n"{}". Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n"{{" and "}}".\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= +\n conversion ::= "r" | "s" | "a"\n format_spec ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point "\'!\'", and a *format_spec*, which is\npreceded by a colon "\':\'". These specify a non-default format for the\nreplacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings "\'10\'" or\n"\':-]\'") within a format string. The *arg_name* can be followed by any\nnumber of index or attribute expressions. An expression of the form\n"\'.name\'" selects the named attribute using "getattr()", while an\nexpression of the form "\'[index]\'" does an index lookup using\n"__getitem__()".\n\nChanged in version 3.1: The positional argument specifiers can be\nomitted, so "\'{} {}\'" is equivalent to "\'{0} {1}\'".\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the "__format__()"\nmethod of the value itself. However, in some cases it is desirable to\nforce a type to be formatted as a string, overriding its own\ndefinition of formatting. By converting the value to a string before\ncalling "__format__()", the normal formatting logic is bypassed.\n\nThree conversion flags are currently supported: "\'!s\'" which calls\n"str()" on the value, "\'!r\'" which calls "repr()" and "\'!a\'" which\ncalls "ascii()".\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n "More {!a}" # Calls ascii() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in "format()" function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string ("""") produces\nthe same result as if you had called "str()" on the value. A non-empty\nformat string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= \n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nIf a valid *align* value is specified, it can be preceded by a *fill*\ncharacter that can be any character and defaults to a space if\nomitted. Note that it is not possible to use "{" and "}" as *fill*\nchar while using the "str.format()" method; this limitation however\ndoesn\'t affect the "format()" function.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | "\'<\'" | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | "\'>\'" | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | "\'=\'" | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | "\'^\'" | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | "\'+\'" | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | "\'-\'" | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe "\'#\'" option causes the "alternate form" to be used for the\nconversion. The alternate form is defined differently for different\ntypes. This option is only valid for integer, float, complex and\nDecimal types. For integers, when binary, octal, or hexadecimal output\nis used, this option adds the prefix respective "\'0b\'", "\'0o\'", or\n"\'0x\'" to the output value. For floats, complex and Decimal the\nalternate form causes the result of the conversion to always contain a\ndecimal-point character, even if no digits follow it. Normally, a\ndecimal-point character appears in the result of these conversions\nonly if a digit follows it. In addition, for "\'g\'" and "\'G\'"\nconversions, trailing zeros are not removed from the result.\n\nThe "\',\'" option signals the use of a comma for a thousands separator.\nFor a locale aware separator, use the "\'n\'" integer presentation type\ninstead.\n\nChanged in version 3.1: Added the "\',\'" option (see also **PEP 378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero ("\'0\'") character enables sign-\naware zero-padding for numeric types. This is equivalent to a *fill*\ncharacter of "\'0\'" with an *alignment* type of "\'=\'".\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with "\'f\'" and "\'F\'", or before and after the decimal point\nfor a floating point value formatted with "\'g\'" or "\'G\'". For non-\nnumber types the field indicates the maximum field size - in other\nwords, how many characters will be used from the field content. The\n*precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | "\'s\'" | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as "\'s\'". |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | "\'b\'" | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | "\'c\'" | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | "\'d\'" | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | "\'o\'" | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | "\'x\'" | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | "\'X\'" | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | "\'n\'" | Number. This is the same as "\'d\'", except that it uses the |\n | | current locale setting to insert the appropriate number |\n | | separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as "\'d\'". |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except "\'n\'"\nand None). When doing so, "float()" is used to convert the integer to\na floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | "\'e\'" | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n | | The default precision is "6". |\n +-----------+------------------------------------------------------------+\n | "\'E\'" | Exponent notation. Same as "\'e\'" except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | "\'f\'" | Fixed point. Displays the number as a fixed-point number. |\n | | The default precision is "6". |\n +-----------+------------------------------------------------------------+\n | "\'F\'" | Fixed point. Same as "\'f\'", but converts "nan" to "NAN" |\n | | and "inf" to "INF". |\n +-----------+------------------------------------------------------------+\n | "\'g\'" | General format. For a given precision "p >= 1", this |\n | | rounds the number to "p" significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type "\'e\'" and precision "p-1" |\n | | would have exponent "exp". Then if "-4 <= exp < p", the |\n | | number is formatted with presentation type "\'f\'" and |\n | | precision "p-1-exp". Otherwise, the number is formatted |\n | | with presentation type "\'e\'" and precision "p-1". In both |\n | | cases insignificant trailing zeros are removed from the |\n | | significand, and the decimal point is also removed if |\n | | there are no remaining digits following it. Positive and |\n | | negative infinity, positive and negative zero, and nans, |\n | | are formatted as "inf", "-inf", "0", "-0" and "nan" |\n | | respectively, regardless of the precision. A precision of |\n | | "0" is treated as equivalent to a precision of "1". The |\n | | default precision is "6". |\n +-----------+------------------------------------------------------------+\n | "\'G\'" | General format. Same as "\'g\'" except switches to "\'E\'" if |\n | | the number gets too large. The representations of infinity |\n | | and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | "\'n\'" | Number. This is the same as "\'g\'", except that it uses the |\n | | current locale setting to insert the appropriate number |\n | | separator characters. |\n +-----------+------------------------------------------------------------+\n | "\'%\'" | Percentage. Multiplies the number by 100 and displays in |\n | | fixed ("\'f\'") format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | Similar to "\'g\'", except that fixed-point notation, when |\n | | used, has at least one digit past the decimal point. The |\n | | default precision is as high as needed to represent the |\n | | particular value. The overall effect is to match the |\n | | output of "str()" as altered by the other format |\n | | modifiers. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old "%"-formatting.\n\nIn most of the cases the syntax is similar to the old "%"-formatting,\nwith the addition of the "{}" and with ":" used instead of "%". For\nexample, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 3.1+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point:\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing "%s" and "%r":\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing "%+f", "%-f", and "% f" and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing "%x" and "%o" and converting the value to different bases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 86.36%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12): #doctest: +NORMALIZE_WHITESPACE\n ... for base in \'dXob\':\n ... print(\'{0:{width}{base}}\'.format(num, base=base, width=width), end=\' \')\n ... print()\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', - 'function': u'\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n | "*" [parameter] ("," defparameter)* ["," "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated from left to right when the\nfunction definition is executed.** This means that the expression is\nevaluated once, when the function is defined, and that the same "pre-\ncomputed" value is used for each call. This is especially important\nto understand when a default parameter is a mutable object, such as a\nlist or a dictionary: if the function modifies the object (e.g. by\nappending an item to a list), the default value is in effect modified.\nThis is generally not what was intended. A way around this is to use\n"None" as the default, and explicitly test for it in the body of the\nfunction, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple. If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name. Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier". Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist. These annotations can be any valid Python expression and are\nevaluated when the function definition is executed. Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction. The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda\nexpressions, described in section *Lambdas*. Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression. The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around. Free variables used\nin the nested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n The original specification for function annotations.\n', - 'global': u'\nThe "global" statement\n**********************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe "global" statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without "global", although free variables may refer to\nglobals without being declared global.\n\nNames listed in a "global" statement must not be used in the same code\nblock textually preceding that "global" statement.\n\nNames listed in a "global" statement must not be defined as formal\nparameters or in a "for" loop control target, "class" definition,\nfunction definition, or "import" statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the two restrictions, but programs should not abuse this\nfreedom, as future implementations may enforce them or silently change\nthe meaning of the program.\n\n**Programmer\'s note:** the "global" is a directive to the parser. It\napplies only to code parsed at the same time as the "global"\nstatement. In particular, a "global" statement contained in a string\nor code object supplied to the built-in "exec()" function does not\naffect the code block *containing* the function call, and code\ncontained in such a string is unaffected by "global" statements in the\ncode containing the function call. The same applies to the "eval()"\nand "compile()" functions.\n', - 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n"_*"\n Not imported by "from module import *". The special identifier "_"\n is used in the interactive interpreter to store the result of the\n last evaluation; it is stored in the "builtins" module. When not\n in interactive mode, "_" has no special meaning and is not defined.\n See section *The import statement*.\n\n Note: The name "_" is often used in conjunction with\n internationalization; refer to the documentation for the\n "gettext" module for more information on this convention.\n\n"__*__"\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of "__*__" names, in any context, that does not\n follow explicitly documented use, is subject to breakage without\n warning.\n\n"__*"\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', - 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions.\n\nThe syntax of identifiers in Python is based on the Unicode standard\nannex UAX-31, with elaboration and changes as defined below; see also\n**PEP 3131** for further details.\n\nWithin the ASCII range (U+0001..U+007F), the valid characters for\nidentifiers are the same as in Python 2.x: the uppercase and lowercase\nletters "A" through "Z", the underscore "_" and, except for the first\ncharacter, the digits "0" through "9".\n\nPython 3.0 introduces additional characters from outside the ASCII\nrange (see **PEP 3131**). For these characters, the classification\nuses the version of the Unicode Character Database as included in the\n"unicodedata" module.\n\nIdentifiers are unlimited in length. Case is significant.\n\n identifier ::= xid_start xid_continue*\n id_start ::= \n id_continue ::= \n xid_start ::= \n xid_continue ::= \n\nThe Unicode category codes mentioned above stand for:\n\n* *Lu* - uppercase letters\n\n* *Ll* - lowercase letters\n\n* *Lt* - titlecase letters\n\n* *Lm* - modifier letters\n\n* *Lo* - other letters\n\n* *Nl* - letter numbers\n\n* *Mn* - nonspacing marks\n\n* *Mc* - spacing combining marks\n\n* *Nd* - decimal numbers\n\n* *Pc* - connector punctuations\n\n* *Other_ID_Start* - explicit list of characters in PropList.txt to\n support backwards compatibility\n\n* *Other_ID_Continue* - likewise\n\nAll identifiers are converted into the normal form NFKC while parsing;\ncomparison of identifiers is based on NFKC.\n\nA non-normative HTML file listing all valid identifier characters for\nUnicode 4.1 can be found at http://www.dcl.hpi.uni-\npotsdam.de/home/loewis/table-3131.html.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n False class finally is return\n None continue for lambda try\n True def from nonlocal while\n and del global not with\n as elif if or yield\n assert else import pass\n break except in raise\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n"_*"\n Not imported by "from module import *". The special identifier "_"\n is used in the interactive interpreter to store the result of the\n last evaluation; it is stored in the "builtins" module. When not\n in interactive mode, "_" has no special meaning and is not defined.\n See section *The import statement*.\n\n Note: The name "_" is often used in conjunction with\n internationalization; refer to the documentation for the\n "gettext" module for more information on this convention.\n\n"__*__"\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of "__*__" names, in any context, that does not\n follow explicitly documented use, is subject to breakage without\n warning.\n\n"__*"\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', - 'if': u'\nThe "if" statement\n******************\n\nThe "if" statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n', - 'imaginary': u'\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range. To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., "(3+4j)". Some examples of imaginary literals:\n\n 3.14j 10.j 10j .001j 1e100j 3.14e-10j\n', - 'import': u'\nThe "import" statement\n**********************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nThe basic import statement (no "from" clause) is executed in two\nsteps:\n\n1. find a module, loading and initializing it if necessary\n\n2. define a name or names in the local namespace for the scope\n where the "import" statement occurs.\n\nWhen the statement contains multiple clauses (separated by commas) the\ntwo steps are carried out separately for each clause, just as though\nthe clauses had been separated out into individiual import statements.\n\nThe details of the first step, finding and loading modules are\ndescribed in greater detail in the section on the *import system*,\nwhich also describes the various types of packages and modules that\ncan be imported, as well as all the hooks that can be used to\ncustomize the import system. Note that failures in this step may\nindicate either that the module could not be located, *or* that an\nerror occurred while initializing the module, which includes execution\nof the module\'s code.\n\nIf the requested module is retrieved successfully, it will be made\navailable in the local namespace in one of three ways:\n\n* If the module name is followed by "as", then the name following\n "as" is bound directly to the imported module.\n\n* If no other name is specified, and the module being imported is a\n top level module, the module\'s name is bound in the local namespace\n as a reference to the imported module\n\n* If the module being imported is *not* a top level module, then the\n name of the top level package that contains the module is bound in\n the local namespace as a reference to the top level package. The\n imported module must be accessed using its full qualified name\n rather than directly\n\nThe "from" form uses a slightly more complex process:\n\n1. find the module specified in the "from" clause, loading and\n initializing it if necessary;\n\n2. for each of the identifiers specified in the "import" clauses:\n\n 1. check if the imported module has an attribute by that name\n\n 2. if not, attempt to import a submodule with that name and then\n check the imported module again for that attribute\n\n 3. if the attribute is not found, "ImportError" is raised.\n\n 4. otherwise, a reference to that value is stored in the local\n namespace, using the name in the "as" clause if it is present,\n otherwise using the attribute name\n\nExamples:\n\n import foo # foo imported and bound locally\n import foo.bar.baz # foo.bar.baz imported, foo bound locally\n import foo.bar.baz as fbb # foo.bar.baz imported and bound as fbb\n from foo.bar import baz # foo.bar.baz imported and bound as baz\n from foo import attr # foo imported and foo.attr bound as attr\n\nIf the list of identifiers is replaced by a star ("\'*\'"), all public\nnames defined in the module are bound in the local namespace for the\nscope where the "import" statement occurs.\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named "__all__"; if defined, it must\nbe a sequence of strings which are names defined or imported by that\nmodule. The names given in "__all__" are all considered public and\nare required to exist. If "__all__" is not defined, the set of public\nnames includes all names found in the module\'s namespace which do not\nbegin with an underscore character ("\'_\'"). "__all__" should contain\nthe entire public API. It is intended to avoid accidentally exporting\nitems that are not part of the API (such as library modules which were\nimported and used within the module).\n\nThe wild card form of import --- "from module import *" --- is only\nallowed at the module level. Attempting to use it in class or\nfunction definitions will raise a "SyntaxError".\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after "from" you\ncan specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n"from . import mod" from a module in the "pkg" package then you will\nend up importing "pkg.mod". If you execute "from ..subpkg2 import mod"\nfrom within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The\nspecification for relative imports is contained within **PEP 328**.\n\n"importlib.import_module()" is provided to support applications that\ndetermine dynamically the modules to be loaded.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python where the feature\nbecomes standard.\n\nThe future statement is intended to ease migration to future versions\nof Python that introduce incompatible changes to the language. It\nallows use of the new features on a per-module basis before the\nrelease in which the feature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 3.0 are "absolute_import",\n"division", "generators", "unicode_literals", "print_function",\n"nested_scopes" and "with_statement". They are all redundant because\nthey are always enabled, and only kept for backwards compatibility.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module "__future__", described later, and it will\nbe imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by calls to the built-in functions "exec()" and\n"compile()" that occur in a module "M" containing a future statement\nwill, by default, use the new syntax or semantics associated with the\nfuture statement. This can be controlled by optional arguments to\n"compile()" --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also: **PEP 236** - Back to the __future__\n\n The original proposal for the __future__ mechanism.\n', - 'in': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like "a < b < c" have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: "True" or "False".\n\nComparisons can be chained arbitrarily, e.g., "x < y <= z" is\nequivalent to "x < y and y <= z", except that "y" is evaluated only\nonce (but in both cases "z" is not evaluated at all when "x < y" is\nfound to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then "a op1 b op2 c ... y\nopN z" is equivalent to "a op1 b and b op2 c and ... y opN z", except\nthat each expression is evaluated at most once.\n\nNote that "a op1 b op2 c" doesn\'t imply any kind of comparison between\n*a* and *c*, so that, e.g., "x < y > z" is perfectly legal (though\nperhaps not pretty).\n\nThe operators "<", ">", "==", ">=", "<=", and "!=" compare the values\nof two objects. The objects need not have the same type. If both are\nnumbers, they are converted to a common type. Otherwise, the "==" and\n"!=" operators *always* consider objects of different types to be\nunequal, while the "<", ">", ">=" and "<=" operators raise a\n"TypeError" when comparing objects of different types that do not\nimplement these operators for the given pair of types. You can\ncontrol comparison behavior of objects of non-built-in types by\ndefining rich comparison methods like "__gt__()", described in section\n*Basic customization*.\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* The values "float(\'NaN\')" and "Decimal(\'NaN\')" are special. They\n are identical to themselves, "x is x" but are not equal to\n themselves, "x != x". Additionally, comparing any value to a\n not-a-number value will return "False". For example, both "3 <\n float(\'NaN\')" and "float(\'NaN\') < 3" will return "False".\n\n* Bytes objects are compared lexicographically using the numeric\n values of their elements.\n\n* Strings are compared lexicographically using the numeric\n equivalents (the result of the built-in function "ord()") of their\n characters. [3] String and bytes object can\'t be compared!\n\n* Tuples and lists are compared lexicographically using comparison\n of corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, "[1,2,x] <= [1,2,y]" has the same\n value as "x <= y". If the corresponding element does not exist, the\n shorter sequence is ordered first (for example, "[1,2] < [1,2,3]").\n\n* Mappings (dictionaries) compare equal if and only if they have the\n same "(key, value)" pairs. Order comparisons "(\'<\', \'<=\', \'>=\',\n \'>\')" raise "TypeError".\n\n* Sets and frozensets define comparison operators to mean subset and\n superset tests. Those relations do not define total orderings (the\n two sets "{1,2}" and {2,3} are not equal, nor subsets of one\n another, nor supersets of one another). Accordingly, sets are not\n appropriate arguments for functions which depend on total ordering.\n For example, "min()", "max()", and "sorted()" produce undefined\n results given a list of sets as inputs.\n\n* Most other objects of built-in types compare unequal unless they\n are the same object; the choice whether one object is considered\n smaller or larger than another one is made arbitrarily but\n consistently within one execution of a program.\n\nComparison of objects of differing types depends on whether either of\nthe types provide explicit support for the comparison. Most numeric\ntypes can be compared with one another. When cross-type comparison is\nnot supported, the comparison method returns "NotImplemented".\n\nThe operators "in" and "not in" test for membership. "x in s"\nevaluates to true if *x* is a member of *s*, and false otherwise. "x\nnot in s" returns the negation of "x in s". All built-in sequences\nand set types support this as well as dictionary, for which "in" tests\nwhether the dictionary has a given key. For container types such as\nlist, tuple, set, frozenset, dict, or collections.deque, the\nexpression "x in y" is equivalent to "any(x is e or x == e for e in\ny)".\n\nFor the string and bytes types, "x in y" is true if and only if *x* is\na substring of *y*. An equivalent test is "y.find(x) != -1". Empty\nstrings are always considered to be a substring of any other string,\nso """ in "abc"" will return "True".\n\nFor user-defined classes which define the "__contains__()" method, "x\nin y" is true if and only if "y.__contains__(x)" is true.\n\nFor user-defined classes which do not define "__contains__()" but do\ndefine "__iter__()", "x in y" is true if some value "z" with "x == z"\nis produced while iterating over "y". If an exception is raised\nduring the iteration, it is as if "in" raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n"__getitem__()", "x in y" is true if and only if there is a non-\nnegative integer index *i* such that "x == y[i]", and all lower\ninteger indices do not raise "IndexError" exception. (If any other\nexception is raised, it is as if "in" raised that exception).\n\nThe operator "not in" is defined to have the inverse true value of\n"in".\n\nThe operators "is" and "is not" test for object identity: "x is y" is\ntrue if and only if *x* and *y* are the same object. "x is not y"\nyields the inverse truth value. [4]\n', - 'integers': u'\nInteger literals\n****************\n\nInteger literals are described by the following lexical definitions:\n\n integer ::= decimalinteger | octinteger | hexinteger | bininteger\n decimalinteger ::= nonzerodigit digit* | "0"+\n nonzerodigit ::= "1"..."9"\n digit ::= "0"..."9"\n octinteger ::= "0" ("o" | "O") octdigit+\n hexinteger ::= "0" ("x" | "X") hexdigit+\n bininteger ::= "0" ("b" | "B") bindigit+\n octdigit ::= "0"..."7"\n hexdigit ::= digit | "a"..."f" | "A"..."F"\n bindigit ::= "0" | "1"\n\nThere is no limit for the length of integer literals apart from what\ncan be stored in available memory.\n\nNote that leading zeros in a non-zero decimal number are not allowed.\nThis is for disambiguation with C-style octal literals, which Python\nused before version 3.0.\n\nSome examples of integer literals:\n\n 7 2147483647 0o177 0b100110111\n 3 79228162514264337593543950336 0o377 0xdeadbeef\n', - 'lambda': u'\nLambdas\n*******\n\n lambda_expr ::= "lambda" [parameter_list]: expression\n lambda_expr_nocond ::= "lambda" [parameter_list]: expression_nocond\n\nLambda expressions (sometimes called lambda forms) are used to create\nanonymous functions. The expression "lambda arguments: expression"\nyields a function object. The unnamed object behaves like a function\nobject defined with\n\n def (arguments):\n return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda expressions cannot contain\nstatements or annotations.\n', - 'lists': u'\nList displays\n*************\n\nA list display is a possibly empty series of expressions enclosed in\nsquare brackets:\n\n list_display ::= "[" [expression_list | comprehension] "]"\n\nA list display yields a new list object, the contents being specified\nby either a list of expressions or a comprehension. When a comma-\nseparated list of expressions is supplied, its elements are evaluated\nfrom left to right and placed into the list object in that order.\nWhen a comprehension is supplied, the list is constructed from the\nelements resulting from the comprehension.\n', - 'naming': u'\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\nas a command line argument to the interpreter) is a code block. A\nscript command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The string argument passed\nto the built-in functions "eval()" and "exec()" is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes comprehensions and generator\nexpressions since they are implemented using a function scope. This\nmeans that the following will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block,\nunless declared as "nonlocal". If a name is bound at the module\nlevel, it is a global variable. (The variables of the module code\nblock are local and global.) If a variable is used in a code block\nbut not defined there, it is a *free variable*.\n\nWhen a name is not found at all, a "NameError" exception is raised.\nIf the name refers to a local variable that has not been bound, an\n"UnboundLocalError" exception is raised. "UnboundLocalError" is a\nsubclass of "NameError".\n\nThe following constructs bind names: formal parameters to functions,\n"import" statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, "for" loop header, or after\n"as" in a "with" statement or "except" clause. The "import" statement\nof the form "from ... import *" binds all names defined in the\nimported module, except those beginning with an underscore. This form\nmay only be used at the module level.\n\nA target occurring in a "del" statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the "global" statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module "builtins". The global namespace is searched first. If\nthe name is not found there, the builtins namespace is searched. The\n"global" statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name "__builtins__" in its global\nnamespace; this should be a dictionary or a module (in the latter case\nthe module\'s dictionary is used). By default, when in the "__main__"\nmodule, "__builtins__" is the built-in module "builtins"; when in any\nother module, "__builtins__" is an alias for the dictionary of the\n"builtins" module itself. "__builtins__" can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n"__builtins__"; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should "import"\nthe "builtins" module and modify its attributes appropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n"__main__".\n\nThe "global" statement has the same scope as a name binding operation\nin the same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names. Names may be resolved in the local\nand global namespaces of the caller. Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace. [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace. If only one namespace is\nspecified, it is used for both.\n', - 'nonlocal': u'\nThe "nonlocal" statement\n************************\n\n nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n\nThe "nonlocal" statement causes the listed identifiers to refer to\npreviously bound variables in the nearest enclosing scope excluding\nglobals. This is important because the default behavior for binding is\nto search the local namespace first. The statement allows\nencapsulated code to rebind variables outside of the local scope\nbesides the global (module) scope.\n\nNames listed in a "nonlocal" statement, unlike those listed in a\n"global" statement, must refer to pre-existing bindings in an\nenclosing scope (the scope in which a new binding should be created\ncannot be determined unambiguously).\n\nNames listed in a "nonlocal" statement must not collide with pre-\nexisting bindings in the local scope.\n\nSee also: **PEP 3104** - Access to Names in Outer Scopes\n\n The specification for the "nonlocal" statement.\n', - 'numbers': u'\nNumeric literals\n****************\n\nThere are three types of numeric literals: integers, floating point\nnumbers, and imaginary numbers. There are no complex literals\n(complex numbers can be formed by adding a real number and an\nimaginary number).\n\nNote that numeric literals do not include a sign; a phrase like "-1"\nis actually an expression composed of the unary operator \'"-"\' and the\nliteral "1".\n', - 'numeric-types': u'\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__matmul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to\n evaluate the expression "x + y", where *x* is an instance of a\n class that has an "__add__()" method, "x.__add__(y)" is called.\n The "__divmod__()" method should be the equivalent to using\n "__floordiv__()" and "__mod__()"; it should not be related to\n "__truediv__()". Note that "__pow__()" should be defined to accept\n an optional third argument if the ternary version of the built-in\n "pow()" function is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rmatmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped)\n operands. These functions are only called if the left operand does\n not support the corresponding operation and the operands are of\n different types. [2] For instance, to evaluate the expression "x -\n y", where *y* is an instance of a class that has an "__rsub__()"\n method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n *NotImplemented*.\n\n Note that ternary "pow()" will not try calling "__rpow__()" (the\n coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left\n operand\'s type and that subclass provides the reflected method\n for the operation, this method will be called before the left\n operand\'s non-reflected method. This behavior allows subclasses\n to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__imatmul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=",\n "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to\n do the operation in-place (modifying *self*) and return the result\n (which could be, but does not have to be, *self*). If a specific\n method is not defined, the augmented assignment falls back to the\n normal methods. For instance, if *x* is an instance of a class\n with an "__iadd__()" method, "x += y" is equivalent to "x =\n x.__iadd__(y)" . Otherwise, "x.__add__(y)" and "y.__radd__(x)" are\n considered, as with the evaluation of "x + y". In certain\n situations, augmented assignment can result in unexpected errors\n (see *Why does a_tuple[i] += [\'item\'] raise an exception when the\n addition works?*), but this behavior is in fact part of the data\n model.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations ("-", "+",\n "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions "complex()", "int()",\n "float()" and "round()". Should return a value of the appropriate\n type.\n\nobject.__index__(self)\n\n Called to implement "operator.index()", and whenever Python needs\n to losslessly convert the numeric object to an integer object (such\n as in slicing, or in the built-in "bin()", "hex()" and "oct()"\n functions). Presence of this method indicates that the numeric\n object is an integer type. Must return an integer.\n\n Note: In order to have a coherent integer type class, when\n "__index__()" is defined "__int__()" should also be defined, and\n both should return the same value.\n', - 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'"is"\' operator compares the\nidentity of two objects; the "id()" function returns an integer\nrepresenting its identity.\n\n**CPython implementation detail:** For CPython, "id(x)" is the memory\naddress where "x" is stored.\n\nAn object\'s type determines the operations that the object supports\n(e.g., "does it have a length?") and also defines the possible values\nfor objects of that type. The "type()" function returns an object\'s\ntype (which is an object itself). Like its identity, an object\'s\n*type* is also unchangeable. [1]\n\nThe *value* of some objects can change. Objects whose value can\nchange are said to be *mutable*; objects whose value is unchangeable\nonce they are created are called *immutable*. (The value of an\nimmutable container object that contains a reference to a mutable\nobject can change when the latter\'s value is changed; however the\ncontainer is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the "gc" module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (so\nyou should always close files explicitly).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'"try"..."except"\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a "close()" method. Programs\nare strongly recommended to explicitly close such objects. The\n\'"try"..."finally"\' statement and the \'"with"\' statement provide\nconvenient ways to do this.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after "a = 1; b = 1",\n"a" and "b" may or may not refer to the same object with the value\none, depending on the implementation, but after "c = []; d = []", "c"\nand "d" are guaranteed to refer to two different, unique, newly\ncreated empty lists. (Note that "c = d = []" assigns the same object\nto both "c" and "d".)\n', - 'operator-summary': u'\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedence in Python, from\nlowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for exponentiation, which\ngroups from right to left).\n\nNote that comparisons, membership tests, and identity tests, all have\nthe same precedence and have a left-to-right chaining feature as\ndescribed in the *Comparisons* section.\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| "lambda" | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| "if" -- "else" | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| "or" | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| "and" | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| "not" "x" | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| "in", "not in", "is", "is not", "<", "<=", ">", | Comparisons, including membership |\n| ">=", "!=", "==" | tests and identity tests |\n+-------------------------------------------------+---------------------------------------+\n| "|" | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| "^" | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| "&" | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| "<<", ">>" | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| "+", "-" | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| "*", "@", "/", "//", "%" | Multiplication, matrix multiplication |\n| | division, remainder [5] |\n+-------------------------------------------------+---------------------------------------+\n| "+x", "-x", "~x" | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| "**" | Exponentiation [6] |\n+-------------------------------------------------+---------------------------------------+\n| "await" "x" | Await expression |\n+-------------------------------------------------+---------------------------------------+\n| "x[index]", "x[index:index]", | Subscription, slicing, call, |\n| "x(arguments...)", "x.attribute" | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| "(expressions...)", "[expressions...]", "{key: | Binding or tuple display, list |\n| value...}", "{expressions...}" | display, dictionary display, set |\n| | display |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] While "abs(x%y) < abs(y)" is true mathematically, for floats\n it may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that "-1e-100 % 1e100" have the same\n sign as "1e100", the computed result is "-1e-100 + 1e100", which\n is numerically exactly equal to "1e100". The function\n "math.fmod()" returns a result whose sign matches the sign of the\n first argument instead, and so returns "-1e-100" in this case.\n Which approach is more appropriate depends on the application.\n\n[2] If x is very close to an exact integer multiple of y, it\'s\n possible for "x//y" to be one larger than "(x-x%y)//y" due to\n rounding. In such cases, Python returns the latter result, in\n order to preserve that "divmod(x,y)[0] * y + x % y" be very close\n to "x".\n\n[3] While comparisons between strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ""\\u00C7"" and ""\\u0327\\u0043"" compare differently, even\n though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using "unicodedata.normalize()".\n\n[4] Due to automatic garbage-collection, free lists, and the\n dynamic nature of descriptors, you may notice seemingly unusual\n behaviour in certain uses of the "is" operator, like those\n involving comparisons between instance methods, or constants.\n Check their documentation for more info.\n\n[5] The "%" operator is also used for string formatting; the same\n precedence applies.\n\n[6] The power operator "**" binds less tightly than an arithmetic\n or bitwise unary operator on its right, that is, "2**-1" is "0.5".\n', - 'pass': u'\nThe "pass" statement\n********************\n\n pass_stmt ::= "pass"\n\n"pass" is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n', - 'power': u'\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= await ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): "-1**2" results in "-1".\n\nThe power operator has the same semantics as the built-in "pow()"\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type, and the result is of that type.\n\nFor int operands, the result has the same type as the operands unless\nthe second argument is negative; in that case, all arguments are\nconverted to float and a float result is delivered. For example,\n"10**2" returns "100", but "10**-2" returns "0.01".\n\nRaising "0.0" to a negative power results in a "ZeroDivisionError".\nRaising a negative number to a fractional power results in a "complex"\nnumber. (In earlier versions it raised a "ValueError".)\n', - 'raise': u'\nThe "raise" statement\n*********************\n\n raise_stmt ::= "raise" [expression ["from" expression]]\n\nIf no expressions are present, "raise" re-raises the last exception\nthat was active in the current scope. If no exception is active in\nthe current scope, a "RuntimeError" exception is raised indicating\nthat this is an error.\n\nOtherwise, "raise" evaluates the first expression as the exception\nobject. It must be either a subclass or an instance of\n"BaseException". If it is a class, the exception instance will be\nobtained when needed by instantiating the class with no arguments.\n\nThe *type* of the exception is the exception instance\'s class, the\n*value* is the instance itself.\n\nA traceback object is normally created automatically when an exception\nis raised and attached to it as the "__traceback__" attribute, which\nis writable. You can create an exception and set your own traceback in\none step using the "with_traceback()" exception method (which returns\nthe same exception instance, with its traceback set to its argument),\nlike so:\n\n raise Exception("foo occurred").with_traceback(tracebackobj)\n\nThe "from" clause is used for exception chaining: if given, the second\n*expression* must be another exception class or instance, which will\nthen be attached to the raised exception as the "__cause__" attribute\n(which is writable). If the raised exception is not handled, both\nexceptions will be printed:\n\n >>> try:\n ... print(1 / 0)\n ... except Exception as exc:\n ... raise RuntimeError("Something bad happened") from exc\n ...\n Traceback (most recent call last):\n File "", line 2, in \n ZeroDivisionError: int division or modulo by zero\n\n The above exception was the direct cause of the following exception:\n\n Traceback (most recent call last):\n File "", line 4, in \n RuntimeError: Something bad happened\n\nA similar mechanism works implicitly if an exception is raised inside\nan exception handler or a "finally" clause: the previous exception is\nthen attached as the new exception\'s "__context__" attribute:\n\n >>> try:\n ... print(1 / 0)\n ... except:\n ... raise RuntimeError("Something bad happened")\n ...\n Traceback (most recent call last):\n File "", line 2, in \n ZeroDivisionError: int division or modulo by zero\n\n During handling of the above exception, another exception occurred:\n\n Traceback (most recent call last):\n File "", line 4, in \n RuntimeError: Something bad happened\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information about handling exceptions is in section\n*The try statement*.\n', - 'return': u'\nThe "return" statement\n**********************\n\n return_stmt ::= "return" [expression_list]\n\n"return" may only occur syntactically nested in a function definition,\nnot within a nested class definition.\n\nIf an expression list is present, it is evaluated, else "None" is\nsubstituted.\n\n"return" leaves the current function call with the expression list (or\n"None") as return value.\n\nWhen "return" passes control out of a "try" statement with a "finally"\nclause, that "finally" clause is executed before really leaving the\nfunction.\n\nIn a generator function, the "return" statement indicates that the\ngenerator is done and will cause "StopIteration" to be raised. The\nreturned value (if any) is used as an argument to construct\n"StopIteration" and becomes the "StopIteration.value" attribute.\n', - 'sequence-types': u'\nEmulating container types\n*************************\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items. It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects. Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators. It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values. It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function "len()". Should return\n the length of the object, an integer ">=" 0. Also, an object that\n doesn\'t define a "__bool__()" method and whose "__len__()" method\n returns zero is considered to be false in a Boolean context.\n\nobject.__length_hint__(self)\n\n Called to implement "operator.length_hint()". Should return an\n estimated length for the object (which may be greater or less than\n the actual length). The length must be an integer ">=" 0. This\n method is purely an optimization and is never required for\n correctness.\n\n New in version 3.4.\n\nNote: Slicing is done exclusively with the following three methods.\n A call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of "self[key]". For sequence types,\n the accepted keys should be integers and slice objects. Note that\n the special interpretation of negative indexes (if the class wishes\n to emulate a sequence type) is up to the "__getitem__()" method. If\n *key* is of an inappropriate type, "TypeError" may be raised; if of\n a value outside the set of indexes for the sequence (after any\n special interpretation of negative values), "IndexError" should be\n raised. For mapping types, if *key* is missing (not in the\n container), "KeyError" should be raised.\n\n Note: "for" loops expect that an "IndexError" will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__missing__(self, key)\n\n Called by "dict"."__getitem__()" to implement "self[key]" for dict\n subclasses when key is not in the dictionary.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the "reversed()" built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the "__reversed__()" method is not provided, the "reversed()"\n built-in will fall back to using the sequence protocol ("__len__()"\n and "__getitem__()"). Objects that support the sequence protocol\n should only provide "__reversed__()" if they can provide an\n implementation that is more efficient than the one provided by\n "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define "__contains__()", the membership test\n first tries iteration via "__iter__()", then the old sequence\n iteration protocol via "__getitem__()", see *this section in the\n language reference*.\n', - 'shifting': u'\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept integers as arguments. They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as floor division by "pow(2,n)".\nA left shift by *n* bits is defined as multiplication with "pow(2,n)".\n\nNote: In the current implementation, the right-hand operand is\n required to be at most "sys.maxsize". If the right-hand operand is\n larger than "sys.maxsize" an "OverflowError" exception is raised.\n', - 'slicings': u'\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or "del" statements. The syntax for a slicing:\n\n slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice\n proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" [stride] ]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice).\n\nThe semantics for a slicing are as follows. The primary is indexed\n(using the same "__getitem__()" method as normal subscription) with a\nkey that is constructed from the slice list, as follows. If the slice\nlist contains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of a proper slice is a\nslice object (see section *The standard type hierarchy*) whose\n"start", "stop" and "step" attributes are the values of the\nexpressions given as lower bound, upper bound and stride,\nrespectively, substituting "None" for missing expressions.\n', - 'specialattrs': u'\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the "dir()" built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object\'s\n (writable) attributes.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nclass.__qualname__\n\n The *qualified name* of the class or type.\n\n New in version 3.3.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in "__mro__".\n\nclass.__subclasses__()\n\n Each class keeps a list of weak references to its immediate\n subclasses. This method returns a list of all those references\n still alive. Example:\n\n >>> int.__subclasses__()\n []\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found\n in the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list "[1, 2]" is considered equal to\n "[1.0, 2.0]", and similarly for tuples.\n\n[3] They must have since the parser can\'t tell the type of the\n operands.\n\n[4] Cased characters are those with general category property\n being one of "Lu" (Letter, uppercase), "Ll" (Letter, lowercase),\n or "Lt" (Letter, titlecase).\n\n[5] To format only a tuple you should therefore provide a\n singleton tuple whose only element is the tuple to be formatted.\n', - 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named "__getitem__()", and "x" is an instance of this class,\nthen "x[i]" is roughly equivalent to "type(x).__getitem__(x, i)".\nExcept where mentioned, attempts to execute an operation raise an\nexception when no appropriate method is defined (typically\n"AttributeError" or "TypeError").\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n"NodeList" interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. "__new__()" is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of "__new__()" should be the new object instance (usually an\n instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s "__new__()" method using\n "super(currentclass, cls).__new__(cls[, ...])" with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If "__new__()" returns an instance of *cls*, then the new\n instance\'s "__init__()" method will be invoked like\n "__init__(self[, ...])", where *self* is the new instance and the\n remaining arguments are the same as were passed to "__new__()".\n\n If "__new__()" does not return an instance of *cls*, then the new\n instance\'s "__init__()" method will not be invoked.\n\n "__new__()" is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called after the instance has been created (by "__new__()"), but\n before it is returned to the caller. The arguments are those\n passed to the class constructor expression. If a base class has an\n "__init__()" method, the derived class\'s "__init__()" method, if\n any, must explicitly call it to ensure proper initialization of the\n base class part of the instance; for example:\n "BaseClass.__init__(self, [args...])".\n\n Because "__new__()" and "__init__()" work together in constructing\n objects ("__new__()" to create it, and "__init__()" to customise\n it), no non-"None" value may be returned by "__init__()"; doing so\n will cause a "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a "__del__()" method, the\n derived class\'s "__del__()" method, if any, must explicitly call it\n to ensure proper deletion of the base class part of the instance.\n Note that it is possible (though not recommended!) for the\n "__del__()" method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n "__del__()" methods are called for objects that still exist when\n the interpreter exits.\n\n Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n decrements the reference count for "x" by one, and the latter is\n only called when "x"\'s reference count reaches zero. Some common\n situations that may prevent the reference count of an object from\n going to zero include: circular references between objects (e.g.,\n a doubly-linked list or a tree data structure with parent and\n child pointers); a reference to the object on the stack frame of\n a function that caught an exception (the traceback stored in\n "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n to the object on the stack frame that raised an unhandled\n exception in interactive mode (the traceback stored in\n "sys.last_traceback" keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the second can be resolved by freeing the reference to the\n traceback object when it is no longer useful, and the third can\n be resolved by storing "None" in "sys.last_traceback". Circular\n references which are garbage are detected and cleaned up when the\n cyclic garbage collector is enabled (it\'s on by default). Refer\n to the documentation for the "gc" module for more information\n about this topic.\n\n Warning: Due to the precarious circumstances under which\n "__del__()" methods are invoked, exceptions that occur during\n their execution are ignored, and a warning is printed to\n "sys.stderr" instead. Also, when "__del__()" is invoked in\n response to a module being deleted (e.g., when execution of the\n program is done), other globals referenced by the "__del__()"\n method may already have been deleted or in the process of being\n torn down (e.g. the import machinery shutting down). For this\n reason, "__del__()" methods should do the absolute minimum needed\n to maintain external invariants. Starting with version 1.5,\n Python guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n Called by the "repr()" built-in function to compute the "official"\n string representation of an object. If at all possible, this\n should look like a valid Python expression that could be used to\n recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n "<...some useful description...>" should be returned. The return\n value must be a string object. If a class defines "__repr__()" but\n not "__str__()", then "__repr__()" is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by "str(object)" and the built-in functions "format()" and\n "print()" to compute the "informal" or nicely printable string\n representation of an object. The return value must be a *string*\n object.\n\n This method differs from "object.__repr__()" in that there is no\n expectation that "__str__()" return a valid Python expression: a\n more convenient or concise representation can be used.\n\n The default implementation defined by the built-in type "object"\n calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n Called by "bytes()" to compute a byte-string representation of an\n object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n Called by the "format()" built-in function (and by extension, the\n "str.format()" method of class "str") to produce a "formatted"\n string representation of an object. The "format_spec" argument is a\n string that contains a description of the formatting options\n desired. The interpretation of the "format_spec" argument is up to\n the type implementing "__format__()", however most classes will\n either delegate formatting to one of the built-in types, or use a\n similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\n Changed in version 3.4: The __format__ method of "object" itself\n raises a "TypeError" if passed any non-empty string.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: "xy" calls\n "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n A rich comparison method may return the singleton "NotImplemented"\n if it does not implement the operation for a given pair of\n arguments. By convention, "False" and "True" are returned for a\n successful comparison. However, these methods can return any value,\n so if the comparison operator is used in a Boolean context (e.g.,\n in the condition of an "if" statement), Python will call "bool()"\n on the value to determine if the result is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of "x==y" does not imply that "x!=y" is false.\n Accordingly, when defining "__eq__()", one should also define\n "__ne__()" so that the operators will behave as expected. See the\n paragraph on "__hash__()" for some important notes on creating\n *hashable* objects which support custom comparison operations and\n are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n and "__eq__()" and "__ne__()" are their own reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see "functools.total_ordering()".\n\nobject.__hash__(self)\n\n Called by built-in function "hash()" and for operations on members\n of hashed collections including "set", "frozenset", and "dict".\n "__hash__()" should return an integer. The only required property\n is that objects which compare equal have the same hash value; it is\n advised to somehow mix together (e.g. using exclusive or) the hash\n values for the components of the object that also play a part in\n comparison of objects.\n\n Note: "hash()" truncates the value returned from an object\'s\n custom "__hash__()" method to the size of a "Py_ssize_t". This\n is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n builds. If an object\'s "__hash__()" must interoperate on builds\n of different bit sizes, be sure to check the width on all\n supported builds. An easy way to do this is with "python -c\n "import sys; print(sys.hash_info.width)""\n\n If a class does not define an "__eq__()" method it should not\n define a "__hash__()" operation either; if it defines "__eq__()"\n but not "__hash__()", its instances will not be usable as items in\n hashable collections. If a class defines mutable objects and\n implements an "__eq__()" method, it should not implement\n "__hash__()", since the implementation of hashable collections\n requires that a key\'s hash value is immutable (if the object\'s hash\n value changes, it will be in the wrong hash bucket).\n\n User-defined classes have "__eq__()" and "__hash__()" methods by\n default; with them, all objects compare unequal (except with\n themselves) and "x.__hash__()" returns an appropriate value such\n that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n A class that overrides "__eq__()" and does not define "__hash__()"\n will have its "__hash__()" implicitly set to "None". When the\n "__hash__()" method of a class is "None", instances of the class\n will raise an appropriate "TypeError" when a program attempts to\n retrieve their hash value, and will also be correctly identified as\n unhashable when checking "isinstance(obj, collections.Hashable").\n\n If a class that overrides "__eq__()" needs to retain the\n implementation of "__hash__()" from a parent class, the interpreter\n must be told this explicitly by setting "__hash__ =\n .__hash__".\n\n If a class that does not override "__eq__()" wishes to suppress\n hash support, it should include "__hash__ = None" in the class\n definition. A class which defines its own "__hash__()" that\n explicitly raises a "TypeError" would be incorrectly identified as\n hashable by an "isinstance(obj, collections.Hashable)" call.\n\n Note: By default, the "__hash__()" values of str, bytes and\n datetime objects are "salted" with an unpredictable random value.\n Although they remain constant within an individual Python\n process, they are not predictable between repeated invocations of\n Python.This is intended to provide protection against a denial-\n of-service caused by carefully-chosen inputs that exploit the\n worst case performance of a dict insertion, O(n^2) complexity.\n See http://www.ocert.org/advisories/ocert-2011-003.html for\n details.Changing hash values affects the iteration order of\n dicts, sets and other mappings. Python has never made guarantees\n about this ordering (and it typically varies between 32-bit and\n 64-bit builds).See also "PYTHONHASHSEED".\n\n Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n "bool()"; should return "False" or "True". When this method is not\n defined, "__len__()" is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither "__len__()" nor "__bool__()", all its instances are\n considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of "x.name") for\nclass instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for "self"). "name" is the attribute name. This\n method should return the (computed) attribute value or raise an\n "AttributeError" exception.\n\n Note that if the attribute is found through the normal mechanism,\n "__getattr__()" is not called. (This is an intentional asymmetry\n between "__getattr__()" and "__setattr__()".) This is done both for\n efficiency reasons and because otherwise "__getattr__()" would have\n no way to access other attributes of the instance. Note that at\n least for instance variables, you can fake total control by not\n inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n "__getattribute__()" method below for a way to actually get total\n control over attribute access.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines "__getattr__()",\n the latter will not be called unless "__getattribute__()" either\n calls it explicitly or raises an "AttributeError". This method\n should return the (computed) attribute value or raise an\n "AttributeError" exception. In order to avoid infinite recursion in\n this method, its implementation should always call the base class\n method with the same name to access any attributes it needs, for\n example, "object.__getattribute__(self, name)".\n\n Note: This method may still be bypassed when looking up special\n methods as the result of implicit invocation via language syntax\n or built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If "__setattr__()" wants to assign to an instance attribute, it\n should call the base class method with the same name, for example,\n "object.__setattr__(self, name, value)".\n\nobject.__delattr__(self, name)\n\n Like "__setattr__()" but for attribute deletion instead of\n assignment. This should only be implemented if "del obj.name" is\n meaningful for the object.\n\nobject.__dir__(self)\n\n Called when "dir()" is called on the object. A sequence must be\n returned. "dir()" converts the returned sequence to a list and\n sorts it.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' "__dict__".\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or "None" when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an "AttributeError"\n exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\nThe attribute "__objclass__" is interpreted by the "inspect" module as\nspecifying the class where this object was defined (setting this\nappropriately can assist in runtime introspection of dynamic class\nattributes). For callables, it may indicate that an instance of the\ngiven type (or a subclass) is expected or required as the first\npositional argument (for example, CPython sets this attribute for\nunbound methods that are implemented in C).\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: "__get__()", "__set__()", and\n"__delete__()". If any of those methods are defined for an object, it\nis said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, "a.x" has a\nlookup chain starting with "a.__dict__[\'x\']", then\n"type(a).__dict__[\'x\']", and continuing through the base classes of\n"type(a)" excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, "a.x". How\nthe arguments are assembled depends on "a":\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: "x.__get__(a)".\n\nInstance Binding\n If binding to an object instance, "a.x" is transformed into the\n call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n\nClass Binding\n If binding to a class, "A.x" is transformed into the call:\n "A.__dict__[\'x\'].__get__(None, A)".\n\nSuper Binding\n If "a" is an instance of "super", then the binding "super(B,\n obj).m()" searches "obj.__class__.__mro__" for the base class "A"\n immediately preceding "B" and then invokes the descriptor with the\n call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of "__get__()", "__set__()" and "__delete__()". If it\ndoes not define "__get__()", then accessing the attribute will return\nthe descriptor object itself unless there is a value in the object\'s\ninstance dictionary. If the descriptor defines "__set__()" and/or\n"__delete__()", it is a data descriptor; if it defines neither, it is\na non-data descriptor. Normally, data descriptors define both\n"__get__()" and "__set__()", while non-data descriptors have just the\n"__get__()" method. Data descriptors with "__set__()" and "__get__()"\ndefined always override a redefinition in an instance dictionary. In\ncontrast, non-data descriptors can be overridden by instances.\n\nPython methods (including "staticmethod()" and "classmethod()") are\nimplemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe "property()" function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage. This wastes space for objects having very few instance\nvariables. The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable. Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. *__slots__*\n reserves space for the declared variables and prevents the\n automatic creation of *__dict__* and *__weakref__* for each\n instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises "AttributeError". If\n dynamic assignment of new variables is desired, then add\n "\'__dict__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* Without a *__weakref__* variable for each instance, classes\n defining *__slots__* do not support weak references to its\n instances. If weak reference support is needed, then add\n "\'__weakref__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the\n instance variable defined by the base class slot is inaccessible\n (except by retrieving its descriptor directly from the base class).\n This renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as "int", "bytes" and "tuple".\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings\n may also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using "type()". The class body is\nexecuted in a new namespace and the class name is bound locally to the\nresult of "type(name, bases, namespace)".\n\nThe class creation process can be customised by passing the\n"metaclass" keyword argument in the class definition line, or by\ninheriting from an existing class that included such an argument. In\nthe following example, both "MyClass" and "MySubclass" are instances\nof "Meta":\n\n class Meta(type):\n pass\n\n class MyClass(metaclass=Meta):\n pass\n\n class MySubclass(MyClass):\n pass\n\nAny other keyword arguments that are specified in the class definition\nare passed through to all metaclass operations described below.\n\nWhen a class definition is executed, the following steps occur:\n\n* the appropriate metaclass is determined\n\n* the class namespace is prepared\n\n* the class body is executed\n\n* the class object is created\n\n\nDetermining the appropriate metaclass\n-------------------------------------\n\nThe appropriate metaclass for a class definition is determined as\nfollows:\n\n* if no bases and no explicit metaclass are given, then "type()" is\n used\n\n* if an explicit metaclass is given and it is *not* an instance of\n "type()", then it is used directly as the metaclass\n\n* if an instance of "type()" is given as the explicit metaclass, or\n bases are defined, then the most derived metaclass is used\n\nThe most derived metaclass is selected from the explicitly specified\nmetaclass (if any) and the metaclasses (i.e. "type(cls)") of all\nspecified base classes. The most derived metaclass is one which is a\nsubtype of *all* of these candidate metaclasses. If none of the\ncandidate metaclasses meets that criterion, then the class definition\nwill fail with "TypeError".\n\n\nPreparing the class namespace\n-----------------------------\n\nOnce the appropriate metaclass has been identified, then the class\nnamespace is prepared. If the metaclass has a "__prepare__" attribute,\nit is called as "namespace = metaclass.__prepare__(name, bases,\n**kwds)" (where the additional keyword arguments, if any, come from\nthe class definition).\n\nIf the metaclass has no "__prepare__" attribute, then the class\nnamespace is initialised as an empty "dict()" instance.\n\nSee also: **PEP 3115** - Metaclasses in Python 3000\n\n Introduced the "__prepare__" namespace hook\n\n\nExecuting the class body\n------------------------\n\nThe class body is executed (approximately) as "exec(body, globals(),\nnamespace)". The key difference from a normal call to "exec()" is that\nlexical scoping allows the class body (including any methods) to\nreference names from the current and outer scopes when the class\ndefinition occurs inside a function.\n\nHowever, even when the class definition occurs inside the function,\nmethods defined inside the class still cannot see names defined at the\nclass scope. Class variables must be accessed through the first\nparameter of instance or class methods, and cannot be accessed at all\nfrom static methods.\n\n\nCreating the class object\n-------------------------\n\nOnce the class namespace has been populated by executing the class\nbody, the class object is created by calling "metaclass(name, bases,\nnamespace, **kwds)" (the additional keywords passed here are the same\nas those passed to "__prepare__").\n\nThis class object is the one that will be referenced by the zero-\nargument form of "super()". "__class__" is an implicit closure\nreference created by the compiler if any methods in a class body refer\nto either "__class__" or "super". This allows the zero argument form\nof "super()" to correctly identify the class being defined based on\nlexical scoping, while the class or instance that was used to make the\ncurrent call is identified based on the first argument passed to the\nmethod.\n\nAfter the class object is created, it is passed to the class\ndecorators included in the class definition (if any) and the resulting\nobject is bound in the local namespace as the defined class.\n\nSee also: **PEP 3135** - New super\n\n Describes the implicit "__class__" closure reference\n\n\nMetaclass example\n-----------------\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored include logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n"collections.OrderedDict" to remember the order that class variables\nare defined:\n\n class OrderedClass(type):\n\n @classmethod\n def __prepare__(metacls, name, bases, **kwds):\n return collections.OrderedDict()\n\n def __new__(cls, name, bases, namespace, **kwds):\n result = type.__new__(cls, name, bases, dict(namespace))\n result.members = tuple(namespace)\n return result\n\n class A(metaclass=OrderedClass):\n def one(self): pass\n def two(self): pass\n def three(self): pass\n def four(self): pass\n\n >>> A.members\n (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s "__prepare__()" method which returns an\nempty "collections.OrderedDict". That mapping records the methods and\nattributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s "__new__()" method gets\ninvoked. That method builds the new type and it saves the ordered\ndictionary keys in an attribute called "members".\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n"isinstance()" and "issubclass()" built-in functions.\n\nIn particular, the metaclass "abc.ABCMeta" implements these methods in\norder to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n "isinstance(instance, class)".\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n "issubclass(subclass, class)".\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also: **PEP 3119** - Introducing Abstract Base Classes\n\n Includes the specification for customizing "isinstance()" and\n "issubclass()" behavior through "__instancecheck__()" and\n "__subclasscheck__()", with motivation for this functionality in\n the context of adding Abstract Base Classes (see the "abc"\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, "x(arg1, arg2, ...)" is a shorthand for\n "x.__call__(arg1, arg2, ...)".\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items. It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects. Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators. It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values. It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function "len()". Should return\n the length of the object, an integer ">=" 0. Also, an object that\n doesn\'t define a "__bool__()" method and whose "__len__()" method\n returns zero is considered to be false in a Boolean context.\n\nobject.__length_hint__(self)\n\n Called to implement "operator.length_hint()". Should return an\n estimated length for the object (which may be greater or less than\n the actual length). The length must be an integer ">=" 0. This\n method is purely an optimization and is never required for\n correctness.\n\n New in version 3.4.\n\nNote: Slicing is done exclusively with the following three methods.\n A call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of "self[key]". For sequence types,\n the accepted keys should be integers and slice objects. Note that\n the special interpretation of negative indexes (if the class wishes\n to emulate a sequence type) is up to the "__getitem__()" method. If\n *key* is of an inappropriate type, "TypeError" may be raised; if of\n a value outside the set of indexes for the sequence (after any\n special interpretation of negative values), "IndexError" should be\n raised. For mapping types, if *key* is missing (not in the\n container), "KeyError" should be raised.\n\n Note: "for" loops expect that an "IndexError" will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__missing__(self, key)\n\n Called by "dict"."__getitem__()" to implement "self[key]" for dict\n subclasses when key is not in the dictionary.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the "reversed()" built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the "__reversed__()" method is not provided, the "reversed()"\n built-in will fall back to using the sequence protocol ("__len__()"\n and "__getitem__()"). Objects that support the sequence protocol\n should only provide "__reversed__()" if they can provide an\n implementation that is more efficient than the one provided by\n "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define "__contains__()", the membership test\n first tries iteration via "__iter__()", then the old sequence\n iteration protocol via "__getitem__()", see *this section in the\n language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__matmul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to\n evaluate the expression "x + y", where *x* is an instance of a\n class that has an "__add__()" method, "x.__add__(y)" is called.\n The "__divmod__()" method should be the equivalent to using\n "__floordiv__()" and "__mod__()"; it should not be related to\n "__truediv__()". Note that "__pow__()" should be defined to accept\n an optional third argument if the ternary version of the built-in\n "pow()" function is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rmatmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped)\n operands. These functions are only called if the left operand does\n not support the corresponding operation and the operands are of\n different types. [2] For instance, to evaluate the expression "x -\n y", where *y* is an instance of a class that has an "__rsub__()"\n method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n *NotImplemented*.\n\n Note that ternary "pow()" will not try calling "__rpow__()" (the\n coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left\n operand\'s type and that subclass provides the reflected method\n for the operation, this method will be called before the left\n operand\'s non-reflected method. This behavior allows subclasses\n to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__imatmul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=",\n "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to\n do the operation in-place (modifying *self*) and return the result\n (which could be, but does not have to be, *self*). If a specific\n method is not defined, the augmented assignment falls back to the\n normal methods. For instance, if *x* is an instance of a class\n with an "__iadd__()" method, "x += y" is equivalent to "x =\n x.__iadd__(y)" . Otherwise, "x.__add__(y)" and "y.__radd__(x)" are\n considered, as with the evaluation of "x + y". In certain\n situations, augmented assignment can result in unexpected errors\n (see *Why does a_tuple[i] += [\'item\'] raise an exception when the\n addition works?*), but this behavior is in fact part of the data\n model.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations ("-", "+",\n "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions "complex()", "int()",\n "float()" and "round()". Should return a value of the appropriate\n type.\n\nobject.__index__(self)\n\n Called to implement "operator.index()", and whenever Python needs\n to losslessly convert the numeric object to an integer object (such\n as in slicing, or in the built-in "bin()", "hex()" and "oct()"\n functions). Presence of this method indicates that the numeric\n object is an integer type. Must return an integer.\n\n Note: In order to have a coherent integer type class, when\n "__index__()" is defined "__int__()" should also be defined, and\n both should return the same value.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a "with" statement. The context manager\nhandles the entry into, and the exit from, the desired runtime context\nfor the execution of the block of code. Context managers are normally\ninvoked using the "with" statement (described in section *The with\nstatement*), but can also be used by directly invoking their methods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The "with"\n statement will bind this method\'s return value to the target(s)\n specified in the "as" clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be "None".\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that "__exit__()" methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception:\n\n >>> class C:\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as "__hash__()" and "__repr__()" that are implemented by\nall objects, including type objects. If the implicit lookup of these\nmethods used the conventional lookup process, they would fail when\ninvoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe "__getattribute__()" method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print("Metaclass getattribute invoked")\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object, metaclass=Meta):\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print("Class getattribute invoked")\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the "__getattribute__()" machinery in this fashion provides\nsignificant scope for speed optimisations within the interpreter, at\nthe cost of some flexibility in the handling of special methods (the\nspecial method *must* be set on the class object itself in order to be\nconsistently invoked by the interpreter).\n', - 'string-methods': u'\nString Methods\n**************\n\nStrings implement all of the *common* sequence operations, along with\nthe additional methods described below.\n\nStrings also support two styles of string formatting, one providing a\nlarge degree of flexibility and customization (see "str.format()",\n*Format String Syntax* and *String Formatting*) and the other based on\nC "printf" style formatting that handles a narrower range of types and\nis slightly harder to use correctly, but is often faster for the cases\nit can handle (*printf-style String Formatting*).\n\nThe *Text Processing Services* section of the standard library covers\na number of other modules that provide various text related utilities\n(including regular expression support in the "re" module).\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.casefold()\n\n Return a casefolded copy of the string. Casefolded strings may be\n used for caseless matching.\n\n Casefolding is similar to lowercasing but more aggressive because\n it is intended to remove all case distinctions in a string. For\n example, the German lowercase letter "\'\xdf\'" is equivalent to ""ss"".\n Since it is already lowercase, "lower()" would do nothing to "\'\xdf\'";\n "casefold()" converts it to ""ss"".\n\n The casefolding algorithm is described in section 3.13 of the\n Unicode Standard.\n\n New in version 3.3.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is an ASCII space). The\n original string is returned if *width* is less than or equal to\n "len(s)".\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n Return an encoded version of the string as a bytes object. Default\n encoding is "\'utf-8\'". *errors* may be given to set a different\n error handling scheme. The default for *errors* is "\'strict\'",\n meaning that encoding errors raise a "UnicodeError". Other possible\n values are "\'ignore\'", "\'replace\'", "\'xmlcharrefreplace\'",\n "\'backslashreplace\'" and any other name registered via\n "codecs.register_error()", see section *Error Handlers*. For a list\n of possible encodings, see section *Standard Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return "True" if the string ends with the specified *suffix*,\n otherwise return "False". *suffix* can also be a tuple of suffixes\n to look for. With optional *start*, test beginning at that\n position. With optional *end*, stop comparing at that position.\n\nstr.expandtabs(tabsize=8)\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. Tab positions occur every *tabsize* characters\n (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n To expand the string, the current column is set to zero and the\n string is examined character by character. If the character is a\n tab ("\\t"), one or more space characters are inserted in the result\n until the current column is equal to the next tab position. (The\n tab character itself is not copied.) If the character is a newline\n ("\\n") or return ("\\r"), it is copied and the current column is\n reset to zero. Any other character is copied unchanged and the\n current column is incremented by one regardless of how the\n character is represented when printed.\n\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n \'01 012 0123 01234\'\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n \'01 012 0123 01234\'\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice "s[start:end]".\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return "-1" if *sub* is not found.\n\n Note: The "find()" method should be used only if you need to know\n the position of *sub*. To check if *sub* is a substring or not,\n use the "in" operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces "{}". Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n Similar to "str.format(**mapping)", except that "mapping" is used\n directly and not copied to a "dict". This is useful if for example\n "mapping" is a dict subclass:\n\n >>> class Default(dict):\n ... def __missing__(self, key):\n ... return key\n ...\n >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n \'Guido was born in country\'\n\n New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n Like "find()", but raise "ValueError" when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character "c"\n is alphanumeric if one of the following returns "True":\n "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()".\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that can be used to\n form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\n Use "keyword.iskeyword()" to test for reserved identifiers such as\n "def" and "class".\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when "repr()" is\n invoked on a string. It has no bearing on the handling of strings\n written to "sys.stdout" or "sys.stderr".)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A "TypeError" will be raised if there are\n any non-string values in *iterable*, including "bytes" objects.\n The separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is an ASCII\n space). The original string is returned if *width* is less than or\n equal to "len(s)".\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n The lowercasing algorithm used is described in section 3.13 of the\n Unicode Standard.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or "None", the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n "str.translate()".\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within "s[start:end]".\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return "-1" on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like "rfind()" but raises "ValueError" when the substring *sub* is\n not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is an ASCII\n space). The original string is returned if *width* is less than or\n equal to "len(s)".\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit(sep=None, maxsplit=-1)\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n "None", any whitespace string is a separator. Except for splitting\n from the right, "rsplit()" behaves like "split()" which is\n described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or "None", the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split(sep=None, maxsplit=-1)\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most "maxsplit+1"\n elements). If *maxsplit* is not specified or "-1", then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', \'2\']"). The *sep* argument\n may consist of multiple characters (for example,\n "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', \'3\']"). Splitting an\n empty string with a specified separator returns "[\'\']".\n\n For example:\n\n >>> \'1,2,3\'.split(\',\')\n [\'1\', \'2\', \'3\']\n >>> \'1,2,3\'.split(\',\', maxsplit=1)\n [\'1\', \'2,3\']\n >>> \'1,2,,3,\'.split(\',\')\n [\'1\', \'2\', \'\', \'3\', \'\']\n\n If *sep* is not specified or is "None", a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a "None" separator returns "[]".\n\n For example:\n\n >>> \'1 2 3\'.split()\n [\'1\', \'2\', \'3\']\n >>> \'1 2 3\'.split(maxsplit=1)\n [\'1\', \'2 3\']\n >>> \' 1 2 3 \'.split()\n [\'1\', \'2\', \'3\']\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n This method splits on the following line boundaries. In\n particular, the boundaries are a superset of *universal newlines*.\n\n +-------------------------+-------------------------------+\n | Representation | Description |\n +=========================+===============================+\n | "\\n" | Line Feed |\n +-------------------------+-------------------------------+\n | "\\r" | Carriage Return |\n +-------------------------+-------------------------------+\n | "\\r\\n" | Carriage Return + Line Feed |\n +-------------------------+-------------------------------+\n | "\\v" or "\\x0b" | Line Tabulation |\n +-------------------------+-------------------------------+\n | "\\f" or "\\x0c" | Form Feed |\n +-------------------------+-------------------------------+\n | "\\x1c" | File Separator |\n +-------------------------+-------------------------------+\n | "\\x1d" | Group Separator |\n +-------------------------+-------------------------------+\n | "\\x1e" | Record Separator |\n +-------------------------+-------------------------------+\n | "\\x85" | Next Line (C1 Control Code) |\n +-------------------------+-------------------------------+\n | "\\u2028" | Line Separator |\n +-------------------------+-------------------------------+\n | "\\u2029" | Paragraph Separator |\n +-------------------------+-------------------------------+\n\n Changed in version 3.2: "\\v" and "\\f" added to list of line\n boundaries.\n\n For example:\n\n >>> \'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()\n [\'ab c\', \'\', \'de fg\', \'kl\']\n >>> \'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines(keepends=True)\n [\'ab c\\n\', \'\\n\', \'de fg\\r\', \'kl\\r\\n\']\n\n Unlike "split()" when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line:\n\n >>> "".splitlines()\n []\n >>> "One line\\n".splitlines()\n [\'One line\']\n\n For comparison, "split(\'\\n\')" gives:\n\n >>> \'\'.split(\'\\n\')\n [\'\']\n >>> \'Two lines\\n\'.split(\'\\n\')\n [\'Two lines\', \'\']\n\nstr.startswith(prefix[, start[, end]])\n\n Return "True" if string starts with the *prefix*, otherwise return\n "False". *prefix* can also be a tuple of prefixes to look for.\n With optional *start*, test string beginning at that position.\n With optional *end*, stop comparing string at that position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or "None", the *chars*\n argument defaults to removing whitespace. The *chars* argument is\n not a prefix or suffix; rather, all combinations of its values are\n stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n The outermost leading and trailing *chars* argument values are\n stripped from the string. Characters are removed from the leading\n end until reaching a string character that is not contained in the\n set of characters in *chars*. A similar action takes place on the\n trailing end. For example:\n\n >>> comment_string = \'#....... Section 3.2.1 Issue #32 .......\'\n >>> comment_string.strip(\'.#! \')\n \'Section 3.2.1 Issue #32\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa. Note that it is not necessarily true that\n "s.swapcase().swapcase() == s".\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n For example:\n\n >>> \'Hello world\'.title()\n \'Hello World\'\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or "None". Unmapped\n characters are left untouched. Characters mapped to "None" are\n deleted.\n\n You can use "str.maketrans()" to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom\n character mapping codec using the "codecs" module (see\n "encodings.cp1251" for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that "str.upper().isupper()" might be\n "False" if "s" contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n The uppercasing algorithm used is described in section 3.13 of the\n Unicode Standard.\n\nstr.zfill(width)\n\n Return a copy of the string left filled with ASCII "\'0\'" digits to\n make a string of length *width*. A leading sign prefix\n ("\'+\'"/"\'-\'") is handled by inserting the padding *after* the sign\n character rather than before. The original string is returned if\n *width* is less than or equal to "len(s)".\n\n For example:\n\n >>> "42".zfill(5)\n \'00042\'\n >>> "-42".zfill(5)\n \'-0042\'\n', - 'strings': u'\nString and Bytes literals\n*************************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "R" | "U"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'" | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | stringescapeseq\n longstringitem ::= longstringchar | stringescapeseq\n shortstringchar ::= \n longstringchar ::= \n stringescapeseq ::= "\\" \n\n bytesliteral ::= bytesprefix(shortbytes | longbytes)\n bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB"\n shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' shortbytesitem* \'"\'\n longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' longbytesitem* \'"""\'\n shortbytesitem ::= shortbyteschar | bytesescapeseq\n longbytesitem ::= longbyteschar | bytesescapeseq\n shortbyteschar ::= \n longbyteschar ::= \n bytesescapeseq ::= "\\" \n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the "stringprefix" or "bytesprefix"\nand the rest of the literal. The source character set is defined by\nthe encoding declaration; it is UTF-8 if no encoding declaration is\ngiven in the source file; see section *Encoding declarations*.\n\nIn plain English: Both types of literals can be enclosed in matching\nsingle quotes ("\'") or double quotes ("""). They can also be enclosed\nin matching groups of three single or double quotes (these are\ngenerally referred to as *triple-quoted strings*). The backslash\n("\\") character is used to escape characters that otherwise have a\nspecial meaning, such as newline, backslash itself, or the quote\ncharacter.\n\nBytes literals are always prefixed with "\'b\'" or "\'B\'"; they produce\nan instance of the "bytes" type instead of the "str" type. They may\nonly contain ASCII characters; bytes with a numeric value of 128 or\ngreater must be expressed with escapes.\n\nAs of Python 3.3 it is possible again to prefix string literals with a\n"u" prefix to simplify maintenance of dual 2.x and 3.x codebases.\n\nBoth string and bytes literals may optionally be prefixed with a\nletter "\'r\'" or "\'R\'"; such strings are called *raw strings* and treat\nbackslashes as literal characters. As a result, in string literals,\n"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated specially.\nGiven that Python 2.x\'s raw unicode literals behave differently than\nPython 3.x\'s the "\'ur\'" syntax is not supported.\n\nNew in version 3.3: The "\'rb\'" prefix of raw bytes literals has been\nadded as a synonym of "\'br\'".\n\nNew in version 3.3: Support for the unicode legacy literal\n("u\'value\'") was reintroduced to simplify the maintenance of dual\nPython 2.x and 3.x codebases. See **PEP 414** for more information.\n\nIn triple-quoted literals, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the literal. (A "quote" is the character used to open the\nliteral, i.e. either "\'" or """.)\n\nUnless an "\'r\'" or "\'R\'" prefix is present, escape sequences in string\nand bytes literals are interpreted according to rules similar to those\nused by Standard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| "\\newline" | Backslash and newline ignored | |\n+-------------------+-----------------------------------+---------+\n| "\\\\" | Backslash ("\\") | |\n+-------------------+-----------------------------------+---------+\n| "\\\'" | Single quote ("\'") | |\n+-------------------+-----------------------------------+---------+\n| "\\"" | Double quote (""") | |\n+-------------------+-----------------------------------+---------+\n| "\\a" | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| "\\b" | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| "\\f" | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| "\\n" | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| "\\r" | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| "\\t" | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| "\\v" | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| "\\ooo" | Character with octal value *ooo* | (1,3) |\n+-------------------+-----------------------------------+---------+\n| "\\xhh" | Character with hex value *hh* | (2,3) |\n+-------------------+-----------------------------------+---------+\n\nEscape sequences only recognized in string literals are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| "\\N{name}" | Character named *name* in the | (4) |\n| | Unicode database | |\n+-------------------+-----------------------------------+---------+\n| "\\uxxxx" | Character with 16-bit hex value | (5) |\n| | *xxxx* | |\n+-------------------+-----------------------------------+---------+\n| "\\Uxxxxxxxx" | Character with 32-bit hex value | (6) |\n| | *xxxxxxxx* | |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. As in Standard C, up to three octal digits are accepted.\n\n2. Unlike in Standard C, exactly two hex digits are required.\n\n3. In a bytes literal, hexadecimal and octal escapes denote the\n byte with the given value. In a string literal, these escapes\n denote a Unicode character with the given value.\n\n4. Changed in version 3.3: Support for name aliases [1] has been\n added.\n\n5. Individual code units which form parts of a surrogate pair can\n be encoded using this escape sequence. Exactly four hex digits are\n required.\n\n6. Any Unicode character can be encoded this way. Exactly eight\n hex digits are required.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the result*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences only recognized in string\nliterals fall into the category of unrecognized escapes for bytes\nliterals.\n\nEven in a raw literal, quotes can be escaped with a backslash, but the\nbackslash remains in the result; for example, "r"\\""" is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; "r"\\"" is not a valid string literal (even a raw string cannot\nend in an odd number of backslashes). Specifically, *a raw literal\ncannot end in a single backslash* (since the backslash would escape\nthe following quote character). Note also that a single backslash\nfollowed by a newline is interpreted as those two characters as part\nof the literal, *not* as a line continuation.\n', - 'subscriptions': u'\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object that supports subscription\n(lists or dictionaries for example). User-defined objects can support\nsubscription by defining a "__getitem__()" method.\n\nFor built-in objects, there are two types of objects that support\nsubscription:\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to\nan integer or a slice (as discussed in the following section).\n\nThe formal syntax makes no special provision for negative indices in\nsequences; however, built-in sequences all provide a "__getitem__()"\nmethod that interprets negative indices by adding the length of the\nsequence to the index (so that "x[-1]" selects the last item of "x").\nThe resulting value must be a nonnegative integer less than the number\nof items in the sequence, and the subscription selects the item whose\nindex is that value (counting from zero). Since the support for\nnegative indices and slicing occurs in the object\'s "__getitem__()"\nmethod, subclasses overriding this method will need to explicitly add\nthat support.\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', - 'truth': u'\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an "if" or\n"while" condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* "None"\n\n* "False"\n\n* zero of any numeric type, for example, "0", "0.0", "0j".\n\n* any empty sequence, for example, "\'\'", "()", "[]".\n\n* any empty mapping, for example, "{}".\n\n* instances of user-defined classes, if the class defines a\n "__bool__()" or "__len__()" method, when that method returns the\n integer zero or "bool" value "False". [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn "0" or "False" for false and "1" or "True" for true, unless\notherwise stated. (Important exception: the Boolean operations "or"\nand "and" always return one of their operands.)\n', - 'try': u'\nThe "try" statement\n*******************\n\nThe "try" statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" identifier]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe "except" clause(s) specify one or more exception handlers. When no\nexception occurs in the "try" clause, no exception handler is\nexecuted. When an exception occurs in the "try" suite, a search for an\nexception handler is started. This search inspects the except clauses\nin turn until one is found that matches the exception. An expression-\nless except clause, if present, must be last; it matches any\nexception. For an except clause with an expression, that expression\nis evaluated, and the clause matches the exception if the resulting\nobject is "compatible" with the exception. An object is compatible\nwith an exception if it is the class or a base class of the exception\nobject or a tuple containing an item compatible with the exception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire "try" statement raised\nthe exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the "as" keyword in that except clause, if\npresent, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using "as target", it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the "sys" module and can be accessed via\n"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of the\nexception class, the exception instance and a traceback object (see\nsection *The standard type hierarchy*) identifying the point in the\nprogram where the exception occurred. "sys.exc_info()" values are\nrestored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional "else" clause is executed if and when control flows off\nthe end of the "try" clause. [2] Exceptions in the "else" clause are\nnot handled by the preceding "except" clauses.\n\nIf "finally" is present, it specifies a \'cleanup\' handler. The "try"\nclause is executed, including any "except" and "else" clauses. If an\nexception occurs in any of the clauses and is not handled, the\nexception is temporarily saved. The "finally" clause is executed. If\nthere is a saved exception it is re-raised at the end of the "finally"\nclause. If the "finally" clause raises another exception, the saved\nexception is set as the context of the new exception. If the "finally"\nclause executes a "return" or "break" statement, the saved exception\nis discarded:\n\n >>> def f():\n ... try:\n ... 1/0\n ... finally:\n ... return 42\n ...\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the "finally" clause.\n\nWhen a "return", "break" or "continue" statement is executed in the\n"try" suite of a "try"..."finally" statement, the "finally" clause is\nalso executed \'on the way out.\' A "continue" statement is illegal in\nthe "finally" clause. (The reason is a problem with the current\nimplementation --- this restriction may be lifted in the future).\n\nThe return value of a function is determined by the last "return"\nstatement executed. Since the "finally" clause always executes, a\n"return" statement executed in the "finally" clause will always be the\nlast one executed:\n\n >>> def foo():\n ... try:\n ... return \'try\'\n ... finally:\n ... return \'finally\'\n ...\n >>> foo()\n \'finally\'\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the "raise" statement to\ngenerate exceptions may be found in section *The raise statement*.\n', - 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name "None". It\n is used to signify the absence of a value in many situations, e.g.,\n it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n "NotImplemented". Numeric methods and rich comparison methods\n should return this value if they do not implement the operation for\n the operands provided. (The interpreter will then try the\n reflected operation, or some other fallback, depending on the\n operator.) Its truth value is true.\n\n See *Implementing the arithmetic operations* for more details.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the literal "..." or the\n built-in name "Ellipsis". Its truth value is true.\n\n"numbers.Number"\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n "numbers.Integral"\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are two types of integers:\n\n Integers ("int")\n\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans ("bool")\n These represent the truth values False and True. The two\n objects representing the values "False" and "True" are the\n only Boolean objects. The Boolean type is a subtype of the\n integer type, and Boolean values behave like the values 0 and\n 1, respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ""False"" or\n ""True"" are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers.\n\n "numbers.Real" ("float")\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these are\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n "numbers.Complex" ("complex")\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number "z" can be retrieved through the read-only\n attributes "z.real" and "z.imag".\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function "len()" returns the number of items\n of a sequence. When the length of a sequence is *n*, the index set\n contains the numbers 0, 1, ..., *n*-1. Item *i* of sequence *a* is\n selected by "a[i]".\n\n Sequences also support slicing: "a[i:j]" selects all items with\n index *k* such that *i* "<=" *k* "<" *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: "a[i:j:k]" selects all items of *a* with index *x* where\n "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n A string is a sequence of values that represent Unicode code\n points. All the code points in the range "U+0000 - U+10FFFF"\n can be represented in a string. Python doesn\'t have a "char"\n type; instead, every code point in the string is represented\n as a string object with length "1". The built-in function\n "ord()" converts a code point from its string form to an\n integer in the range "0 - 10FFFF"; "chr()" converts an\n integer in the range "0 - 10FFFF" to the corresponding length\n "1" string object. "str.encode()" can be used to convert a\n "str" to "bytes" using the given text encoding, and\n "bytes.decode()" can be used to achieve the opposite.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Bytes\n A bytes object is an immutable array. The items are 8-bit\n bytes, represented by integers in the range 0 <= x < 256.\n Bytes literals (like "b\'abc\'") and the built-in function\n "bytes()" can be used to construct bytes objects. Also,\n bytes objects can be decoded to strings via the "decode()"\n method.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and "del" (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in "bytearray()" constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module "array" provides an additional example of a\n mutable sequence type, as does the "collections" module.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function "len()"\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., "1" and\n "1.0"), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n "set()" constructor and can be modified afterwards by several\n methods, such as "add()".\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in "frozenset()" constructor. As a frozenset is immutable\n and *hashable*, it can be used again as an element of another\n set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation "a[k]" selects the item indexed by "k"\n from the mapping "a"; this can be used in expressions and as the\n target of assignments or "del" statements. The built-in function\n "len()" returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., "1" and "1.0")\n then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the "{...}"\n notation (see section *Dictionary displays*).\n\n The extension modules "dbm.ndbm" and "dbm.gnu" provide\n additional examples of mapping types, as does the "collections"\n module.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +---------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +===========================+=================================+=============+\n | "__doc__" | The function\'s documentation | Writable |\n | | string, or "None" if | |\n | | unavailable; not inherited by | |\n | | subclasses | |\n +---------------------------+---------------------------------+-------------+\n | "__name__" | The function\'s name | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__qualname__" | The function\'s *qualified name* | Writable |\n | | New in version 3.3. | |\n +---------------------------+---------------------------------+-------------+\n | "__module__" | The name of the module the | Writable |\n | | function was defined in, or | |\n | | "None" if unavailable. | |\n +---------------------------+---------------------------------+-------------+\n | "__defaults__" | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or "None" if no arguments have | |\n | | a default value | |\n +---------------------------+---------------------------------+-------------+\n | "__code__" | The code object representing | Writable |\n | | the compiled function body. | |\n +---------------------------+---------------------------------+-------------+\n | "__globals__" | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +---------------------------+---------------------------------+-------------+\n | "__dict__" | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +---------------------------+---------------------------------+-------------+\n | "__closure__" | "None" or a tuple of cells that | Read-only |\n | | contain bindings for the | |\n | | function\'s free variables. | |\n +---------------------------+---------------------------------+-------------+\n | "__annotations__" | A dict containing annotations | Writable |\n | | of parameters. The keys of the | |\n | | dict are the parameter names, | |\n | | and "\'return\'" for the return | |\n | | annotation, if provided. | |\n +---------------------------+---------------------------------+-------------+\n | "__kwdefaults__" | A dict containing defaults for | Writable |\n | | keyword-only parameters. | |\n +---------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n Instance methods\n An instance method object combines a class, a class instance and\n any callable object (normally a user-defined function).\n\n Special read-only attributes: "__self__" is the class instance\n object, "__func__" is the function object; "__doc__" is the\n method\'s documentation (same as "__func__.__doc__"); "__name__"\n is the method name (same as "__func__.__name__"); "__module__"\n is the name of the module the method was defined in, or "None"\n if unavailable.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object or a class\n method object.\n\n When an instance method object is created by retrieving a user-\n defined function object from a class via one of its instances,\n its "__self__" attribute is the instance, and the method object\n is said to be bound. The new method\'s "__func__" attribute is\n the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the "__func__"\n attribute of the new instance is not the original method object\n but its "__func__" attribute.\n\n When an instance method object is created by retrieving a class\n method object from a class or instance, its "__self__" attribute\n is the class itself, and its "__func__" attribute is the\n function object underlying the class method.\n\n When an instance method object is called, the underlying\n function ("__func__") is called, inserting the class instance\n ("__self__") in front of the argument list. For instance, when\n "C" is a class which contains a definition for a function "f()",\n and "x" is an instance of "C", calling "x.f(1)" is equivalent to\n calling "C.f(x, 1)".\n\n When an instance method object is derived from a class method\n object, the "class instance" stored in "__self__" will actually\n be the class itself, so that calling either "x.f(1)" or "C.f(1)"\n is equivalent to calling "f(C,1)" where "f" is the underlying\n function.\n\n Note that the transformation from function object to instance\n method object happens each time the attribute is retrieved from\n the instance. In some cases, a fruitful optimization is to\n assign the attribute to a local variable and call that local\n variable. Also notice that this transformation only happens for\n user-defined functions; other callable objects (and all non-\n callable objects) are retrieved without transformation. It is\n also important to note that user-defined functions which are\n attributes of a class instance are not converted to bound\n methods; this *only* happens when the function is an attribute\n of the class.\n\n Generator functions\n A function or method which uses the "yield" statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s "iterator.__next__()" method will cause the\n function to execute until it provides a value using the "yield"\n statement. When the function executes a "return" statement or\n falls off the end, a "StopIteration" exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Coroutine functions\n A function or method which is defined using "async def" is\n called a *coroutine function*. Such a function, when called,\n returns a *coroutine* object. It may contain "await"\n expressions, as well as "async with" and "async for" statements.\n See also *Coroutines* section.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are "len()" and "math.sin()"\n ("math" is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: "__doc__" is the function\'s documentation\n string, or "None" if unavailable; "__name__" is the function\'s\n name; "__self__" is set to "None" (but see the next item);\n "__module__" is the name of the module the function was defined\n in or "None" if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n "alist.append()", assuming *alist* is a list object. In this\n case, the special read-only attribute "__self__" is set to the\n object denoted by *alist*.\n\n Classes\n Classes are callable. These objects normally act as factories\n for new instances of themselves, but variations are possible for\n class types that override "__new__()". The arguments of the\n call are passed to "__new__()" and, in the typical case, to\n "__init__()" to initialize the new instance.\n\n Class Instances\n Instances of arbitrary classes can be made callable by defining\n a "__call__()" method in their class.\n\nModules\n Modules are a basic organizational unit of Python code, and are\n created by the *import system* as invoked either by the "import"\n statement (see "import"), or by calling functions such as\n "importlib.import_module()" and built-in "__import__()". A module\n object has a namespace implemented by a dictionary object (this is\n the dictionary referenced by the "__globals__" attribute of\n functions defined in the module). Attribute references are\n translated to lookups in this dictionary, e.g., "m.x" is equivalent\n to "m.__dict__["x"]". A module object does not contain the code\n object used to initialize the module (since it isn\'t needed once\n the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n\n Special read-only attribute: "__dict__" is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: "__name__" is the module\'s name;\n "__doc__" is the module\'s documentation string, or "None" if\n unavailable; "__file__" is the pathname of the file from which the\n module was loaded, if it was loaded from a file. The "__file__"\n attribute may be missing for certain types of modules, such as C\n modules that are statically linked into the interpreter; for\n extension modules loaded dynamically from a shared library, it is\n the pathname of the shared library file.\n\nCustom classes\n Custom class types are typically created by class definitions (see\n section *Class definitions*). A class has a namespace implemented\n by a dictionary object. Class attribute references are translated\n to lookups in this dictionary, e.g., "C.x" is translated to\n "C.__dict__["x"]" (although there are a number of hooks which allow\n for other means of locating attributes). When the attribute name is\n not found there, the attribute search continues in the base\n classes. This search of the base classes uses the C3 method\n resolution order which behaves correctly even in the presence of\n \'diamond\' inheritance structures where there are multiple\n inheritance paths leading back to a common ancestor. Additional\n details on the C3 MRO used by Python can be found in the\n documentation accompanying the 2.3 release at\n https://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class "C", say) would yield a\n class method object, it is transformed into an instance method\n object whose "__self__" attributes is "C". When it would yield a\n static method object, it is transformed into the object wrapped by\n the static method object. See section *Implementing Descriptors*\n for another way in which attributes retrieved from a class may\n differ from those actually contained in its "__dict__".\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: "__name__" is the class name; "__module__" is\n the module name in which the class was defined; "__dict__" is the\n dictionary containing the class\'s namespace; "__bases__" is a tuple\n (possibly empty or a singleton) containing the base classes, in the\n order of their occurrence in the base class list; "__doc__" is the\n class\'s documentation string, or None if undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object, it is transformed into an instance method object\n whose "__self__" attribute is the instance. Static method and\n class method objects are also transformed; see above under\n "Classes". See section *Implementing Descriptors* for another way\n in which attributes of a class retrieved via its instances may\n differ from the objects actually stored in the class\'s "__dict__".\n If no class attribute is found, and the object\'s class has a\n "__getattr__()" method, that is called to satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n "__setattr__()" or "__delattr__()" method, this is called instead\n of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: "__dict__" is the attribute dictionary;\n "__class__" is the instance\'s class.\n\nI/O objects (also known as file objects)\n A *file object* represents an open file. Various shortcuts are\n available to create file objects: the "open()" built-in function,\n and also "os.popen()", "os.fdopen()", and the "makefile()" method\n of socket objects (and perhaps by other functions or methods\n provided by extension modules).\n\n The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n initialized to file objects corresponding to the interpreter\'s\n standard input, output and error streams; they are all open in text\n mode and therefore follow the interface defined by the\n "io.TextIOBase" abstract class.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: "co_name" gives the function name;\n "co_argcount" is the number of positional arguments (including\n arguments with default values); "co_nlocals" is the number of\n local variables used by the function (including arguments);\n "co_varnames" is a tuple containing the names of the local\n variables (starting with the argument names); "co_cellvars" is a\n tuple containing the names of local variables that are\n referenced by nested functions; "co_freevars" is a tuple\n containing the names of free variables; "co_code" is a string\n representing the sequence of bytecode instructions; "co_consts"\n is a tuple containing the literals used by the bytecode;\n "co_names" is a tuple containing the names used by the bytecode;\n "co_filename" is the filename from which the code was compiled;\n "co_firstlineno" is the first line number of the function;\n "co_lnotab" is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); "co_stacksize" is the required stack size\n (including local variables); "co_flags" is an integer encoding a\n number of flags for the interpreter.\n\n The following flag bits are defined for "co_flags": bit "0x04"\n is set if the function uses the "*arguments" syntax to accept an\n arbitrary number of positional arguments; bit "0x08" is set if\n the function uses the "**keywords" syntax to accept arbitrary\n keyword arguments; bit "0x20" is set if the function is a\n generator.\n\n Future feature declarations ("from __future__ import division")\n also use bits in "co_flags" to indicate whether a code object\n was compiled with a particular feature enabled: bit "0x2000" is\n set if the function was compiled with future division enabled;\n bits "0x10" and "0x1000" were used in earlier versions of\n Python.\n\n Other bits in "co_flags" are reserved for internal use.\n\n If a code object represents a function, the first item in\n "co_consts" is the documentation string of the function, or\n "None" if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: "f_back" is to the previous stack\n frame (towards the caller), or "None" if this is the bottom\n stack frame; "f_code" is the code object being executed in this\n frame; "f_locals" is the dictionary used to look up local\n variables; "f_globals" is used for global variables;\n "f_builtins" is used for built-in (intrinsic) names; "f_lasti"\n gives the precise instruction (this is an index into the\n bytecode string of the code object).\n\n Special writable attributes: "f_trace", if not "None", is a\n function called at the start of each source code line (this is\n used by the debugger); "f_lineno" is the current line number of\n the frame --- writing to this from within a trace function jumps\n to the given line (only for the bottom-most frame). A debugger\n can implement a Jump command (aka Set Next Statement) by writing\n to f_lineno.\n\n Frame objects support one method:\n\n frame.clear()\n\n This method clears all references to local variables held by\n the frame. Also, if the frame belonged to a generator, the\n generator is finalized. This helps break reference cycles\n involving frame objects (for example when catching an\n exception and storing its traceback for later use).\n\n "RuntimeError" is raised if the frame is currently executing.\n\n New in version 3.4.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as the third item of the\n tuple returned by "sys.exc_info()". When the program contains no\n suitable handler, the stack trace is written (nicely formatted)\n to the standard error stream; if the interpreter is interactive,\n it is also made available to the user as "sys.last_traceback".\n\n Special read-only attributes: "tb_next" is the next level in the\n stack trace (towards the frame where the exception occurred), or\n "None" if there is no next level; "tb_frame" points to the\n execution frame of the current level; "tb_lineno" gives the line\n number where the exception occurred; "tb_lasti" indicates the\n precise instruction. The line number and last instruction in\n the traceback may differ from the line number of its frame\n object if the exception occurred in a "try" statement with no\n matching except clause or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices for "__getitem__()"\n methods. They are also created by the built-in "slice()"\n function.\n\n Special read-only attributes: "start" is the lower bound; "stop"\n is the upper bound; "step" is the step value; each is "None" if\n omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the slice that the slice object\n would describe if applied to a sequence of *length* items.\n It returns a tuple of three integers; respectively these are\n the *start* and *stop* indices and the *step* or stride\n length of the slice. Missing or out-of-bounds indices are\n handled in a manner consistent with regular slices.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n "staticmethod()" constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in "classmethod()" constructor.\n', - 'typesfunctions': u'\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: "func(argument-list)".\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n', - 'typesmapping': u'\nMapping Types --- "dict"\n************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built-\nin "list", "set", and "tuple" classes, and the "collections" module.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as "1" and "1.0") then they can be used interchangeably to index\nthe same dictionary entry. (Note however, that since computers store\nfloating-point numbers as approximations it is usually unwise to use\nthem as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of "key:\nvalue" pairs within braces, for example: "{\'jack\': 4098, \'sjoerd\':\n4127}" or "{4098: \'jack\', 4127: \'sjoerd\'}", or by the "dict"\nconstructor.\n\nclass class dict(**kwarg)\nclass class dict(mapping, **kwarg)\nclass class dict(iterable, **kwarg)\n\n Return a new dictionary initialized from an optional positional\n argument and a possibly empty set of keyword arguments.\n\n If no positional argument is given, an empty dictionary is created.\n If a positional argument is given and it is a mapping object, a\n dictionary is created with the same key-value pairs as the mapping\n object. Otherwise, the positional argument must be an *iterable*\n object. Each item in the iterable must itself be an iterable with\n exactly two objects. The first object of each item becomes a key\n in the new dictionary, and the second object the corresponding\n value. If a key occurs more than once, the last value for that key\n becomes the corresponding value in the new dictionary.\n\n If keyword arguments are given, the keyword arguments and their\n values are added to the dictionary created from the positional\n argument. If a key being added is already present, the value from\n the keyword argument replaces the value from the positional\n argument.\n\n To illustrate, the following examples all return a dictionary equal\n to "{"one": 1, "two": 2, "three": 3}":\n\n >>> a = dict(one=1, two=2, three=3)\n >>> b = {\'one\': 1, \'two\': 2, \'three\': 3}\n >>> c = dict(zip([\'one\', \'two\', \'three\'], [1, 2, 3]))\n >>> d = dict([(\'two\', 2), (\'one\', 1), (\'three\', 3)])\n >>> e = dict({\'three\': 3, \'one\': 1, \'two\': 2})\n >>> a == b == c == d == e\n True\n\n Providing keyword arguments as in the first example only works for\n keys that are valid Python identifiers. Otherwise, any valid keys\n can be used.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a "KeyError" if\n *key* is not in the map.\n\n If a subclass of dict defines a method "__missing__()" and *key*\n is not present, the "d[key]" operation calls that method with\n the key *key* as argument. The "d[key]" operation then returns\n or raises whatever is returned or raised by the\n "__missing__(key)" call. No other operations or methods invoke\n "__missing__()". If "__missing__()" is not defined, "KeyError"\n is raised. "__missing__()" must be a method; it cannot be an\n instance variable:\n\n >>> class Counter(dict):\n ... def __missing__(self, key):\n ... return 0\n >>> c = Counter()\n >>> c[\'red\']\n 0\n >>> c[\'red\'] += 1\n >>> c[\'red\']\n 1\n\n The example above shows part of the implementation of\n "collections.Counter". A different "__missing__" method is used\n by "collections.defaultdict".\n\n d[key] = value\n\n Set "d[key]" to *value*.\n\n del d[key]\n\n Remove "d[key]" from *d*. Raises a "KeyError" if *key* is not\n in the map.\n\n key in d\n\n Return "True" if *d* has a key *key*, else "False".\n\n key not in d\n\n Equivalent to "not key in d".\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for "iter(d.keys())".\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n classmethod fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n "fromkeys()" is a class method that returns a new dictionary.\n *value* defaults to "None".\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to "None", so\n that this method never raises a "KeyError".\n\n items()\n\n Return a new view of the dictionary\'s items ("(key, value)"\n pairs). See the *documentation of view objects*.\n\n keys()\n\n Return a new view of the dictionary\'s keys. See the\n *documentation of view objects*.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a "KeyError" is raised.\n\n popitem()\n\n Remove and return an arbitrary "(key, value)" pair from the\n dictionary.\n\n "popitem()" is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling "popitem()" raises a "KeyError".\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to "None".\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return "None".\n\n "update()" accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: "d.update(red=1,\n blue=2)".\n\n values()\n\n Return a new view of the dictionary\'s values. See the\n *documentation of view objects*.\n\nSee also: "types.MappingProxyType" can be used to create a read-only\n view of a "dict".\n\n\nDictionary view objects\n=======================\n\nThe objects returned by "dict.keys()", "dict.values()" and\n"dict.items()" are *view objects*. They provide a dynamic view on the\ndictionary\'s entries, which means that when the dictionary changes,\nthe view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of "(key, value)") in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of "(value, key)" pairs using\n "zip()": "pairs = zip(d.values(), d.keys())". Another way to\n create the same list is "pairs = [(v, k) for (k, v) in d.items()]".\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a "RuntimeError" or fail to iterate over all entries.\n\nx in dictview\n\n Return "True" if *x* is in the underlying dictionary\'s keys, values\n or items (in the latter case, *x* should be a "(key, value)"\n tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that "(key, value)" pairs are unique\nand hashable, then the items view is also set-like. (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class "collections.abc.Set" are available (for example, "==",\n"<", or "^").\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.keys()\n >>> values = dishes.values()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n >>> keys ^ {\'sausage\', \'juice\'}\n {\'juice\', \'sausage\', \'bacon\', \'spam\'}\n', - 'typesmethods': u'\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as "append()" on lists)\nand class instance methods. Built-in methods are described with the\ntypes that support them.\n\nIf you access a method (a function defined in a class namespace)\nthrough an instance, you get a special object: a *bound method* (also\ncalled *instance method*) object. When called, it will add the "self"\nargument to the argument list. Bound methods have two special read-\nonly attributes: "m.__self__" is the object on which the method\noperates, and "m.__func__" is the function implementing the method.\nCalling "m(arg-1, arg-2, ..., arg-n)" is completely equivalent to\ncalling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n\nLike function objects, bound method objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object ("meth.__func__"), setting method\nattributes on bound methods is disallowed. Attempting to set an\nattribute on a method results in an "AttributeError" being raised. In\norder to set a method attribute, you need to explicitly set it on the\nunderlying function object:\n\n >>> class C:\n ... def method(self):\n ... pass\n ...\n >>> c = C()\n >>> c.method.whoami = \'my name is method\' # can\'t set on the method\n Traceback (most recent call last):\n File "", line 1, in \n AttributeError: \'method\' object has no attribute \'whoami\'\n >>> c.method.__func__.whoami = \'my name is method\'\n >>> c.method.whoami\n \'my name is method\'\n\nSee *The standard type hierarchy* for more information.\n', - 'typesmodules': u'\nModules\n*******\n\nThe only special operation on a module is attribute access: "m.name",\nwhere *m* is a module and *name* accesses a name defined in *m*\'s\nsymbol table. Module attributes can be assigned to. (Note that the\n"import" statement is not, strictly speaking, an operation on a module\nobject; "import foo" does not require a module object named *foo* to\nexist, rather it requires an (external) *definition* for a module\nnamed *foo* somewhere.)\n\nA special attribute of every module is "__dict__". This is the\ndictionary containing the module\'s symbol table. Modifying this\ndictionary will actually change the module\'s symbol table, but direct\nassignment to the "__dict__" attribute is not possible (you can write\n"m.__dict__[\'a\'] = 1", which defines "m.a" to be "1", but you can\'t\nwrite "m.__dict__ = {}"). Modifying "__dict__" directly is not\nrecommended.\n\nModules built into the interpreter are written like this: "". If loaded from a file, they are written as\n"".\n', - 'typesseq': u'\nSequence Types --- "list", "tuple", "range"\n*******************************************\n\nThere are three basic sequence types: lists, tuples, and range\nobjects. Additional sequence types tailored for processing of *binary\ndata* and *text strings* are described in dedicated sections.\n\n\nCommon Sequence Operations\n==========================\n\nThe operations in the following table are supported by most sequence\ntypes, both mutable and immutable. The "collections.abc.Sequence" ABC\nis provided to make it easier to correctly implement these operations\non custom sequence types.\n\nThis table lists the sequence operations sorted in ascending priority.\nIn the table, *s* and *t* are sequences of the same type, *n*, *i*,\n*j* and *k* are integers and *x* is an arbitrary object that meets any\ntype and value restrictions imposed by *s*.\n\nThe "in" and "not in" operations have the same priorities as the\ncomparison operations. The "+" (concatenation) and "*" (repetition)\noperations have the same priority as the corresponding numeric\noperations.\n\n+----------------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+============================+==================================+============+\n| "x in s" | "True" if an item of *s* is | (1) |\n| | equal to *x*, else "False" | |\n+----------------------------+----------------------------------+------------+\n| "x not in s" | "False" if an item of *s* is | (1) |\n| | equal to *x*, else "True" | |\n+----------------------------+----------------------------------+------------+\n| "s + t" | the concatenation of *s* and *t* | (6)(7) |\n+----------------------------+----------------------------------+------------+\n| "s * n" or "n * s" | *n* shallow copies of *s* | (2)(7) |\n| | concatenated | |\n+----------------------------+----------------------------------+------------+\n| "s[i]" | *i*th item of *s*, origin 0 | (3) |\n+----------------------------+----------------------------------+------------+\n| "s[i:j]" | slice of *s* from *i* to *j* | (3)(4) |\n+----------------------------+----------------------------------+------------+\n| "s[i:j:k]" | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+----------------------------+----------------------------------+------------+\n| "len(s)" | length of *s* | |\n+----------------------------+----------------------------------+------------+\n| "min(s)" | smallest item of *s* | |\n+----------------------------+----------------------------------+------------+\n| "max(s)" | largest item of *s* | |\n+----------------------------+----------------------------------+------------+\n| "s.index(x[, i[, j]])" | index of the first occurrence of | (8) |\n| | *x* in *s* (at or after index | |\n| | *i* and before index *j*) | |\n+----------------------------+----------------------------------+------------+\n| "s.count(x)" | total number of occurrences of | |\n| | *x* in *s* | |\n+----------------------------+----------------------------------+------------+\n\nSequences of the same type also support comparisons. In particular,\ntuples and lists are compared lexicographically by comparing\ncorresponding elements. This means that to compare equal, every\nelement must compare equal and the two sequences must be of the same\ntype and have the same length. (For full details see *Comparisons* in\nthe language reference.)\n\nNotes:\n\n1. While the "in" and "not in" operations are used only for simple\n containment testing in the general case, some specialised sequences\n (such as "str", "bytes" and "bytearray") also use them for\n subsequence testing:\n\n >>> "gg" in "eggs"\n True\n\n2. Values of *n* less than "0" are treated as "0" (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that "[[]]" is a one-element list containing\n an empty list, so all three elements of "[[]] * 3" are (pointers\n to) this single empty list. Modifying any of the elements of\n "lists" modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of\n the string: "len(s) + i" or "len(s) + j" is substituted. But note\n that "-0" is still "0".\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that "i <= k < j". If *i* or *j* is\n greater than "len(s)", use "len(s)". If *i* is omitted or "None",\n use "0". If *j* is omitted or "None", use "len(s)". If *i* is\n greater than or equal to *j*, the slice is empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index "x = i + n*k" such that "0 <= n <\n (j-i)/k". In other words, the indices are "i", "i+k", "i+2*k",\n "i+3*k" and so on, stopping when *j* is reached (but never\n including *j*). If *i* or *j* is greater than "len(s)", use\n "len(s)". If *i* or *j* are omitted or "None", they become "end"\n values (which end depends on the sign of *k*). Note, *k* cannot be\n zero. If *k* is "None", it is treated like "1".\n\n6. Concatenating immutable sequences always results in a new\n object. This means that building up a sequence by repeated\n concatenation will have a quadratic runtime cost in the total\n sequence length. To get a linear runtime cost, you must switch to\n one of the alternatives below:\n\n * if concatenating "str" objects, you can build a list and use\n "str.join()" at the end or else write to a "io.StringIO" instance\n and retrieve its value when complete\n\n * if concatenating "bytes" objects, you can similarly use\n "bytes.join()" or "io.BytesIO", or you can do in-place\n concatenation with a "bytearray" object. "bytearray" objects are\n mutable and have an efficient overallocation mechanism\n\n * if concatenating "tuple" objects, extend a "list" instead\n\n * for other types, investigate the relevant class documentation\n\n7. Some sequence types (such as "range") only support item\n sequences that follow specific patterns, and hence don\'t support\n sequence concatenation or repetition.\n\n8. "index" raises "ValueError" when *x* is not found in *s*. When\n supported, the additional arguments to the index method allow\n efficient searching of subsections of the sequence. Passing the\n extra arguments is roughly equivalent to using "s[i:j].index(x)",\n only without copying any data and with the returned index being\n relative to the start of the sequence rather than the start of the\n slice.\n\n\nImmutable Sequence Types\n========================\n\nThe only operation that immutable sequence types generally implement\nthat is not also implemented by mutable sequence types is support for\nthe "hash()" built-in.\n\nThis support allows immutable sequences, such as "tuple" instances, to\nbe used as "dict" keys and stored in "set" and "frozenset" instances.\n\nAttempting to hash an immutable sequence that contains unhashable\nvalues will result in "TypeError".\n\n\nMutable Sequence Types\n======================\n\nThe operations in the following table are defined on mutable sequence\ntypes. The "collections.abc.MutableSequence" ABC is provided to make\nit easier to correctly implement these operations on custom sequence\ntypes.\n\nIn the table *s* is an instance of a mutable sequence type, *t* is any\niterable object and *x* is an arbitrary object that meets any type and\nvalue restrictions imposed by *s* (for example, "bytearray" only\naccepts integers that meet the value restriction "0 <= x <= 255").\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| "s[i] = x" | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j] = t" | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j]" | same as "s[i:j] = []" | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j:k]" | removes the elements of | |\n| | "s[i:j:k]" from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.append(x)" | appends *x* to the end of the | |\n| | sequence (same as | |\n| | "s[len(s):len(s)] = [x]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.clear()" | removes all items from "s" (same | (5) |\n| | as "del s[:]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.copy()" | creates a shallow copy of "s" | (5) |\n| | (same as "s[:]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.extend(t)" | extends *s* with the contents of | |\n| | *t* (same as "s[len(s):len(s)] = | |\n| | t") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.insert(i, x)" | inserts *x* into *s* at the | |\n| | index given by *i* (same as | |\n| | "s[i:i] = [x]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.pop([i])" | retrieves the item at *i* and | (2) |\n| | also removes it from *s* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.remove(x)" | remove the first item from *s* | (3) |\n| | where "s[i] == x" | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.reverse()" | reverses the items of *s* in | (4) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The optional argument *i* defaults to "-1", so that by default\n the last item is removed and returned.\n\n3. "remove" raises "ValueError" when *x* is not found in *s*.\n\n4. The "reverse()" method modifies the sequence in place for\n economy of space when reversing a large sequence. To remind users\n that it operates by side effect, it does not return the reversed\n sequence.\n\n5. "clear()" and "copy()" are included for consistency with the\n interfaces of mutable containers that don\'t support slicing\n operations (such as "dict" and "set")\n\n New in version 3.3: "clear()" and "copy()" methods.\n\n\nLists\n=====\n\nLists are mutable sequences, typically used to store collections of\nhomogeneous items (where the precise degree of similarity will vary by\napplication).\n\nclass class list([iterable])\n\n Lists may be constructed in several ways:\n\n * Using a pair of square brackets to denote the empty list: "[]"\n\n * Using square brackets, separating items with commas: "[a]",\n "[a, b, c]"\n\n * Using a list comprehension: "[x for x in iterable]"\n\n * Using the type constructor: "list()" or "list(iterable)"\n\n The constructor builds a list whose items are the same and in the\n same order as *iterable*\'s items. *iterable* may be either a\n sequence, a container that supports iteration, or an iterator\n object. If *iterable* is already a list, a copy is made and\n returned, similar to "iterable[:]". For example, "list(\'abc\')"\n returns "[\'a\', \'b\', \'c\']" and "list( (1, 2, 3) )" returns "[1, 2,\n 3]". If no argument is given, the constructor creates a new empty\n list, "[]".\n\n Many other operations also produce lists, including the "sorted()"\n built-in.\n\n Lists implement all of the *common* and *mutable* sequence\n operations. Lists also provide the following additional method:\n\n sort(*, key=None, reverse=None)\n\n This method sorts the list in place, using only "<" comparisons\n between items. Exceptions are not suppressed - if any comparison\n operations fail, the entire sort operation will fail (and the\n list will likely be left in a partially modified state).\n\n "sort()" accepts two arguments that can only be passed by\n keyword (*keyword-only arguments*):\n\n *key* specifies a function of one argument that is used to\n extract a comparison key from each list element (for example,\n "key=str.lower"). The key corresponding to each item in the list\n is calculated once and then used for the entire sorting process.\n The default value of "None" means that list items are sorted\n directly without calculating a separate key value.\n\n The "functools.cmp_to_key()" utility is available to convert a\n 2.x style *cmp* function to a *key* function.\n\n *reverse* is a boolean value. If set to "True", then the list\n elements are sorted as if each comparison were reversed.\n\n This method modifies the sequence in place for economy of space\n when sorting a large sequence. To remind users that it operates\n by side effect, it does not return the sorted sequence (use\n "sorted()" to explicitly request a new sorted list instance).\n\n The "sort()" method is guaranteed to be stable. A sort is\n stable if it guarantees not to change the relative order of\n elements that compare equal --- this is helpful for sorting in\n multiple passes (for example, sort by department, then by salary\n grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises "ValueError" if it can detect\n that the list has been mutated during a sort.\n\n\nTuples\n======\n\nTuples are immutable sequences, typically used to store collections of\nheterogeneous data (such as the 2-tuples produced by the "enumerate()"\nbuilt-in). Tuples are also used for cases where an immutable sequence\nof homogeneous data is needed (such as allowing storage in a "set" or\n"dict" instance).\n\nclass class tuple([iterable])\n\n Tuples may be constructed in a number of ways:\n\n * Using a pair of parentheses to denote the empty tuple: "()"\n\n * Using a trailing comma for a singleton tuple: "a," or "(a,)"\n\n * Separating items with commas: "a, b, c" or "(a, b, c)"\n\n * Using the "tuple()" built-in: "tuple()" or "tuple(iterable)"\n\n The constructor builds a tuple whose items are the same and in the\n same order as *iterable*\'s items. *iterable* may be either a\n sequence, a container that supports iteration, or an iterator\n object. If *iterable* is already a tuple, it is returned\n unchanged. For example, "tuple(\'abc\')" returns "(\'a\', \'b\', \'c\')"\n and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument is\n given, the constructor creates a new empty tuple, "()".\n\n Note that it is actually the comma which makes a tuple, not the\n parentheses. The parentheses are optional, except in the empty\n tuple case, or when they are needed to avoid syntactic ambiguity.\n For example, "f(a, b, c)" is a function call with three arguments,\n while "f((a, b, c))" is a function call with a 3-tuple as the sole\n argument.\n\n Tuples implement all of the *common* sequence operations.\n\nFor heterogeneous collections of data where access by name is clearer\nthan access by index, "collections.namedtuple()" may be a more\nappropriate choice than a simple tuple object.\n\n\nRanges\n======\n\nThe "range" type represents an immutable sequence of numbers and is\ncommonly used for looping a specific number of times in "for" loops.\n\nclass class range(stop)\nclass class range(start, stop[, step])\n\n The arguments to the range constructor must be integers (either\n built-in "int" or any object that implements the "__index__"\n special method). If the *step* argument is omitted, it defaults to\n "1". If the *start* argument is omitted, it defaults to "0". If\n *step* is zero, "ValueError" is raised.\n\n For a positive *step*, the contents of a range "r" are determined\n by the formula "r[i] = start + step*i" where "i >= 0" and "r[i] <\n stop".\n\n For a negative *step*, the contents of the range are still\n determined by the formula "r[i] = start + step*i", but the\n constraints are "i >= 0" and "r[i] > stop".\n\n A range object will be empty if "r[0]" does not meet the value\n constraint. Ranges do support negative indices, but these are\n interpreted as indexing from the end of the sequence determined by\n the positive indices.\n\n Ranges containing absolute values larger than "sys.maxsize" are\n permitted but some features (such as "len()") may raise\n "OverflowError".\n\n Range examples:\n\n >>> list(range(10))\n [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n >>> list(range(1, 11))\n [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n >>> list(range(0, 30, 5))\n [0, 5, 10, 15, 20, 25]\n >>> list(range(0, 10, 3))\n [0, 3, 6, 9]\n >>> list(range(0, -10, -1))\n [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n >>> list(range(0))\n []\n >>> list(range(1, 0))\n []\n\n Ranges implement all of the *common* sequence operations except\n concatenation and repetition (due to the fact that range objects\n can only represent sequences that follow a strict pattern and\n repetition and concatenation will usually violate that pattern).\n\nThe advantage of the "range" type over a regular "list" or "tuple" is\nthat a "range" object will always take the same (small) amount of\nmemory, no matter the size of the range it represents (as it only\nstores the "start", "stop" and "step" values, calculating individual\nitems and subranges as needed).\n\nRange objects implement the "collections.abc.Sequence" ABC, and\nprovide features such as containment tests, element index lookup,\nslicing and support for negative indices (see *Sequence Types ---\nlist, tuple, range*):\n\n>>> r = range(0, 20, 2)\n>>> r\nrange(0, 20, 2)\n>>> 11 in r\nFalse\n>>> 10 in r\nTrue\n>>> r.index(10)\n5\n>>> r[5]\n10\n>>> r[:5]\nrange(0, 10, 2)\n>>> r[-1]\n18\n\nTesting range objects for equality with "==" and "!=" compares them as\nsequences. That is, two range objects are considered equal if they\nrepresent the same sequence of values. (Note that two range objects\nthat compare equal might have different "start", "stop" and "step"\nattributes, for example "range(0) == range(2, 1, 3)" or "range(0, 3,\n2) == range(0, 4, 2)".)\n\nChanged in version 3.2: Implement the Sequence ABC. Support slicing\nand negative indices. Test "int" objects for membership in constant\ntime instead of iterating through all items.\n\nChanged in version 3.3: Define \'==\' and \'!=\' to compare range objects\nbased on the sequence of values they define (instead of comparing\nbased on object identity).\n\nNew in version 3.3: The "start", "stop" and "step" attributes.\n', - 'typesseq-mutable': u'\nMutable Sequence Types\n**********************\n\nThe operations in the following table are defined on mutable sequence\ntypes. The "collections.abc.MutableSequence" ABC is provided to make\nit easier to correctly implement these operations on custom sequence\ntypes.\n\nIn the table *s* is an instance of a mutable sequence type, *t* is any\niterable object and *x* is an arbitrary object that meets any type and\nvalue restrictions imposed by *s* (for example, "bytearray" only\naccepts integers that meet the value restriction "0 <= x <= 255").\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| "s[i] = x" | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j] = t" | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j]" | same as "s[i:j] = []" | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j:k]" | removes the elements of | |\n| | "s[i:j:k]" from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.append(x)" | appends *x* to the end of the | |\n| | sequence (same as | |\n| | "s[len(s):len(s)] = [x]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.clear()" | removes all items from "s" (same | (5) |\n| | as "del s[:]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.copy()" | creates a shallow copy of "s" | (5) |\n| | (same as "s[:]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.extend(t)" | extends *s* with the contents of | |\n| | *t* (same as "s[len(s):len(s)] = | |\n| | t") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.insert(i, x)" | inserts *x* into *s* at the | |\n| | index given by *i* (same as | |\n| | "s[i:i] = [x]") | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.pop([i])" | retrieves the item at *i* and | (2) |\n| | also removes it from *s* | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.remove(x)" | remove the first item from *s* | (3) |\n| | where "s[i] == x" | |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.reverse()" | reverses the items of *s* in | (4) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The optional argument *i* defaults to "-1", so that by default\n the last item is removed and returned.\n\n3. "remove" raises "ValueError" when *x* is not found in *s*.\n\n4. The "reverse()" method modifies the sequence in place for\n economy of space when reversing a large sequence. To remind users\n that it operates by side effect, it does not return the reversed\n sequence.\n\n5. "clear()" and "copy()" are included for consistency with the\n interfaces of mutable containers that don\'t support slicing\n operations (such as "dict" and "set")\n\n New in version 3.3: "clear()" and "copy()" methods.\n', - 'unary': u'\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary "-" (minus) operator yields the negation of its numeric\nargument.\n\nThe unary "+" (plus) operator yields its numeric argument unchanged.\n\nThe unary "~" (invert) operator yields the bitwise inversion of its\ninteger argument. The bitwise inversion of "x" is defined as\n"-(x+1)". It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n"TypeError" exception is raised.\n', - 'while': u'\nThe "while" statement\n*********************\n\nThe "while" statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the "else" clause, if present, is executed\nand the loop terminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and goes back\nto testing the expression.\n', - 'with': u'\nThe "with" statement\n********************\n\nThe "with" statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common "try"..."except"..."finally"\nusage patterns to be encapsulated for convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the "with" statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the "with_item")\n is evaluated to obtain a context manager.\n\n2. The context manager\'s "__exit__()" is loaded for later use.\n\n3. The context manager\'s "__enter__()" method is invoked.\n\n4. If a target was included in the "with" statement, the return\n value from "__enter__()" is assigned to it.\n\n Note: The "with" statement guarantees that if the "__enter__()"\n method returns without an error, then "__exit__()" will always be\n called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s "__exit__()" method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to "__exit__()". Otherwise, three\n "None" arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the "__exit__()" method was false, the exception is reraised.\n If the return value was true, the exception is suppressed, and\n execution continues with the statement following the "with"\n statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from "__exit__()" is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple "with" statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n', - 'yield': u'\nThe "yield" statement\n*********************\n\n yield_stmt ::= yield_expression\n\nA "yield" statement is semantically equivalent to a *yield\nexpression*. The yield statement can be used to omit the parentheses\nthat would otherwise be required in the equivalent yield expression\nstatement. For example, the yield statements\n\n yield \n yield from \n\nare equivalent to the yield expression statements\n\n (yield )\n (yield from )\n\nYield expressions and statements are only used when defining a\n*generator* function, and are only used in the body of the generator\nfunction. Using yield in a function definition is sufficient to cause\nthat definition to create a generator function instead of a normal\nfunction.\n\nFor full details of "yield" semantics, refer to the *Yield\nexpressions* section.\n'} +# Autogenerated by Sphinx on Mon May 16 13:41:38 2016 +topics = {'assert': '\n' + 'The "assert" statement\n' + '**********************\n' + '\n' + 'Assert statements are a convenient way to insert debugging ' + 'assertions\n' + 'into a program:\n' + '\n' + ' assert_stmt ::= "assert" expression ["," expression]\n' + '\n' + 'The simple form, "assert expression", is equivalent to\n' + '\n' + ' if __debug__:\n' + ' if not expression: raise AssertionError\n' + '\n' + 'The extended form, "assert expression1, expression2", is ' + 'equivalent to\n' + '\n' + ' if __debug__:\n' + ' if not expression1: raise AssertionError(expression2)\n' + '\n' + 'These equivalences assume that "__debug__" and "AssertionError" ' + 'refer\n' + 'to the built-in variables with those names. In the current\n' + 'implementation, the built-in variable "__debug__" is "True" under\n' + 'normal circumstances, "False" when optimization is requested ' + '(command\n' + 'line option -O). The current code generator emits no code for an\n' + 'assert statement when optimization is requested at compile time. ' + 'Note\n' + 'that it is unnecessary to include the source code for the ' + 'expression\n' + 'that failed in the error message; it will be displayed as part of ' + 'the\n' + 'stack trace.\n' + '\n' + 'Assignments to "__debug__" are illegal. The value for the ' + 'built-in\n' + 'variable is determined when the interpreter starts.\n', + 'assignment': '\n' + 'Assignment statements\n' + '*********************\n' + '\n' + 'Assignment statements are used to (re)bind names to values and ' + 'to\n' + 'modify attributes or items of mutable objects:\n' + '\n' + ' assignment_stmt ::= (target_list "=")+ (expression_list | ' + 'yield_expression)\n' + ' target_list ::= target ("," target)* [","]\n' + ' target ::= identifier\n' + ' | "(" target_list ")"\n' + ' | "[" target_list "]"\n' + ' | attributeref\n' + ' | subscription\n' + ' | slicing\n' + ' | "*" target\n' + '\n' + '(See section Primaries for the syntax definitions for ' + '*attributeref*,\n' + '*subscription*, and *slicing*.)\n' + '\n' + 'An assignment statement evaluates the expression list ' + '(remember that\n' + 'this can be a single expression or a comma-separated list, the ' + 'latter\n' + 'yielding a tuple) and assigns the single resulting object to ' + 'each of\n' + 'the target lists, from left to right.\n' + '\n' + 'Assignment is defined recursively depending on the form of the ' + 'target\n' + '(list). When a target is part of a mutable object (an ' + 'attribute\n' + 'reference, subscription or slicing), the mutable object must\n' + 'ultimately perform the assignment and decide about its ' + 'validity, and\n' + 'may raise an exception if the assignment is unacceptable. The ' + 'rules\n' + 'observed by various types and the exceptions raised are given ' + 'with the\n' + 'definition of the object types (see section The standard type\n' + 'hierarchy).\n' + '\n' + 'Assignment of an object to a target list, optionally enclosed ' + 'in\n' + 'parentheses or square brackets, is recursively defined as ' + 'follows.\n' + '\n' + '* If the target list is a single target: The object is ' + 'assigned to\n' + ' that target.\n' + '\n' + '* If the target list is a comma-separated list of targets: ' + 'The\n' + ' object must be an iterable with the same number of items as ' + 'there\n' + ' are targets in the target list, and the items are assigned, ' + 'from\n' + ' left to right, to the corresponding targets.\n' + '\n' + ' * If the target list contains one target prefixed with an\n' + ' asterisk, called a "starred" target: The object must be a ' + 'sequence\n' + ' with at least as many items as there are targets in the ' + 'target\n' + ' list, minus one. The first items of the sequence are ' + 'assigned,\n' + ' from left to right, to the targets before the starred ' + 'target. The\n' + ' final items of the sequence are assigned to the targets ' + 'after the\n' + ' starred target. A list of the remaining items in the ' + 'sequence is\n' + ' then assigned to the starred target (the list can be ' + 'empty).\n' + '\n' + ' * Else: The object must be a sequence with the same number ' + 'of\n' + ' items as there are targets in the target list, and the ' + 'items are\n' + ' assigned, from left to right, to the corresponding ' + 'targets.\n' + '\n' + 'Assignment of an object to a single target is recursively ' + 'defined as\n' + 'follows.\n' + '\n' + '* If the target is an identifier (name):\n' + '\n' + ' * If the name does not occur in a "global" or "nonlocal" ' + 'statement\n' + ' in the current code block: the name is bound to the object ' + 'in the\n' + ' current local namespace.\n' + '\n' + ' * Otherwise: the name is bound to the object in the global\n' + ' namespace or the outer namespace determined by ' + '"nonlocal",\n' + ' respectively.\n' + '\n' + ' The name is rebound if it was already bound. This may cause ' + 'the\n' + ' reference count for the object previously bound to the name ' + 'to reach\n' + ' zero, causing the object to be deallocated and its ' + 'destructor (if it\n' + ' has one) to be called.\n' + '\n' + '* If the target is a target list enclosed in parentheses or ' + 'in\n' + ' square brackets: The object must be an iterable with the ' + 'same number\n' + ' of items as there are targets in the target list, and its ' + 'items are\n' + ' assigned, from left to right, to the corresponding targets.\n' + '\n' + '* If the target is an attribute reference: The primary ' + 'expression in\n' + ' the reference is evaluated. It should yield an object with\n' + ' assignable attributes; if this is not the case, "TypeError" ' + 'is\n' + ' raised. That object is then asked to assign the assigned ' + 'object to\n' + ' the given attribute; if it cannot perform the assignment, it ' + 'raises\n' + ' an exception (usually but not necessarily ' + '"AttributeError").\n' + '\n' + ' Note: If the object is a class instance and the attribute ' + 'reference\n' + ' occurs on both sides of the assignment operator, the RHS ' + 'expression,\n' + ' "a.x" can access either an instance attribute or (if no ' + 'instance\n' + ' attribute exists) a class attribute. The LHS target "a.x" ' + 'is always\n' + ' set as an instance attribute, creating it if necessary. ' + 'Thus, the\n' + ' two occurrences of "a.x" do not necessarily refer to the ' + 'same\n' + ' attribute: if the RHS expression refers to a class ' + 'attribute, the\n' + ' LHS creates a new instance attribute as the target of the\n' + ' assignment:\n' + '\n' + ' class Cls:\n' + ' x = 3 # class variable\n' + ' inst = Cls()\n' + ' inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x ' + 'as 3\n' + '\n' + ' This description does not necessarily apply to descriptor\n' + ' attributes, such as properties created with "property()".\n' + '\n' + '* If the target is a subscription: The primary expression in ' + 'the\n' + ' reference is evaluated. It should yield either a mutable ' + 'sequence\n' + ' object (such as a list) or a mapping object (such as a ' + 'dictionary).\n' + ' Next, the subscript expression is evaluated.\n' + '\n' + ' If the primary is a mutable sequence object (such as a ' + 'list), the\n' + ' subscript must yield an integer. If it is negative, the ' + "sequence's\n" + ' length is added to it. The resulting value must be a ' + 'nonnegative\n' + " integer less than the sequence's length, and the sequence is " + 'asked\n' + ' to assign the assigned object to its item with that index. ' + 'If the\n' + ' index is out of range, "IndexError" is raised (assignment to ' + 'a\n' + ' subscripted sequence cannot add new items to a list).\n' + '\n' + ' If the primary is a mapping object (such as a dictionary), ' + 'the\n' + " subscript must have a type compatible with the mapping's key " + 'type,\n' + ' and the mapping is then asked to create a key/datum pair ' + 'which maps\n' + ' the subscript to the assigned object. This can either ' + 'replace an\n' + ' existing key/value pair with the same key value, or insert a ' + 'new\n' + ' key/value pair (if no key with the same value existed).\n' + '\n' + ' For user-defined objects, the "__setitem__()" method is ' + 'called with\n' + ' appropriate arguments.\n' + '\n' + '* If the target is a slicing: The primary expression in the\n' + ' reference is evaluated. It should yield a mutable sequence ' + 'object\n' + ' (such as a list). The assigned object should be a sequence ' + 'object\n' + ' of the same type. Next, the lower and upper bound ' + 'expressions are\n' + ' evaluated, insofar they are present; defaults are zero and ' + 'the\n' + " sequence's length. The bounds should evaluate to integers. " + 'If\n' + " either bound is negative, the sequence's length is added to " + 'it. The\n' + ' resulting bounds are clipped to lie between zero and the ' + "sequence's\n" + ' length, inclusive. Finally, the sequence object is asked to ' + 'replace\n' + ' the slice with the items of the assigned sequence. The ' + 'length of\n' + ' the slice may be different from the length of the assigned ' + 'sequence,\n' + ' thus changing the length of the target sequence, if the ' + 'target\n' + ' sequence allows it.\n' + '\n' + '**CPython implementation detail:** In the current ' + 'implementation, the\n' + 'syntax for targets is taken to be the same as for expressions, ' + 'and\n' + 'invalid syntax is rejected during the code generation phase, ' + 'causing\n' + 'less detailed error messages.\n' + '\n' + 'Although the definition of assignment implies that overlaps ' + 'between\n' + "the left-hand side and the right-hand side are 'simultaneous' " + '(for\n' + 'example "a, b = b, a" swaps two variables), overlaps *within* ' + 'the\n' + 'collection of assigned-to variables occur left-to-right, ' + 'sometimes\n' + 'resulting in confusion. For instance, the following program ' + 'prints\n' + '"[0, 2]":\n' + '\n' + ' x = [0, 1]\n' + ' i = 0\n' + ' i, x[i] = 1, 2 # i is updated, then x[i] is ' + 'updated\n' + ' print(x)\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 3132** - Extended Iterable Unpacking\n' + ' The specification for the "*target" feature.\n' + '\n' + '\n' + 'Augmented assignment statements\n' + '===============================\n' + '\n' + 'Augmented assignment is the combination, in a single ' + 'statement, of a\n' + 'binary operation and an assignment statement:\n' + '\n' + ' augmented_assignment_stmt ::= augtarget augop ' + '(expression_list | yield_expression)\n' + ' augtarget ::= identifier | attributeref | ' + 'subscription | slicing\n' + ' augop ::= "+=" | "-=" | "*=" | "@=" | ' + '"/=" | "//=" | "%=" | "**="\n' + ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' + '\n' + '(See section Primaries for the syntax definitions of the last ' + 'three\n' + 'symbols.)\n' + '\n' + 'An augmented assignment evaluates the target (which, unlike ' + 'normal\n' + 'assignment statements, cannot be an unpacking) and the ' + 'expression\n' + 'list, performs the binary operation specific to the type of ' + 'assignment\n' + 'on the two operands, and assigns the result to the original ' + 'target.\n' + 'The target is only evaluated once.\n' + '\n' + 'An augmented assignment expression like "x += 1" can be ' + 'rewritten as\n' + '"x = x + 1" to achieve a similar, but not exactly equal ' + 'effect. In the\n' + 'augmented version, "x" is only evaluated once. Also, when ' + 'possible,\n' + 'the actual operation is performed *in-place*, meaning that ' + 'rather than\n' + 'creating a new object and assigning that to the target, the ' + 'old object\n' + 'is modified instead.\n' + '\n' + 'Unlike normal assignments, augmented assignments evaluate the ' + 'left-\n' + 'hand side *before* evaluating the right-hand side. For ' + 'example, "a[i]\n' + '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' + 'performs\n' + 'the addition, and lastly, it writes the result back to ' + '"a[i]".\n' + '\n' + 'With the exception of assigning to tuples and multiple targets ' + 'in a\n' + 'single statement, the assignment done by augmented assignment\n' + 'statements is handled the same way as normal assignments. ' + 'Similarly,\n' + 'with the exception of the possible *in-place* behavior, the ' + 'binary\n' + 'operation performed by augmented assignment is the same as the ' + 'normal\n' + 'binary operations.\n' + '\n' + 'For targets which are attribute references, the same caveat ' + 'about\n' + 'class and instance attributes applies as for regular ' + 'assignments.\n', + 'atom-identifiers': '\n' + 'Identifiers (Names)\n' + '*******************\n' + '\n' + 'An identifier occurring as an atom is a name. See ' + 'section Identifiers\n' + 'and keywords for lexical definition and section Naming ' + 'and binding for\n' + 'documentation of naming and binding.\n' + '\n' + 'When the name is bound to an object, evaluation of the ' + 'atom yields\n' + 'that object. When a name is not bound, an attempt to ' + 'evaluate it\n' + 'raises a "NameError" exception.\n' + '\n' + '**Private name mangling:** When an identifier that ' + 'textually occurs in\n' + 'a class definition begins with two or more underscore ' + 'characters and\n' + 'does not end in two or more underscores, it is ' + 'considered a *private\n' + 'name* of that class. Private names are transformed to a ' + 'longer form\n' + 'before code is generated for them. The transformation ' + 'inserts the\n' + 'class name, with leading underscores removed and a ' + 'single underscore\n' + 'inserted, in front of the name. For example, the ' + 'identifier "__spam"\n' + 'occurring in a class named "Ham" will be transformed to ' + '"_Ham__spam".\n' + 'This transformation is independent of the syntactical ' + 'context in which\n' + 'the identifier is used. If the transformed name is ' + 'extremely long\n' + '(longer than 255 characters), implementation defined ' + 'truncation may\n' + 'happen. If the class name consists only of underscores, ' + 'no\n' + 'transformation is done.\n', + 'atom-literals': '\n' + 'Literals\n' + '********\n' + '\n' + 'Python supports string and bytes literals and various ' + 'numeric\n' + 'literals:\n' + '\n' + ' literal ::= stringliteral | bytesliteral\n' + ' | integer | floatnumber | imagnumber\n' + '\n' + 'Evaluation of a literal yields an object of the given type ' + '(string,\n' + 'bytes, integer, floating point number, complex number) with ' + 'the given\n' + 'value. The value may be approximated in the case of ' + 'floating point\n' + 'and imaginary (complex) literals. See section Literals for ' + 'details.\n' + '\n' + 'All literals correspond to immutable data types, and hence ' + 'the\n' + "object's identity is less important than its value. " + 'Multiple\n' + 'evaluations of literals with the same value (either the ' + 'same\n' + 'occurrence in the program text or a different occurrence) ' + 'may obtain\n' + 'the same object or a different object with the same ' + 'value.\n', + 'attribute-access': '\n' + 'Customizing attribute access\n' + '****************************\n' + '\n' + 'The following methods can be defined to customize the ' + 'meaning of\n' + 'attribute access (use of, assignment to, or deletion of ' + '"x.name") for\n' + 'class instances.\n' + '\n' + 'object.__getattr__(self, name)\n' + '\n' + ' Called when an attribute lookup has not found the ' + 'attribute in the\n' + ' usual places (i.e. it is not an instance attribute ' + 'nor is it found\n' + ' in the class tree for "self"). "name" is the ' + 'attribute name. This\n' + ' method should return the (computed) attribute value ' + 'or raise an\n' + ' "AttributeError" exception.\n' + '\n' + ' Note that if the attribute is found through the ' + 'normal mechanism,\n' + ' "__getattr__()" is not called. (This is an ' + 'intentional asymmetry\n' + ' between "__getattr__()" and "__setattr__()".) This is ' + 'done both for\n' + ' efficiency reasons and because otherwise ' + '"__getattr__()" would have\n' + ' no way to access other attributes of the instance. ' + 'Note that at\n' + ' least for instance variables, you can fake total ' + 'control by not\n' + ' inserting any values in the instance attribute ' + 'dictionary (but\n' + ' instead inserting them in another object). See the\n' + ' "__getattribute__()" method below for a way to ' + 'actually get total\n' + ' control over attribute access.\n' + '\n' + 'object.__getattribute__(self, name)\n' + '\n' + ' Called unconditionally to implement attribute ' + 'accesses for\n' + ' instances of the class. If the class also defines ' + '"__getattr__()",\n' + ' the latter will not be called unless ' + '"__getattribute__()" either\n' + ' calls it explicitly or raises an "AttributeError". ' + 'This method\n' + ' should return the (computed) attribute value or raise ' + 'an\n' + ' "AttributeError" exception. In order to avoid ' + 'infinite recursion in\n' + ' this method, its implementation should always call ' + 'the base class\n' + ' method with the same name to access any attributes it ' + 'needs, for\n' + ' example, "object.__getattribute__(self, name)".\n' + '\n' + ' Note: This method may still be bypassed when looking ' + 'up special\n' + ' methods as the result of implicit invocation via ' + 'language syntax\n' + ' or built-in functions. See Special method lookup.\n' + '\n' + 'object.__setattr__(self, name, value)\n' + '\n' + ' Called when an attribute assignment is attempted. ' + 'This is called\n' + ' instead of the normal mechanism (i.e. store the value ' + 'in the\n' + ' instance dictionary). *name* is the attribute name, ' + '*value* is the\n' + ' value to be assigned to it.\n' + '\n' + ' If "__setattr__()" wants to assign to an instance ' + 'attribute, it\n' + ' should call the base class method with the same name, ' + 'for example,\n' + ' "object.__setattr__(self, name, value)".\n' + '\n' + 'object.__delattr__(self, name)\n' + '\n' + ' Like "__setattr__()" but for attribute deletion ' + 'instead of\n' + ' assignment. This should only be implemented if "del ' + 'obj.name" is\n' + ' meaningful for the object.\n' + '\n' + 'object.__dir__(self)\n' + '\n' + ' Called when "dir()" is called on the object. A ' + 'sequence must be\n' + ' returned. "dir()" converts the returned sequence to a ' + 'list and\n' + ' sorts it.\n' + '\n' + '\n' + 'Implementing Descriptors\n' + '========================\n' + '\n' + 'The following methods only apply when an instance of the ' + 'class\n' + 'containing the method (a so-called *descriptor* class) ' + 'appears in an\n' + '*owner* class (the descriptor must be in either the ' + "owner's class\n" + 'dictionary or in the class dictionary for one of its ' + 'parents). In the\n' + 'examples below, "the attribute" refers to the attribute ' + 'whose name is\n' + "the key of the property in the owner class' " + '"__dict__".\n' + '\n' + 'object.__get__(self, instance, owner)\n' + '\n' + ' Called to get the attribute of the owner class (class ' + 'attribute\n' + ' access) or of an instance of that class (instance ' + 'attribute\n' + ' access). *owner* is always the owner class, while ' + '*instance* is the\n' + ' instance that the attribute was accessed through, or ' + '"None" when\n' + ' the attribute is accessed through the *owner*. This ' + 'method should\n' + ' return the (computed) attribute value or raise an ' + '"AttributeError"\n' + ' exception.\n' + '\n' + 'object.__set__(self, instance, value)\n' + '\n' + ' Called to set the attribute on an instance *instance* ' + 'of the owner\n' + ' class to a new value, *value*.\n' + '\n' + 'object.__delete__(self, instance)\n' + '\n' + ' Called to delete the attribute on an instance ' + '*instance* of the\n' + ' owner class.\n' + '\n' + 'The attribute "__objclass__" is interpreted by the ' + '"inspect" module as\n' + 'specifying the class where this object was defined ' + '(setting this\n' + 'appropriately can assist in runtime introspection of ' + 'dynamic class\n' + 'attributes). For callables, it may indicate that an ' + 'instance of the\n' + 'given type (or a subclass) is expected or required as ' + 'the first\n' + 'positional argument (for example, CPython sets this ' + 'attribute for\n' + 'unbound methods that are implemented in C).\n' + '\n' + '\n' + 'Invoking Descriptors\n' + '====================\n' + '\n' + 'In general, a descriptor is an object attribute with ' + '"binding\n' + 'behavior", one whose attribute access has been ' + 'overridden by methods\n' + 'in the descriptor protocol: "__get__()", "__set__()", ' + 'and\n' + '"__delete__()". If any of those methods are defined for ' + 'an object, it\n' + 'is said to be a descriptor.\n' + '\n' + 'The default behavior for attribute access is to get, ' + 'set, or delete\n' + "the attribute from an object's dictionary. For instance, " + '"a.x" has a\n' + 'lookup chain starting with "a.__dict__[\'x\']", then\n' + '"type(a).__dict__[\'x\']", and continuing through the ' + 'base classes of\n' + '"type(a)" excluding metaclasses.\n' + '\n' + 'However, if the looked-up value is an object defining ' + 'one of the\n' + 'descriptor methods, then Python may override the default ' + 'behavior and\n' + 'invoke the descriptor method instead. Where this occurs ' + 'in the\n' + 'precedence chain depends on which descriptor methods ' + 'were defined and\n' + 'how they were called.\n' + '\n' + 'The starting point for descriptor invocation is a ' + 'binding, "a.x". How\n' + 'the arguments are assembled depends on "a":\n' + '\n' + 'Direct Call\n' + ' The simplest and least common call is when user code ' + 'directly\n' + ' invokes a descriptor method: "x.__get__(a)".\n' + '\n' + 'Instance Binding\n' + ' If binding to an object instance, "a.x" is ' + 'transformed into the\n' + ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' + '\n' + 'Class Binding\n' + ' If binding to a class, "A.x" is transformed into the ' + 'call:\n' + ' "A.__dict__[\'x\'].__get__(None, A)".\n' + '\n' + 'Super Binding\n' + ' If "a" is an instance of "super", then the binding ' + '"super(B,\n' + ' obj).m()" searches "obj.__class__.__mro__" for the ' + 'base class "A"\n' + ' immediately preceding "B" and then invokes the ' + 'descriptor with the\n' + ' call: "A.__dict__[\'m\'].__get__(obj, ' + 'obj.__class__)".\n' + '\n' + 'For instance bindings, the precedence of descriptor ' + 'invocation depends\n' + 'on the which descriptor methods are defined. A ' + 'descriptor can define\n' + 'any combination of "__get__()", "__set__()" and ' + '"__delete__()". If it\n' + 'does not define "__get__()", then accessing the ' + 'attribute will return\n' + 'the descriptor object itself unless there is a value in ' + "the object's\n" + 'instance dictionary. If the descriptor defines ' + '"__set__()" and/or\n' + '"__delete__()", it is a data descriptor; if it defines ' + 'neither, it is\n' + 'a non-data descriptor. Normally, data descriptors ' + 'define both\n' + '"__get__()" and "__set__()", while non-data descriptors ' + 'have just the\n' + '"__get__()" method. Data descriptors with "__set__()" ' + 'and "__get__()"\n' + 'defined always override a redefinition in an instance ' + 'dictionary. In\n' + 'contrast, non-data descriptors can be overridden by ' + 'instances.\n' + '\n' + 'Python methods (including "staticmethod()" and ' + '"classmethod()") are\n' + 'implemented as non-data descriptors. Accordingly, ' + 'instances can\n' + 'redefine and override methods. This allows individual ' + 'instances to\n' + 'acquire behaviors that differ from other instances of ' + 'the same class.\n' + '\n' + 'The "property()" function is implemented as a data ' + 'descriptor.\n' + 'Accordingly, instances cannot override the behavior of a ' + 'property.\n' + '\n' + '\n' + '__slots__\n' + '=========\n' + '\n' + 'By default, instances of classes have a dictionary for ' + 'attribute\n' + 'storage. This wastes space for objects having very few ' + 'instance\n' + 'variables. The space consumption can become acute when ' + 'creating large\n' + 'numbers of instances.\n' + '\n' + 'The default can be overridden by defining *__slots__* in ' + 'a class\n' + 'definition. The *__slots__* declaration takes a sequence ' + 'of instance\n' + 'variables and reserves just enough space in each ' + 'instance to hold a\n' + 'value for each variable. Space is saved because ' + '*__dict__* is not\n' + 'created for each instance.\n' + '\n' + 'object.__slots__\n' + '\n' + ' This class variable can be assigned a string, ' + 'iterable, or sequence\n' + ' of strings with variable names used by instances. ' + '*__slots__*\n' + ' reserves space for the declared variables and ' + 'prevents the\n' + ' automatic creation of *__dict__* and *__weakref__* ' + 'for each\n' + ' instance.\n' + '\n' + '\n' + 'Notes on using *__slots__*\n' + '--------------------------\n' + '\n' + '* When inheriting from a class without *__slots__*, the ' + '*__dict__*\n' + ' attribute of that class will always be accessible, so ' + 'a *__slots__*\n' + ' definition in the subclass is meaningless.\n' + '\n' + '* Without a *__dict__* variable, instances cannot be ' + 'assigned new\n' + ' variables not listed in the *__slots__* definition. ' + 'Attempts to\n' + ' assign to an unlisted variable name raises ' + '"AttributeError". If\n' + ' dynamic assignment of new variables is desired, then ' + 'add\n' + ' "\'__dict__\'" to the sequence of strings in the ' + '*__slots__*\n' + ' declaration.\n' + '\n' + '* Without a *__weakref__* variable for each instance, ' + 'classes\n' + ' defining *__slots__* do not support weak references to ' + 'its\n' + ' instances. If weak reference support is needed, then ' + 'add\n' + ' "\'__weakref__\'" to the sequence of strings in the ' + '*__slots__*\n' + ' declaration.\n' + '\n' + '* *__slots__* are implemented at the class level by ' + 'creating\n' + ' descriptors (Implementing Descriptors) for each ' + 'variable name. As a\n' + ' result, class attributes cannot be used to set default ' + 'values for\n' + ' instance variables defined by *__slots__*; otherwise, ' + 'the class\n' + ' attribute would overwrite the descriptor assignment.\n' + '\n' + '* The action of a *__slots__* declaration is limited to ' + 'the class\n' + ' where it is defined. As a result, subclasses will ' + 'have a *__dict__*\n' + ' unless they also define *__slots__* (which must only ' + 'contain names\n' + ' of any *additional* slots).\n' + '\n' + '* If a class defines a slot also defined in a base ' + 'class, the\n' + ' instance variable defined by the base class slot is ' + 'inaccessible\n' + ' (except by retrieving its descriptor directly from the ' + 'base class).\n' + ' This renders the meaning of the program undefined. In ' + 'the future, a\n' + ' check may be added to prevent this.\n' + '\n' + '* Nonempty *__slots__* does not work for classes derived ' + 'from\n' + ' "variable-length" built-in types such as "int", ' + '"bytes" and "tuple".\n' + '\n' + '* Any non-string iterable may be assigned to ' + '*__slots__*. Mappings\n' + ' may also be used; however, in the future, special ' + 'meaning may be\n' + ' assigned to the values corresponding to each key.\n' + '\n' + '* *__class__* assignment works only if both classes have ' + 'the same\n' + ' *__slots__*.\n', + 'attribute-references': '\n' + 'Attribute references\n' + '********************\n' + '\n' + 'An attribute reference is a primary followed by a ' + 'period and a name:\n' + '\n' + ' attributeref ::= primary "." identifier\n' + '\n' + 'The primary must evaluate to an object of a type ' + 'that supports\n' + 'attribute references, which most objects do. This ' + 'object is then\n' + 'asked to produce the attribute whose name is the ' + 'identifier. This\n' + 'production can be customized by overriding the ' + '"__getattr__()" method.\n' + 'If this attribute is not available, the exception ' + '"AttributeError" is\n' + 'raised. Otherwise, the type and value of the object ' + 'produced is\n' + 'determined by the object. Multiple evaluations of ' + 'the same attribute\n' + 'reference may yield different objects.\n', + 'augassign': '\n' + 'Augmented assignment statements\n' + '*******************************\n' + '\n' + 'Augmented assignment is the combination, in a single statement, ' + 'of a\n' + 'binary operation and an assignment statement:\n' + '\n' + ' augmented_assignment_stmt ::= augtarget augop ' + '(expression_list | yield_expression)\n' + ' augtarget ::= identifier | attributeref | ' + 'subscription | slicing\n' + ' augop ::= "+=" | "-=" | "*=" | "@=" | ' + '"/=" | "//=" | "%=" | "**="\n' + ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' + '\n' + '(See section Primaries for the syntax definitions of the last ' + 'three\n' + 'symbols.)\n' + '\n' + 'An augmented assignment evaluates the target (which, unlike ' + 'normal\n' + 'assignment statements, cannot be an unpacking) and the ' + 'expression\n' + 'list, performs the binary operation specific to the type of ' + 'assignment\n' + 'on the two operands, and assigns the result to the original ' + 'target.\n' + 'The target is only evaluated once.\n' + '\n' + 'An augmented assignment expression like "x += 1" can be ' + 'rewritten as\n' + '"x = x + 1" to achieve a similar, but not exactly equal effect. ' + 'In the\n' + 'augmented version, "x" is only evaluated once. Also, when ' + 'possible,\n' + 'the actual operation is performed *in-place*, meaning that ' + 'rather than\n' + 'creating a new object and assigning that to the target, the old ' + 'object\n' + 'is modified instead.\n' + '\n' + 'Unlike normal assignments, augmented assignments evaluate the ' + 'left-\n' + 'hand side *before* evaluating the right-hand side. For ' + 'example, "a[i]\n' + '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' + 'performs\n' + 'the addition, and lastly, it writes the result back to "a[i]".\n' + '\n' + 'With the exception of assigning to tuples and multiple targets ' + 'in a\n' + 'single statement, the assignment done by augmented assignment\n' + 'statements is handled the same way as normal assignments. ' + 'Similarly,\n' + 'with the exception of the possible *in-place* behavior, the ' + 'binary\n' + 'operation performed by augmented assignment is the same as the ' + 'normal\n' + 'binary operations.\n' + '\n' + 'For targets which are attribute references, the same caveat ' + 'about\n' + 'class and instance attributes applies as for regular ' + 'assignments.\n', + 'binary': '\n' + 'Binary arithmetic operations\n' + '****************************\n' + '\n' + 'The binary arithmetic operations have the conventional priority\n' + 'levels. Note that some of these operations also apply to certain ' + 'non-\n' + 'numeric types. Apart from the power operator, there are only two\n' + 'levels, one for multiplicative operators and one for additive\n' + 'operators:\n' + '\n' + ' m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr |\n' + ' m_expr "//" u_expr| m_expr "/" u_expr |\n' + ' m_expr "%" u_expr\n' + ' a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n' + '\n' + 'The "*" (multiplication) operator yields the product of its ' + 'arguments.\n' + 'The arguments must either both be numbers, or one argument must be ' + 'an\n' + 'integer and the other must be a sequence. In the former case, the\n' + 'numbers are converted to a common type and then multiplied ' + 'together.\n' + 'In the latter case, sequence repetition is performed; a negative\n' + 'repetition factor yields an empty sequence.\n' + '\n' + 'The "@" (at) operator is intended to be used for matrix\n' + 'multiplication. No builtin Python types implement this operator.\n' + '\n' + 'New in version 3.5.\n' + '\n' + 'The "/" (division) and "//" (floor division) operators yield the\n' + 'quotient of their arguments. The numeric arguments are first\n' + 'converted to a common type. Division of integers yields a float, ' + 'while\n' + 'floor division of integers results in an integer; the result is ' + 'that\n' + "of mathematical division with the 'floor' function applied to the\n" + 'result. Division by zero raises the "ZeroDivisionError" ' + 'exception.\n' + '\n' + 'The "%" (modulo) operator yields the remainder from the division ' + 'of\n' + 'the first argument by the second. The numeric arguments are ' + 'first\n' + 'converted to a common type. A zero right argument raises the\n' + '"ZeroDivisionError" exception. The arguments may be floating ' + 'point\n' + 'numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals ' + '"4*0.7 +\n' + '0.34".) The modulo operator always yields a result with the same ' + 'sign\n' + 'as its second operand (or zero); the absolute value of the result ' + 'is\n' + 'strictly smaller than the absolute value of the second operand ' + '[1].\n' + '\n' + 'The floor division and modulo operators are connected by the ' + 'following\n' + 'identity: "x == (x//y)*y + (x%y)". Floor division and modulo are ' + 'also\n' + 'connected with the built-in function "divmod()": "divmod(x, y) ==\n' + '(x//y, x%y)". [2].\n' + '\n' + 'In addition to performing the modulo operation on numbers, the ' + '"%"\n' + 'operator is also overloaded by string objects to perform ' + 'old-style\n' + 'string formatting (also known as interpolation). The syntax for\n' + 'string formatting is described in the Python Library Reference,\n' + 'section printf-style String Formatting.\n' + '\n' + 'The floor division operator, the modulo operator, and the ' + '"divmod()"\n' + 'function are not defined for complex numbers. Instead, convert to ' + 'a\n' + 'floating point number using the "abs()" function if appropriate.\n' + '\n' + 'The "+" (addition) operator yields the sum of its arguments. The\n' + 'arguments must either both be numbers or both be sequences of the ' + 'same\n' + 'type. In the former case, the numbers are converted to a common ' + 'type\n' + 'and then added together. In the latter case, the sequences are\n' + 'concatenated.\n' + '\n' + 'The "-" (subtraction) operator yields the difference of its ' + 'arguments.\n' + 'The numeric arguments are first converted to a common type.\n', + 'bitwise': '\n' + 'Binary bitwise operations\n' + '*************************\n' + '\n' + 'Each of the three bitwise operations has a different priority ' + 'level:\n' + '\n' + ' and_expr ::= shift_expr | and_expr "&" shift_expr\n' + ' xor_expr ::= and_expr | xor_expr "^" and_expr\n' + ' or_expr ::= xor_expr | or_expr "|" xor_expr\n' + '\n' + 'The "&" operator yields the bitwise AND of its arguments, which ' + 'must\n' + 'be integers.\n' + '\n' + 'The "^" operator yields the bitwise XOR (exclusive OR) of its\n' + 'arguments, which must be integers.\n' + '\n' + 'The "|" operator yields the bitwise (inclusive) OR of its ' + 'arguments,\n' + 'which must be integers.\n', + 'bltin-code-objects': '\n' + 'Code Objects\n' + '************\n' + '\n' + 'Code objects are used by the implementation to ' + 'represent "pseudo-\n' + 'compiled" executable Python code such as a function ' + 'body. They differ\n' + "from function objects because they don't contain a " + 'reference to their\n' + 'global execution environment. Code objects are ' + 'returned by the built-\n' + 'in "compile()" function and can be extracted from ' + 'function objects\n' + 'through their "__code__" attribute. See also the ' + '"code" module.\n' + '\n' + 'A code object can be executed or evaluated by passing ' + 'it (instead of a\n' + 'source string) to the "exec()" or "eval()" built-in ' + 'functions.\n' + '\n' + 'See The standard type hierarchy for more ' + 'information.\n', + 'bltin-ellipsis-object': '\n' + 'The Ellipsis Object\n' + '*******************\n' + '\n' + 'This object is commonly used by slicing (see ' + 'Slicings). It supports\n' + 'no special operations. There is exactly one ' + 'ellipsis object, named\n' + '"Ellipsis" (a built-in name). "type(Ellipsis)()" ' + 'produces the\n' + '"Ellipsis" singleton.\n' + '\n' + 'It is written as "Ellipsis" or "...".\n', + 'bltin-null-object': '\n' + 'The Null Object\n' + '***************\n' + '\n' + "This object is returned by functions that don't " + 'explicitly return a\n' + 'value. It supports no special operations. There is ' + 'exactly one null\n' + 'object, named "None" (a built-in name). "type(None)()" ' + 'produces the\n' + 'same singleton.\n' + '\n' + 'It is written as "None".\n', + 'bltin-type-objects': '\n' + 'Type Objects\n' + '************\n' + '\n' + 'Type objects represent the various object types. An ' + "object's type is\n" + 'accessed by the built-in function "type()". There are ' + 'no special\n' + 'operations on types. The standard module "types" ' + 'defines names for\n' + 'all standard built-in types.\n' + '\n' + 'Types are written like this: "".\n', + 'booleans': '\n' + 'Boolean operations\n' + '******************\n' + '\n' + ' or_test ::= and_test | or_test "or" and_test\n' + ' and_test ::= not_test | and_test "and" not_test\n' + ' not_test ::= comparison | "not" not_test\n' + '\n' + 'In the context of Boolean operations, and also when expressions ' + 'are\n' + 'used by control flow statements, the following values are ' + 'interpreted\n' + 'as false: "False", "None", numeric zero of all types, and empty\n' + 'strings and containers (including strings, tuples, lists,\n' + 'dictionaries, sets and frozensets). All other values are ' + 'interpreted\n' + 'as true. User-defined objects can customize their truth value ' + 'by\n' + 'providing a "__bool__()" method.\n' + '\n' + 'The operator "not" yields "True" if its argument is false, ' + '"False"\n' + 'otherwise.\n' + '\n' + 'The expression "x and y" first evaluates *x*; if *x* is false, ' + 'its\n' + 'value is returned; otherwise, *y* is evaluated and the resulting ' + 'value\n' + 'is returned.\n' + '\n' + 'The expression "x or y" first evaluates *x*; if *x* is true, its ' + 'value\n' + 'is returned; otherwise, *y* is evaluated and the resulting value ' + 'is\n' + 'returned.\n' + '\n' + '(Note that neither "and" nor "or" restrict the value and type ' + 'they\n' + 'return to "False" and "True", but rather return the last ' + 'evaluated\n' + 'argument. This is sometimes useful, e.g., if "s" is a string ' + 'that\n' + 'should be replaced by a default value if it is empty, the ' + 'expression\n' + '"s or \'foo\'" yields the desired value. Because "not" has to ' + 'create a\n' + 'new value, it returns a boolean value regardless of the type of ' + 'its\n' + 'argument (for example, "not \'foo\'" produces "False" rather ' + 'than "\'\'".)\n', + 'break': '\n' + 'The "break" statement\n' + '*********************\n' + '\n' + ' break_stmt ::= "break"\n' + '\n' + '"break" may only occur syntactically nested in a "for" or "while"\n' + 'loop, but not nested in a function or class definition within that\n' + 'loop.\n' + '\n' + 'It terminates the nearest enclosing loop, skipping the optional ' + '"else"\n' + 'clause if the loop has one.\n' + '\n' + 'If a "for" loop is terminated by "break", the loop control target\n' + 'keeps its current value.\n' + '\n' + 'When "break" passes control out of a "try" statement with a ' + '"finally"\n' + 'clause, that "finally" clause is executed before really leaving ' + 'the\n' + 'loop.\n', + 'callable-types': '\n' + 'Emulating callable objects\n' + '**************************\n' + '\n' + 'object.__call__(self[, args...])\n' + '\n' + ' Called when the instance is "called" as a function; if ' + 'this method\n' + ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' + ' "x.__call__(arg1, arg2, ...)".\n', + 'calls': '\n' + 'Calls\n' + '*****\n' + '\n' + 'A call calls a callable object (e.g., a *function*) with a ' + 'possibly\n' + 'empty series of *arguments*:\n' + '\n' + ' call ::= primary "(" [argument_list [","] | ' + 'comprehension] ")"\n' + ' argument_list ::= positional_arguments ["," ' + 'keyword_arguments]\n' + ' ["," "*" expression] ["," ' + 'keyword_arguments]\n' + ' ["," "**" expression]\n' + ' | keyword_arguments ["," "*" expression]\n' + ' ["," keyword_arguments] ["," "**" ' + 'expression]\n' + ' | "*" expression ["," keyword_arguments] ["," ' + '"**" expression]\n' + ' | "**" expression\n' + ' positional_arguments ::= expression ("," expression)*\n' + ' keyword_arguments ::= keyword_item ("," keyword_item)*\n' + ' keyword_item ::= identifier "=" expression\n' + '\n' + 'An optional trailing comma may be present after the positional and\n' + 'keyword arguments but does not affect the semantics.\n' + '\n' + 'The primary must evaluate to a callable object (user-defined\n' + 'functions, built-in functions, methods of built-in objects, class\n' + 'objects, methods of class instances, and all objects having a\n' + '"__call__()" method are callable). All argument expressions are\n' + 'evaluated before the call is attempted. Please refer to section\n' + 'Function definitions for the syntax of formal *parameter* lists.\n' + '\n' + 'If keyword arguments are present, they are first converted to\n' + 'positional arguments, as follows. First, a list of unfilled slots ' + 'is\n' + 'created for the formal parameters. If there are N positional\n' + 'arguments, they are placed in the first N slots. Next, for each\n' + 'keyword argument, the identifier is used to determine the\n' + 'corresponding slot (if the identifier is the same as the first ' + 'formal\n' + 'parameter name, the first slot is used, and so on). If the slot ' + 'is\n' + 'already filled, a "TypeError" exception is raised. Otherwise, the\n' + 'value of the argument is placed in the slot, filling it (even if ' + 'the\n' + 'expression is "None", it fills the slot). When all arguments have\n' + 'been processed, the slots that are still unfilled are filled with ' + 'the\n' + 'corresponding default value from the function definition. ' + '(Default\n' + 'values are calculated, once, when the function is defined; thus, a\n' + 'mutable object such as a list or dictionary used as default value ' + 'will\n' + "be shared by all calls that don't specify an argument value for " + 'the\n' + 'corresponding slot; this should usually be avoided.) If there are ' + 'any\n' + 'unfilled slots for which no default value is specified, a ' + '"TypeError"\n' + 'exception is raised. Otherwise, the list of filled slots is used ' + 'as\n' + 'the argument list for the call.\n' + '\n' + '**CPython implementation detail:** An implementation may provide\n' + 'built-in functions whose positional parameters do not have names, ' + 'even\n' + "if they are 'named' for the purpose of documentation, and which\n" + 'therefore cannot be supplied by keyword. In CPython, this is the ' + 'case\n' + 'for functions implemented in C that use "PyArg_ParseTuple()" to ' + 'parse\n' + 'their arguments.\n' + '\n' + 'If there are more positional arguments than there are formal ' + 'parameter\n' + 'slots, a "TypeError" exception is raised, unless a formal ' + 'parameter\n' + 'using the syntax "*identifier" is present; in this case, that ' + 'formal\n' + 'parameter receives a tuple containing the excess positional ' + 'arguments\n' + '(or an empty tuple if there were no excess positional arguments).\n' + '\n' + 'If any keyword argument does not correspond to a formal parameter\n' + 'name, a "TypeError" exception is raised, unless a formal parameter\n' + 'using the syntax "**identifier" is present; in this case, that ' + 'formal\n' + 'parameter receives a dictionary containing the excess keyword\n' + 'arguments (using the keywords as keys and the argument values as\n' + 'corresponding values), or a (new) empty dictionary if there were ' + 'no\n' + 'excess keyword arguments.\n' + '\n' + 'If the syntax "*expression" appears in the function call, ' + '"expression"\n' + 'must evaluate to an iterable. Elements from this iterable are ' + 'treated\n' + 'as if they were additional positional arguments; if there are\n' + 'positional arguments *x1*, ..., *xN*, and "expression" evaluates to ' + 'a\n' + 'sequence *y1*, ..., *yM*, this is equivalent to a call with M+N\n' + 'positional arguments *x1*, ..., *xN*, *y1*, ..., *yM*.\n' + '\n' + 'A consequence of this is that although the "*expression" syntax ' + 'may\n' + 'appear *after* some keyword arguments, it is processed *before* ' + 'the\n' + 'keyword arguments (and the "**expression" argument, if any -- see\n' + 'below). So:\n' + '\n' + ' >>> def f(a, b):\n' + ' ... print(a, b)\n' + ' ...\n' + ' >>> f(b=1, *(2,))\n' + ' 2 1\n' + ' >>> f(a=1, *(2,))\n' + ' Traceback (most recent call last):\n' + ' File "", line 1, in ?\n' + " TypeError: f() got multiple values for keyword argument 'a'\n" + ' >>> f(1, *(2,))\n' + ' 1 2\n' + '\n' + 'It is unusual for both keyword arguments and the "*expression" ' + 'syntax\n' + 'to be used in the same call, so in practice this confusion does ' + 'not\n' + 'arise.\n' + '\n' + 'If the syntax "**expression" appears in the function call,\n' + '"expression" must evaluate to a mapping, the contents of which are\n' + 'treated as additional keyword arguments. In the case of a keyword\n' + 'appearing in both "expression" and as an explicit keyword argument, ' + 'a\n' + '"TypeError" exception is raised.\n' + '\n' + 'Formal parameters using the syntax "*identifier" or "**identifier"\n' + 'cannot be used as positional argument slots or as keyword argument\n' + 'names.\n' + '\n' + 'A call always returns some value, possibly "None", unless it raises ' + 'an\n' + 'exception. How this value is computed depends on the type of the\n' + 'callable object.\n' + '\n' + 'If it is---\n' + '\n' + 'a user-defined function:\n' + ' The code block for the function is executed, passing it the\n' + ' argument list. The first thing the code block will do is bind ' + 'the\n' + ' formal parameters to the arguments; this is described in ' + 'section\n' + ' Function definitions. When the code block executes a "return"\n' + ' statement, this specifies the return value of the function ' + 'call.\n' + '\n' + 'a built-in function or method:\n' + ' The result is up to the interpreter; see Built-in Functions for ' + 'the\n' + ' descriptions of built-in functions and methods.\n' + '\n' + 'a class object:\n' + ' A new instance of that class is returned.\n' + '\n' + 'a class instance method:\n' + ' The corresponding user-defined function is called, with an ' + 'argument\n' + ' list that is one longer than the argument list of the call: the\n' + ' instance becomes the first argument.\n' + '\n' + 'a class instance:\n' + ' The class must define a "__call__()" method; the effect is then ' + 'the\n' + ' same as if that method was called.\n', + 'class': '\n' + 'Class definitions\n' + '*****************\n' + '\n' + 'A class definition defines a class object (see section The ' + 'standard\n' + 'type hierarchy):\n' + '\n' + ' classdef ::= [decorators] "class" classname [inheritance] ":" ' + 'suite\n' + ' inheritance ::= "(" [parameter_list] ")"\n' + ' classname ::= identifier\n' + '\n' + 'A class definition is an executable statement. The inheritance ' + 'list\n' + 'usually gives a list of base classes (see Customizing class ' + 'creation\n' + 'for more advanced uses), so each item in the list should evaluate ' + 'to a\n' + 'class object which allows subclassing. Classes without an ' + 'inheritance\n' + 'list inherit, by default, from the base class "object"; hence,\n' + '\n' + ' class Foo:\n' + ' pass\n' + '\n' + 'is equivalent to\n' + '\n' + ' class Foo(object):\n' + ' pass\n' + '\n' + "The class's suite is then executed in a new execution frame (see\n" + 'Naming and binding), using a newly created local namespace and the\n' + 'original global namespace. (Usually, the suite contains mostly\n' + "function definitions.) When the class's suite finishes execution, " + 'its\n' + 'execution frame is discarded but its local namespace is saved. [4] ' + 'A\n' + 'class object is then created using the inheritance list for the ' + 'base\n' + 'classes and the saved local namespace for the attribute ' + 'dictionary.\n' + 'The class name is bound to this class object in the original local\n' + 'namespace.\n' + '\n' + 'Class creation can be customized heavily using metaclasses.\n' + '\n' + 'Classes can also be decorated: just like when decorating ' + 'functions,\n' + '\n' + ' @f1(arg)\n' + ' @f2\n' + ' class Foo: pass\n' + '\n' + 'is equivalent to\n' + '\n' + ' class Foo: pass\n' + ' Foo = f1(arg)(f2(Foo))\n' + '\n' + 'The evaluation rules for the decorator expressions are the same as ' + 'for\n' + 'function decorators. The result must be a class object, which is ' + 'then\n' + 'bound to the class name.\n' + '\n' + "**Programmer's note:** Variables defined in the class definition " + 'are\n' + 'class attributes; they are shared by instances. Instance ' + 'attributes\n' + 'can be set in a method with "self.name = value". Both class and\n' + 'instance attributes are accessible through the notation ' + '""self.name"",\n' + 'and an instance attribute hides a class attribute with the same ' + 'name\n' + 'when accessed in this way. Class attributes can be used as ' + 'defaults\n' + 'for instance attributes, but using mutable values there can lead ' + 'to\n' + 'unexpected results. Descriptors can be used to create instance\n' + 'variables with different implementation details.\n' + '\n' + 'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n' + ' Class Decorators\n', + 'comparisons': '\n' + 'Comparisons\n' + '***********\n' + '\n' + 'Unlike C, all comparison operations in Python have the same ' + 'priority,\n' + 'which is lower than that of any arithmetic, shifting or ' + 'bitwise\n' + 'operation. Also unlike C, expressions like "a < b < c" have ' + 'the\n' + 'interpretation that is conventional in mathematics:\n' + '\n' + ' comparison ::= or_expr ( comp_operator or_expr )*\n' + ' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n' + ' | "is" ["not"] | ["not"] "in"\n' + '\n' + 'Comparisons yield boolean values: "True" or "False".\n' + '\n' + 'Comparisons can be chained arbitrarily, e.g., "x < y <= z" ' + 'is\n' + 'equivalent to "x < y and y <= z", except that "y" is ' + 'evaluated only\n' + 'once (but in both cases "z" is not evaluated at all when "x < ' + 'y" is\n' + 'found to be false).\n' + '\n' + 'Formally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and ' + '*op1*,\n' + '*op2*, ..., *opN* are comparison operators, then "a op1 b op2 ' + 'c ... y\n' + 'opN z" is equivalent to "a op1 b and b op2 c and ... y opN ' + 'z", except\n' + 'that each expression is evaluated at most once.\n' + '\n' + 'Note that "a op1 b op2 c" doesn\'t imply any kind of ' + 'comparison between\n' + '*a* and *c*, so that, e.g., "x < y > z" is perfectly legal ' + '(though\n' + 'perhaps not pretty).\n' + '\n' + '\n' + 'Value comparisons\n' + '=================\n' + '\n' + 'The operators "<", ">", "==", ">=", "<=", and "!=" compare ' + 'the values\n' + 'of two objects. The objects do not need to have the same ' + 'type.\n' + '\n' + 'Chapter Objects, values and types states that objects have a ' + 'value (in\n' + 'addition to type and identity). The value of an object is a ' + 'rather\n' + 'abstract notion in Python: For example, there is no canonical ' + 'access\n' + "method for an object's value. Also, there is no requirement " + 'that the\n' + 'value of an object should be constructed in a particular way, ' + 'e.g.\n' + 'comprised of all its data attributes. Comparison operators ' + 'implement a\n' + 'particular notion of what the value of an object is. One can ' + 'think of\n' + 'them as defining the value of an object indirectly, by means ' + 'of their\n' + 'comparison implementation.\n' + '\n' + 'Because all types are (direct or indirect) subtypes of ' + '"object", they\n' + 'inherit the default comparison behavior from "object". Types ' + 'can\n' + 'customize their comparison behavior by implementing *rich ' + 'comparison\n' + 'methods* like "__lt__()", described in Basic customization.\n' + '\n' + 'The default behavior for equality comparison ("==" and "!=") ' + 'is based\n' + 'on the identity of the objects. Hence, equality comparison ' + 'of\n' + 'instances with the same identity results in equality, and ' + 'equality\n' + 'comparison of instances with different identities results in\n' + 'inequality. A motivation for this default behavior is the ' + 'desire that\n' + 'all objects should be reflexive (i.e. "x is y" implies "x == ' + 'y").\n' + '\n' + 'A default order comparison ("<", ">", "<=", and ">=") is not ' + 'provided;\n' + 'an attempt raises "TypeError". A motivation for this default ' + 'behavior\n' + 'is the lack of a similar invariant as for equality.\n' + '\n' + 'The behavior of the default equality comparison, that ' + 'instances with\n' + 'different identities are always unequal, may be in contrast ' + 'to what\n' + 'types will need that have a sensible definition of object ' + 'value and\n' + 'value-based equality. Such types will need to customize ' + 'their\n' + 'comparison behavior, and in fact, a number of built-in types ' + 'have done\n' + 'that.\n' + '\n' + 'The following list describes the comparison behavior of the ' + 'most\n' + 'important built-in types.\n' + '\n' + '* Numbers of built-in numeric types (Numeric Types --- int, ' + 'float,\n' + ' complex) and of the standard library types ' + '"fractions.Fraction" and\n' + ' "decimal.Decimal" can be compared within and across their ' + 'types,\n' + ' with the restriction that complex numbers do not support ' + 'order\n' + ' comparison. Within the limits of the types involved, they ' + 'compare\n' + ' mathematically (algorithmically) correct without loss of ' + 'precision.\n' + '\n' + ' The not-a-number values "float(\'NaN\')" and ' + '"Decimal(\'NaN\')" are\n' + ' special. They are identical to themselves ("x is x" is ' + 'true) but\n' + ' are not equal to themselves ("x == x" is false). ' + 'Additionally,\n' + ' comparing any number to a not-a-number value will return ' + '"False".\n' + ' For example, both "3 < float(\'NaN\')" and "float(\'NaN\') ' + '< 3" will\n' + ' return "False".\n' + '\n' + '* Binary sequences (instances of "bytes" or "bytearray") can ' + 'be\n' + ' compared within and across their types. They compare\n' + ' lexicographically using the numeric values of their ' + 'elements.\n' + '\n' + '* Strings (instances of "str") compare lexicographically ' + 'using the\n' + ' numerical Unicode code points (the result of the built-in ' + 'function\n' + ' "ord()") of their characters. [3]\n' + '\n' + ' Strings and binary sequences cannot be directly compared.\n' + '\n' + '* Sequences (instances of "tuple", "list", or "range") can ' + 'be\n' + ' compared only within each of their types, with the ' + 'restriction that\n' + ' ranges do not support order comparison. Equality ' + 'comparison across\n' + ' these types results in unequality, and ordering comparison ' + 'across\n' + ' these types raises "TypeError".\n' + '\n' + ' Sequences compare lexicographically using comparison of\n' + ' corresponding elements, whereby reflexivity of the elements ' + 'is\n' + ' enforced.\n' + '\n' + ' In enforcing reflexivity of elements, the comparison of ' + 'collections\n' + ' assumes that for a collection element "x", "x == x" is ' + 'always true.\n' + ' Based on that assumption, element identity is compared ' + 'first, and\n' + ' element comparison is performed only for distinct ' + 'elements. This\n' + ' approach yields the same result as a strict element ' + 'comparison\n' + ' would, if the compared elements are reflexive. For ' + 'non-reflexive\n' + ' elements, the result is different than for strict element\n' + ' comparison, and may be surprising: The non-reflexive ' + 'not-a-number\n' + ' values for example result in the following comparison ' + 'behavior when\n' + ' used in a list:\n' + '\n' + " >>> nan = float('NaN')\n" + ' >>> nan is nan\n' + ' True\n' + ' >>> nan == nan\n' + ' False <-- the defined non-reflexive ' + 'behavior of NaN\n' + ' >>> [nan] == [nan]\n' + ' True <-- list enforces reflexivity and ' + 'tests identity first\n' + '\n' + ' Lexicographical comparison between built-in collections ' + 'works as\n' + ' follows:\n' + '\n' + ' * For two collections to compare equal, they must be of the ' + 'same\n' + ' type, have the same length, and each pair of ' + 'corresponding\n' + ' elements must compare equal (for example, "[1,2] == ' + '(1,2)" is\n' + ' false because the type is not the same).\n' + '\n' + ' * Collections that support order comparison are ordered the ' + 'same\n' + ' as their first unequal elements (for example, "[1,2,x] <= ' + '[1,2,y]"\n' + ' has the same value as "x <= y"). If a corresponding ' + 'element does\n' + ' not exist, the shorter collection is ordered first (for ' + 'example,\n' + ' "[1,2] < [1,2,3]" is true).\n' + '\n' + '* Mappings (instances of "dict") compare equal if and only if ' + 'they\n' + ' have equal *(key, value)* pairs. Equality comparison of the ' + 'keys and\n' + ' elements enforces reflexivity.\n' + '\n' + ' Order comparisons ("<", ">", "<=", and ">=") raise ' + '"TypeError".\n' + '\n' + '* Sets (instances of "set" or "frozenset") can be compared ' + 'within\n' + ' and across their types.\n' + '\n' + ' They define order comparison operators to mean subset and ' + 'superset\n' + ' tests. Those relations do not define total orderings (for ' + 'example,\n' + ' the two sets "{1,2}" and "{2,3}" are not equal, nor subsets ' + 'of one\n' + ' another, nor supersets of one another). Accordingly, sets ' + 'are not\n' + ' appropriate arguments for functions which depend on total ' + 'ordering\n' + ' (for example, "min()", "max()", and "sorted()" produce ' + 'undefined\n' + ' results given a list of sets as inputs).\n' + '\n' + ' Comparison of sets enforces reflexivity of its elements.\n' + '\n' + '* Most other built-in types have no comparison methods ' + 'implemented,\n' + ' so they inherit the default comparison behavior.\n' + '\n' + 'User-defined classes that customize their comparison behavior ' + 'should\n' + 'follow some consistency rules, if possible:\n' + '\n' + '* Equality comparison should be reflexive. In other words, ' + 'identical\n' + ' objects should compare equal:\n' + '\n' + ' "x is y" implies "x == y"\n' + '\n' + '* Comparison should be symmetric. In other words, the ' + 'following\n' + ' expressions should have the same result:\n' + '\n' + ' "x == y" and "y == x"\n' + '\n' + ' "x != y" and "y != x"\n' + '\n' + ' "x < y" and "y > x"\n' + '\n' + ' "x <= y" and "y >= x"\n' + '\n' + '* Comparison should be transitive. The following ' + '(non-exhaustive)\n' + ' examples illustrate that:\n' + '\n' + ' "x > y and y > z" implies "x > z"\n' + '\n' + ' "x < y and y <= z" implies "x < z"\n' + '\n' + '* Inverse comparison should result in the boolean negation. ' + 'In other\n' + ' words, the following expressions should have the same ' + 'result:\n' + '\n' + ' "x == y" and "not x != y"\n' + '\n' + ' "x < y" and "not x >= y" (for total ordering)\n' + '\n' + ' "x > y" and "not x <= y" (for total ordering)\n' + '\n' + ' The last two expressions apply to totally ordered ' + 'collections (e.g.\n' + ' to sequences, but not to sets or mappings). See also the\n' + ' "total_ordering()" decorator.\n' + '\n' + 'Python does not enforce these consistency rules. In fact, ' + 'the\n' + 'not-a-number values are an example for not following these ' + 'rules.\n' + '\n' + '\n' + 'Membership test operations\n' + '==========================\n' + '\n' + 'The operators "in" and "not in" test for membership. "x in ' + 's"\n' + 'evaluates to true if *x* is a member of *s*, and false ' + 'otherwise. "x\n' + 'not in s" returns the negation of "x in s". All built-in ' + 'sequences\n' + 'and set types support this as well as dictionary, for which ' + '"in" tests\n' + 'whether the dictionary has a given key. For container types ' + 'such as\n' + 'list, tuple, set, frozenset, dict, or collections.deque, the\n' + 'expression "x in y" is equivalent to "any(x is e or x == e ' + 'for e in\n' + 'y)".\n' + '\n' + 'For the string and bytes types, "x in y" is true if and only ' + 'if *x* is\n' + 'a substring of *y*. An equivalent test is "y.find(x) != ' + '-1". Empty\n' + 'strings are always considered to be a substring of any other ' + 'string,\n' + 'so """ in "abc"" will return "True".\n' + '\n' + 'For user-defined classes which define the "__contains__()" ' + 'method, "x\n' + 'in y" is true if and only if "y.__contains__(x)" is true.\n' + '\n' + 'For user-defined classes which do not define "__contains__()" ' + 'but do\n' + 'define "__iter__()", "x in y" is true if some value "z" with ' + '"x == z"\n' + 'is produced while iterating over "y". If an exception is ' + 'raised\n' + 'during the iteration, it is as if "in" raised that ' + 'exception.\n' + '\n' + 'Lastly, the old-style iteration protocol is tried: if a class ' + 'defines\n' + '"__getitem__()", "x in y" is true if and only if there is a ' + 'non-\n' + 'negative integer index *i* such that "x == y[i]", and all ' + 'lower\n' + 'integer indices do not raise "IndexError" exception. (If any ' + 'other\n' + 'exception is raised, it is as if "in" raised that ' + 'exception).\n' + '\n' + 'The operator "not in" is defined to have the inverse true ' + 'value of\n' + '"in".\n' + '\n' + '\n' + 'Identity comparisons\n' + '====================\n' + '\n' + 'The operators "is" and "is not" test for object identity: "x ' + 'is y" is\n' + 'true if and only if *x* and *y* are the same object. "x is ' + 'not y"\n' + 'yields the inverse truth value. [4]\n', + 'compound': '\n' + 'Compound statements\n' + '*******************\n' + '\n' + 'Compound statements contain (groups of) other statements; they ' + 'affect\n' + 'or control the execution of those other statements in some way. ' + 'In\n' + 'general, compound statements span multiple lines, although in ' + 'simple\n' + 'incarnations a whole compound statement may be contained in one ' + 'line.\n' + '\n' + 'The "if", "while" and "for" statements implement traditional ' + 'control\n' + 'flow constructs. "try" specifies exception handlers and/or ' + 'cleanup\n' + 'code for a group of statements, while the "with" statement ' + 'allows the\n' + 'execution of initialization and finalization code around a block ' + 'of\n' + 'code. Function and class definitions are also syntactically ' + 'compound\n' + 'statements.\n' + '\n' + "A compound statement consists of one or more 'clauses.' A " + 'clause\n' + "consists of a header and a 'suite.' The clause headers of a\n" + 'particular compound statement are all at the same indentation ' + 'level.\n' + 'Each clause header begins with a uniquely identifying keyword ' + 'and ends\n' + 'with a colon. A suite is a group of statements controlled by a\n' + 'clause. A suite can be one or more semicolon-separated simple\n' + 'statements on the same line as the header, following the ' + "header's\n" + 'colon, or it can be one or more indented statements on ' + 'subsequent\n' + 'lines. Only the latter form of a suite can contain nested ' + 'compound\n' + "statements; the following is illegal, mostly because it wouldn't " + 'be\n' + 'clear to which "if" clause a following "else" clause would ' + 'belong:\n' + '\n' + ' if test1: if test2: print(x)\n' + '\n' + 'Also note that the semicolon binds tighter than the colon in ' + 'this\n' + 'context, so that in the following example, either all or none of ' + 'the\n' + '"print()" calls are executed:\n' + '\n' + ' if x < y < z: print(x); print(y); print(z)\n' + '\n' + 'Summarizing:\n' + '\n' + ' compound_stmt ::= if_stmt\n' + ' | while_stmt\n' + ' | for_stmt\n' + ' | try_stmt\n' + ' | with_stmt\n' + ' | funcdef\n' + ' | classdef\n' + ' | async_with_stmt\n' + ' | async_for_stmt\n' + ' | async_funcdef\n' + ' suite ::= stmt_list NEWLINE | NEWLINE INDENT ' + 'statement+ DEDENT\n' + ' statement ::= stmt_list NEWLINE | compound_stmt\n' + ' stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n' + '\n' + 'Note that statements always end in a "NEWLINE" possibly followed ' + 'by a\n' + '"DEDENT". Also note that optional continuation clauses always ' + 'begin\n' + 'with a keyword that cannot start a statement, thus there are no\n' + 'ambiguities (the \'dangling "else"\' problem is solved in Python ' + 'by\n' + 'requiring nested "if" statements to be indented).\n' + '\n' + 'The formatting of the grammar rules in the following sections ' + 'places\n' + 'each clause on a separate line for clarity.\n' + '\n' + '\n' + 'The "if" statement\n' + '==================\n' + '\n' + 'The "if" statement is used for conditional execution:\n' + '\n' + ' if_stmt ::= "if" expression ":" suite\n' + ' ( "elif" expression ":" suite )*\n' + ' ["else" ":" suite]\n' + '\n' + 'It selects exactly one of the suites by evaluating the ' + 'expressions one\n' + 'by one until one is found to be true (see section Boolean ' + 'operations\n' + 'for the definition of true and false); then that suite is ' + 'executed\n' + '(and no other part of the "if" statement is executed or ' + 'evaluated).\n' + 'If all expressions are false, the suite of the "else" clause, ' + 'if\n' + 'present, is executed.\n' + '\n' + '\n' + 'The "while" statement\n' + '=====================\n' + '\n' + 'The "while" statement is used for repeated execution as long as ' + 'an\n' + 'expression is true:\n' + '\n' + ' while_stmt ::= "while" expression ":" suite\n' + ' ["else" ":" suite]\n' + '\n' + 'This repeatedly tests the expression and, if it is true, ' + 'executes the\n' + 'first suite; if the expression is false (which may be the first ' + 'time\n' + 'it is tested) the suite of the "else" clause, if present, is ' + 'executed\n' + 'and the loop terminates.\n' + '\n' + 'A "break" statement executed in the first suite terminates the ' + 'loop\n' + 'without executing the "else" clause\'s suite. A "continue" ' + 'statement\n' + 'executed in the first suite skips the rest of the suite and goes ' + 'back\n' + 'to testing the expression.\n' + '\n' + '\n' + 'The "for" statement\n' + '===================\n' + '\n' + 'The "for" statement is used to iterate over the elements of a ' + 'sequence\n' + '(such as a string, tuple or list) or other iterable object:\n' + '\n' + ' for_stmt ::= "for" target_list "in" expression_list ":" ' + 'suite\n' + ' ["else" ":" suite]\n' + '\n' + 'The expression list is evaluated once; it should yield an ' + 'iterable\n' + 'object. An iterator is created for the result of the\n' + '"expression_list". The suite is then executed once for each ' + 'item\n' + 'provided by the iterator, in the order returned by the ' + 'iterator. Each\n' + 'item in turn is assigned to the target list using the standard ' + 'rules\n' + 'for assignments (see Assignment statements), and then the suite ' + 'is\n' + 'executed. When the items are exhausted (which is immediately ' + 'when the\n' + 'sequence is empty or an iterator raises a "StopIteration" ' + 'exception),\n' + 'the suite in the "else" clause, if present, is executed, and the ' + 'loop\n' + 'terminates.\n' + '\n' + 'A "break" statement executed in the first suite terminates the ' + 'loop\n' + 'without executing the "else" clause\'s suite. A "continue" ' + 'statement\n' + 'executed in the first suite skips the rest of the suite and ' + 'continues\n' + 'with the next item, or with the "else" clause if there is no ' + 'next\n' + 'item.\n' + '\n' + 'The for-loop makes assignments to the variables(s) in the target ' + 'list.\n' + 'This overwrites all previous assignments to those variables ' + 'including\n' + 'those made in the suite of the for-loop:\n' + '\n' + ' for i in range(10):\n' + ' print(i)\n' + ' i = 5 # this will not affect the for-loop\n' + ' # because i will be overwritten with ' + 'the next\n' + ' # index in the range\n' + '\n' + 'Names in the target list are not deleted when the loop is ' + 'finished,\n' + 'but if the sequence is empty, they will not have been assigned ' + 'to at\n' + 'all by the loop. Hint: the built-in function "range()" returns ' + 'an\n' + "iterator of integers suitable to emulate the effect of Pascal's " + '"for i\n' + ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, ' + '2]".\n' + '\n' + 'Note: There is a subtlety when the sequence is being modified by ' + 'the\n' + ' loop (this can only occur for mutable sequences, i.e. lists). ' + 'An\n' + ' internal counter is used to keep track of which item is used ' + 'next,\n' + ' and this is incremented on each iteration. When this counter ' + 'has\n' + ' reached the length of the sequence the loop terminates. This ' + 'means\n' + ' that if the suite deletes the current (or a previous) item ' + 'from the\n' + ' sequence, the next item will be skipped (since it gets the ' + 'index of\n' + ' the current item which has already been treated). Likewise, ' + 'if the\n' + ' suite inserts an item in the sequence before the current item, ' + 'the\n' + ' current item will be treated again the next time through the ' + 'loop.\n' + ' This can lead to nasty bugs that can be avoided by making a\n' + ' temporary copy using a slice of the whole sequence, e.g.,\n' + '\n' + ' for x in a[:]:\n' + ' if x < 0: a.remove(x)\n' + '\n' + '\n' + 'The "try" statement\n' + '===================\n' + '\n' + 'The "try" statement specifies exception handlers and/or cleanup ' + 'code\n' + 'for a group of statements:\n' + '\n' + ' try_stmt ::= try1_stmt | try2_stmt\n' + ' try1_stmt ::= "try" ":" suite\n' + ' ("except" [expression ["as" identifier]] ":" ' + 'suite)+\n' + ' ["else" ":" suite]\n' + ' ["finally" ":" suite]\n' + ' try2_stmt ::= "try" ":" suite\n' + ' "finally" ":" suite\n' + '\n' + 'The "except" clause(s) specify one or more exception handlers. ' + 'When no\n' + 'exception occurs in the "try" clause, no exception handler is\n' + 'executed. When an exception occurs in the "try" suite, a search ' + 'for an\n' + 'exception handler is started. This search inspects the except ' + 'clauses\n' + 'in turn until one is found that matches the exception. An ' + 'expression-\n' + 'less except clause, if present, must be last; it matches any\n' + 'exception. For an except clause with an expression, that ' + 'expression\n' + 'is evaluated, and the clause matches the exception if the ' + 'resulting\n' + 'object is "compatible" with the exception. An object is ' + 'compatible\n' + 'with an exception if it is the class or a base class of the ' + 'exception\n' + 'object or a tuple containing an item compatible with the ' + 'exception.\n' + '\n' + 'If no except clause matches the exception, the search for an ' + 'exception\n' + 'handler continues in the surrounding code and on the invocation ' + 'stack.\n' + '[1]\n' + '\n' + 'If the evaluation of an expression in the header of an except ' + 'clause\n' + 'raises an exception, the original search for a handler is ' + 'canceled and\n' + 'a search starts for the new exception in the surrounding code ' + 'and on\n' + 'the call stack (it is treated as if the entire "try" statement ' + 'raised\n' + 'the exception).\n' + '\n' + 'When a matching except clause is found, the exception is ' + 'assigned to\n' + 'the target specified after the "as" keyword in that except ' + 'clause, if\n' + "present, and the except clause's suite is executed. All except\n" + 'clauses must have an executable block. When the end of this ' + 'block is\n' + 'reached, execution continues normally after the entire try ' + 'statement.\n' + '(This means that if two nested handlers exist for the same ' + 'exception,\n' + 'and the exception occurs in the try clause of the inner handler, ' + 'the\n' + 'outer handler will not handle the exception.)\n' + '\n' + 'When an exception has been assigned using "as target", it is ' + 'cleared\n' + 'at the end of the except clause. This is as if\n' + '\n' + ' except E as N:\n' + ' foo\n' + '\n' + 'was translated to\n' + '\n' + ' except E as N:\n' + ' try:\n' + ' foo\n' + ' finally:\n' + ' del N\n' + '\n' + 'This means the exception must be assigned to a different name to ' + 'be\n' + 'able to refer to it after the except clause. Exceptions are ' + 'cleared\n' + 'because with the traceback attached to them, they form a ' + 'reference\n' + 'cycle with the stack frame, keeping all locals in that frame ' + 'alive\n' + 'until the next garbage collection occurs.\n' + '\n' + "Before an except clause's suite is executed, details about the\n" + 'exception are stored in the "sys" module and can be accessed ' + 'via\n' + '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting ' + 'of the\n' + 'exception class, the exception instance and a traceback object ' + '(see\n' + 'section The standard type hierarchy) identifying the point in ' + 'the\n' + 'program where the exception occurred. "sys.exc_info()" values ' + 'are\n' + 'restored to their previous values (before the call) when ' + 'returning\n' + 'from a function that handled an exception.\n' + '\n' + 'The optional "else" clause is executed if and when control flows ' + 'off\n' + 'the end of the "try" clause. [2] Exceptions in the "else" clause ' + 'are\n' + 'not handled by the preceding "except" clauses.\n' + '\n' + 'If "finally" is present, it specifies a \'cleanup\' handler. ' + 'The "try"\n' + 'clause is executed, including any "except" and "else" clauses. ' + 'If an\n' + 'exception occurs in any of the clauses and is not handled, the\n' + 'exception is temporarily saved. The "finally" clause is ' + 'executed. If\n' + 'there is a saved exception it is re-raised at the end of the ' + '"finally"\n' + 'clause. If the "finally" clause raises another exception, the ' + 'saved\n' + 'exception is set as the context of the new exception. If the ' + '"finally"\n' + 'clause executes a "return" or "break" statement, the saved ' + 'exception\n' + 'is discarded:\n' + '\n' + ' >>> def f():\n' + ' ... try:\n' + ' ... 1/0\n' + ' ... finally:\n' + ' ... return 42\n' + ' ...\n' + ' >>> f()\n' + ' 42\n' + '\n' + 'The exception information is not available to the program ' + 'during\n' + 'execution of the "finally" clause.\n' + '\n' + 'When a "return", "break" or "continue" statement is executed in ' + 'the\n' + '"try" suite of a "try"..."finally" statement, the "finally" ' + 'clause is\n' + 'also executed \'on the way out.\' A "continue" statement is ' + 'illegal in\n' + 'the "finally" clause. (The reason is a problem with the current\n' + 'implementation --- this restriction may be lifted in the ' + 'future).\n' + '\n' + 'The return value of a function is determined by the last ' + '"return"\n' + 'statement executed. Since the "finally" clause always executes, ' + 'a\n' + '"return" statement executed in the "finally" clause will always ' + 'be the\n' + 'last one executed:\n' + '\n' + ' >>> def foo():\n' + ' ... try:\n' + " ... return 'try'\n" + ' ... finally:\n' + " ... return 'finally'\n" + ' ...\n' + ' >>> foo()\n' + " 'finally'\n" + '\n' + 'Additional information on exceptions can be found in section\n' + 'Exceptions, and information on using the "raise" statement to ' + 'generate\n' + 'exceptions may be found in section The raise statement.\n' + '\n' + '\n' + 'The "with" statement\n' + '====================\n' + '\n' + 'The "with" statement is used to wrap the execution of a block ' + 'with\n' + 'methods defined by a context manager (see section With ' + 'Statement\n' + 'Context Managers). This allows common ' + '"try"..."except"..."finally"\n' + 'usage patterns to be encapsulated for convenient reuse.\n' + '\n' + ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' + ' with_item ::= expression ["as" target]\n' + '\n' + 'The execution of the "with" statement with one "item" proceeds ' + 'as\n' + 'follows:\n' + '\n' + '1. The context expression (the expression given in the ' + '"with_item")\n' + ' is evaluated to obtain a context manager.\n' + '\n' + '2. The context manager\'s "__exit__()" is loaded for later use.\n' + '\n' + '3. The context manager\'s "__enter__()" method is invoked.\n' + '\n' + '4. If a target was included in the "with" statement, the return\n' + ' value from "__enter__()" is assigned to it.\n' + '\n' + ' Note: The "with" statement guarantees that if the ' + '"__enter__()"\n' + ' method returns without an error, then "__exit__()" will ' + 'always be\n' + ' called. Thus, if an error occurs during the assignment to ' + 'the\n' + ' target list, it will be treated the same as an error ' + 'occurring\n' + ' within the suite would be. See step 6 below.\n' + '\n' + '5. The suite is executed.\n' + '\n' + '6. The context manager\'s "__exit__()" method is invoked. If ' + 'an\n' + ' exception caused the suite to be exited, its type, value, ' + 'and\n' + ' traceback are passed as arguments to "__exit__()". Otherwise, ' + 'three\n' + ' "None" arguments are supplied.\n' + '\n' + ' If the suite was exited due to an exception, and the return ' + 'value\n' + ' from the "__exit__()" method was false, the exception is ' + 'reraised.\n' + ' If the return value was true, the exception is suppressed, ' + 'and\n' + ' execution continues with the statement following the "with"\n' + ' statement.\n' + '\n' + ' If the suite was exited for any reason other than an ' + 'exception, the\n' + ' return value from "__exit__()" is ignored, and execution ' + 'proceeds\n' + ' at the normal location for the kind of exit that was taken.\n' + '\n' + 'With more than one item, the context managers are processed as ' + 'if\n' + 'multiple "with" statements were nested:\n' + '\n' + ' with A() as a, B() as b:\n' + ' suite\n' + '\n' + 'is equivalent to\n' + '\n' + ' with A() as a:\n' + ' with B() as b:\n' + ' suite\n' + '\n' + 'Changed in version 3.1: Support for multiple context ' + 'expressions.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 343** - The "with" statement\n' + ' The specification, background, and examples for the Python ' + '"with"\n' + ' statement.\n' + '\n' + '\n' + 'Function definitions\n' + '====================\n' + '\n' + 'A function definition defines a user-defined function object ' + '(see\n' + 'section The standard type hierarchy):\n' + '\n' + ' funcdef ::= [decorators] "def" funcname "(" ' + '[parameter_list] ")" ["->" expression] ":" suite\n' + ' decorators ::= decorator+\n' + ' decorator ::= "@" dotted_name ["(" ' + '[parameter_list [","]] ")"] NEWLINE\n' + ' dotted_name ::= identifier ("." identifier)*\n' + ' parameter_list ::= defparameter ("," defparameter)* ' + '["," [parameter_list_starargs]]\n' + ' | parameter_list_starargs\n' + ' parameter_list_starargs ::= "*" [parameter] ("," ' + 'defparameter)* ["," ["**" parameter [","]]]\n' + ' | "**" parameter [","]\n' + ' parameter ::= identifier [":" expression]\n' + ' defparameter ::= parameter ["=" expression]\n' + ' funcname ::= identifier\n' + '\n' + 'A function definition is an executable statement. Its execution ' + 'binds\n' + 'the function name in the current local namespace to a function ' + 'object\n' + '(a wrapper around the executable code for the function). This\n' + 'function object contains a reference to the current global ' + 'namespace\n' + 'as the global namespace to be used when the function is called.\n' + '\n' + 'The function definition does not execute the function body; this ' + 'gets\n' + 'executed only when the function is called. [3]\n' + '\n' + 'A function definition may be wrapped by one or more *decorator*\n' + 'expressions. Decorator expressions are evaluated when the ' + 'function is\n' + 'defined, in the scope that contains the function definition. ' + 'The\n' + 'result must be a callable, which is invoked with the function ' + 'object\n' + 'as the only argument. The returned value is bound to the ' + 'function name\n' + 'instead of the function object. Multiple decorators are applied ' + 'in\n' + 'nested fashion. For example, the following code\n' + '\n' + ' @f1(arg)\n' + ' @f2\n' + ' def func(): pass\n' + '\n' + 'is equivalent to\n' + '\n' + ' def func(): pass\n' + ' func = f1(arg)(f2(func))\n' + '\n' + 'When one or more *parameters* have the form *parameter* "="\n' + '*expression*, the function is said to have "default parameter ' + 'values."\n' + 'For a parameter with a default value, the corresponding ' + '*argument* may\n' + "be omitted from a call, in which case the parameter's default " + 'value is\n' + 'substituted. If a parameter has a default value, all following\n' + 'parameters up until the ""*"" must also have a default value --- ' + 'this\n' + 'is a syntactic restriction that is not expressed by the ' + 'grammar.\n' + '\n' + '**Default parameter values are evaluated from left to right when ' + 'the\n' + 'function definition is executed.** This means that the ' + 'expression is\n' + 'evaluated once, when the function is defined, and that the same ' + '"pre-\n' + 'computed" value is used for each call. This is especially ' + 'important\n' + 'to understand when a default parameter is a mutable object, such ' + 'as a\n' + 'list or a dictionary: if the function modifies the object (e.g. ' + 'by\n' + 'appending an item to a list), the default value is in effect ' + 'modified.\n' + 'This is generally not what was intended. A way around this is ' + 'to use\n' + '"None" as the default, and explicitly test for it in the body of ' + 'the\n' + 'function, e.g.:\n' + '\n' + ' def whats_on_the_telly(penguin=None):\n' + ' if penguin is None:\n' + ' penguin = []\n' + ' penguin.append("property of the zoo")\n' + ' return penguin\n' + '\n' + 'Function call semantics are described in more detail in section ' + 'Calls.\n' + 'A function call always assigns values to all parameters ' + 'mentioned in\n' + 'the parameter list, either from position arguments, from ' + 'keyword\n' + 'arguments, or from default values. If the form ""*identifier"" ' + 'is\n' + 'present, it is initialized to a tuple receiving any excess ' + 'positional\n' + 'parameters, defaulting to the empty tuple. If the form\n' + '""**identifier"" is present, it is initialized to a new ' + 'dictionary\n' + 'receiving any excess keyword arguments, defaulting to a new ' + 'empty\n' + 'dictionary. Parameters after ""*"" or ""*identifier"" are ' + 'keyword-only\n' + 'parameters and may only be passed used keyword arguments.\n' + '\n' + 'Parameters may have annotations of the form "": expression"" ' + 'following\n' + 'the parameter name. Any parameter may have an annotation even ' + 'those\n' + 'of the form "*identifier" or "**identifier". Functions may ' + 'have\n' + '"return" annotation of the form ""-> expression"" after the ' + 'parameter\n' + 'list. These annotations can be any valid Python expression and ' + 'are\n' + 'evaluated when the function definition is executed. Annotations ' + 'may\n' + 'be evaluated in a different order than they appear in the source ' + 'code.\n' + 'The presence of annotations does not change the semantics of a\n' + 'function. The annotation values are available as values of a\n' + "dictionary keyed by the parameters' names in the " + '"__annotations__"\n' + 'attribute of the function object.\n' + '\n' + 'It is also possible to create anonymous functions (functions not ' + 'bound\n' + 'to a name), for immediate use in expressions. This uses lambda\n' + 'expressions, described in section Lambdas. Note that the ' + 'lambda\n' + 'expression is merely a shorthand for a simplified function ' + 'definition;\n' + 'a function defined in a ""def"" statement can be passed around ' + 'or\n' + 'assigned to another name just like a function defined by a ' + 'lambda\n' + 'expression. The ""def"" form is actually more powerful since ' + 'it\n' + 'allows the execution of multiple statements and annotations.\n' + '\n' + "**Programmer's note:** Functions are first-class objects. A " + '""def""\n' + 'statement executed inside a function definition defines a local\n' + 'function that can be returned or passed around. Free variables ' + 'used\n' + 'in the nested function can access the local variables of the ' + 'function\n' + 'containing the def. See section Naming and binding for ' + 'details.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 3107** - Function Annotations\n' + ' The original specification for function annotations.\n' + '\n' + '\n' + 'Class definitions\n' + '=================\n' + '\n' + 'A class definition defines a class object (see section The ' + 'standard\n' + 'type hierarchy):\n' + '\n' + ' classdef ::= [decorators] "class" classname [inheritance] ' + '":" suite\n' + ' inheritance ::= "(" [parameter_list] ")"\n' + ' classname ::= identifier\n' + '\n' + 'A class definition is an executable statement. The inheritance ' + 'list\n' + 'usually gives a list of base classes (see Customizing class ' + 'creation\n' + 'for more advanced uses), so each item in the list should ' + 'evaluate to a\n' + 'class object which allows subclassing. Classes without an ' + 'inheritance\n' + 'list inherit, by default, from the base class "object"; hence,\n' + '\n' + ' class Foo:\n' + ' pass\n' + '\n' + 'is equivalent to\n' + '\n' + ' class Foo(object):\n' + ' pass\n' + '\n' + "The class's suite is then executed in a new execution frame " + '(see\n' + 'Naming and binding), using a newly created local namespace and ' + 'the\n' + 'original global namespace. (Usually, the suite contains mostly\n' + "function definitions.) When the class's suite finishes " + 'execution, its\n' + 'execution frame is discarded but its local namespace is saved. ' + '[4] A\n' + 'class object is then created using the inheritance list for the ' + 'base\n' + 'classes and the saved local namespace for the attribute ' + 'dictionary.\n' + 'The class name is bound to this class object in the original ' + 'local\n' + 'namespace.\n' + '\n' + 'Class creation can be customized heavily using metaclasses.\n' + '\n' + 'Classes can also be decorated: just like when decorating ' + 'functions,\n' + '\n' + ' @f1(arg)\n' + ' @f2\n' + ' class Foo: pass\n' + '\n' + 'is equivalent to\n' + '\n' + ' class Foo: pass\n' + ' Foo = f1(arg)(f2(Foo))\n' + '\n' + 'The evaluation rules for the decorator expressions are the same ' + 'as for\n' + 'function decorators. The result must be a class object, which ' + 'is then\n' + 'bound to the class name.\n' + '\n' + "**Programmer's note:** Variables defined in the class definition " + 'are\n' + 'class attributes; they are shared by instances. Instance ' + 'attributes\n' + 'can be set in a method with "self.name = value". Both class ' + 'and\n' + 'instance attributes are accessible through the notation ' + '""self.name"",\n' + 'and an instance attribute hides a class attribute with the same ' + 'name\n' + 'when accessed in this way. Class attributes can be used as ' + 'defaults\n' + 'for instance attributes, but using mutable values there can lead ' + 'to\n' + 'unexpected results. Descriptors can be used to create instance\n' + 'variables with different implementation details.\n' + '\n' + 'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n' + ' Class Decorators\n' + '\n' + '\n' + 'Coroutines\n' + '==========\n' + '\n' + 'New in version 3.5.\n' + '\n' + '\n' + 'Coroutine function definition\n' + '-----------------------------\n' + '\n' + ' async_funcdef ::= [decorators] "async" "def" funcname "(" ' + '[parameter_list] ")" ["->" expression] ":" suite\n' + '\n' + 'Execution of Python coroutines can be suspended and resumed at ' + 'many\n' + 'points (see *coroutine*). In the body of a coroutine, any ' + '"await" and\n' + '"async" identifiers become reserved keywords; "await" ' + 'expressions,\n' + '"async for" and "async with" can only be used in coroutine ' + 'bodies.\n' + '\n' + 'Functions defined with "async def" syntax are always coroutine\n' + 'functions, even if they do not contain "await" or "async" ' + 'keywords.\n' + '\n' + 'It is a "SyntaxError" to use "yield" expressions in "async def"\n' + 'coroutines.\n' + '\n' + 'An example of a coroutine function:\n' + '\n' + ' async def func(param1, param2):\n' + ' do_stuff()\n' + ' await some_coroutine()\n' + '\n' + '\n' + 'The "async for" statement\n' + '-------------------------\n' + '\n' + ' async_for_stmt ::= "async" for_stmt\n' + '\n' + 'An *asynchronous iterable* is able to call asynchronous code in ' + 'its\n' + '*iter* implementation, and *asynchronous iterator* can call\n' + 'asynchronous code in its *next* method.\n' + '\n' + 'The "async for" statement allows convenient iteration over\n' + 'asynchronous iterators.\n' + '\n' + 'The following code:\n' + '\n' + ' async for TARGET in ITER:\n' + ' BLOCK\n' + ' else:\n' + ' BLOCK2\n' + '\n' + 'Is semantically equivalent to:\n' + '\n' + ' iter = (ITER)\n' + ' iter = await type(iter).__aiter__(iter)\n' + ' running = True\n' + ' while running:\n' + ' try:\n' + ' TARGET = await type(iter).__anext__(iter)\n' + ' except StopAsyncIteration:\n' + ' running = False\n' + ' else:\n' + ' BLOCK\n' + ' else:\n' + ' BLOCK2\n' + '\n' + 'See also "__aiter__()" and "__anext__()" for details.\n' + '\n' + 'It is a "SyntaxError" to use "async for" statement outside of ' + 'an\n' + '"async def" function.\n' + '\n' + '\n' + 'The "async with" statement\n' + '--------------------------\n' + '\n' + ' async_with_stmt ::= "async" with_stmt\n' + '\n' + 'An *asynchronous context manager* is a *context manager* that is ' + 'able\n' + 'to suspend execution in its *enter* and *exit* methods.\n' + '\n' + 'The following code:\n' + '\n' + ' async with EXPR as VAR:\n' + ' BLOCK\n' + '\n' + 'Is semantically equivalent to:\n' + '\n' + ' mgr = (EXPR)\n' + ' aexit = type(mgr).__aexit__\n' + ' aenter = type(mgr).__aenter__(mgr)\n' + ' exc = True\n' + '\n' + ' VAR = await aenter\n' + ' try:\n' + ' BLOCK\n' + ' except:\n' + ' if not await aexit(mgr, *sys.exc_info()):\n' + ' raise\n' + ' else:\n' + ' await aexit(mgr, None, None, None)\n' + '\n' + 'See also "__aenter__()" and "__aexit__()" for details.\n' + '\n' + 'It is a "SyntaxError" to use "async with" statement outside of ' + 'an\n' + '"async def" function.\n' + '\n' + 'See also: **PEP 492** - Coroutines with async and await syntax\n' + '\n' + '-[ Footnotes ]-\n' + '\n' + '[1] The exception is propagated to the invocation stack unless\n' + ' there is a "finally" clause which happens to raise another\n' + ' exception. That new exception causes the old one to be ' + 'lost.\n' + '\n' + '[2] Currently, control "flows off the end" except in the case ' + 'of\n' + ' an exception or the execution of a "return", "continue", or\n' + ' "break" statement.\n' + '\n' + '[3] A string literal appearing as the first statement in the\n' + ' function body is transformed into the function\'s "__doc__"\n' + " attribute and therefore the function's *docstring*.\n" + '\n' + '[4] A string literal appearing as the first statement in the ' + 'class\n' + ' body is transformed into the namespace\'s "__doc__" item ' + 'and\n' + " therefore the class's *docstring*.\n", + 'context-managers': '\n' + 'With Statement Context Managers\n' + '*******************************\n' + '\n' + 'A *context manager* is an object that defines the ' + 'runtime context to\n' + 'be established when executing a "with" statement. The ' + 'context manager\n' + 'handles the entry into, and the exit from, the desired ' + 'runtime context\n' + 'for the execution of the block of code. Context ' + 'managers are normally\n' + 'invoked using the "with" statement (described in section ' + 'The with\n' + 'statement), but can also be used by directly invoking ' + 'their methods.\n' + '\n' + 'Typical uses of context managers include saving and ' + 'restoring various\n' + 'kinds of global state, locking and unlocking resources, ' + 'closing opened\n' + 'files, etc.\n' + '\n' + 'For more information on context managers, see Context ' + 'Manager Types.\n' + '\n' + 'object.__enter__(self)\n' + '\n' + ' Enter the runtime context related to this object. The ' + '"with"\n' + " statement will bind this method's return value to the " + 'target(s)\n' + ' specified in the "as" clause of the statement, if ' + 'any.\n' + '\n' + 'object.__exit__(self, exc_type, exc_value, traceback)\n' + '\n' + ' Exit the runtime context related to this object. The ' + 'parameters\n' + ' describe the exception that caused the context to be ' + 'exited. If the\n' + ' context was exited without an exception, all three ' + 'arguments will\n' + ' be "None".\n' + '\n' + ' If an exception is supplied, and the method wishes to ' + 'suppress the\n' + ' exception (i.e., prevent it from being propagated), ' + 'it should\n' + ' return a true value. Otherwise, the exception will be ' + 'processed\n' + ' normally upon exit from this method.\n' + '\n' + ' Note that "__exit__()" methods should not reraise the ' + 'passed-in\n' + " exception; this is the caller's responsibility.\n" + '\n' + 'See also:\n' + '\n' + ' **PEP 343** - The "with" statement\n' + ' The specification, background, and examples for the ' + 'Python "with"\n' + ' statement.\n', + 'continue': '\n' + 'The "continue" statement\n' + '************************\n' + '\n' + ' continue_stmt ::= "continue"\n' + '\n' + '"continue" may only occur syntactically nested in a "for" or ' + '"while"\n' + 'loop, but not nested in a function or class definition or ' + '"finally"\n' + 'clause within that loop. It continues with the next cycle of ' + 'the\n' + 'nearest enclosing loop.\n' + '\n' + 'When "continue" passes control out of a "try" statement with a\n' + '"finally" clause, that "finally" clause is executed before ' + 'really\n' + 'starting the next loop cycle.\n', + 'conversions': '\n' + 'Arithmetic conversions\n' + '**********************\n' + '\n' + 'When a description of an arithmetic operator below uses the ' + 'phrase\n' + '"the numeric arguments are converted to a common type," this ' + 'means\n' + 'that the operator implementation for built-in types works as ' + 'follows:\n' + '\n' + '* If either argument is a complex number, the other is ' + 'converted to\n' + ' complex;\n' + '\n' + '* otherwise, if either argument is a floating point number, ' + 'the\n' + ' other is converted to floating point;\n' + '\n' + '* otherwise, both must be integers and no conversion is ' + 'necessary.\n' + '\n' + 'Some additional rules apply for certain operators (e.g., a ' + 'string as a\n' + "left argument to the '%' operator). Extensions must define " + 'their own\n' + 'conversion behavior.\n', + 'customization': '\n' + 'Basic customization\n' + '*******************\n' + '\n' + 'object.__new__(cls[, ...])\n' + '\n' + ' Called to create a new instance of class *cls*. ' + '"__new__()" is a\n' + ' static method (special-cased so you need not declare it ' + 'as such)\n' + ' that takes the class of which an instance was requested ' + 'as its\n' + ' first argument. The remaining arguments are those ' + 'passed to the\n' + ' object constructor expression (the call to the class). ' + 'The return\n' + ' value of "__new__()" should be the new object instance ' + '(usually an\n' + ' instance of *cls*).\n' + '\n' + ' Typical implementations create a new instance of the ' + 'class by\n' + ' invoking the superclass\'s "__new__()" method using\n' + ' "super(currentclass, cls).__new__(cls[, ...])" with ' + 'appropriate\n' + ' arguments and then modifying the newly-created instance ' + 'as\n' + ' necessary before returning it.\n' + '\n' + ' If "__new__()" returns an instance of *cls*, then the ' + 'new\n' + ' instance\'s "__init__()" method will be invoked like\n' + ' "__init__(self[, ...])", where *self* is the new ' + 'instance and the\n' + ' remaining arguments are the same as were passed to ' + '"__new__()".\n' + '\n' + ' If "__new__()" does not return an instance of *cls*, ' + 'then the new\n' + ' instance\'s "__init__()" method will not be invoked.\n' + '\n' + ' "__new__()" is intended mainly to allow subclasses of ' + 'immutable\n' + ' types (like int, str, or tuple) to customize instance ' + 'creation. It\n' + ' is also commonly overridden in custom metaclasses in ' + 'order to\n' + ' customize class creation.\n' + '\n' + 'object.__init__(self[, ...])\n' + '\n' + ' Called after the instance has been created (by ' + '"__new__()"), but\n' + ' before it is returned to the caller. The arguments are ' + 'those\n' + ' passed to the class constructor expression. If a base ' + 'class has an\n' + ' "__init__()" method, the derived class\'s "__init__()" ' + 'method, if\n' + ' any, must explicitly call it to ensure proper ' + 'initialization of the\n' + ' base class part of the instance; for example:\n' + ' "BaseClass.__init__(self, [args...])".\n' + '\n' + ' Because "__new__()" and "__init__()" work together in ' + 'constructing\n' + ' objects ("__new__()" to create it, and "__init__()" to ' + 'customise\n' + ' it), no non-"None" value may be returned by ' + '"__init__()"; doing so\n' + ' will cause a "TypeError" to be raised at runtime.\n' + '\n' + 'object.__del__(self)\n' + '\n' + ' Called when the instance is about to be destroyed. This ' + 'is also\n' + ' called a destructor. If a base class has a "__del__()" ' + 'method, the\n' + ' derived class\'s "__del__()" method, if any, must ' + 'explicitly call it\n' + ' to ensure proper deletion of the base class part of the ' + 'instance.\n' + ' Note that it is possible (though not recommended!) for ' + 'the\n' + ' "__del__()" method to postpone destruction of the ' + 'instance by\n' + ' creating a new reference to it. It may then be called ' + 'at a later\n' + ' time when this new reference is deleted. It is not ' + 'guaranteed that\n' + ' "__del__()" methods are called for objects that still ' + 'exist when\n' + ' the interpreter exits.\n' + '\n' + ' Note: "del x" doesn\'t directly call "x.__del__()" --- ' + 'the former\n' + ' decrements the reference count for "x" by one, and the ' + 'latter is\n' + ' only called when "x"\'s reference count reaches zero. ' + 'Some common\n' + ' situations that may prevent the reference count of an ' + 'object from\n' + ' going to zero include: circular references between ' + 'objects (e.g.,\n' + ' a doubly-linked list or a tree data structure with ' + 'parent and\n' + ' child pointers); a reference to the object on the ' + 'stack frame of\n' + ' a function that caught an exception (the traceback ' + 'stored in\n' + ' "sys.exc_info()[2]" keeps the stack frame alive); or a ' + 'reference\n' + ' to the object on the stack frame that raised an ' + 'unhandled\n' + ' exception in interactive mode (the traceback stored ' + 'in\n' + ' "sys.last_traceback" keeps the stack frame alive). ' + 'The first\n' + ' situation can only be remedied by explicitly breaking ' + 'the cycles;\n' + ' the second can be resolved by freeing the reference to ' + 'the\n' + ' traceback object when it is no longer useful, and the ' + 'third can\n' + ' be resolved by storing "None" in "sys.last_traceback". ' + 'Circular\n' + ' references which are garbage are detected and cleaned ' + 'up when the\n' + " cyclic garbage collector is enabled (it's on by " + 'default). Refer\n' + ' to the documentation for the "gc" module for more ' + 'information\n' + ' about this topic.\n' + '\n' + ' Warning: Due to the precarious circumstances under ' + 'which\n' + ' "__del__()" methods are invoked, exceptions that occur ' + 'during\n' + ' their execution are ignored, and a warning is printed ' + 'to\n' + ' "sys.stderr" instead. Also, when "__del__()" is ' + 'invoked in\n' + ' response to a module being deleted (e.g., when ' + 'execution of the\n' + ' program is done), other globals referenced by the ' + '"__del__()"\n' + ' method may already have been deleted or in the process ' + 'of being\n' + ' torn down (e.g. the import machinery shutting down). ' + 'For this\n' + ' reason, "__del__()" methods should do the absolute ' + 'minimum needed\n' + ' to maintain external invariants. Starting with ' + 'version 1.5,\n' + ' Python guarantees that globals whose name begins with ' + 'a single\n' + ' underscore are deleted from their module before other ' + 'globals are\n' + ' deleted; if no other references to such globals exist, ' + 'this may\n' + ' help in assuring that imported modules are still ' + 'available at the\n' + ' time when the "__del__()" method is called.\n' + '\n' + 'object.__repr__(self)\n' + '\n' + ' Called by the "repr()" built-in function to compute the ' + '"official"\n' + ' string representation of an object. If at all possible, ' + 'this\n' + ' should look like a valid Python expression that could be ' + 'used to\n' + ' recreate an object with the same value (given an ' + 'appropriate\n' + ' environment). If this is not possible, a string of the ' + 'form\n' + ' "<...some useful description...>" should be returned. ' + 'The return\n' + ' value must be a string object. If a class defines ' + '"__repr__()" but\n' + ' not "__str__()", then "__repr__()" is also used when an ' + '"informal"\n' + ' string representation of instances of that class is ' + 'required.\n' + '\n' + ' This is typically used for debugging, so it is important ' + 'that the\n' + ' representation is information-rich and unambiguous.\n' + '\n' + 'object.__str__(self)\n' + '\n' + ' Called by "str(object)" and the built-in functions ' + '"format()" and\n' + ' "print()" to compute the "informal" or nicely printable ' + 'string\n' + ' representation of an object. The return value must be a ' + 'string\n' + ' object.\n' + '\n' + ' This method differs from "object.__repr__()" in that ' + 'there is no\n' + ' expectation that "__str__()" return a valid Python ' + 'expression: a\n' + ' more convenient or concise representation can be used.\n' + '\n' + ' The default implementation defined by the built-in type ' + '"object"\n' + ' calls "object.__repr__()".\n' + '\n' + 'object.__bytes__(self)\n' + '\n' + ' Called by "bytes()" to compute a byte-string ' + 'representation of an\n' + ' object. This should return a "bytes" object.\n' + '\n' + 'object.__format__(self, format_spec)\n' + '\n' + ' Called by the "format()" built-in function, and by ' + 'extension,\n' + ' evaluation of formatted string literals and the ' + '"str.format()"\n' + ' method, to produce a "formatted" string representation ' + 'of an\n' + ' object. The "format_spec" argument is a string that ' + 'contains a\n' + ' description of the formatting options desired. The ' + 'interpretation\n' + ' of the "format_spec" argument is up to the type ' + 'implementing\n' + ' "__format__()", however most classes will either ' + 'delegate\n' + ' formatting to one of the built-in types, or use a ' + 'similar\n' + ' formatting option syntax.\n' + '\n' + ' See Format Specification Mini-Language for a description ' + 'of the\n' + ' standard formatting syntax.\n' + '\n' + ' The return value must be a string object.\n' + '\n' + ' Changed in version 3.4: The __format__ method of ' + '"object" itself\n' + ' raises a "TypeError" if passed any non-empty string.\n' + '\n' + 'object.__lt__(self, other)\n' + 'object.__le__(self, other)\n' + 'object.__eq__(self, other)\n' + 'object.__ne__(self, other)\n' + 'object.__gt__(self, other)\n' + 'object.__ge__(self, other)\n' + '\n' + ' These are the so-called "rich comparison" methods. The\n' + ' correspondence between operator symbols and method names ' + 'is as\n' + ' follows: "xy" calls\n' + ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' + '\n' + ' A rich comparison method may return the singleton ' + '"NotImplemented"\n' + ' if it does not implement the operation for a given pair ' + 'of\n' + ' arguments. By convention, "False" and "True" are ' + 'returned for a\n' + ' successful comparison. However, these methods can return ' + 'any value,\n' + ' so if the comparison operator is used in a Boolean ' + 'context (e.g.,\n' + ' in the condition of an "if" statement), Python will call ' + '"bool()"\n' + ' on the value to determine if the result is true or ' + 'false.\n' + '\n' + ' By default, "__ne__()" delegates to "__eq__()" and ' + 'inverts the\n' + ' result unless it is "NotImplemented". There are no ' + 'other implied\n' + ' relationships among the comparison operators, for ' + 'example, the\n' + ' truth of "(x.__hash__".\n' + '\n' + ' If a class that does not override "__eq__()" wishes to ' + 'suppress\n' + ' hash support, it should include "__hash__ = None" in the ' + 'class\n' + ' definition. A class which defines its own "__hash__()" ' + 'that\n' + ' explicitly raises a "TypeError" would be incorrectly ' + 'identified as\n' + ' hashable by an "isinstance(obj, collections.Hashable)" ' + 'call.\n' + '\n' + ' Note: By default, the "__hash__()" values of str, bytes ' + 'and\n' + ' datetime objects are "salted" with an unpredictable ' + 'random value.\n' + ' Although they remain constant within an individual ' + 'Python\n' + ' process, they are not predictable between repeated ' + 'invocations of\n' + ' Python.This is intended to provide protection against ' + 'a denial-\n' + ' of-service caused by carefully-chosen inputs that ' + 'exploit the\n' + ' worst case performance of a dict insertion, O(n^2) ' + 'complexity.\n' + ' See ' + 'http://www.ocert.org/advisories/ocert-2011-003.html for\n' + ' details.Changing hash values affects the iteration ' + 'order of\n' + ' dicts, sets and other mappings. Python has never made ' + 'guarantees\n' + ' about this ordering (and it typically varies between ' + '32-bit and\n' + ' 64-bit builds).See also "PYTHONHASHSEED".\n' + '\n' + ' Changed in version 3.3: Hash randomization is enabled by ' + 'default.\n' + '\n' + 'object.__bool__(self)\n' + '\n' + ' Called to implement truth value testing and the built-in ' + 'operation\n' + ' "bool()"; should return "False" or "True". When this ' + 'method is not\n' + ' defined, "__len__()" is called, if it is defined, and ' + 'the object is\n' + ' considered true if its result is nonzero. If a class ' + 'defines\n' + ' neither "__len__()" nor "__bool__()", all its instances ' + 'are\n' + ' considered true.\n', + 'debugger': '\n' + '"pdb" --- The Python Debugger\n' + '*****************************\n' + '\n' + '**Source code:** Lib/pdb.py\n' + '\n' + '======================================================================\n' + '\n' + 'The module "pdb" defines an interactive source code debugger ' + 'for\n' + 'Python programs. It supports setting (conditional) breakpoints ' + 'and\n' + 'single stepping at the source line level, inspection of stack ' + 'frames,\n' + 'source code listing, and evaluation of arbitrary Python code in ' + 'the\n' + 'context of any stack frame. It also supports post-mortem ' + 'debugging\n' + 'and can be called under program control.\n' + '\n' + 'The debugger is extensible -- it is actually defined as the ' + 'class\n' + '"Pdb". This is currently undocumented but easily understood by ' + 'reading\n' + 'the source. The extension interface uses the modules "bdb" and ' + '"cmd".\n' + '\n' + 'The debugger\'s prompt is "(Pdb)". Typical usage to run a ' + 'program under\n' + 'control of the debugger is:\n' + '\n' + ' >>> import pdb\n' + ' >>> import mymodule\n' + " >>> pdb.run('mymodule.test()')\n" + ' > (0)?()\n' + ' (Pdb) continue\n' + ' > (1)?()\n' + ' (Pdb) continue\n' + " NameError: 'spam'\n" + ' > (1)?()\n' + ' (Pdb)\n' + '\n' + 'Changed in version 3.3: Tab-completion via the "readline" module ' + 'is\n' + 'available for commands and command arguments, e.g. the current ' + 'global\n' + 'and local names are offered as arguments of the "p" command.\n' + '\n' + '"pdb.py" can also be invoked as a script to debug other ' + 'scripts. For\n' + 'example:\n' + '\n' + ' python3 -m pdb myscript.py\n' + '\n' + 'When invoked as a script, pdb will automatically enter ' + 'post-mortem\n' + 'debugging if the program being debugged exits abnormally. After ' + 'post-\n' + 'mortem debugging (or after normal exit of the program), pdb ' + 'will\n' + "restart the program. Automatic restarting preserves pdb's state " + '(such\n' + 'as breakpoints) and in most cases is more useful than quitting ' + 'the\n' + "debugger upon program's exit.\n" + '\n' + 'New in version 3.2: "pdb.py" now accepts a "-c" option that ' + 'executes\n' + 'commands as if given in a ".pdbrc" file, see Debugger Commands.\n' + '\n' + 'The typical usage to break into the debugger from a running ' + 'program is\n' + 'to insert\n' + '\n' + ' import pdb; pdb.set_trace()\n' + '\n' + 'at the location you want to break into the debugger. You can ' + 'then\n' + 'step through the code following this statement, and continue ' + 'running\n' + 'without the debugger using the "continue" command.\n' + '\n' + 'The typical usage to inspect a crashed program is:\n' + '\n' + ' >>> import pdb\n' + ' >>> import mymodule\n' + ' >>> mymodule.test()\n' + ' Traceback (most recent call last):\n' + ' File "", line 1, in ?\n' + ' File "./mymodule.py", line 4, in test\n' + ' test2()\n' + ' File "./mymodule.py", line 3, in test2\n' + ' print(spam)\n' + ' NameError: spam\n' + ' >>> pdb.pm()\n' + ' > ./mymodule.py(3)test2()\n' + ' -> print(spam)\n' + ' (Pdb)\n' + '\n' + 'The module defines the following functions; each enters the ' + 'debugger\n' + 'in a slightly different way:\n' + '\n' + 'pdb.run(statement, globals=None, locals=None)\n' + '\n' + ' Execute the *statement* (given as a string or a code object) ' + 'under\n' + ' debugger control. The debugger prompt appears before any ' + 'code is\n' + ' executed; you can set breakpoints and type "continue", or you ' + 'can\n' + ' step through the statement using "step" or "next" (all these\n' + ' commands are explained below). The optional *globals* and ' + '*locals*\n' + ' arguments specify the environment in which the code is ' + 'executed; by\n' + ' default the dictionary of the module "__main__" is used. ' + '(See the\n' + ' explanation of the built-in "exec()" or "eval()" functions.)\n' + '\n' + 'pdb.runeval(expression, globals=None, locals=None)\n' + '\n' + ' Evaluate the *expression* (given as a string or a code ' + 'object)\n' + ' under debugger control. When "runeval()" returns, it returns ' + 'the\n' + ' value of the expression. Otherwise this function is similar ' + 'to\n' + ' "run()".\n' + '\n' + 'pdb.runcall(function, *args, **kwds)\n' + '\n' + ' Call the *function* (a function or method object, not a ' + 'string)\n' + ' with the given arguments. When "runcall()" returns, it ' + 'returns\n' + ' whatever the function call returned. The debugger prompt ' + 'appears\n' + ' as soon as the function is entered.\n' + '\n' + 'pdb.set_trace()\n' + '\n' + ' Enter the debugger at the calling stack frame. This is ' + 'useful to\n' + ' hard-code a breakpoint at a given point in a program, even if ' + 'the\n' + ' code is not otherwise being debugged (e.g. when an assertion\n' + ' fails).\n' + '\n' + 'pdb.post_mortem(traceback=None)\n' + '\n' + ' Enter post-mortem debugging of the given *traceback* object. ' + 'If no\n' + ' *traceback* is given, it uses the one of the exception that ' + 'is\n' + ' currently being handled (an exception must be being handled ' + 'if the\n' + ' default is to be used).\n' + '\n' + 'pdb.pm()\n' + '\n' + ' Enter post-mortem debugging of the traceback found in\n' + ' "sys.last_traceback".\n' + '\n' + 'The "run*" functions and "set_trace()" are aliases for ' + 'instantiating\n' + 'the "Pdb" class and calling the method of the same name. If you ' + 'want\n' + 'to access further features, you have to do this yourself:\n' + '\n' + "class pdb.Pdb(completekey='tab', stdin=None, stdout=None, " + 'skip=None, nosigint=False)\n' + '\n' + ' "Pdb" is the debugger class.\n' + '\n' + ' The *completekey*, *stdin* and *stdout* arguments are passed ' + 'to the\n' + ' underlying "cmd.Cmd" class; see the description there.\n' + '\n' + ' The *skip* argument, if given, must be an iterable of ' + 'glob-style\n' + ' module name patterns. The debugger will not step into frames ' + 'that\n' + ' originate in a module that matches one of these patterns. ' + '[1]\n' + '\n' + ' By default, Pdb sets a handler for the SIGINT signal (which ' + 'is sent\n' + ' when the user presses "Ctrl-C" on the console) when you give ' + 'a\n' + ' "continue" command. This allows you to break into the ' + 'debugger\n' + ' again by pressing "Ctrl-C". If you want Pdb not to touch ' + 'the\n' + ' SIGINT handler, set *nosigint* tot true.\n' + '\n' + ' Example call to enable tracing with *skip*:\n' + '\n' + " import pdb; pdb.Pdb(skip=['django.*']).set_trace()\n" + '\n' + ' New in version 3.1: The *skip* argument.\n' + '\n' + ' New in version 3.2: The *nosigint* argument. Previously, a ' + 'SIGINT\n' + ' handler was never set by Pdb.\n' + '\n' + ' run(statement, globals=None, locals=None)\n' + ' runeval(expression, globals=None, locals=None)\n' + ' runcall(function, *args, **kwds)\n' + ' set_trace()\n' + '\n' + ' See the documentation for the functions explained above.\n' + '\n' + '\n' + 'Debugger Commands\n' + '=================\n' + '\n' + 'The commands recognized by the debugger are listed below. Most\n' + 'commands can be abbreviated to one or two letters as indicated; ' + 'e.g.\n' + '"h(elp)" means that either "h" or "help" can be used to enter ' + 'the help\n' + 'command (but not "he" or "hel", nor "H" or "Help" or "HELP").\n' + 'Arguments to commands must be separated by whitespace (spaces ' + 'or\n' + 'tabs). Optional arguments are enclosed in square brackets ' + '("[]") in\n' + 'the command syntax; the square brackets must not be typed.\n' + 'Alternatives in the command syntax are separated by a vertical ' + 'bar\n' + '("|").\n' + '\n' + 'Entering a blank line repeats the last command entered. ' + 'Exception: if\n' + 'the last command was a "list" command, the next 11 lines are ' + 'listed.\n' + '\n' + "Commands that the debugger doesn't recognize are assumed to be " + 'Python\n' + 'statements and are executed in the context of the program being\n' + 'debugged. Python statements can also be prefixed with an ' + 'exclamation\n' + 'point ("!"). This is a powerful way to inspect the program ' + 'being\n' + 'debugged; it is even possible to change a variable or call a ' + 'function.\n' + 'When an exception occurs in such a statement, the exception name ' + 'is\n' + "printed but the debugger's state is not changed.\n" + '\n' + 'The debugger supports aliases. Aliases can have parameters ' + 'which\n' + 'allows one a certain level of adaptability to the context under\n' + 'examination.\n' + '\n' + 'Multiple commands may be entered on a single line, separated by ' + '";;".\n' + '(A single ";" is not used as it is the separator for multiple ' + 'commands\n' + 'in a line that is passed to the Python parser.) No intelligence ' + 'is\n' + 'applied to separating the commands; the input is split at the ' + 'first\n' + '";;" pair, even if it is in the middle of a quoted string.\n' + '\n' + 'If a file ".pdbrc" exists in the user\'s home directory or in ' + 'the\n' + 'current directory, it is read in and executed as if it had been ' + 'typed\n' + 'at the debugger prompt. This is particularly useful for ' + 'aliases. If\n' + 'both files exist, the one in the home directory is read first ' + 'and\n' + 'aliases defined there can be overridden by the local file.\n' + '\n' + 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' + 'continue debugging, such as "continue" or "next". Previously, ' + 'these\n' + 'commands had no effect.\n' + '\n' + 'h(elp) [command]\n' + '\n' + ' Without argument, print the list of available commands. With ' + 'a\n' + ' *command* as argument, print help about that command. "help ' + 'pdb"\n' + ' displays the full documentation (the docstring of the "pdb"\n' + ' module). Since the *command* argument must be an identifier, ' + '"help\n' + ' exec" must be entered to get help on the "!" command.\n' + '\n' + 'w(here)\n' + '\n' + ' Print a stack trace, with the most recent frame at the ' + 'bottom. An\n' + ' arrow indicates the current frame, which determines the ' + 'context of\n' + ' most commands.\n' + '\n' + 'd(own) [count]\n' + '\n' + ' Move the current frame *count* (default one) levels down in ' + 'the\n' + ' stack trace (to a newer frame).\n' + '\n' + 'u(p) [count]\n' + '\n' + ' Move the current frame *count* (default one) levels up in the ' + 'stack\n' + ' trace (to an older frame).\n' + '\n' + 'b(reak) [([filename:]lineno | function) [, condition]]\n' + '\n' + ' With a *lineno* argument, set a break there in the current ' + 'file.\n' + ' With a *function* argument, set a break at the first ' + 'executable\n' + ' statement within that function. The line number may be ' + 'prefixed\n' + ' with a filename and a colon, to specify a breakpoint in ' + 'another\n' + " file (probably one that hasn't been loaded yet). The file " + 'is\n' + ' searched on "sys.path". Note that each breakpoint is ' + 'assigned a\n' + ' number to which all the other breakpoint commands refer.\n' + '\n' + ' If a second argument is present, it is an expression which ' + 'must\n' + ' evaluate to true before the breakpoint is honored.\n' + '\n' + ' Without argument, list all breaks, including for each ' + 'breakpoint,\n' + ' the number of times that breakpoint has been hit, the ' + 'current\n' + ' ignore count, and the associated condition if any.\n' + '\n' + 'tbreak [([filename:]lineno | function) [, condition]]\n' + '\n' + ' Temporary breakpoint, which is removed automatically when it ' + 'is\n' + ' first hit. The arguments are the same as for "break".\n' + '\n' + 'cl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n' + '\n' + ' With a *filename:lineno* argument, clear all the breakpoints ' + 'at\n' + ' this line. With a space separated list of breakpoint numbers, ' + 'clear\n' + ' those breakpoints. Without argument, clear all breaks (but ' + 'first\n' + ' ask confirmation).\n' + '\n' + 'disable [bpnumber [bpnumber ...]]\n' + '\n' + ' Disable the breakpoints given as a space separated list of\n' + ' breakpoint numbers. Disabling a breakpoint means it cannot ' + 'cause\n' + ' the program to stop execution, but unlike clearing a ' + 'breakpoint, it\n' + ' remains in the list of breakpoints and can be (re-)enabled.\n' + '\n' + 'enable [bpnumber [bpnumber ...]]\n' + '\n' + ' Enable the breakpoints specified.\n' + '\n' + 'ignore bpnumber [count]\n' + '\n' + ' Set the ignore count for the given breakpoint number. If ' + 'count is\n' + ' omitted, the ignore count is set to 0. A breakpoint becomes ' + 'active\n' + ' when the ignore count is zero. When non-zero, the count is\n' + ' decremented each time the breakpoint is reached and the ' + 'breakpoint\n' + ' is not disabled and any associated condition evaluates to ' + 'true.\n' + '\n' + 'condition bpnumber [condition]\n' + '\n' + ' Set a new *condition* for the breakpoint, an expression which ' + 'must\n' + ' evaluate to true before the breakpoint is honored. If ' + '*condition*\n' + ' is absent, any existing condition is removed; i.e., the ' + 'breakpoint\n' + ' is made unconditional.\n' + '\n' + 'commands [bpnumber]\n' + '\n' + ' Specify a list of commands for breakpoint number *bpnumber*. ' + 'The\n' + ' commands themselves appear on the following lines. Type a ' + 'line\n' + ' containing just "end" to terminate the commands. An example:\n' + '\n' + ' (Pdb) commands 1\n' + ' (com) p some_variable\n' + ' (com) end\n' + ' (Pdb)\n' + '\n' + ' To remove all commands from a breakpoint, type commands and ' + 'follow\n' + ' it immediately with "end"; that is, give no commands.\n' + '\n' + ' With no *bpnumber* argument, commands refers to the last ' + 'breakpoint\n' + ' set.\n' + '\n' + ' You can use breakpoint commands to start your program up ' + 'again.\n' + ' Simply use the continue command, or step, or any other ' + 'command that\n' + ' resumes execution.\n' + '\n' + ' Specifying any command resuming execution (currently ' + 'continue,\n' + ' step, next, return, jump, quit and their abbreviations) ' + 'terminates\n' + ' the command list (as if that command was immediately followed ' + 'by\n' + ' end). This is because any time you resume execution (even ' + 'with a\n' + ' simple next or step), you may encounter another ' + 'breakpoint--which\n' + ' could have its own command list, leading to ambiguities about ' + 'which\n' + ' list to execute.\n' + '\n' + " If you use the 'silent' command in the command list, the " + 'usual\n' + ' message about stopping at a breakpoint is not printed. This ' + 'may be\n' + ' desirable for breakpoints that are to print a specific ' + 'message and\n' + ' then continue. If none of the other commands print anything, ' + 'you\n' + ' see no sign that the breakpoint was reached.\n' + '\n' + 's(tep)\n' + '\n' + ' Execute the current line, stop at the first possible ' + 'occasion\n' + ' (either in a function that is called or on the next line in ' + 'the\n' + ' current function).\n' + '\n' + 'n(ext)\n' + '\n' + ' Continue execution until the next line in the current ' + 'function is\n' + ' reached or it returns. (The difference between "next" and ' + '"step"\n' + ' is that "step" stops inside a called function, while "next"\n' + ' executes called functions at (nearly) full speed, only ' + 'stopping at\n' + ' the next line in the current function.)\n' + '\n' + 'unt(il) [lineno]\n' + '\n' + ' Without argument, continue execution until the line with a ' + 'number\n' + ' greater than the current one is reached.\n' + '\n' + ' With a line number, continue execution until a line with a ' + 'number\n' + ' greater or equal to that is reached. In both cases, also ' + 'stop when\n' + ' the current frame returns.\n' + '\n' + ' Changed in version 3.2: Allow giving an explicit line ' + 'number.\n' + '\n' + 'r(eturn)\n' + '\n' + ' Continue execution until the current function returns.\n' + '\n' + 'c(ont(inue))\n' + '\n' + ' Continue execution, only stop when a breakpoint is ' + 'encountered.\n' + '\n' + 'j(ump) lineno\n' + '\n' + ' Set the next line that will be executed. Only available in ' + 'the\n' + ' bottom-most frame. This lets you jump back and execute code ' + 'again,\n' + " or jump forward to skip code that you don't want to run.\n" + '\n' + ' It should be noted that not all jumps are allowed -- for ' + 'instance\n' + ' it is not possible to jump into the middle of a "for" loop or ' + 'out\n' + ' of a "finally" clause.\n' + '\n' + 'l(ist) [first[, last]]\n' + '\n' + ' List source code for the current file. Without arguments, ' + 'list 11\n' + ' lines around the current line or continue the previous ' + 'listing.\n' + ' With "." as argument, list 11 lines around the current line. ' + 'With\n' + ' one argument, list 11 lines around at that line. With two\n' + ' arguments, list the given range; if the second argument is ' + 'less\n' + ' than the first, it is interpreted as a count.\n' + '\n' + ' The current line in the current frame is indicated by "->". ' + 'If an\n' + ' exception is being debugged, the line where the exception ' + 'was\n' + ' originally raised or propagated is indicated by ">>", if it ' + 'differs\n' + ' from the current line.\n' + '\n' + ' New in version 3.2: The ">>" marker.\n' + '\n' + 'll | longlist\n' + '\n' + ' List all source code for the current function or frame.\n' + ' Interesting lines are marked as for "list".\n' + '\n' + ' New in version 3.2.\n' + '\n' + 'a(rgs)\n' + '\n' + ' Print the argument list of the current function.\n' + '\n' + 'p expression\n' + '\n' + ' Evaluate the *expression* in the current context and print ' + 'its\n' + ' value.\n' + '\n' + ' Note: "print()" can also be used, but is not a debugger ' + 'command\n' + ' --- this executes the Python "print()" function.\n' + '\n' + 'pp expression\n' + '\n' + ' Like the "p" command, except the value of the expression is ' + 'pretty-\n' + ' printed using the "pprint" module.\n' + '\n' + 'whatis expression\n' + '\n' + ' Print the type of the *expression*.\n' + '\n' + 'source expression\n' + '\n' + ' Try to get source code for the given object and display it.\n' + '\n' + ' New in version 3.2.\n' + '\n' + 'display [expression]\n' + '\n' + ' Display the value of the expression if it changed, each time\n' + ' execution stops in the current frame.\n' + '\n' + ' Without expression, list all display expressions for the ' + 'current\n' + ' frame.\n' + '\n' + ' New in version 3.2.\n' + '\n' + 'undisplay [expression]\n' + '\n' + ' Do not display the expression any more in the current frame.\n' + ' Without expression, clear all display expressions for the ' + 'current\n' + ' frame.\n' + '\n' + ' New in version 3.2.\n' + '\n' + 'interact\n' + '\n' + ' Start an interative interpreter (using the "code" module) ' + 'whose\n' + ' global namespace contains all the (global and local) names ' + 'found in\n' + ' the current scope.\n' + '\n' + ' New in version 3.2.\n' + '\n' + 'alias [name [command]]\n' + '\n' + ' Create an alias called *name* that executes *command*. The ' + 'command\n' + ' must *not* be enclosed in quotes. Replaceable parameters can ' + 'be\n' + ' indicated by "%1", "%2", and so on, while "%*" is replaced by ' + 'all\n' + ' the parameters. If no command is given, the current alias ' + 'for\n' + ' *name* is shown. If no arguments are given, all aliases are ' + 'listed.\n' + '\n' + ' Aliases may be nested and can contain anything that can be ' + 'legally\n' + ' typed at the pdb prompt. Note that internal pdb commands ' + '*can* be\n' + ' overridden by aliases. Such a command is then hidden until ' + 'the\n' + ' alias is removed. Aliasing is recursively applied to the ' + 'first\n' + ' word of the command line; all other words in the line are ' + 'left\n' + ' alone.\n' + '\n' + ' As an example, here are two useful aliases (especially when ' + 'placed\n' + ' in the ".pdbrc" file):\n' + '\n' + ' # Print instance variables (usage "pi classInst")\n' + ' alias pi for k in %1.__dict__.keys(): ' + 'print("%1.",k,"=",%1.__dict__[k])\n' + ' # Print instance variables in self\n' + ' alias ps pi self\n' + '\n' + 'unalias name\n' + '\n' + ' Delete the specified alias.\n' + '\n' + '! statement\n' + '\n' + ' Execute the (one-line) *statement* in the context of the ' + 'current\n' + ' stack frame. The exclamation point can be omitted unless the ' + 'first\n' + ' word of the statement resembles a debugger command. To set ' + 'a\n' + ' global variable, you can prefix the assignment command with ' + 'a\n' + ' "global" statement on the same line, e.g.:\n' + '\n' + " (Pdb) global list_options; list_options = ['-l']\n" + ' (Pdb)\n' + '\n' + 'run [args ...]\n' + 'restart [args ...]\n' + '\n' + ' Restart the debugged Python program. If an argument is ' + 'supplied,\n' + ' it is split with "shlex" and the result is used as the new\n' + ' "sys.argv". History, breakpoints, actions and debugger ' + 'options are\n' + ' preserved. "restart" is an alias for "run".\n' + '\n' + 'q(uit)\n' + '\n' + ' Quit from the debugger. The program being executed is ' + 'aborted.\n' + '\n' + '-[ Footnotes ]-\n' + '\n' + '[1] Whether a frame is considered to originate in a certain ' + 'module\n' + ' is determined by the "__name__" in the frame globals.\n', + 'del': '\n' + 'The "del" statement\n' + '*******************\n' + '\n' + ' del_stmt ::= "del" target_list\n' + '\n' + 'Deletion is recursively defined very similar to the way assignment ' + 'is\n' + 'defined. Rather than spelling it out in full details, here are some\n' + 'hints.\n' + '\n' + 'Deletion of a target list recursively deletes each target, from left\n' + 'to right.\n' + '\n' + 'Deletion of a name removes the binding of that name from the local ' + 'or\n' + 'global namespace, depending on whether the name occurs in a "global"\n' + 'statement in the same code block. If the name is unbound, a\n' + '"NameError" exception will be raised.\n' + '\n' + 'Deletion of attribute references, subscriptions and slicings is ' + 'passed\n' + 'to the primary object involved; deletion of a slicing is in general\n' + 'equivalent to assignment of an empty slice of the right type (but ' + 'even\n' + 'this is determined by the sliced object).\n' + '\n' + 'Changed in version 3.2: Previously it was illegal to delete a name\n' + 'from the local namespace if it occurs as a free variable in a nested\n' + 'block.\n', + 'dict': '\n' + 'Dictionary displays\n' + '*******************\n' + '\n' + 'A dictionary display is a possibly empty series of key/datum pairs\n' + 'enclosed in curly braces:\n' + '\n' + ' dict_display ::= "{" [key_datum_list | dict_comprehension] ' + '"}"\n' + ' key_datum_list ::= key_datum ("," key_datum)* [","]\n' + ' key_datum ::= expression ":" expression\n' + ' dict_comprehension ::= expression ":" expression comp_for\n' + '\n' + 'A dictionary display yields a new dictionary object.\n' + '\n' + 'If a comma-separated sequence of key/datum pairs is given, they are\n' + 'evaluated from left to right to define the entries of the ' + 'dictionary:\n' + 'each key object is used as a key into the dictionary to store the\n' + 'corresponding datum. This means that you can specify the same key\n' + "multiple times in the key/datum list, and the final dictionary's " + 'value\n' + 'for that key will be the last one given.\n' + '\n' + 'A dict comprehension, in contrast to list and set comprehensions,\n' + 'needs two expressions separated with a colon followed by the usual\n' + '"for" and "if" clauses. When the comprehension is run, the ' + 'resulting\n' + 'key and value elements are inserted in the new dictionary in the ' + 'order\n' + 'they are produced.\n' + '\n' + 'Restrictions on the types of the key values are listed earlier in\n' + 'section The standard type hierarchy. (To summarize, the key type\n' + 'should be *hashable*, which excludes all mutable objects.) Clashes\n' + 'between duplicate keys are not detected; the last datum (textually\n' + 'rightmost in the display) stored for a given key value prevails.\n', + 'dynamic-features': '\n' + 'Interaction with dynamic features\n' + '*********************************\n' + '\n' + 'Name resolution of free variables occurs at runtime, not ' + 'at compile\n' + 'time. This means that the following code will print 42:\n' + '\n' + ' i = 10\n' + ' def f():\n' + ' print(i)\n' + ' i = 42\n' + ' f()\n' + '\n' + 'There are several cases where Python statements are ' + 'illegal when used\n' + 'in conjunction with nested scopes that contain free ' + 'variables.\n' + '\n' + 'If a variable is referenced in an enclosing scope, it is ' + 'illegal to\n' + 'delete the name. An error will be reported at compile ' + 'time.\n' + '\n' + 'The "eval()" and "exec()" functions do not have access ' + 'to the full\n' + 'environment for resolving names. Names may be resolved ' + 'in the local\n' + 'and global namespaces of the caller. Free variables are ' + 'not resolved\n' + 'in the nearest enclosing namespace, but in the global ' + 'namespace. [1]\n' + 'The "exec()" and "eval()" functions have optional ' + 'arguments to\n' + 'override the global and local namespace. If only one ' + 'namespace is\n' + 'specified, it is used for both.\n', + 'else': '\n' + 'The "if" statement\n' + '******************\n' + '\n' + 'The "if" statement is used for conditional execution:\n' + '\n' + ' if_stmt ::= "if" expression ":" suite\n' + ' ( "elif" expression ":" suite )*\n' + ' ["else" ":" suite]\n' + '\n' + 'It selects exactly one of the suites by evaluating the expressions ' + 'one\n' + 'by one until one is found to be true (see section Boolean ' + 'operations\n' + 'for the definition of true and false); then that suite is executed\n' + '(and no other part of the "if" statement is executed or evaluated).\n' + 'If all expressions are false, the suite of the "else" clause, if\n' + 'present, is executed.\n', + 'exceptions': '\n' + 'Exceptions\n' + '**********\n' + '\n' + 'Exceptions are a means of breaking out of the normal flow of ' + 'control\n' + 'of a code block in order to handle errors or other ' + 'exceptional\n' + 'conditions. An exception is *raised* at the point where the ' + 'error is\n' + 'detected; it may be *handled* by the surrounding code block or ' + 'by any\n' + 'code block that directly or indirectly invoked the code block ' + 'where\n' + 'the error occurred.\n' + '\n' + 'The Python interpreter raises an exception when it detects a ' + 'run-time\n' + 'error (such as division by zero). A Python program can also\n' + 'explicitly raise an exception with the "raise" statement. ' + 'Exception\n' + 'handlers are specified with the "try" ... "except" statement. ' + 'The\n' + '"finally" clause of such a statement can be used to specify ' + 'cleanup\n' + 'code which does not handle the exception, but is executed ' + 'whether an\n' + 'exception occurred or not in the preceding code.\n' + '\n' + 'Python uses the "termination" model of error handling: an ' + 'exception\n' + 'handler can find out what happened and continue execution at ' + 'an outer\n' + 'level, but it cannot repair the cause of the error and retry ' + 'the\n' + 'failing operation (except by re-entering the offending piece ' + 'of code\n' + 'from the top).\n' + '\n' + 'When an exception is not handled at all, the interpreter ' + 'terminates\n' + 'execution of the program, or returns to its interactive main ' + 'loop. In\n' + 'either case, it prints a stack backtrace, except when the ' + 'exception is\n' + '"SystemExit".\n' + '\n' + 'Exceptions are identified by class instances. The "except" ' + 'clause is\n' + 'selected depending on the class of the instance: it must ' + 'reference the\n' + 'class of the instance or a base class thereof. The instance ' + 'can be\n' + 'received by the handler and can carry additional information ' + 'about the\n' + 'exceptional condition.\n' + '\n' + 'Note: Exception messages are not part of the Python API. ' + 'Their\n' + ' contents may change from one version of Python to the next ' + 'without\n' + ' warning and should not be relied on by code which will run ' + 'under\n' + ' multiple versions of the interpreter.\n' + '\n' + 'See also the description of the "try" statement in section The ' + 'try\n' + 'statement and "raise" statement in section The raise ' + 'statement.\n' + '\n' + '-[ Footnotes ]-\n' + '\n' + '[1] This limitation occurs because the code that is executed ' + 'by\n' + ' these operations is not available at the time the module ' + 'is\n' + ' compiled.\n', + 'execmodel': '\n' + 'Execution model\n' + '***************\n' + '\n' + '\n' + 'Structure of a program\n' + '======================\n' + '\n' + 'A Python program is constructed from code blocks. A *block* is ' + 'a piece\n' + 'of Python program text that is executed as a unit. The ' + 'following are\n' + 'blocks: a module, a function body, and a class definition. ' + 'Each\n' + 'command typed interactively is a block. A script file (a file ' + 'given\n' + 'as standard input to the interpreter or specified as a command ' + 'line\n' + 'argument to the interpreter) is a code block. A script command ' + '(a\n' + 'command specified on the interpreter command line with the ' + "'**-c**'\n" + 'option) is a code block. The string argument passed to the ' + 'built-in\n' + 'functions "eval()" and "exec()" is a code block.\n' + '\n' + 'A code block is executed in an *execution frame*. A frame ' + 'contains\n' + 'some administrative information (used for debugging) and ' + 'determines\n' + "where and how execution continues after the code block's " + 'execution has\n' + 'completed.\n' + '\n' + '\n' + 'Naming and binding\n' + '==================\n' + '\n' + '\n' + 'Binding of names\n' + '----------------\n' + '\n' + '*Names* refer to objects. Names are introduced by name ' + 'binding\n' + 'operations.\n' + '\n' + 'The following constructs bind names: formal parameters to ' + 'functions,\n' + '"import" statements, class and function definitions (these bind ' + 'the\n' + 'class or function name in the defining block), and targets that ' + 'are\n' + 'identifiers if occurring in an assignment, "for" loop header, ' + 'or after\n' + '"as" in a "with" statement or "except" clause. The "import" ' + 'statement\n' + 'of the form "from ... import *" binds all names defined in the\n' + 'imported module, except those beginning with an underscore. ' + 'This form\n' + 'may only be used at the module level.\n' + '\n' + 'A target occurring in a "del" statement is also considered ' + 'bound for\n' + 'this purpose (though the actual semantics are to unbind the ' + 'name).\n' + '\n' + 'Each assignment or import statement occurs within a block ' + 'defined by a\n' + 'class or function definition or at the module level (the ' + 'top-level\n' + 'code block).\n' + '\n' + 'If a name is bound in a block, it is a local variable of that ' + 'block,\n' + 'unless declared as "nonlocal" or "global". If a name is bound ' + 'at the\n' + 'module level, it is a global variable. (The variables of the ' + 'module\n' + 'code block are local and global.) If a variable is used in a ' + 'code\n' + 'block but not defined there, it is a *free variable*.\n' + '\n' + 'Each occurrence of a name in the program text refers to the ' + '*binding*\n' + 'of that name established by the following name resolution ' + 'rules.\n' + '\n' + '\n' + 'Resolution of names\n' + '-------------------\n' + '\n' + 'A *scope* defines the visibility of a name within a block. If ' + 'a local\n' + 'variable is defined in a block, its scope includes that block. ' + 'If the\n' + 'definition occurs in a function block, the scope extends to any ' + 'blocks\n' + 'contained within the defining one, unless a contained block ' + 'introduces\n' + 'a different binding for the name.\n' + '\n' + 'When a name is used in a code block, it is resolved using the ' + 'nearest\n' + 'enclosing scope. The set of all such scopes visible to a code ' + 'block\n' + "is called the block's *environment*.\n" + '\n' + 'When a name is not found at all, a "NameError" exception is ' + 'raised. If\n' + 'the current scope is a function scope, and the name refers to a ' + 'local\n' + 'variable that has not yet been bound to a value at the point ' + 'where the\n' + 'name is used, an "UnboundLocalError" exception is raised.\n' + '"UnboundLocalError" is a subclass of "NameError".\n' + '\n' + 'If a name binding operation occurs anywhere within a code ' + 'block, all\n' + 'uses of the name within the block are treated as references to ' + 'the\n' + 'current block. This can lead to errors when a name is used ' + 'within a\n' + 'block before it is bound. This rule is subtle. Python lacks\n' + 'declarations and allows name binding operations to occur ' + 'anywhere\n' + 'within a code block. The local variables of a code block can ' + 'be\n' + 'determined by scanning the entire text of the block for name ' + 'binding\n' + 'operations.\n' + '\n' + 'If the "global" statement occurs within a block, all uses of ' + 'the name\n' + 'specified in the statement refer to the binding of that name in ' + 'the\n' + 'top-level namespace. Names are resolved in the top-level ' + 'namespace by\n' + 'searching the global namespace, i.e. the namespace of the ' + 'module\n' + 'containing the code block, and the builtins namespace, the ' + 'namespace\n' + 'of the module "builtins". The global namespace is searched ' + 'first. If\n' + 'the name is not found there, the builtins namespace is ' + 'searched. The\n' + '"global" statement must precede all uses of the name.\n' + '\n' + 'The "global" statement has the same scope as a name binding ' + 'operation\n' + 'in the same block. If the nearest enclosing scope for a free ' + 'variable\n' + 'contains a global statement, the free variable is treated as a ' + 'global.\n' + '\n' + 'The "nonlocal" statement causes corresponding names to refer ' + 'to\n' + 'previously bound variables in the nearest enclosing function ' + 'scope.\n' + '"SyntaxError" is raised at compile time if the given name does ' + 'not\n' + 'exist in any enclosing function scope.\n' + '\n' + 'The namespace for a module is automatically created the first ' + 'time a\n' + 'module is imported. The main module for a script is always ' + 'called\n' + '"__main__".\n' + '\n' + 'Class definition blocks and arguments to "exec()" and "eval()" ' + 'are\n' + 'special in the context of name resolution. A class definition ' + 'is an\n' + 'executable statement that may use and define names. These ' + 'references\n' + 'follow the normal rules for name resolution with an exception ' + 'that\n' + 'unbound local variables are looked up in the global namespace. ' + 'The\n' + 'namespace of the class definition becomes the attribute ' + 'dictionary of\n' + 'the class. The scope of names defined in a class block is ' + 'limited to\n' + 'the class block; it does not extend to the code blocks of ' + 'methods --\n' + 'this includes comprehensions and generator expressions since ' + 'they are\n' + 'implemented using a function scope. This means that the ' + 'following\n' + 'will fail:\n' + '\n' + ' class A:\n' + ' a = 42\n' + ' b = list(a + i for i in range(10))\n' + '\n' + '\n' + 'Builtins and restricted execution\n' + '---------------------------------\n' + '\n' + 'The builtins namespace associated with the execution of a code ' + 'block\n' + 'is actually found by looking up the name "__builtins__" in its ' + 'global\n' + 'namespace; this should be a dictionary or a module (in the ' + 'latter case\n' + "the module's dictionary is used). By default, when in the " + '"__main__"\n' + 'module, "__builtins__" is the built-in module "builtins"; when ' + 'in any\n' + 'other module, "__builtins__" is an alias for the dictionary of ' + 'the\n' + '"builtins" module itself. "__builtins__" can be set to a ' + 'user-created\n' + 'dictionary to create a weak form of restricted execution.\n' + '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. ' + 'Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' + '\n' + 'Interaction with dynamic features\n' + '---------------------------------\n' + '\n' + 'Name resolution of free variables occurs at runtime, not at ' + 'compile\n' + 'time. This means that the following code will print 42:\n' + '\n' + ' i = 10\n' + ' def f():\n' + ' print(i)\n' + ' i = 42\n' + ' f()\n' + '\n' + 'There are several cases where Python statements are illegal ' + 'when used\n' + 'in conjunction with nested scopes that contain free variables.\n' + '\n' + 'If a variable is referenced in an enclosing scope, it is ' + 'illegal to\n' + 'delete the name. An error will be reported at compile time.\n' + '\n' + 'The "eval()" and "exec()" functions do not have access to the ' + 'full\n' + 'environment for resolving names. Names may be resolved in the ' + 'local\n' + 'and global namespaces of the caller. Free variables are not ' + 'resolved\n' + 'in the nearest enclosing namespace, but in the global ' + 'namespace. [1]\n' + 'The "exec()" and "eval()" functions have optional arguments to\n' + 'override the global and local namespace. If only one namespace ' + 'is\n' + 'specified, it is used for both.\n' + '\n' + '\n' + 'Exceptions\n' + '==========\n' + '\n' + 'Exceptions are a means of breaking out of the normal flow of ' + 'control\n' + 'of a code block in order to handle errors or other exceptional\n' + 'conditions. An exception is *raised* at the point where the ' + 'error is\n' + 'detected; it may be *handled* by the surrounding code block or ' + 'by any\n' + 'code block that directly or indirectly invoked the code block ' + 'where\n' + 'the error occurred.\n' + '\n' + 'The Python interpreter raises an exception when it detects a ' + 'run-time\n' + 'error (such as division by zero). A Python program can also\n' + 'explicitly raise an exception with the "raise" statement. ' + 'Exception\n' + 'handlers are specified with the "try" ... "except" statement. ' + 'The\n' + '"finally" clause of such a statement can be used to specify ' + 'cleanup\n' + 'code which does not handle the exception, but is executed ' + 'whether an\n' + 'exception occurred or not in the preceding code.\n' + '\n' + 'Python uses the "termination" model of error handling: an ' + 'exception\n' + 'handler can find out what happened and continue execution at an ' + 'outer\n' + 'level, but it cannot repair the cause of the error and retry ' + 'the\n' + 'failing operation (except by re-entering the offending piece of ' + 'code\n' + 'from the top).\n' + '\n' + 'When an exception is not handled at all, the interpreter ' + 'terminates\n' + 'execution of the program, or returns to its interactive main ' + 'loop. In\n' + 'either case, it prints a stack backtrace, except when the ' + 'exception is\n' + '"SystemExit".\n' + '\n' + 'Exceptions are identified by class instances. The "except" ' + 'clause is\n' + 'selected depending on the class of the instance: it must ' + 'reference the\n' + 'class of the instance or a base class thereof. The instance ' + 'can be\n' + 'received by the handler and can carry additional information ' + 'about the\n' + 'exceptional condition.\n' + '\n' + 'Note: Exception messages are not part of the Python API. ' + 'Their\n' + ' contents may change from one version of Python to the next ' + 'without\n' + ' warning and should not be relied on by code which will run ' + 'under\n' + ' multiple versions of the interpreter.\n' + '\n' + 'See also the description of the "try" statement in section The ' + 'try\n' + 'statement and "raise" statement in section The raise ' + 'statement.\n' + '\n' + '-[ Footnotes ]-\n' + '\n' + '[1] This limitation occurs because the code that is executed ' + 'by\n' + ' these operations is not available at the time the module ' + 'is\n' + ' compiled.\n', + 'exprlists': '\n' + 'Expression lists\n' + '****************\n' + '\n' + ' expression_list ::= expression ( "," expression )* [","]\n' + '\n' + 'An expression list containing at least one comma yields a ' + 'tuple. The\n' + 'length of the tuple is the number of expressions in the list. ' + 'The\n' + 'expressions are evaluated from left to right.\n' + '\n' + 'The trailing comma is required only to create a single tuple ' + '(a.k.a. a\n' + '*singleton*); it is optional in all other cases. A single ' + 'expression\n' + "without a trailing comma doesn't create a tuple, but rather " + 'yields the\n' + 'value of that expression. (To create an empty tuple, use an ' + 'empty pair\n' + 'of parentheses: "()".)\n', + 'floating': '\n' + 'Floating point literals\n' + '***********************\n' + '\n' + 'Floating point literals are described by the following lexical\n' + 'definitions:\n' + '\n' + ' floatnumber ::= pointfloat | exponentfloat\n' + ' pointfloat ::= [intpart] fraction | intpart "."\n' + ' exponentfloat ::= (intpart | pointfloat) exponent\n' + ' intpart ::= digit+\n' + ' fraction ::= "." digit+\n' + ' exponent ::= ("e" | "E") ["+" | "-"] digit+\n' + '\n' + 'Note that the integer and exponent parts are always interpreted ' + 'using\n' + 'radix 10. For example, "077e010" is legal, and denotes the same ' + 'number\n' + 'as "77e10". The allowed range of floating point literals is\n' + 'implementation-dependent. Some examples of floating point ' + 'literals:\n' + '\n' + ' 3.14 10. .001 1e100 3.14e-10 0e0\n' + '\n' + 'Note that numeric literals do not include a sign; a phrase like ' + '"-1"\n' + 'is actually an expression composed of the unary operator "-" and ' + 'the\n' + 'literal "1".\n', + 'for': '\n' + 'The "for" statement\n' + '*******************\n' + '\n' + 'The "for" statement is used to iterate over the elements of a ' + 'sequence\n' + '(such as a string, tuple or list) or other iterable object:\n' + '\n' + ' for_stmt ::= "for" target_list "in" expression_list ":" suite\n' + ' ["else" ":" suite]\n' + '\n' + 'The expression list is evaluated once; it should yield an iterable\n' + 'object. An iterator is created for the result of the\n' + '"expression_list". The suite is then executed once for each item\n' + 'provided by the iterator, in the order returned by the iterator. ' + 'Each\n' + 'item in turn is assigned to the target list using the standard rules\n' + 'for assignments (see Assignment statements), and then the suite is\n' + 'executed. When the items are exhausted (which is immediately when ' + 'the\n' + 'sequence is empty or an iterator raises a "StopIteration" ' + 'exception),\n' + 'the suite in the "else" clause, if present, is executed, and the ' + 'loop\n' + 'terminates.\n' + '\n' + 'A "break" statement executed in the first suite terminates the loop\n' + 'without executing the "else" clause\'s suite. A "continue" ' + 'statement\n' + 'executed in the first suite skips the rest of the suite and ' + 'continues\n' + 'with the next item, or with the "else" clause if there is no next\n' + 'item.\n' + '\n' + 'The for-loop makes assignments to the variables(s) in the target ' + 'list.\n' + 'This overwrites all previous assignments to those variables ' + 'including\n' + 'those made in the suite of the for-loop:\n' + '\n' + ' for i in range(10):\n' + ' print(i)\n' + ' i = 5 # this will not affect the for-loop\n' + ' # because i will be overwritten with the ' + 'next\n' + ' # index in the range\n' + '\n' + 'Names in the target list are not deleted when the loop is finished,\n' + 'but if the sequence is empty, they will not have been assigned to at\n' + 'all by the loop. Hint: the built-in function "range()" returns an\n' + 'iterator of integers suitable to emulate the effect of Pascal\'s "for ' + 'i\n' + ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n' + '\n' + 'Note: There is a subtlety when the sequence is being modified by the\n' + ' loop (this can only occur for mutable sequences, i.e. lists). An\n' + ' internal counter is used to keep track of which item is used next,\n' + ' and this is incremented on each iteration. When this counter has\n' + ' reached the length of the sequence the loop terminates. This ' + 'means\n' + ' that if the suite deletes the current (or a previous) item from ' + 'the\n' + ' sequence, the next item will be skipped (since it gets the index ' + 'of\n' + ' the current item which has already been treated). Likewise, if ' + 'the\n' + ' suite inserts an item in the sequence before the current item, the\n' + ' current item will be treated again the next time through the loop.\n' + ' This can lead to nasty bugs that can be avoided by making a\n' + ' temporary copy using a slice of the whole sequence, e.g.,\n' + '\n' + ' for x in a[:]:\n' + ' if x < 0: a.remove(x)\n', + 'formatstrings': '\n' + 'Format String Syntax\n' + '********************\n' + '\n' + 'The "str.format()" method and the "Formatter" class share ' + 'the same\n' + 'syntax for format strings (although in the case of ' + '"Formatter",\n' + 'subclasses can define their own format string syntax). The ' + 'syntax is\n' + 'related to that of formatted string literals, but there ' + 'are\n' + 'differences.\n' + '\n' + 'Format strings contain "replacement fields" surrounded by ' + 'curly braces\n' + '"{}". Anything that is not contained in braces is ' + 'considered literal\n' + 'text, which is copied unchanged to the output. If you need ' + 'to include\n' + 'a brace character in the literal text, it can be escaped by ' + 'doubling:\n' + '"{{" and "}}".\n' + '\n' + 'The grammar for a replacement field is as follows:\n' + '\n' + ' replacement_field ::= "{" [field_name] ["!" ' + 'conversion] [":" format_spec] "}"\n' + ' field_name ::= arg_name ("." attribute_name | ' + '"[" element_index "]")*\n' + ' arg_name ::= [identifier | integer]\n' + ' attribute_name ::= identifier\n' + ' element_index ::= integer | index_string\n' + ' index_string ::= +\n' + ' conversion ::= "r" | "s" | "a"\n' + ' format_spec ::= \n' + '\n' + 'In less formal terms, the replacement field can start with ' + 'a\n' + '*field_name* that specifies the object whose value is to be ' + 'formatted\n' + 'and inserted into the output instead of the replacement ' + 'field. The\n' + '*field_name* is optionally followed by a *conversion* ' + 'field, which is\n' + 'preceded by an exclamation point "\'!\'", and a ' + '*format_spec*, which is\n' + 'preceded by a colon "\':\'". These specify a non-default ' + 'format for the\n' + 'replacement value.\n' + '\n' + 'See also the Format Specification Mini-Language section.\n' + '\n' + 'The *field_name* itself begins with an *arg_name* that is ' + 'either a\n' + "number or a keyword. If it's a number, it refers to a " + 'positional\n' + "argument, and if it's a keyword, it refers to a named " + 'keyword\n' + 'argument. If the numerical arg_names in a format string ' + 'are 0, 1, 2,\n' + '... in sequence, they can all be omitted (not just some) ' + 'and the\n' + 'numbers 0, 1, 2, ... will be automatically inserted in that ' + 'order.\n' + 'Because *arg_name* is not quote-delimited, it is not ' + 'possible to\n' + 'specify arbitrary dictionary keys (e.g., the strings ' + '"\'10\'" or\n' + '"\':-]\'") within a format string. The *arg_name* can be ' + 'followed by any\n' + 'number of index or attribute expressions. An expression of ' + 'the form\n' + '"\'.name\'" selects the named attribute using "getattr()", ' + 'while an\n' + 'expression of the form "\'[index]\'" does an index lookup ' + 'using\n' + '"__getitem__()".\n' + '\n' + 'Changed in version 3.1: The positional argument specifiers ' + 'can be\n' + 'omitted, so "\'{} {}\'" is equivalent to "\'{0} {1}\'".\n' + '\n' + 'Some simple format string examples:\n' + '\n' + ' "First, thou shalt count to {0}" # References first ' + 'positional argument\n' + ' "Bring me a {}" # Implicitly ' + 'references the first positional argument\n' + ' "From {} to {}" # Same as "From {0} to ' + '{1}"\n' + ' "My quest is {name}" # References keyword ' + "argument 'name'\n" + ' "Weight in tons {0.weight}" # \'weight\' attribute ' + 'of first positional arg\n' + ' "Units destroyed: {players[0]}" # First element of ' + "keyword argument 'players'.\n" + '\n' + 'The *conversion* field causes a type coercion before ' + 'formatting.\n' + 'Normally, the job of formatting a value is done by the ' + '"__format__()"\n' + 'method of the value itself. However, in some cases it is ' + 'desirable to\n' + 'force a type to be formatted as a string, overriding its ' + 'own\n' + 'definition of formatting. By converting the value to a ' + 'string before\n' + 'calling "__format__()", the normal formatting logic is ' + 'bypassed.\n' + '\n' + 'Three conversion flags are currently supported: "\'!s\'" ' + 'which calls\n' + '"str()" on the value, "\'!r\'" which calls "repr()" and ' + '"\'!a\'" which\n' + 'calls "ascii()".\n' + '\n' + 'Some examples:\n' + '\n' + ' "Harold\'s a clever {0!s}" # Calls str() on the ' + 'argument first\n' + ' "Bring out the holy {name!r}" # Calls repr() on the ' + 'argument first\n' + ' "More {!a}" # Calls ascii() on the ' + 'argument first\n' + '\n' + 'The *format_spec* field contains a specification of how the ' + 'value\n' + 'should be presented, including such details as field width, ' + 'alignment,\n' + 'padding, decimal precision and so on. Each value type can ' + 'define its\n' + 'own "formatting mini-language" or interpretation of the ' + '*format_spec*.\n' + '\n' + 'Most built-in types support a common formatting ' + 'mini-language, which\n' + 'is described in the next section.\n' + '\n' + 'A *format_spec* field can also include nested replacement ' + 'fields\n' + 'within it. These nested replacement fields may contain a ' + 'field name,\n' + 'conversion flag and format specification, but deeper ' + 'nesting is not\n' + 'allowed. The replacement fields within the format_spec ' + 'are\n' + 'substituted before the *format_spec* string is interpreted. ' + 'This\n' + 'allows the formatting of a value to be dynamically ' + 'specified.\n' + '\n' + 'See the Format examples section for some examples.\n' + '\n' + '\n' + 'Format Specification Mini-Language\n' + '==================================\n' + '\n' + '"Format specifications" are used within replacement fields ' + 'contained\n' + 'within a format string to define how individual values are ' + 'presented\n' + '(see Format String Syntax and Formatted string literals). ' + 'They can\n' + 'also be passed directly to the built-in "format()" ' + 'function. Each\n' + 'formattable type may define how the format specification is ' + 'to be\n' + 'interpreted.\n' + '\n' + 'Most built-in types implement the following options for ' + 'format\n' + 'specifications, although some of the formatting options are ' + 'only\n' + 'supported by the numeric types.\n' + '\n' + 'A general convention is that an empty format string ("""") ' + 'produces\n' + 'the same result as if you had called "str()" on the value. ' + 'A non-empty\n' + 'format string typically modifies the result.\n' + '\n' + 'The general form of a *standard format specifier* is:\n' + '\n' + ' format_spec ::= ' + '[[fill]align][sign][#][0][width][,][.precision][type]\n' + ' fill ::= \n' + ' align ::= "<" | ">" | "=" | "^"\n' + ' sign ::= "+" | "-" | " "\n' + ' width ::= integer\n' + ' precision ::= integer\n' + ' type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" ' + '| "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n' + '\n' + 'If a valid *align* value is specified, it can be preceded ' + 'by a *fill*\n' + 'character that can be any character and defaults to a space ' + 'if\n' + 'omitted. It is not possible to use a literal curly brace ' + '(""{"" or\n' + '""}"") as the *fill* character in a formatted string ' + 'literal or when\n' + 'using the "str.format()" method. However, it is possible ' + 'to insert a\n' + 'curly brace with a nested replacement field. This ' + "limitation doesn't\n" + 'affect the "format()" function.\n' + '\n' + 'The meaning of the various alignment options is as ' + 'follows:\n' + '\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | Option | ' + 'Meaning ' + '|\n' + ' ' + '+===========+============================================================+\n' + ' | "\'<\'" | Forces the field to be left-aligned ' + 'within the available |\n' + ' | | space (this is the default for most ' + 'objects). |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'>\'" | Forces the field to be right-aligned ' + 'within the available |\n' + ' | | space (this is the default for ' + 'numbers). |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'=\'" | Forces the padding to be placed after ' + 'the sign (if any) |\n' + ' | | but before the digits. This is used for ' + 'printing fields |\n' + " | | in the form '+000000120'. This alignment " + 'option is only |\n' + ' | | valid for numeric types. It becomes the ' + "default when '0' |\n" + ' | | immediately precedes the field ' + 'width. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'^\'" | Forces the field to be centered within ' + 'the available |\n' + ' | | ' + 'space. ' + '|\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + '\n' + 'Note that unless a minimum field width is defined, the ' + 'field width\n' + 'will always be the same size as the data to fill it, so ' + 'that the\n' + 'alignment option has no meaning in this case.\n' + '\n' + 'The *sign* option is only valid for number types, and can ' + 'be one of\n' + 'the following:\n' + '\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | Option | ' + 'Meaning ' + '|\n' + ' ' + '+===========+============================================================+\n' + ' | "\'+\'" | indicates that a sign should be used for ' + 'both positive as |\n' + ' | | well as negative ' + 'numbers. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'-\'" | indicates that a sign should be used ' + 'only for negative |\n' + ' | | numbers (this is the default ' + 'behavior). |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | space | indicates that a leading space should be ' + 'used on positive |\n' + ' | | numbers, and a minus sign on negative ' + 'numbers. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + '\n' + 'The "\'#\'" option causes the "alternate form" to be used ' + 'for the\n' + 'conversion. The alternate form is defined differently for ' + 'different\n' + 'types. This option is only valid for integer, float, ' + 'complex and\n' + 'Decimal types. For integers, when binary, octal, or ' + 'hexadecimal output\n' + 'is used, this option adds the prefix respective "\'0b\'", ' + '"\'0o\'", or\n' + '"\'0x\'" to the output value. For floats, complex and ' + 'Decimal the\n' + 'alternate form causes the result of the conversion to ' + 'always contain a\n' + 'decimal-point character, even if no digits follow it. ' + 'Normally, a\n' + 'decimal-point character appears in the result of these ' + 'conversions\n' + 'only if a digit follows it. In addition, for "\'g\'" and ' + '"\'G\'"\n' + 'conversions, trailing zeros are not removed from the ' + 'result.\n' + '\n' + 'The "\',\'" option signals the use of a comma for a ' + 'thousands separator.\n' + 'For a locale aware separator, use the "\'n\'" integer ' + 'presentation type\n' + 'instead.\n' + '\n' + 'Changed in version 3.1: Added the "\',\'" option (see also ' + '**PEP 378**).\n' + '\n' + '*width* is a decimal integer defining the minimum field ' + 'width. If not\n' + 'specified, then the field width will be determined by the ' + 'content.\n' + '\n' + 'When no explicit alignment is given, preceding the *width* ' + 'field by a\n' + 'zero ("\'0\'") character enables sign-aware zero-padding ' + 'for numeric\n' + 'types. This is equivalent to a *fill* character of "\'0\'" ' + 'with an\n' + '*alignment* type of "\'=\'".\n' + '\n' + 'The *precision* is a decimal number indicating how many ' + 'digits should\n' + 'be displayed after the decimal point for a floating point ' + 'value\n' + 'formatted with "\'f\'" and "\'F\'", or before and after the ' + 'decimal point\n' + 'for a floating point value formatted with "\'g\'" or ' + '"\'G\'". For non-\n' + 'number types the field indicates the maximum field size - ' + 'in other\n' + 'words, how many characters will be used from the field ' + 'content. The\n' + '*precision* is not allowed for integer values.\n' + '\n' + 'Finally, the *type* determines how the data should be ' + 'presented.\n' + '\n' + 'The available string presentation types are:\n' + '\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | Type | ' + 'Meaning ' + '|\n' + ' ' + '+===========+============================================================+\n' + ' | "\'s\'" | String format. This is the default type ' + 'for strings and |\n' + ' | | may be ' + 'omitted. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | None | The same as ' + '"\'s\'". |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + '\n' + 'The available integer presentation types are:\n' + '\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | Type | ' + 'Meaning ' + '|\n' + ' ' + '+===========+============================================================+\n' + ' | "\'b\'" | Binary format. Outputs the number in ' + 'base 2. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'c\'" | Character. Converts the integer to the ' + 'corresponding |\n' + ' | | unicode character before ' + 'printing. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'d\'" | Decimal Integer. Outputs the number in ' + 'base 10. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'o\'" | Octal format. Outputs the number in base ' + '8. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'x\'" | Hex format. Outputs the number in base ' + '16, using lower- |\n' + ' | | case letters for the digits above ' + '9. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'X\'" | Hex format. Outputs the number in base ' + '16, using upper- |\n' + ' | | case letters for the digits above ' + '9. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'n\'" | Number. This is the same as "\'d\'", ' + 'except that it uses the |\n' + ' | | current locale setting to insert the ' + 'appropriate number |\n' + ' | | separator ' + 'characters. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | None | The same as ' + '"\'d\'". |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + '\n' + 'In addition to the above presentation types, integers can ' + 'be formatted\n' + 'with the floating point presentation types listed below ' + '(except "\'n\'"\n' + 'and None). When doing so, "float()" is used to convert the ' + 'integer to\n' + 'a floating point number before formatting.\n' + '\n' + 'The available presentation types for floating point and ' + 'decimal values\n' + 'are:\n' + '\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | Type | ' + 'Meaning ' + '|\n' + ' ' + '+===========+============================================================+\n' + ' | "\'e\'" | Exponent notation. Prints the number in ' + 'scientific |\n' + " | | notation using the letter 'e' to indicate " + 'the exponent. |\n' + ' | | The default precision is ' + '"6". |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'E\'" | Exponent notation. Same as "\'e\'" ' + 'except it uses an upper |\n' + " | | case 'E' as the separator " + 'character. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'f\'" | Fixed point. Displays the number as a ' + 'fixed-point number. |\n' + ' | | The default precision is ' + '"6". |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'F\'" | Fixed point. Same as "\'f\'", but ' + 'converts "nan" to "NAN" |\n' + ' | | and "inf" to ' + '"INF". |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'g\'" | General format. For a given precision ' + '"p >= 1", this |\n' + ' | | rounds the number to "p" significant ' + 'digits and then |\n' + ' | | formats the result in either fixed-point ' + 'format or in |\n' + ' | | scientific notation, depending on its ' + 'magnitude. The |\n' + ' | | precise rules are as follows: suppose that ' + 'the result |\n' + ' | | formatted with presentation type "\'e\'" ' + 'and precision "p-1" |\n' + ' | | would have exponent "exp". Then if "-4 <= ' + 'exp < p", the |\n' + ' | | number is formatted with presentation type ' + '"\'f\'" and |\n' + ' | | precision "p-1-exp". Otherwise, the ' + 'number is formatted |\n' + ' | | with presentation type "\'e\'" and ' + 'precision "p-1". In both |\n' + ' | | cases insignificant trailing zeros are ' + 'removed from the |\n' + ' | | significand, and the decimal point is also ' + 'removed if |\n' + ' | | there are no remaining digits following ' + 'it. Positive and |\n' + ' | | negative infinity, positive and negative ' + 'zero, and nans, |\n' + ' | | are formatted as "inf", "-inf", "0", "-0" ' + 'and "nan" |\n' + ' | | respectively, regardless of the ' + 'precision. A precision of |\n' + ' | | "0" is treated as equivalent to a ' + 'precision of "1". The |\n' + ' | | default precision is ' + '"6". |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'G\'" | General format. Same as "\'g\'" except ' + 'switches to "\'E\'" if |\n' + ' | | the number gets too large. The ' + 'representations of infinity |\n' + ' | | and NaN are uppercased, ' + 'too. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'n\'" | Number. This is the same as "\'g\'", ' + 'except that it uses the |\n' + ' | | current locale setting to insert the ' + 'appropriate number |\n' + ' | | separator ' + 'characters. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | "\'%\'" | Percentage. Multiplies the number by 100 ' + 'and displays in |\n' + ' | | fixed ("\'f\'") format, followed by a ' + 'percent sign. |\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + ' | None | Similar to "\'g\'", except that ' + 'fixed-point notation, when |\n' + ' | | used, has at least one digit past the ' + 'decimal point. The |\n' + ' | | default precision is as high as needed to ' + 'represent the |\n' + ' | | particular value. The overall effect is to ' + 'match the |\n' + ' | | output of "str()" as altered by the other ' + 'format |\n' + ' | | ' + 'modifiers. ' + '|\n' + ' ' + '+-----------+------------------------------------------------------------+\n' + '\n' + '\n' + 'Format examples\n' + '===============\n' + '\n' + 'This section contains examples of the "str.format()" syntax ' + 'and\n' + 'comparison with the old "%"-formatting.\n' + '\n' + 'In most of the cases the syntax is similar to the old ' + '"%"-formatting,\n' + 'with the addition of the "{}" and with ":" used instead of ' + '"%". For\n' + 'example, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n' + '\n' + 'The new format syntax also supports new and different ' + 'options, shown\n' + 'in the follow examples.\n' + '\n' + 'Accessing arguments by position:\n' + '\n' + " >>> '{0}, {1}, {2}'.format('a', 'b', 'c')\n" + " 'a, b, c'\n" + " >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only\n" + " 'a, b, c'\n" + " >>> '{2}, {1}, {0}'.format('a', 'b', 'c')\n" + " 'c, b, a'\n" + " >>> '{2}, {1}, {0}'.format(*'abc') # unpacking " + 'argument sequence\n' + " 'c, b, a'\n" + " >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' " + 'indices can be repeated\n' + " 'abracadabra'\n" + '\n' + 'Accessing arguments by name:\n' + '\n' + " >>> 'Coordinates: {latitude}, " + "{longitude}'.format(latitude='37.24N', " + "longitude='-115.81W')\n" + " 'Coordinates: 37.24N, -115.81W'\n" + " >>> coord = {'latitude': '37.24N', 'longitude': " + "'-115.81W'}\n" + " >>> 'Coordinates: {latitude}, " + "{longitude}'.format(**coord)\n" + " 'Coordinates: 37.24N, -115.81W'\n" + '\n' + "Accessing arguments' attributes:\n" + '\n' + ' >>> c = 3-5j\n' + " >>> ('The complex number {0} is formed from the real " + "part {0.real} '\n" + " ... 'and the imaginary part {0.imag}.').format(c)\n" + " 'The complex number (3-5j) is formed from the real part " + "3.0 and the imaginary part -5.0.'\n" + ' >>> class Point:\n' + ' ... def __init__(self, x, y):\n' + ' ... self.x, self.y = x, y\n' + ' ... def __str__(self):\n' + " ... return 'Point({self.x}, " + "{self.y})'.format(self=self)\n" + ' ...\n' + ' >>> str(Point(4, 2))\n' + " 'Point(4, 2)'\n" + '\n' + "Accessing arguments' items:\n" + '\n' + ' >>> coord = (3, 5)\n' + " >>> 'X: {0[0]}; Y: {0[1]}'.format(coord)\n" + " 'X: 3; Y: 5'\n" + '\n' + 'Replacing "%s" and "%r":\n' + '\n' + ' >>> "repr() shows quotes: {!r}; str() doesn\'t: ' + '{!s}".format(\'test1\', \'test2\')\n' + ' "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n' + '\n' + 'Aligning the text and specifying a width:\n' + '\n' + " >>> '{:<30}'.format('left aligned')\n" + " 'left aligned '\n" + " >>> '{:>30}'.format('right aligned')\n" + " ' right aligned'\n" + " >>> '{:^30}'.format('centered')\n" + " ' centered '\n" + " >>> '{:*^30}'.format('centered') # use '*' as a fill " + 'char\n' + " '***********centered***********'\n" + '\n' + 'Replacing "%+f", "%-f", and "% f" and specifying a sign:\n' + '\n' + " >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it " + 'always\n' + " '+3.140000; -3.140000'\n" + " >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space " + 'for positive numbers\n' + " ' 3.140000; -3.140000'\n" + " >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the " + "minus -- same as '{:f}; {:f}'\n" + " '3.140000; -3.140000'\n" + '\n' + 'Replacing "%x" and "%o" and converting the value to ' + 'different bases:\n' + '\n' + ' >>> # format also supports binary numbers\n' + ' >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: ' + '{0:b}".format(42)\n' + " 'int: 42; hex: 2a; oct: 52; bin: 101010'\n" + ' >>> # with 0x, 0o, or 0b as prefix:\n' + ' >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: ' + '{0:#b}".format(42)\n' + " 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010'\n" + '\n' + 'Using the comma as a thousands separator:\n' + '\n' + " >>> '{:,}'.format(1234567890)\n" + " '1,234,567,890'\n" + '\n' + 'Expressing a percentage:\n' + '\n' + ' >>> points = 19\n' + ' >>> total = 22\n' + " >>> 'Correct answers: {:.2%}'.format(points/total)\n" + " 'Correct answers: 86.36%'\n" + '\n' + 'Using type-specific formatting:\n' + '\n' + ' >>> import datetime\n' + ' >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n' + " >>> '{:%Y-%m-%d %H:%M:%S}'.format(d)\n" + " '2010-07-04 12:15:58'\n" + '\n' + 'Nesting arguments and more complex examples:\n' + '\n' + " >>> for align, text in zip('<^>', ['left', 'center', " + "'right']):\n" + " ... '{0:{fill}{align}16}'.format(text, fill=align, " + 'align=align)\n' + ' ...\n' + " 'left<<<<<<<<<<<<'\n" + " '^^^^^center^^^^^'\n" + " '>>>>>>>>>>>right'\n" + ' >>>\n' + ' >>> octets = [192, 168, 0, 1]\n' + " >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets)\n" + " 'C0A80001'\n" + ' >>> int(_, 16)\n' + ' 3232235521\n' + ' >>>\n' + ' >>> width = 5\n' + ' >>> for num in range(5,12): #doctest: ' + '+NORMALIZE_WHITESPACE\n' + " ... for base in 'dXob':\n" + " ... print('{0:{width}{base}}'.format(num, " + "base=base, width=width), end=' ')\n" + ' ... print()\n' + ' ...\n' + ' 5 5 5 101\n' + ' 6 6 6 110\n' + ' 7 7 7 111\n' + ' 8 8 10 1000\n' + ' 9 9 11 1001\n' + ' 10 A 12 1010\n' + ' 11 B 13 1011\n', + 'function': '\n' + 'Function definitions\n' + '********************\n' + '\n' + 'A function definition defines a user-defined function object ' + '(see\n' + 'section The standard type hierarchy):\n' + '\n' + ' funcdef ::= [decorators] "def" funcname "(" ' + '[parameter_list] ")" ["->" expression] ":" suite\n' + ' decorators ::= decorator+\n' + ' decorator ::= "@" dotted_name ["(" ' + '[parameter_list [","]] ")"] NEWLINE\n' + ' dotted_name ::= identifier ("." identifier)*\n' + ' parameter_list ::= defparameter ("," defparameter)* ' + '["," [parameter_list_starargs]]\n' + ' | parameter_list_starargs\n' + ' parameter_list_starargs ::= "*" [parameter] ("," ' + 'defparameter)* ["," ["**" parameter [","]]]\n' + ' | "**" parameter [","]\n' + ' parameter ::= identifier [":" expression]\n' + ' defparameter ::= parameter ["=" expression]\n' + ' funcname ::= identifier\n' + '\n' + 'A function definition is an executable statement. Its execution ' + 'binds\n' + 'the function name in the current local namespace to a function ' + 'object\n' + '(a wrapper around the executable code for the function). This\n' + 'function object contains a reference to the current global ' + 'namespace\n' + 'as the global namespace to be used when the function is called.\n' + '\n' + 'The function definition does not execute the function body; this ' + 'gets\n' + 'executed only when the function is called. [3]\n' + '\n' + 'A function definition may be wrapped by one or more *decorator*\n' + 'expressions. Decorator expressions are evaluated when the ' + 'function is\n' + 'defined, in the scope that contains the function definition. ' + 'The\n' + 'result must be a callable, which is invoked with the function ' + 'object\n' + 'as the only argument. The returned value is bound to the ' + 'function name\n' + 'instead of the function object. Multiple decorators are applied ' + 'in\n' + 'nested fashion. For example, the following code\n' + '\n' + ' @f1(arg)\n' + ' @f2\n' + ' def func(): pass\n' + '\n' + 'is equivalent to\n' + '\n' + ' def func(): pass\n' + ' func = f1(arg)(f2(func))\n' + '\n' + 'When one or more *parameters* have the form *parameter* "="\n' + '*expression*, the function is said to have "default parameter ' + 'values."\n' + 'For a parameter with a default value, the corresponding ' + '*argument* may\n' + "be omitted from a call, in which case the parameter's default " + 'value is\n' + 'substituted. If a parameter has a default value, all following\n' + 'parameters up until the ""*"" must also have a default value --- ' + 'this\n' + 'is a syntactic restriction that is not expressed by the ' + 'grammar.\n' + '\n' + '**Default parameter values are evaluated from left to right when ' + 'the\n' + 'function definition is executed.** This means that the ' + 'expression is\n' + 'evaluated once, when the function is defined, and that the same ' + '"pre-\n' + 'computed" value is used for each call. This is especially ' + 'important\n' + 'to understand when a default parameter is a mutable object, such ' + 'as a\n' + 'list or a dictionary: if the function modifies the object (e.g. ' + 'by\n' + 'appending an item to a list), the default value is in effect ' + 'modified.\n' + 'This is generally not what was intended. A way around this is ' + 'to use\n' + '"None" as the default, and explicitly test for it in the body of ' + 'the\n' + 'function, e.g.:\n' + '\n' + ' def whats_on_the_telly(penguin=None):\n' + ' if penguin is None:\n' + ' penguin = []\n' + ' penguin.append("property of the zoo")\n' + ' return penguin\n' + '\n' + 'Function call semantics are described in more detail in section ' + 'Calls.\n' + 'A function call always assigns values to all parameters ' + 'mentioned in\n' + 'the parameter list, either from position arguments, from ' + 'keyword\n' + 'arguments, or from default values. If the form ""*identifier"" ' + 'is\n' + 'present, it is initialized to a tuple receiving any excess ' + 'positional\n' + 'parameters, defaulting to the empty tuple. If the form\n' + '""**identifier"" is present, it is initialized to a new ' + 'dictionary\n' + 'receiving any excess keyword arguments, defaulting to a new ' + 'empty\n' + 'dictionary. Parameters after ""*"" or ""*identifier"" are ' + 'keyword-only\n' + 'parameters and may only be passed used keyword arguments.\n' + '\n' + 'Parameters may have annotations of the form "": expression"" ' + 'following\n' + 'the parameter name. Any parameter may have an annotation even ' + 'those\n' + 'of the form "*identifier" or "**identifier". Functions may ' + 'have\n' + '"return" annotation of the form ""-> expression"" after the ' + 'parameter\n' + 'list. These annotations can be any valid Python expression and ' + 'are\n' + 'evaluated when the function definition is executed. Annotations ' + 'may\n' + 'be evaluated in a different order than they appear in the source ' + 'code.\n' + 'The presence of annotations does not change the semantics of a\n' + 'function. The annotation values are available as values of a\n' + "dictionary keyed by the parameters' names in the " + '"__annotations__"\n' + 'attribute of the function object.\n' + '\n' + 'It is also possible to create anonymous functions (functions not ' + 'bound\n' + 'to a name), for immediate use in expressions. This uses lambda\n' + 'expressions, described in section Lambdas. Note that the ' + 'lambda\n' + 'expression is merely a shorthand for a simplified function ' + 'definition;\n' + 'a function defined in a ""def"" statement can be passed around ' + 'or\n' + 'assigned to another name just like a function defined by a ' + 'lambda\n' + 'expression. The ""def"" form is actually more powerful since ' + 'it\n' + 'allows the execution of multiple statements and annotations.\n' + '\n' + "**Programmer's note:** Functions are first-class objects. A " + '""def""\n' + 'statement executed inside a function definition defines a local\n' + 'function that can be returned or passed around. Free variables ' + 'used\n' + 'in the nested function can access the local variables of the ' + 'function\n' + 'containing the def. See section Naming and binding for ' + 'details.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 3107** - Function Annotations\n' + ' The original specification for function annotations.\n', + 'global': '\n' + 'The "global" statement\n' + '**********************\n' + '\n' + ' global_stmt ::= "global" identifier ("," identifier)*\n' + '\n' + 'The "global" statement is a declaration which holds for the ' + 'entire\n' + 'current code block. It means that the listed identifiers are to ' + 'be\n' + 'interpreted as globals. It would be impossible to assign to a ' + 'global\n' + 'variable without "global", although free variables may refer to\n' + 'globals without being declared global.\n' + '\n' + 'Names listed in a "global" statement must not be used in the same ' + 'code\n' + 'block textually preceding that "global" statement.\n' + '\n' + 'Names listed in a "global" statement must not be defined as ' + 'formal\n' + 'parameters or in a "for" loop control target, "class" definition,\n' + 'function definition, or "import" statement.\n' + '\n' + '**CPython implementation detail:** The current implementation does ' + 'not\n' + 'enforce the two restrictions, but programs should not abuse this\n' + 'freedom, as future implementations may enforce them or silently ' + 'change\n' + 'the meaning of the program.\n' + '\n' + '**Programmer\'s note:** the "global" is a directive to the ' + 'parser. It\n' + 'applies only to code parsed at the same time as the "global"\n' + 'statement. In particular, a "global" statement contained in a ' + 'string\n' + 'or code object supplied to the built-in "exec()" function does ' + 'not\n' + 'affect the code block *containing* the function call, and code\n' + 'contained in such a string is unaffected by "global" statements in ' + 'the\n' + 'code containing the function call. The same applies to the ' + '"eval()"\n' + 'and "compile()" functions.\n', + 'id-classes': '\n' + 'Reserved classes of identifiers\n' + '*******************************\n' + '\n' + 'Certain classes of identifiers (besides keywords) have ' + 'special\n' + 'meanings. These classes are identified by the patterns of ' + 'leading and\n' + 'trailing underscore characters:\n' + '\n' + '"_*"\n' + ' Not imported by "from module import *". The special ' + 'identifier "_"\n' + ' is used in the interactive interpreter to store the result ' + 'of the\n' + ' last evaluation; it is stored in the "builtins" module. ' + 'When not\n' + ' in interactive mode, "_" has no special meaning and is not ' + 'defined.\n' + ' See section The import statement.\n' + '\n' + ' Note: The name "_" is often used in conjunction with\n' + ' internationalization; refer to the documentation for the\n' + ' "gettext" module for more information on this ' + 'convention.\n' + '\n' + '"__*__"\n' + ' System-defined names. These names are defined by the ' + 'interpreter\n' + ' and its implementation (including the standard library). ' + 'Current\n' + ' system names are discussed in the Special method names ' + 'section and\n' + ' elsewhere. More will likely be defined in future versions ' + 'of\n' + ' Python. *Any* use of "__*__" names, in any context, that ' + 'does not\n' + ' follow explicitly documented use, is subject to breakage ' + 'without\n' + ' warning.\n' + '\n' + '"__*"\n' + ' Class-private names. Names in this category, when used ' + 'within the\n' + ' context of a class definition, are re-written to use a ' + 'mangled form\n' + ' to help avoid name clashes between "private" attributes of ' + 'base and\n' + ' derived classes. See section Identifiers (Names).\n', + 'identifiers': '\n' + 'Identifiers and keywords\n' + '************************\n' + '\n' + 'Identifiers (also referred to as *names*) are described by ' + 'the\n' + 'following lexical definitions.\n' + '\n' + 'The syntax of identifiers in Python is based on the Unicode ' + 'standard\n' + 'annex UAX-31, with elaboration and changes as defined below; ' + 'see also\n' + '**PEP 3131** for further details.\n' + '\n' + 'Within the ASCII range (U+0001..U+007F), the valid characters ' + 'for\n' + 'identifiers are the same as in Python 2.x: the uppercase and ' + 'lowercase\n' + 'letters "A" through "Z", the underscore "_" and, except for ' + 'the first\n' + 'character, the digits "0" through "9".\n' + '\n' + 'Python 3.0 introduces additional characters from outside the ' + 'ASCII\n' + 'range (see **PEP 3131**). For these characters, the ' + 'classification\n' + 'uses the version of the Unicode Character Database as ' + 'included in the\n' + '"unicodedata" module.\n' + '\n' + 'Identifiers are unlimited in length. Case is significant.\n' + '\n' + ' identifier ::= xid_start xid_continue*\n' + ' id_start ::= \n' + ' id_continue ::= \n' + ' xid_start ::= \n' + ' xid_continue ::= \n' + '\n' + 'The Unicode category codes mentioned above stand for:\n' + '\n' + '* *Lu* - uppercase letters\n' + '\n' + '* *Ll* - lowercase letters\n' + '\n' + '* *Lt* - titlecase letters\n' + '\n' + '* *Lm* - modifier letters\n' + '\n' + '* *Lo* - other letters\n' + '\n' + '* *Nl* - letter numbers\n' + '\n' + '* *Mn* - nonspacing marks\n' + '\n' + '* *Mc* - spacing combining marks\n' + '\n' + '* *Nd* - decimal numbers\n' + '\n' + '* *Pc* - connector punctuations\n' + '\n' + '* *Other_ID_Start* - explicit list of characters in ' + 'PropList.txt to\n' + ' support backwards compatibility\n' + '\n' + '* *Other_ID_Continue* - likewise\n' + '\n' + 'All identifiers are converted into the normal form NFKC while ' + 'parsing;\n' + 'comparison of identifiers is based on NFKC.\n' + '\n' + 'A non-normative HTML file listing all valid identifier ' + 'characters for\n' + 'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n' + 'potsdam.de/home/loewis/table-3131.html.\n' + '\n' + '\n' + 'Keywords\n' + '========\n' + '\n' + 'The following identifiers are used as reserved words, or ' + '*keywords* of\n' + 'the language, and cannot be used as ordinary identifiers. ' + 'They must\n' + 'be spelled exactly as written here:\n' + '\n' + ' False class finally is return\n' + ' None continue for lambda try\n' + ' True def from nonlocal while\n' + ' and del global not with\n' + ' as elif if or yield\n' + ' assert else import pass\n' + ' break except in raise\n' + '\n' + '\n' + 'Reserved classes of identifiers\n' + '===============================\n' + '\n' + 'Certain classes of identifiers (besides keywords) have ' + 'special\n' + 'meanings. These classes are identified by the patterns of ' + 'leading and\n' + 'trailing underscore characters:\n' + '\n' + '"_*"\n' + ' Not imported by "from module import *". The special ' + 'identifier "_"\n' + ' is used in the interactive interpreter to store the result ' + 'of the\n' + ' last evaluation; it is stored in the "builtins" module. ' + 'When not\n' + ' in interactive mode, "_" has no special meaning and is not ' + 'defined.\n' + ' See section The import statement.\n' + '\n' + ' Note: The name "_" is often used in conjunction with\n' + ' internationalization; refer to the documentation for ' + 'the\n' + ' "gettext" module for more information on this ' + 'convention.\n' + '\n' + '"__*__"\n' + ' System-defined names. These names are defined by the ' + 'interpreter\n' + ' and its implementation (including the standard library). ' + 'Current\n' + ' system names are discussed in the Special method names ' + 'section and\n' + ' elsewhere. More will likely be defined in future versions ' + 'of\n' + ' Python. *Any* use of "__*__" names, in any context, that ' + 'does not\n' + ' follow explicitly documented use, is subject to breakage ' + 'without\n' + ' warning.\n' + '\n' + '"__*"\n' + ' Class-private names. Names in this category, when used ' + 'within the\n' + ' context of a class definition, are re-written to use a ' + 'mangled form\n' + ' to help avoid name clashes between "private" attributes of ' + 'base and\n' + ' derived classes. See section Identifiers (Names).\n', + 'if': '\n' + 'The "if" statement\n' + '******************\n' + '\n' + 'The "if" statement is used for conditional execution:\n' + '\n' + ' if_stmt ::= "if" expression ":" suite\n' + ' ( "elif" expression ":" suite )*\n' + ' ["else" ":" suite]\n' + '\n' + 'It selects exactly one of the suites by evaluating the expressions ' + 'one\n' + 'by one until one is found to be true (see section Boolean operations\n' + 'for the definition of true and false); then that suite is executed\n' + '(and no other part of the "if" statement is executed or evaluated).\n' + 'If all expressions are false, the suite of the "else" clause, if\n' + 'present, is executed.\n', + 'imaginary': '\n' + 'Imaginary literals\n' + '******************\n' + '\n' + 'Imaginary literals are described by the following lexical ' + 'definitions:\n' + '\n' + ' imagnumber ::= (floatnumber | intpart) ("j" | "J")\n' + '\n' + 'An imaginary literal yields a complex number with a real part ' + 'of 0.0.\n' + 'Complex numbers are represented as a pair of floating point ' + 'numbers\n' + 'and have the same restrictions on their range. To create a ' + 'complex\n' + 'number with a nonzero real part, add a floating point number to ' + 'it,\n' + 'e.g., "(3+4j)". Some examples of imaginary literals:\n' + '\n' + ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j\n', + 'import': '\n' + 'The "import" statement\n' + '**********************\n' + '\n' + ' import_stmt ::= "import" module ["as" name] ( "," module ' + '["as" name] )*\n' + ' | "from" relative_module "import" identifier ' + '["as" name]\n' + ' ( "," identifier ["as" name] )*\n' + ' | "from" relative_module "import" "(" ' + 'identifier ["as" name]\n' + ' ( "," identifier ["as" name] )* [","] ")"\n' + ' | "from" module "import" "*"\n' + ' module ::= (identifier ".")* identifier\n' + ' relative_module ::= "."* module | "."+\n' + ' name ::= identifier\n' + '\n' + 'The basic import statement (no "from" clause) is executed in two\n' + 'steps:\n' + '\n' + '1. find a module, loading and initializing it if necessary\n' + '\n' + '2. define a name or names in the local namespace for the scope\n' + ' where the "import" statement occurs.\n' + '\n' + 'When the statement contains multiple clauses (separated by commas) ' + 'the\n' + 'two steps are carried out separately for each clause, just as ' + 'though\n' + 'the clauses had been separated out into individiual import ' + 'statements.\n' + '\n' + 'The details of the first step, finding and loading modules are\n' + 'described in greater detail in the section on the import system, ' + 'which\n' + 'also describes the various types of packages and modules that can ' + 'be\n' + 'imported, as well as all the hooks that can be used to customize ' + 'the\n' + 'import system. Note that failures in this step may indicate ' + 'either\n' + 'that the module could not be located, *or* that an error occurred\n' + 'while initializing the module, which includes execution of the\n' + "module's code.\n" + '\n' + 'If the requested module is retrieved successfully, it will be ' + 'made\n' + 'available in the local namespace in one of three ways:\n' + '\n' + '* If the module name is followed by "as", then the name following\n' + ' "as" is bound directly to the imported module.\n' + '\n' + '* If no other name is specified, and the module being imported is ' + 'a\n' + " top level module, the module's name is bound in the local " + 'namespace\n' + ' as a reference to the imported module\n' + '\n' + '* If the module being imported is *not* a top level module, then ' + 'the\n' + ' name of the top level package that contains the module is bound ' + 'in\n' + ' the local namespace as a reference to the top level package. ' + 'The\n' + ' imported module must be accessed using its full qualified name\n' + ' rather than directly\n' + '\n' + 'The "from" form uses a slightly more complex process:\n' + '\n' + '1. find the module specified in the "from" clause, loading and\n' + ' initializing it if necessary;\n' + '\n' + '2. for each of the identifiers specified in the "import" clauses:\n' + '\n' + ' 1. check if the imported module has an attribute by that name\n' + '\n' + ' 2. if not, attempt to import a submodule with that name and ' + 'then\n' + ' check the imported module again for that attribute\n' + '\n' + ' 3. if the attribute is not found, "ImportError" is raised.\n' + '\n' + ' 4. otherwise, a reference to that value is stored in the local\n' + ' namespace, using the name in the "as" clause if it is ' + 'present,\n' + ' otherwise using the attribute name\n' + '\n' + 'Examples:\n' + '\n' + ' import foo # foo imported and bound locally\n' + ' import foo.bar.baz # foo.bar.baz imported, foo bound ' + 'locally\n' + ' import foo.bar.baz as fbb # foo.bar.baz imported and bound as ' + 'fbb\n' + ' from foo.bar import baz # foo.bar.baz imported and bound as ' + 'baz\n' + ' from foo import attr # foo imported and foo.attr bound as ' + 'attr\n' + '\n' + 'If the list of identifiers is replaced by a star ("\'*\'"), all ' + 'public\n' + 'names defined in the module are bound in the local namespace for ' + 'the\n' + 'scope where the "import" statement occurs.\n' + '\n' + 'The *public names* defined by a module are determined by checking ' + 'the\n' + 'module\'s namespace for a variable named "__all__"; if defined, it ' + 'must\n' + 'be a sequence of strings which are names defined or imported by ' + 'that\n' + 'module. The names given in "__all__" are all considered public ' + 'and\n' + 'are required to exist. If "__all__" is not defined, the set of ' + 'public\n' + "names includes all names found in the module's namespace which do " + 'not\n' + 'begin with an underscore character ("\'_\'"). "__all__" should ' + 'contain\n' + 'the entire public API. It is intended to avoid accidentally ' + 'exporting\n' + 'items that are not part of the API (such as library modules which ' + 'were\n' + 'imported and used within the module).\n' + '\n' + 'The wild card form of import --- "from module import *" --- is ' + 'only\n' + 'allowed at the module level. Attempting to use it in class or\n' + 'function definitions will raise a "SyntaxError".\n' + '\n' + 'When specifying what module to import you do not have to specify ' + 'the\n' + 'absolute name of the module. When a module or package is ' + 'contained\n' + 'within another package it is possible to make a relative import ' + 'within\n' + 'the same top package without having to mention the package name. ' + 'By\n' + 'using leading dots in the specified module or package after "from" ' + 'you\n' + 'can specify how high to traverse up the current package hierarchy\n' + 'without specifying exact names. One leading dot means the current\n' + 'package where the module making the import exists. Two dots means ' + 'up\n' + 'one package level. Three dots is up two levels, etc. So if you ' + 'execute\n' + '"from . import mod" from a module in the "pkg" package then you ' + 'will\n' + 'end up importing "pkg.mod". If you execute "from ..subpkg2 import ' + 'mod"\n' + 'from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The\n' + 'specification for relative imports is contained within **PEP ' + '328**.\n' + '\n' + '"importlib.import_module()" is provided to support applications ' + 'that\n' + 'determine dynamically the modules to be loaded.\n' + '\n' + '\n' + 'Future statements\n' + '=================\n' + '\n' + 'A *future statement* is a directive to the compiler that a ' + 'particular\n' + 'module should be compiled using syntax or semantics that will be\n' + 'available in a specified future release of Python where the ' + 'feature\n' + 'becomes standard.\n' + '\n' + 'The future statement is intended to ease migration to future ' + 'versions\n' + 'of Python that introduce incompatible changes to the language. ' + 'It\n' + 'allows use of the new features on a per-module basis before the\n' + 'release in which the feature becomes standard.\n' + '\n' + ' future_statement ::= "from" "__future__" "import" feature ["as" ' + 'name]\n' + ' ("," feature ["as" name])*\n' + ' | "from" "__future__" "import" "(" feature ' + '["as" name]\n' + ' ("," feature ["as" name])* [","] ")"\n' + ' feature ::= identifier\n' + ' name ::= identifier\n' + '\n' + 'A future statement must appear near the top of the module. The ' + 'only\n' + 'lines that can appear before a future statement are:\n' + '\n' + '* the module docstring (if any),\n' + '\n' + '* comments,\n' + '\n' + '* blank lines, and\n' + '\n' + '* other future statements.\n' + '\n' + 'The features recognized by Python 3.0 are "absolute_import",\n' + '"division", "generators", "unicode_literals", "print_function",\n' + '"nested_scopes" and "with_statement". They are all redundant ' + 'because\n' + 'they are always enabled, and only kept for backwards ' + 'compatibility.\n' + '\n' + 'A future statement is recognized and treated specially at compile\n' + 'time: Changes to the semantics of core constructs are often\n' + 'implemented by generating different code. It may even be the ' + 'case\n' + 'that a new feature introduces new incompatible syntax (such as a ' + 'new\n' + 'reserved word), in which case the compiler may need to parse the\n' + 'module differently. Such decisions cannot be pushed off until\n' + 'runtime.\n' + '\n' + 'For any given release, the compiler knows which feature names ' + 'have\n' + 'been defined, and raises a compile-time error if a future ' + 'statement\n' + 'contains a feature not known to it.\n' + '\n' + 'The direct runtime semantics are the same as for any import ' + 'statement:\n' + 'there is a standard module "__future__", described later, and it ' + 'will\n' + 'be imported in the usual way at the time the future statement is\n' + 'executed.\n' + '\n' + 'The interesting runtime semantics depend on the specific feature\n' + 'enabled by the future statement.\n' + '\n' + 'Note that there is nothing special about the statement:\n' + '\n' + ' import __future__ [as name]\n' + '\n' + "That is not a future statement; it's an ordinary import statement " + 'with\n' + 'no special semantics or syntax restrictions.\n' + '\n' + 'Code compiled by calls to the built-in functions "exec()" and\n' + '"compile()" that occur in a module "M" containing a future ' + 'statement\n' + 'will, by default, use the new syntax or semantics associated with ' + 'the\n' + 'future statement. This can be controlled by optional arguments ' + 'to\n' + '"compile()" --- see the documentation of that function for ' + 'details.\n' + '\n' + 'A future statement typed at an interactive interpreter prompt ' + 'will\n' + 'take effect for the rest of the interpreter session. If an\n' + 'interpreter is started with the "-i" option, is passed a script ' + 'name\n' + 'to execute, and the script includes a future statement, it will be ' + 'in\n' + 'effect in the interactive session started after the script is\n' + 'executed.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 236** - Back to the __future__\n' + ' The original proposal for the __future__ mechanism.\n', + 'in': '\n' + 'Membership test operations\n' + '**************************\n' + '\n' + 'The operators "in" and "not in" test for membership. "x in s"\n' + 'evaluates to true if *x* is a member of *s*, and false otherwise. "x\n' + 'not in s" returns the negation of "x in s". All built-in sequences\n' + 'and set types support this as well as dictionary, for which "in" ' + 'tests\n' + 'whether the dictionary has a given key. For container types such as\n' + 'list, tuple, set, frozenset, dict, or collections.deque, the\n' + 'expression "x in y" is equivalent to "any(x is e or x == e for e in\n' + 'y)".\n' + '\n' + 'For the string and bytes types, "x in y" is true if and only if *x* ' + 'is\n' + 'a substring of *y*. An equivalent test is "y.find(x) != -1". Empty\n' + 'strings are always considered to be a substring of any other string,\n' + 'so """ in "abc"" will return "True".\n' + '\n' + 'For user-defined classes which define the "__contains__()" method, "x\n' + 'in y" is true if and only if "y.__contains__(x)" is true.\n' + '\n' + 'For user-defined classes which do not define "__contains__()" but do\n' + 'define "__iter__()", "x in y" is true if some value "z" with "x == z"\n' + 'is produced while iterating over "y". If an exception is raised\n' + 'during the iteration, it is as if "in" raised that exception.\n' + '\n' + 'Lastly, the old-style iteration protocol is tried: if a class defines\n' + '"__getitem__()", "x in y" is true if and only if there is a non-\n' + 'negative integer index *i* such that "x == y[i]", and all lower\n' + 'integer indices do not raise "IndexError" exception. (If any other\n' + 'exception is raised, it is as if "in" raised that exception).\n' + '\n' + 'The operator "not in" is defined to have the inverse true value of\n' + '"in".\n', + 'integers': '\n' + 'Integer literals\n' + '****************\n' + '\n' + 'Integer literals are described by the following lexical ' + 'definitions:\n' + '\n' + ' integer ::= decimalinteger | octinteger | hexinteger | ' + 'bininteger\n' + ' decimalinteger ::= nonzerodigit digit* | "0"+\n' + ' nonzerodigit ::= "1"..."9"\n' + ' digit ::= "0"..."9"\n' + ' octinteger ::= "0" ("o" | "O") octdigit+\n' + ' hexinteger ::= "0" ("x" | "X") hexdigit+\n' + ' bininteger ::= "0" ("b" | "B") bindigit+\n' + ' octdigit ::= "0"..."7"\n' + ' hexdigit ::= digit | "a"..."f" | "A"..."F"\n' + ' bindigit ::= "0" | "1"\n' + '\n' + 'There is no limit for the length of integer literals apart from ' + 'what\n' + 'can be stored in available memory.\n' + '\n' + 'Note that leading zeros in a non-zero decimal number are not ' + 'allowed.\n' + 'This is for disambiguation with C-style octal literals, which ' + 'Python\n' + 'used before version 3.0.\n' + '\n' + 'Some examples of integer literals:\n' + '\n' + ' 7 2147483647 0o177 0b100110111\n' + ' 3 79228162514264337593543950336 0o377 0xdeadbeef\n', + 'lambda': '\n' + 'Lambdas\n' + '*******\n' + '\n' + ' lambda_expr ::= "lambda" [parameter_list]: expression\n' + ' lambda_expr_nocond ::= "lambda" [parameter_list]: ' + 'expression_nocond\n' + '\n' + 'Lambda expressions (sometimes called lambda forms) are used to ' + 'create\n' + 'anonymous functions. The expression "lambda arguments: ' + 'expression"\n' + 'yields a function object. The unnamed object behaves like a ' + 'function\n' + 'object defined with\n' + '\n' + ' def (arguments):\n' + ' return expression\n' + '\n' + 'See section Function definitions for the syntax of parameter ' + 'lists.\n' + 'Note that functions created with lambda expressions cannot ' + 'contain\n' + 'statements or annotations.\n', + 'lists': '\n' + 'List displays\n' + '*************\n' + '\n' + 'A list display is a possibly empty series of expressions enclosed ' + 'in\n' + 'square brackets:\n' + '\n' + ' list_display ::= "[" [expression_list | comprehension] "]"\n' + '\n' + 'A list display yields a new list object, the contents being ' + 'specified\n' + 'by either a list of expressions or a comprehension. When a comma-\n' + 'separated list of expressions is supplied, its elements are ' + 'evaluated\n' + 'from left to right and placed into the list object in that order.\n' + 'When a comprehension is supplied, the list is constructed from the\n' + 'elements resulting from the comprehension.\n', + 'naming': '\n' + 'Naming and binding\n' + '******************\n' + '\n' + '\n' + 'Binding of names\n' + '================\n' + '\n' + '*Names* refer to objects. Names are introduced by name binding\n' + 'operations.\n' + '\n' + 'The following constructs bind names: formal parameters to ' + 'functions,\n' + '"import" statements, class and function definitions (these bind ' + 'the\n' + 'class or function name in the defining block), and targets that ' + 'are\n' + 'identifiers if occurring in an assignment, "for" loop header, or ' + 'after\n' + '"as" in a "with" statement or "except" clause. The "import" ' + 'statement\n' + 'of the form "from ... import *" binds all names defined in the\n' + 'imported module, except those beginning with an underscore. This ' + 'form\n' + 'may only be used at the module level.\n' + '\n' + 'A target occurring in a "del" statement is also considered bound ' + 'for\n' + 'this purpose (though the actual semantics are to unbind the ' + 'name).\n' + '\n' + 'Each assignment or import statement occurs within a block defined ' + 'by a\n' + 'class or function definition or at the module level (the ' + 'top-level\n' + 'code block).\n' + '\n' + 'If a name is bound in a block, it is a local variable of that ' + 'block,\n' + 'unless declared as "nonlocal" or "global". If a name is bound at ' + 'the\n' + 'module level, it is a global variable. (The variables of the ' + 'module\n' + 'code block are local and global.) If a variable is used in a ' + 'code\n' + 'block but not defined there, it is a *free variable*.\n' + '\n' + 'Each occurrence of a name in the program text refers to the ' + '*binding*\n' + 'of that name established by the following name resolution rules.\n' + '\n' + '\n' + 'Resolution of names\n' + '===================\n' + '\n' + 'A *scope* defines the visibility of a name within a block. If a ' + 'local\n' + 'variable is defined in a block, its scope includes that block. If ' + 'the\n' + 'definition occurs in a function block, the scope extends to any ' + 'blocks\n' + 'contained within the defining one, unless a contained block ' + 'introduces\n' + 'a different binding for the name.\n' + '\n' + 'When a name is used in a code block, it is resolved using the ' + 'nearest\n' + 'enclosing scope. The set of all such scopes visible to a code ' + 'block\n' + "is called the block's *environment*.\n" + '\n' + 'When a name is not found at all, a "NameError" exception is ' + 'raised. If\n' + 'the current scope is a function scope, and the name refers to a ' + 'local\n' + 'variable that has not yet been bound to a value at the point where ' + 'the\n' + 'name is used, an "UnboundLocalError" exception is raised.\n' + '"UnboundLocalError" is a subclass of "NameError".\n' + '\n' + 'If a name binding operation occurs anywhere within a code block, ' + 'all\n' + 'uses of the name within the block are treated as references to ' + 'the\n' + 'current block. This can lead to errors when a name is used within ' + 'a\n' + 'block before it is bound. This rule is subtle. Python lacks\n' + 'declarations and allows name binding operations to occur anywhere\n' + 'within a code block. The local variables of a code block can be\n' + 'determined by scanning the entire text of the block for name ' + 'binding\n' + 'operations.\n' + '\n' + 'If the "global" statement occurs within a block, all uses of the ' + 'name\n' + 'specified in the statement refer to the binding of that name in ' + 'the\n' + 'top-level namespace. Names are resolved in the top-level ' + 'namespace by\n' + 'searching the global namespace, i.e. the namespace of the module\n' + 'containing the code block, and the builtins namespace, the ' + 'namespace\n' + 'of the module "builtins". The global namespace is searched ' + 'first. If\n' + 'the name is not found there, the builtins namespace is searched. ' + 'The\n' + '"global" statement must precede all uses of the name.\n' + '\n' + 'The "global" statement has the same scope as a name binding ' + 'operation\n' + 'in the same block. If the nearest enclosing scope for a free ' + 'variable\n' + 'contains a global statement, the free variable is treated as a ' + 'global.\n' + '\n' + 'The "nonlocal" statement causes corresponding names to refer to\n' + 'previously bound variables in the nearest enclosing function ' + 'scope.\n' + '"SyntaxError" is raised at compile time if the given name does ' + 'not\n' + 'exist in any enclosing function scope.\n' + '\n' + 'The namespace for a module is automatically created the first time ' + 'a\n' + 'module is imported. The main module for a script is always ' + 'called\n' + '"__main__".\n' + '\n' + 'Class definition blocks and arguments to "exec()" and "eval()" ' + 'are\n' + 'special in the context of name resolution. A class definition is ' + 'an\n' + 'executable statement that may use and define names. These ' + 'references\n' + 'follow the normal rules for name resolution with an exception ' + 'that\n' + 'unbound local variables are looked up in the global namespace. ' + 'The\n' + 'namespace of the class definition becomes the attribute dictionary ' + 'of\n' + 'the class. The scope of names defined in a class block is limited ' + 'to\n' + 'the class block; it does not extend to the code blocks of methods ' + '--\n' + 'this includes comprehensions and generator expressions since they ' + 'are\n' + 'implemented using a function scope. This means that the ' + 'following\n' + 'will fail:\n' + '\n' + ' class A:\n' + ' a = 42\n' + ' b = list(a + i for i in range(10))\n' + '\n' + '\n' + 'Builtins and restricted execution\n' + '=================================\n' + '\n' + 'The builtins namespace associated with the execution of a code ' + 'block\n' + 'is actually found by looking up the name "__builtins__" in its ' + 'global\n' + 'namespace; this should be a dictionary or a module (in the latter ' + 'case\n' + "the module's dictionary is used). By default, when in the " + '"__main__"\n' + 'module, "__builtins__" is the built-in module "builtins"; when in ' + 'any\n' + 'other module, "__builtins__" is an alias for the dictionary of ' + 'the\n' + '"builtins" module itself. "__builtins__" can be set to a ' + 'user-created\n' + 'dictionary to create a weak form of restricted execution.\n' + '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' + '\n' + 'Interaction with dynamic features\n' + '=================================\n' + '\n' + 'Name resolution of free variables occurs at runtime, not at ' + 'compile\n' + 'time. This means that the following code will print 42:\n' + '\n' + ' i = 10\n' + ' def f():\n' + ' print(i)\n' + ' i = 42\n' + ' f()\n' + '\n' + 'There are several cases where Python statements are illegal when ' + 'used\n' + 'in conjunction with nested scopes that contain free variables.\n' + '\n' + 'If a variable is referenced in an enclosing scope, it is illegal ' + 'to\n' + 'delete the name. An error will be reported at compile time.\n' + '\n' + 'The "eval()" and "exec()" functions do not have access to the ' + 'full\n' + 'environment for resolving names. Names may be resolved in the ' + 'local\n' + 'and global namespaces of the caller. Free variables are not ' + 'resolved\n' + 'in the nearest enclosing namespace, but in the global namespace. ' + '[1]\n' + 'The "exec()" and "eval()" functions have optional arguments to\n' + 'override the global and local namespace. If only one namespace ' + 'is\n' + 'specified, it is used for both.\n', + 'nonlocal': '\n' + 'The "nonlocal" statement\n' + '************************\n' + '\n' + ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' + '\n' + 'The "nonlocal" statement causes the listed identifiers to refer ' + 'to\n' + 'previously bound variables in the nearest enclosing scope ' + 'excluding\n' + 'globals. This is important because the default behavior for ' + 'binding is\n' + 'to search the local namespace first. The statement allows\n' + 'encapsulated code to rebind variables outside of the local ' + 'scope\n' + 'besides the global (module) scope.\n' + '\n' + 'Names listed in a "nonlocal" statement, unlike those listed in ' + 'a\n' + '"global" statement, must refer to pre-existing bindings in an\n' + 'enclosing scope (the scope in which a new binding should be ' + 'created\n' + 'cannot be determined unambiguously).\n' + '\n' + 'Names listed in a "nonlocal" statement must not collide with ' + 'pre-\n' + 'existing bindings in the local scope.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 3104** - Access to Names in Outer Scopes\n' + ' The specification for the "nonlocal" statement.\n', + 'numbers': '\n' + 'Numeric literals\n' + '****************\n' + '\n' + 'There are three types of numeric literals: integers, floating ' + 'point\n' + 'numbers, and imaginary numbers. There are no complex literals\n' + '(complex numbers can be formed by adding a real number and an\n' + 'imaginary number).\n' + '\n' + 'Note that numeric literals do not include a sign; a phrase like ' + '"-1"\n' + 'is actually an expression composed of the unary operator \'"-"\' ' + 'and the\n' + 'literal "1".\n', + 'numeric-types': '\n' + 'Emulating numeric types\n' + '***********************\n' + '\n' + 'The following methods can be defined to emulate numeric ' + 'objects.\n' + 'Methods corresponding to operations that are not supported ' + 'by the\n' + 'particular kind of number implemented (e.g., bitwise ' + 'operations for\n' + 'non-integral numbers) should be left undefined.\n' + '\n' + 'object.__add__(self, other)\n' + 'object.__sub__(self, other)\n' + 'object.__mul__(self, other)\n' + 'object.__matmul__(self, other)\n' + 'object.__truediv__(self, other)\n' + 'object.__floordiv__(self, other)\n' + 'object.__mod__(self, other)\n' + 'object.__divmod__(self, other)\n' + 'object.__pow__(self, other[, modulo])\n' + 'object.__lshift__(self, other)\n' + 'object.__rshift__(self, other)\n' + 'object.__and__(self, other)\n' + 'object.__xor__(self, other)\n' + 'object.__or__(self, other)\n' + '\n' + ' These methods are called to implement the binary ' + 'arithmetic\n' + ' operations ("+", "-", "*", "@", "/", "//", "%", ' + '"divmod()",\n' + ' "pow()", "**", "<<", ">>", "&", "^", "|"). For ' + 'instance, to\n' + ' evaluate the expression "x + y", where *x* is an ' + 'instance of a\n' + ' class that has an "__add__()" method, "x.__add__(y)" is ' + 'called.\n' + ' The "__divmod__()" method should be the equivalent to ' + 'using\n' + ' "__floordiv__()" and "__mod__()"; it should not be ' + 'related to\n' + ' "__truediv__()". Note that "__pow__()" should be ' + 'defined to accept\n' + ' an optional third argument if the ternary version of the ' + 'built-in\n' + ' "pow()" function is to be supported.\n' + '\n' + ' If one of those methods does not support the operation ' + 'with the\n' + ' supplied arguments, it should return "NotImplemented".\n' + '\n' + 'object.__radd__(self, other)\n' + 'object.__rsub__(self, other)\n' + 'object.__rmul__(self, other)\n' + 'object.__rmatmul__(self, other)\n' + 'object.__rtruediv__(self, other)\n' + 'object.__rfloordiv__(self, other)\n' + 'object.__rmod__(self, other)\n' + 'object.__rdivmod__(self, other)\n' + 'object.__rpow__(self, other)\n' + 'object.__rlshift__(self, other)\n' + 'object.__rrshift__(self, other)\n' + 'object.__rand__(self, other)\n' + 'object.__rxor__(self, other)\n' + 'object.__ror__(self, other)\n' + '\n' + ' These methods are called to implement the binary ' + 'arithmetic\n' + ' operations ("+", "-", "*", "@", "/", "//", "%", ' + '"divmod()",\n' + ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' + '(swapped)\n' + ' operands. These functions are only called if the left ' + 'operand does\n' + ' not support the corresponding operation and the operands ' + 'are of\n' + ' different types. [2] For instance, to evaluate the ' + 'expression "x -\n' + ' y", where *y* is an instance of a class that has an ' + '"__rsub__()"\n' + ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' + 'returns\n' + ' *NotImplemented*.\n' + '\n' + ' Note that ternary "pow()" will not try calling ' + '"__rpow__()" (the\n' + ' coercion rules would become too complicated).\n' + '\n' + " Note: If the right operand's type is a subclass of the " + 'left\n' + " operand's type and that subclass provides the " + 'reflected method\n' + ' for the operation, this method will be called before ' + 'the left\n' + " operand's non-reflected method. This behavior allows " + 'subclasses\n' + " to override their ancestors' operations.\n" + '\n' + 'object.__iadd__(self, other)\n' + 'object.__isub__(self, other)\n' + 'object.__imul__(self, other)\n' + 'object.__imatmul__(self, other)\n' + 'object.__itruediv__(self, other)\n' + 'object.__ifloordiv__(self, other)\n' + 'object.__imod__(self, other)\n' + 'object.__ipow__(self, other[, modulo])\n' + 'object.__ilshift__(self, other)\n' + 'object.__irshift__(self, other)\n' + 'object.__iand__(self, other)\n' + 'object.__ixor__(self, other)\n' + 'object.__ior__(self, other)\n' + '\n' + ' These methods are called to implement the augmented ' + 'arithmetic\n' + ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' + '"**=",\n' + ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' + 'attempt to\n' + ' do the operation in-place (modifying *self*) and return ' + 'the result\n' + ' (which could be, but does not have to be, *self*). If a ' + 'specific\n' + ' method is not defined, the augmented assignment falls ' + 'back to the\n' + ' normal methods. For instance, if *x* is an instance of ' + 'a class\n' + ' with an "__iadd__()" method, "x += y" is equivalent to ' + '"x =\n' + ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' + '"y.__radd__(x)" are\n' + ' considered, as with the evaluation of "x + y". In ' + 'certain\n' + ' situations, augmented assignment can result in ' + 'unexpected errors\n' + " (see Why does a_tuple[i] += ['item'] raise an exception " + 'when the\n' + ' addition works?), but this behavior is in fact part of ' + 'the data\n' + ' model.\n' + '\n' + 'object.__neg__(self)\n' + 'object.__pos__(self)\n' + 'object.__abs__(self)\n' + 'object.__invert__(self)\n' + '\n' + ' Called to implement the unary arithmetic operations ' + '("-", "+",\n' + ' "abs()" and "~").\n' + '\n' + 'object.__complex__(self)\n' + 'object.__int__(self)\n' + 'object.__float__(self)\n' + 'object.__round__(self[, n])\n' + '\n' + ' Called to implement the built-in functions "complex()", ' + '"int()",\n' + ' "float()" and "round()". Should return a value of the ' + 'appropriate\n' + ' type.\n' + '\n' + 'object.__index__(self)\n' + '\n' + ' Called to implement "operator.index()", and whenever ' + 'Python needs\n' + ' to losslessly convert the numeric object to an integer ' + 'object (such\n' + ' as in slicing, or in the built-in "bin()", "hex()" and ' + '"oct()"\n' + ' functions). Presence of this method indicates that the ' + 'numeric\n' + ' object is an integer type. Must return an integer.\n' + '\n' + ' Note: In order to have a coherent integer type class, ' + 'when\n' + ' "__index__()" is defined "__int__()" should also be ' + 'defined, and\n' + ' both should return the same value.\n', + 'objects': '\n' + 'Objects, values and types\n' + '*************************\n' + '\n' + "*Objects* are Python's abstraction for data. All data in a " + 'Python\n' + 'program is represented by objects or by relations between ' + 'objects. (In\n' + 'a sense, and in conformance to Von Neumann\'s model of a "stored\n' + 'program computer," code is also represented by objects.)\n' + '\n' + "Every object has an identity, a type and a value. An object's\n" + '*identity* never changes once it has been created; you may think ' + 'of it\n' + 'as the object\'s address in memory. The \'"is"\' operator ' + 'compares the\n' + 'identity of two objects; the "id()" function returns an integer\n' + 'representing its identity.\n' + '\n' + '**CPython implementation detail:** For CPython, "id(x)" is the ' + 'memory\n' + 'address where "x" is stored.\n' + '\n' + "An object's type determines the operations that the object " + 'supports\n' + '(e.g., "does it have a length?") and also defines the possible ' + 'values\n' + 'for objects of that type. The "type()" function returns an ' + "object's\n" + 'type (which is an object itself). Like its identity, an ' + "object's\n" + '*type* is also unchangeable. [1]\n' + '\n' + 'The *value* of some objects can change. Objects whose value can\n' + 'change are said to be *mutable*; objects whose value is ' + 'unchangeable\n' + 'once they are created are called *immutable*. (The value of an\n' + 'immutable container object that contains a reference to a ' + 'mutable\n' + "object can change when the latter's value is changed; however " + 'the\n' + 'container is still considered immutable, because the collection ' + 'of\n' + 'objects it contains cannot be changed. So, immutability is not\n' + 'strictly the same as having an unchangeable value, it is more ' + 'subtle.)\n' + "An object's mutability is determined by its type; for instance,\n" + 'numbers, strings and tuples are immutable, while dictionaries ' + 'and\n' + 'lists are mutable.\n' + '\n' + 'Objects are never explicitly destroyed; however, when they ' + 'become\n' + 'unreachable they may be garbage-collected. An implementation is\n' + 'allowed to postpone garbage collection or omit it altogether --- ' + 'it is\n' + 'a matter of implementation quality how garbage collection is\n' + 'implemented, as long as no objects are collected that are still\n' + 'reachable.\n' + '\n' + '**CPython implementation detail:** CPython currently uses a ' + 'reference-\n' + 'counting scheme with (optional) delayed detection of cyclically ' + 'linked\n' + 'garbage, which collects most objects as soon as they become\n' + 'unreachable, but is not guaranteed to collect garbage containing\n' + 'circular references. See the documentation of the "gc" module ' + 'for\n' + 'information on controlling the collection of cyclic garbage. ' + 'Other\n' + 'implementations act differently and CPython may change. Do not ' + 'depend\n' + 'on immediate finalization of objects when they become unreachable ' + '(so\n' + 'you should always close files explicitly).\n' + '\n' + "Note that the use of the implementation's tracing or debugging\n" + 'facilities may keep objects alive that would normally be ' + 'collectable.\n' + 'Also note that catching an exception with a \'"try"..."except"\'\n' + 'statement may keep objects alive.\n' + '\n' + 'Some objects contain references to "external" resources such as ' + 'open\n' + 'files or windows. It is understood that these resources are ' + 'freed\n' + 'when the object is garbage-collected, but since garbage ' + 'collection is\n' + 'not guaranteed to happen, such objects also provide an explicit ' + 'way to\n' + 'release the external resource, usually a "close()" method. ' + 'Programs\n' + 'are strongly recommended to explicitly close such objects. The\n' + '\'"try"..."finally"\' statement and the \'"with"\' statement ' + 'provide\n' + 'convenient ways to do this.\n' + '\n' + 'Some objects contain references to other objects; these are ' + 'called\n' + '*containers*. Examples of containers are tuples, lists and\n' + "dictionaries. The references are part of a container's value. " + 'In\n' + 'most cases, when we talk about the value of a container, we imply ' + 'the\n' + 'values, not the identities of the contained objects; however, ' + 'when we\n' + 'talk about the mutability of a container, only the identities of ' + 'the\n' + 'immediately contained objects are implied. So, if an immutable\n' + 'container (like a tuple) contains a reference to a mutable ' + 'object, its\n' + 'value changes if that mutable object is changed.\n' + '\n' + 'Types affect almost all aspects of object behavior. Even the\n' + 'importance of object identity is affected in some sense: for ' + 'immutable\n' + 'types, operations that compute new values may actually return a\n' + 'reference to any existing object with the same type and value, ' + 'while\n' + 'for mutable objects this is not allowed. E.g., after "a = 1; b = ' + '1",\n' + '"a" and "b" may or may not refer to the same object with the ' + 'value\n' + 'one, depending on the implementation, but after "c = []; d = []", ' + '"c"\n' + 'and "d" are guaranteed to refer to two different, unique, newly\n' + 'created empty lists. (Note that "c = d = []" assigns the same ' + 'object\n' + 'to both "c" and "d".)\n', + 'operator-summary': '\n' + 'Operator precedence\n' + '*******************\n' + '\n' + 'The following table summarizes the operator precedence ' + 'in Python, from\n' + 'lowest precedence (least binding) to highest precedence ' + '(most\n' + 'binding). Operators in the same box have the same ' + 'precedence. Unless\n' + 'the syntax is explicitly given, operators are binary. ' + 'Operators in\n' + 'the same box group left to right (except for ' + 'exponentiation, which\n' + 'groups from right to left).\n' + '\n' + 'Note that comparisons, membership tests, and identity ' + 'tests, all have\n' + 'the same precedence and have a left-to-right chaining ' + 'feature as\n' + 'described in the Comparisons section.\n' + '\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| Operator | ' + 'Description |\n' + '+=================================================+=======================================+\n' + '| "lambda" | ' + 'Lambda expression |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "if" -- "else" | ' + 'Conditional expression |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "or" | ' + 'Boolean OR |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "and" | ' + 'Boolean AND |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "not" "x" | ' + 'Boolean NOT |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "in", "not in", "is", "is not", "<", "<=", ">", | ' + 'Comparisons, including membership |\n' + '| ">=", "!=", "==" | ' + 'tests and identity tests |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "|" | ' + 'Bitwise OR |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "^" | ' + 'Bitwise XOR |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "&" | ' + 'Bitwise AND |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "<<", ">>" | ' + 'Shifts |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "+", "-" | ' + 'Addition and subtraction |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "*", "@", "/", "//", "%" | ' + 'Multiplication, matrix multiplication |\n' + '| | ' + 'division, remainder [5] |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "+x", "-x", "~x" | ' + 'Positive, negative, bitwise NOT |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "**" | ' + 'Exponentiation [6] |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "await" "x" | ' + 'Await expression |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "x[index]", "x[index:index]", | ' + 'Subscription, slicing, call, |\n' + '| "x(arguments...)", "x.attribute" | ' + 'attribute reference |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '| "(expressions...)", "[expressions...]", "{key: | ' + 'Binding or tuple display, list |\n' + '| value...}", "{expressions...}" | ' + 'display, dictionary display, set |\n' + '| | ' + 'display |\n' + '+-------------------------------------------------+---------------------------------------+\n' + '\n' + '-[ Footnotes ]-\n' + '\n' + '[1] While "abs(x%y) < abs(y)" is true mathematically, ' + 'for floats\n' + ' it may not be true numerically due to roundoff. For ' + 'example, and\n' + ' assuming a platform on which a Python float is an ' + 'IEEE 754 double-\n' + ' precision number, in order that "-1e-100 % 1e100" ' + 'have the same\n' + ' sign as "1e100", the computed result is "-1e-100 + ' + '1e100", which\n' + ' is numerically exactly equal to "1e100". The ' + 'function\n' + ' "math.fmod()" returns a result whose sign matches ' + 'the sign of the\n' + ' first argument instead, and so returns "-1e-100" in ' + 'this case.\n' + ' Which approach is more appropriate depends on the ' + 'application.\n' + '\n' + '[2] If x is very close to an exact integer multiple of ' + "y, it's\n" + ' possible for "x//y" to be one larger than ' + '"(x-x%y)//y" due to\n' + ' rounding. In such cases, Python returns the latter ' + 'result, in\n' + ' order to preserve that "divmod(x,y)[0] * y + x % y" ' + 'be very close\n' + ' to "x".\n' + '\n' + '[3] The Unicode standard distinguishes between *code ' + 'points* (e.g.\n' + ' U+0041) and *abstract characters* (e.g. "LATIN ' + 'CAPITAL LETTER A").\n' + ' While most abstract characters in Unicode are only ' + 'represented\n' + ' using one code point, there is a number of abstract ' + 'characters\n' + ' that can in addition be represented using a sequence ' + 'of more than\n' + ' one code point. For example, the abstract character ' + '"LATIN\n' + ' CAPITAL LETTER C WITH CEDILLA" can be represented as ' + 'a single\n' + ' *precomposed character* at code position U+00C7, or ' + 'as a sequence\n' + ' of a *base character* at code position U+0043 (LATIN ' + 'CAPITAL\n' + ' LETTER C), followed by a *combining character* at ' + 'code position\n' + ' U+0327 (COMBINING CEDILLA).\n' + '\n' + ' The comparison operators on strings compare at the ' + 'level of\n' + ' Unicode code points. This may be counter-intuitive ' + 'to humans. For\n' + ' example, ""\\u00C7" == "\\u0043\\u0327"" is "False", ' + 'even though both\n' + ' strings represent the same abstract character "LATIN ' + 'CAPITAL\n' + ' LETTER C WITH CEDILLA".\n' + '\n' + ' To compare strings at the level of abstract ' + 'characters (that is,\n' + ' in a way intuitive to humans), use ' + '"unicodedata.normalize()".\n' + '\n' + '[4] Due to automatic garbage-collection, free lists, and ' + 'the\n' + ' dynamic nature of descriptors, you may notice ' + 'seemingly unusual\n' + ' behaviour in certain uses of the "is" operator, like ' + 'those\n' + ' involving comparisons between instance methods, or ' + 'constants.\n' + ' Check their documentation for more info.\n' + '\n' + '[5] The "%" operator is also used for string formatting; ' + 'the same\n' + ' precedence applies.\n' + '\n' + '[6] The power operator "**" binds less tightly than an ' + 'arithmetic\n' + ' or bitwise unary operator on its right, that is, ' + '"2**-1" is "0.5".\n', + 'pass': '\n' + 'The "pass" statement\n' + '********************\n' + '\n' + ' pass_stmt ::= "pass"\n' + '\n' + '"pass" is a null operation --- when it is executed, nothing ' + 'happens.\n' + 'It is useful as a placeholder when a statement is required\n' + 'syntactically, but no code needs to be executed, for example:\n' + '\n' + ' def f(arg): pass # a function that does nothing (yet)\n' + '\n' + ' class C: pass # a class with no methods (yet)\n', + 'power': '\n' + 'The power operator\n' + '******************\n' + '\n' + 'The power operator binds more tightly than unary operators on its\n' + 'left; it binds less tightly than unary operators on its right. ' + 'The\n' + 'syntax is:\n' + '\n' + ' power ::= ( await_expr | primary ) ["**" u_expr]\n' + '\n' + 'Thus, in an unparenthesized sequence of power and unary operators, ' + 'the\n' + 'operators are evaluated from right to left (this does not ' + 'constrain\n' + 'the evaluation order for the operands): "-1**2" results in "-1".\n' + '\n' + 'The power operator has the same semantics as the built-in "pow()"\n' + 'function, when called with two arguments: it yields its left ' + 'argument\n' + 'raised to the power of its right argument. The numeric arguments ' + 'are\n' + 'first converted to a common type, and the result is of that type.\n' + '\n' + 'For int operands, the result has the same type as the operands ' + 'unless\n' + 'the second argument is negative; in that case, all arguments are\n' + 'converted to float and a float result is delivered. For example,\n' + '"10**2" returns "100", but "10**-2" returns "0.01".\n' + '\n' + 'Raising "0.0" to a negative power results in a ' + '"ZeroDivisionError".\n' + 'Raising a negative number to a fractional power results in a ' + '"complex"\n' + 'number. (In earlier versions it raised a "ValueError".)\n', + 'raise': '\n' + 'The "raise" statement\n' + '*********************\n' + '\n' + ' raise_stmt ::= "raise" [expression ["from" expression]]\n' + '\n' + 'If no expressions are present, "raise" re-raises the last ' + 'exception\n' + 'that was active in the current scope. If no exception is active ' + 'in\n' + 'the current scope, a "RuntimeError" exception is raised indicating\n' + 'that this is an error.\n' + '\n' + 'Otherwise, "raise" evaluates the first expression as the exception\n' + 'object. It must be either a subclass or an instance of\n' + '"BaseException". If it is a class, the exception instance will be\n' + 'obtained when needed by instantiating the class with no arguments.\n' + '\n' + "The *type* of the exception is the exception instance's class, the\n" + '*value* is the instance itself.\n' + '\n' + 'A traceback object is normally created automatically when an ' + 'exception\n' + 'is raised and attached to it as the "__traceback__" attribute, ' + 'which\n' + 'is writable. You can create an exception and set your own traceback ' + 'in\n' + 'one step using the "with_traceback()" exception method (which ' + 'returns\n' + 'the same exception instance, with its traceback set to its ' + 'argument),\n' + 'like so:\n' + '\n' + ' raise Exception("foo occurred").with_traceback(tracebackobj)\n' + '\n' + 'The "from" clause is used for exception chaining: if given, the ' + 'second\n' + '*expression* must be another exception class or instance, which ' + 'will\n' + 'then be attached to the raised exception as the "__cause__" ' + 'attribute\n' + '(which is writable). If the raised exception is not handled, both\n' + 'exceptions will be printed:\n' + '\n' + ' >>> try:\n' + ' ... print(1 / 0)\n' + ' ... except Exception as exc:\n' + ' ... raise RuntimeError("Something bad happened") from exc\n' + ' ...\n' + ' Traceback (most recent call last):\n' + ' File "", line 2, in \n' + ' ZeroDivisionError: int division or modulo by zero\n' + '\n' + ' The above exception was the direct cause of the following ' + 'exception:\n' + '\n' + ' Traceback (most recent call last):\n' + ' File "", line 4, in \n' + ' RuntimeError: Something bad happened\n' + '\n' + 'A similar mechanism works implicitly if an exception is raised ' + 'inside\n' + 'an exception handler or a "finally" clause: the previous exception ' + 'is\n' + 'then attached as the new exception\'s "__context__" attribute:\n' + '\n' + ' >>> try:\n' + ' ... print(1 / 0)\n' + ' ... except:\n' + ' ... raise RuntimeError("Something bad happened")\n' + ' ...\n' + ' Traceback (most recent call last):\n' + ' File "", line 2, in \n' + ' ZeroDivisionError: int division or modulo by zero\n' + '\n' + ' During handling of the above exception, another exception ' + 'occurred:\n' + '\n' + ' Traceback (most recent call last):\n' + ' File "", line 4, in \n' + ' RuntimeError: Something bad happened\n' + '\n' + 'Additional information on exceptions can be found in section\n' + 'Exceptions, and information about handling exceptions is in ' + 'section\n' + 'The try statement.\n', + 'return': '\n' + 'The "return" statement\n' + '**********************\n' + '\n' + ' return_stmt ::= "return" [expression_list]\n' + '\n' + '"return" may only occur syntactically nested in a function ' + 'definition,\n' + 'not within a nested class definition.\n' + '\n' + 'If an expression list is present, it is evaluated, else "None" is\n' + 'substituted.\n' + '\n' + '"return" leaves the current function call with the expression list ' + '(or\n' + '"None") as return value.\n' + '\n' + 'When "return" passes control out of a "try" statement with a ' + '"finally"\n' + 'clause, that "finally" clause is executed before really leaving ' + 'the\n' + 'function.\n' + '\n' + 'In a generator function, the "return" statement indicates that ' + 'the\n' + 'generator is done and will cause "StopIteration" to be raised. ' + 'The\n' + 'returned value (if any) is used as an argument to construct\n' + '"StopIteration" and becomes the "StopIteration.value" attribute.\n', + 'sequence-types': '\n' + 'Emulating container types\n' + '*************************\n' + '\n' + 'The following methods can be defined to implement ' + 'container objects.\n' + 'Containers usually are sequences (such as lists or tuples) ' + 'or mappings\n' + '(like dictionaries), but can represent other containers as ' + 'well. The\n' + 'first set of methods is used either to emulate a sequence ' + 'or to\n' + 'emulate a mapping; the difference is that for a sequence, ' + 'the\n' + 'allowable keys should be the integers *k* for which "0 <= ' + 'k < N" where\n' + '*N* is the length of the sequence, or slice objects, which ' + 'define a\n' + 'range of items. It is also recommended that mappings ' + 'provide the\n' + 'methods "keys()", "values()", "items()", "get()", ' + '"clear()",\n' + '"setdefault()", "pop()", "popitem()", "copy()", and ' + '"update()"\n' + "behaving similar to those for Python's standard dictionary " + 'objects.\n' + 'The "collections" module provides a "MutableMapping" ' + 'abstract base\n' + 'class to help create those methods from a base set of ' + '"__getitem__()",\n' + '"__setitem__()", "__delitem__()", and "keys()". Mutable ' + 'sequences\n' + 'should provide methods "append()", "count()", "index()", ' + '"extend()",\n' + '"insert()", "pop()", "remove()", "reverse()" and "sort()", ' + 'like Python\n' + 'standard list objects. Finally, sequence types should ' + 'implement\n' + 'addition (meaning concatenation) and multiplication ' + '(meaning\n' + 'repetition) by defining the methods "__add__()", ' + '"__radd__()",\n' + '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" ' + 'described\n' + 'below; they should not define other numerical operators. ' + 'It is\n' + 'recommended that both mappings and sequences implement ' + 'the\n' + '"__contains__()" method to allow efficient use of the "in" ' + 'operator;\n' + 'for mappings, "in" should search the mapping\'s keys; for ' + 'sequences, it\n' + 'should search through the values. It is further ' + 'recommended that both\n' + 'mappings and sequences implement the "__iter__()" method ' + 'to allow\n' + 'efficient iteration through the container; for mappings, ' + '"__iter__()"\n' + 'should be the same as "keys()"; for sequences, it should ' + 'iterate\n' + 'through the values.\n' + '\n' + 'object.__len__(self)\n' + '\n' + ' Called to implement the built-in function "len()". ' + 'Should return\n' + ' the length of the object, an integer ">=" 0. Also, an ' + 'object that\n' + ' doesn\'t define a "__bool__()" method and whose ' + '"__len__()" method\n' + ' returns zero is considered to be false in a Boolean ' + 'context.\n' + '\n' + 'object.__length_hint__(self)\n' + '\n' + ' Called to implement "operator.length_hint()". Should ' + 'return an\n' + ' estimated length for the object (which may be greater ' + 'or less than\n' + ' the actual length). The length must be an integer ">=" ' + '0. This\n' + ' method is purely an optimization and is never required ' + 'for\n' + ' correctness.\n' + '\n' + ' New in version 3.4.\n' + '\n' + 'Note: Slicing is done exclusively with the following three ' + 'methods.\n' + ' A call like\n' + '\n' + ' a[1:2] = b\n' + '\n' + ' is translated to\n' + '\n' + ' a[slice(1, 2, None)] = b\n' + '\n' + ' and so forth. Missing slice items are always filled in ' + 'with "None".\n' + '\n' + 'object.__getitem__(self, key)\n' + '\n' + ' Called to implement evaluation of "self[key]". For ' + 'sequence types,\n' + ' the accepted keys should be integers and slice ' + 'objects. Note that\n' + ' the special interpretation of negative indexes (if the ' + 'class wishes\n' + ' to emulate a sequence type) is up to the ' + '"__getitem__()" method. If\n' + ' *key* is of an inappropriate type, "TypeError" may be ' + 'raised; if of\n' + ' a value outside the set of indexes for the sequence ' + '(after any\n' + ' special interpretation of negative values), ' + '"IndexError" should be\n' + ' raised. For mapping types, if *key* is missing (not in ' + 'the\n' + ' container), "KeyError" should be raised.\n' + '\n' + ' Note: "for" loops expect that an "IndexError" will be ' + 'raised for\n' + ' illegal indexes to allow proper detection of the end ' + 'of the\n' + ' sequence.\n' + '\n' + 'object.__missing__(self, key)\n' + '\n' + ' Called by "dict"."__getitem__()" to implement ' + '"self[key]" for dict\n' + ' subclasses when key is not in the dictionary.\n' + '\n' + 'object.__setitem__(self, key, value)\n' + '\n' + ' Called to implement assignment to "self[key]". Same ' + 'note as for\n' + ' "__getitem__()". This should only be implemented for ' + 'mappings if\n' + ' the objects support changes to the values for keys, or ' + 'if new keys\n' + ' can be added, or for sequences if elements can be ' + 'replaced. The\n' + ' same exceptions should be raised for improper *key* ' + 'values as for\n' + ' the "__getitem__()" method.\n' + '\n' + 'object.__delitem__(self, key)\n' + '\n' + ' Called to implement deletion of "self[key]". Same note ' + 'as for\n' + ' "__getitem__()". This should only be implemented for ' + 'mappings if\n' + ' the objects support removal of keys, or for sequences ' + 'if elements\n' + ' can be removed from the sequence. The same exceptions ' + 'should be\n' + ' raised for improper *key* values as for the ' + '"__getitem__()" method.\n' + '\n' + 'object.__iter__(self)\n' + '\n' + ' This method is called when an iterator is required for ' + 'a container.\n' + ' This method should return a new iterator object that ' + 'can iterate\n' + ' over all the objects in the container. For mappings, ' + 'it should\n' + ' iterate over the keys of the container.\n' + '\n' + ' Iterator objects also need to implement this method; ' + 'they are\n' + ' required to return themselves. For more information on ' + 'iterator\n' + ' objects, see Iterator Types.\n' + '\n' + 'object.__reversed__(self)\n' + '\n' + ' Called (if present) by the "reversed()" built-in to ' + 'implement\n' + ' reverse iteration. It should return a new iterator ' + 'object that\n' + ' iterates over all the objects in the container in ' + 'reverse order.\n' + '\n' + ' If the "__reversed__()" method is not provided, the ' + '"reversed()"\n' + ' built-in will fall back to using the sequence protocol ' + '("__len__()"\n' + ' and "__getitem__()"). Objects that support the ' + 'sequence protocol\n' + ' should only provide "__reversed__()" if they can ' + 'provide an\n' + ' implementation that is more efficient than the one ' + 'provided by\n' + ' "reversed()".\n' + '\n' + 'The membership test operators ("in" and "not in") are ' + 'normally\n' + 'implemented as an iteration through a sequence. However, ' + 'container\n' + 'objects can supply the following special method with a ' + 'more efficient\n' + 'implementation, which also does not require the object be ' + 'a sequence.\n' + '\n' + 'object.__contains__(self, item)\n' + '\n' + ' Called to implement membership test operators. Should ' + 'return true\n' + ' if *item* is in *self*, false otherwise. For mapping ' + 'objects, this\n' + ' should consider the keys of the mapping rather than the ' + 'values or\n' + ' the key-item pairs.\n' + '\n' + ' For objects that don\'t define "__contains__()", the ' + 'membership test\n' + ' first tries iteration via "__iter__()", then the old ' + 'sequence\n' + ' iteration protocol via "__getitem__()", see this ' + 'section in the\n' + ' language reference.\n', + 'shifting': '\n' + 'Shifting operations\n' + '*******************\n' + '\n' + 'The shifting operations have lower priority than the arithmetic\n' + 'operations:\n' + '\n' + ' shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n' + '\n' + 'These operators accept integers as arguments. They shift the ' + 'first\n' + 'argument to the left or right by the number of bits given by ' + 'the\n' + 'second argument.\n' + '\n' + 'A right shift by *n* bits is defined as floor division by ' + '"pow(2,n)".\n' + 'A left shift by *n* bits is defined as multiplication with ' + '"pow(2,n)".\n' + '\n' + 'Note: In the current implementation, the right-hand operand is\n' + ' required to be at most "sys.maxsize". If the right-hand ' + 'operand is\n' + ' larger than "sys.maxsize" an "OverflowError" exception is ' + 'raised.\n', + 'slicings': '\n' + 'Slicings\n' + '********\n' + '\n' + 'A slicing selects a range of items in a sequence object (e.g., ' + 'a\n' + 'string, tuple or list). Slicings may be used as expressions or ' + 'as\n' + 'targets in assignment or "del" statements. The syntax for a ' + 'slicing:\n' + '\n' + ' slicing ::= primary "[" slice_list "]"\n' + ' slice_list ::= slice_item ("," slice_item)* [","]\n' + ' slice_item ::= expression | proper_slice\n' + ' proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" ' + '[stride] ]\n' + ' lower_bound ::= expression\n' + ' upper_bound ::= expression\n' + ' stride ::= expression\n' + '\n' + 'There is ambiguity in the formal syntax here: anything that ' + 'looks like\n' + 'an expression list also looks like a slice list, so any ' + 'subscription\n' + 'can be interpreted as a slicing. Rather than further ' + 'complicating the\n' + 'syntax, this is disambiguated by defining that in this case the\n' + 'interpretation as a subscription takes priority over the\n' + 'interpretation as a slicing (this is the case if the slice list\n' + 'contains no proper slice).\n' + '\n' + 'The semantics for a slicing are as follows. The primary is ' + 'indexed\n' + '(using the same "__getitem__()" method as normal subscription) ' + 'with a\n' + 'key that is constructed from the slice list, as follows. If the ' + 'slice\n' + 'list contains at least one comma, the key is a tuple containing ' + 'the\n' + 'conversion of the slice items; otherwise, the conversion of the ' + 'lone\n' + 'slice item is the key. The conversion of a slice item that is ' + 'an\n' + 'expression is that expression. The conversion of a proper slice ' + 'is a\n' + 'slice object (see section The standard type hierarchy) whose ' + '"start",\n' + '"stop" and "step" attributes are the values of the expressions ' + 'given\n' + 'as lower bound, upper bound and stride, respectively, ' + 'substituting\n' + '"None" for missing expressions.\n', + 'specialattrs': '\n' + 'Special Attributes\n' + '******************\n' + '\n' + 'The implementation adds a few special read-only attributes ' + 'to several\n' + 'object types, where they are relevant. Some of these are ' + 'not reported\n' + 'by the "dir()" built-in function.\n' + '\n' + 'object.__dict__\n' + '\n' + ' A dictionary or other mapping object used to store an ' + "object's\n" + ' (writable) attributes.\n' + '\n' + 'instance.__class__\n' + '\n' + ' The class to which a class instance belongs.\n' + '\n' + 'class.__bases__\n' + '\n' + ' The tuple of base classes of a class object.\n' + '\n' + 'class.__name__\n' + '\n' + ' The name of the class or type.\n' + '\n' + 'class.__qualname__\n' + '\n' + ' The *qualified name* of the class or type.\n' + '\n' + ' New in version 3.3.\n' + '\n' + 'class.__mro__\n' + '\n' + ' This attribute is a tuple of classes that are considered ' + 'when\n' + ' looking for base classes during method resolution.\n' + '\n' + 'class.mro()\n' + '\n' + ' This method can be overridden by a metaclass to customize ' + 'the\n' + ' method resolution order for its instances. It is called ' + 'at class\n' + ' instantiation, and its result is stored in "__mro__".\n' + '\n' + 'class.__subclasses__()\n' + '\n' + ' Each class keeps a list of weak references to its ' + 'immediate\n' + ' subclasses. This method returns a list of all those ' + 'references\n' + ' still alive. Example:\n' + '\n' + ' >>> int.__subclasses__()\n' + " []\n" + '\n' + '-[ Footnotes ]-\n' + '\n' + '[1] Additional information on these special methods may be ' + 'found\n' + ' in the Python Reference Manual (Basic customization).\n' + '\n' + '[2] As a consequence, the list "[1, 2]" is considered equal ' + 'to\n' + ' "[1.0, 2.0]", and similarly for tuples.\n' + '\n' + "[3] They must have since the parser can't tell the type of " + 'the\n' + ' operands.\n' + '\n' + '[4] Cased characters are those with general category ' + 'property\n' + ' being one of "Lu" (Letter, uppercase), "Ll" (Letter, ' + 'lowercase),\n' + ' or "Lt" (Letter, titlecase).\n' + '\n' + '[5] To format only a tuple you should therefore provide a\n' + ' singleton tuple whose only element is the tuple to be ' + 'formatted.\n', + 'specialnames': '\n' + 'Special method names\n' + '********************\n' + '\n' + 'A class can implement certain operations that are invoked by ' + 'special\n' + 'syntax (such as arithmetic operations or subscripting and ' + 'slicing) by\n' + "defining methods with special names. This is Python's " + 'approach to\n' + '*operator overloading*, allowing classes to define their own ' + 'behavior\n' + 'with respect to language operators. For instance, if a ' + 'class defines\n' + 'a method named "__getitem__()", and "x" is an instance of ' + 'this class,\n' + 'then "x[i]" is roughly equivalent to "type(x).__getitem__(x, ' + 'i)".\n' + 'Except where mentioned, attempts to execute an operation ' + 'raise an\n' + 'exception when no appropriate method is defined (typically\n' + '"AttributeError" or "TypeError").\n' + '\n' + 'When implementing a class that emulates any built-in type, ' + 'it is\n' + 'important that the emulation only be implemented to the ' + 'degree that it\n' + 'makes sense for the object being modelled. For example, ' + 'some\n' + 'sequences may work well with retrieval of individual ' + 'elements, but\n' + 'extracting a slice may not make sense. (One example of this ' + 'is the\n' + '"NodeList" interface in the W3C\'s Document Object Model.)\n' + '\n' + '\n' + 'Basic customization\n' + '===================\n' + '\n' + 'object.__new__(cls[, ...])\n' + '\n' + ' Called to create a new instance of class *cls*. ' + '"__new__()" is a\n' + ' static method (special-cased so you need not declare it ' + 'as such)\n' + ' that takes the class of which an instance was requested ' + 'as its\n' + ' first argument. The remaining arguments are those passed ' + 'to the\n' + ' object constructor expression (the call to the class). ' + 'The return\n' + ' value of "__new__()" should be the new object instance ' + '(usually an\n' + ' instance of *cls*).\n' + '\n' + ' Typical implementations create a new instance of the ' + 'class by\n' + ' invoking the superclass\'s "__new__()" method using\n' + ' "super(currentclass, cls).__new__(cls[, ...])" with ' + 'appropriate\n' + ' arguments and then modifying the newly-created instance ' + 'as\n' + ' necessary before returning it.\n' + '\n' + ' If "__new__()" returns an instance of *cls*, then the ' + 'new\n' + ' instance\'s "__init__()" method will be invoked like\n' + ' "__init__(self[, ...])", where *self* is the new instance ' + 'and the\n' + ' remaining arguments are the same as were passed to ' + '"__new__()".\n' + '\n' + ' If "__new__()" does not return an instance of *cls*, then ' + 'the new\n' + ' instance\'s "__init__()" method will not be invoked.\n' + '\n' + ' "__new__()" is intended mainly to allow subclasses of ' + 'immutable\n' + ' types (like int, str, or tuple) to customize instance ' + 'creation. It\n' + ' is also commonly overridden in custom metaclasses in ' + 'order to\n' + ' customize class creation.\n' + '\n' + 'object.__init__(self[, ...])\n' + '\n' + ' Called after the instance has been created (by ' + '"__new__()"), but\n' + ' before it is returned to the caller. The arguments are ' + 'those\n' + ' passed to the class constructor expression. If a base ' + 'class has an\n' + ' "__init__()" method, the derived class\'s "__init__()" ' + 'method, if\n' + ' any, must explicitly call it to ensure proper ' + 'initialization of the\n' + ' base class part of the instance; for example:\n' + ' "BaseClass.__init__(self, [args...])".\n' + '\n' + ' Because "__new__()" and "__init__()" work together in ' + 'constructing\n' + ' objects ("__new__()" to create it, and "__init__()" to ' + 'customise\n' + ' it), no non-"None" value may be returned by "__init__()"; ' + 'doing so\n' + ' will cause a "TypeError" to be raised at runtime.\n' + '\n' + 'object.__del__(self)\n' + '\n' + ' Called when the instance is about to be destroyed. This ' + 'is also\n' + ' called a destructor. If a base class has a "__del__()" ' + 'method, the\n' + ' derived class\'s "__del__()" method, if any, must ' + 'explicitly call it\n' + ' to ensure proper deletion of the base class part of the ' + 'instance.\n' + ' Note that it is possible (though not recommended!) for ' + 'the\n' + ' "__del__()" method to postpone destruction of the ' + 'instance by\n' + ' creating a new reference to it. It may then be called at ' + 'a later\n' + ' time when this new reference is deleted. It is not ' + 'guaranteed that\n' + ' "__del__()" methods are called for objects that still ' + 'exist when\n' + ' the interpreter exits.\n' + '\n' + ' Note: "del x" doesn\'t directly call "x.__del__()" --- ' + 'the former\n' + ' decrements the reference count for "x" by one, and the ' + 'latter is\n' + ' only called when "x"\'s reference count reaches zero. ' + 'Some common\n' + ' situations that may prevent the reference count of an ' + 'object from\n' + ' going to zero include: circular references between ' + 'objects (e.g.,\n' + ' a doubly-linked list or a tree data structure with ' + 'parent and\n' + ' child pointers); a reference to the object on the stack ' + 'frame of\n' + ' a function that caught an exception (the traceback ' + 'stored in\n' + ' "sys.exc_info()[2]" keeps the stack frame alive); or a ' + 'reference\n' + ' to the object on the stack frame that raised an ' + 'unhandled\n' + ' exception in interactive mode (the traceback stored in\n' + ' "sys.last_traceback" keeps the stack frame alive). The ' + 'first\n' + ' situation can only be remedied by explicitly breaking ' + 'the cycles;\n' + ' the second can be resolved by freeing the reference to ' + 'the\n' + ' traceback object when it is no longer useful, and the ' + 'third can\n' + ' be resolved by storing "None" in "sys.last_traceback". ' + 'Circular\n' + ' references which are garbage are detected and cleaned ' + 'up when the\n' + " cyclic garbage collector is enabled (it's on by " + 'default). Refer\n' + ' to the documentation for the "gc" module for more ' + 'information\n' + ' about this topic.\n' + '\n' + ' Warning: Due to the precarious circumstances under which\n' + ' "__del__()" methods are invoked, exceptions that occur ' + 'during\n' + ' their execution are ignored, and a warning is printed ' + 'to\n' + ' "sys.stderr" instead. Also, when "__del__()" is invoked ' + 'in\n' + ' response to a module being deleted (e.g., when ' + 'execution of the\n' + ' program is done), other globals referenced by the ' + '"__del__()"\n' + ' method may already have been deleted or in the process ' + 'of being\n' + ' torn down (e.g. the import machinery shutting down). ' + 'For this\n' + ' reason, "__del__()" methods should do the absolute ' + 'minimum needed\n' + ' to maintain external invariants. Starting with version ' + '1.5,\n' + ' Python guarantees that globals whose name begins with a ' + 'single\n' + ' underscore are deleted from their module before other ' + 'globals are\n' + ' deleted; if no other references to such globals exist, ' + 'this may\n' + ' help in assuring that imported modules are still ' + 'available at the\n' + ' time when the "__del__()" method is called.\n' + '\n' + 'object.__repr__(self)\n' + '\n' + ' Called by the "repr()" built-in function to compute the ' + '"official"\n' + ' string representation of an object. If at all possible, ' + 'this\n' + ' should look like a valid Python expression that could be ' + 'used to\n' + ' recreate an object with the same value (given an ' + 'appropriate\n' + ' environment). If this is not possible, a string of the ' + 'form\n' + ' "<...some useful description...>" should be returned. The ' + 'return\n' + ' value must be a string object. If a class defines ' + '"__repr__()" but\n' + ' not "__str__()", then "__repr__()" is also used when an ' + '"informal"\n' + ' string representation of instances of that class is ' + 'required.\n' + '\n' + ' This is typically used for debugging, so it is important ' + 'that the\n' + ' representation is information-rich and unambiguous.\n' + '\n' + 'object.__str__(self)\n' + '\n' + ' Called by "str(object)" and the built-in functions ' + '"format()" and\n' + ' "print()" to compute the "informal" or nicely printable ' + 'string\n' + ' representation of an object. The return value must be a ' + 'string\n' + ' object.\n' + '\n' + ' This method differs from "object.__repr__()" in that ' + 'there is no\n' + ' expectation that "__str__()" return a valid Python ' + 'expression: a\n' + ' more convenient or concise representation can be used.\n' + '\n' + ' The default implementation defined by the built-in type ' + '"object"\n' + ' calls "object.__repr__()".\n' + '\n' + 'object.__bytes__(self)\n' + '\n' + ' Called by "bytes()" to compute a byte-string ' + 'representation of an\n' + ' object. This should return a "bytes" object.\n' + '\n' + 'object.__format__(self, format_spec)\n' + '\n' + ' Called by the "format()" built-in function, and by ' + 'extension,\n' + ' evaluation of formatted string literals and the ' + '"str.format()"\n' + ' method, to produce a "formatted" string representation of ' + 'an\n' + ' object. The "format_spec" argument is a string that ' + 'contains a\n' + ' description of the formatting options desired. The ' + 'interpretation\n' + ' of the "format_spec" argument is up to the type ' + 'implementing\n' + ' "__format__()", however most classes will either ' + 'delegate\n' + ' formatting to one of the built-in types, or use a ' + 'similar\n' + ' formatting option syntax.\n' + '\n' + ' See Format Specification Mini-Language for a description ' + 'of the\n' + ' standard formatting syntax.\n' + '\n' + ' The return value must be a string object.\n' + '\n' + ' Changed in version 3.4: The __format__ method of "object" ' + 'itself\n' + ' raises a "TypeError" if passed any non-empty string.\n' + '\n' + 'object.__lt__(self, other)\n' + 'object.__le__(self, other)\n' + 'object.__eq__(self, other)\n' + 'object.__ne__(self, other)\n' + 'object.__gt__(self, other)\n' + 'object.__ge__(self, other)\n' + '\n' + ' These are the so-called "rich comparison" methods. The\n' + ' correspondence between operator symbols and method names ' + 'is as\n' + ' follows: "xy" calls\n' + ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' + '\n' + ' A rich comparison method may return the singleton ' + '"NotImplemented"\n' + ' if it does not implement the operation for a given pair ' + 'of\n' + ' arguments. By convention, "False" and "True" are returned ' + 'for a\n' + ' successful comparison. However, these methods can return ' + 'any value,\n' + ' so if the comparison operator is used in a Boolean ' + 'context (e.g.,\n' + ' in the condition of an "if" statement), Python will call ' + '"bool()"\n' + ' on the value to determine if the result is true or ' + 'false.\n' + '\n' + ' By default, "__ne__()" delegates to "__eq__()" and ' + 'inverts the\n' + ' result unless it is "NotImplemented". There are no other ' + 'implied\n' + ' relationships among the comparison operators, for ' + 'example, the\n' + ' truth of "(x.__hash__".\n' + '\n' + ' If a class that does not override "__eq__()" wishes to ' + 'suppress\n' + ' hash support, it should include "__hash__ = None" in the ' + 'class\n' + ' definition. A class which defines its own "__hash__()" ' + 'that\n' + ' explicitly raises a "TypeError" would be incorrectly ' + 'identified as\n' + ' hashable by an "isinstance(obj, collections.Hashable)" ' + 'call.\n' + '\n' + ' Note: By default, the "__hash__()" values of str, bytes ' + 'and\n' + ' datetime objects are "salted" with an unpredictable ' + 'random value.\n' + ' Although they remain constant within an individual ' + 'Python\n' + ' process, they are not predictable between repeated ' + 'invocations of\n' + ' Python.This is intended to provide protection against a ' + 'denial-\n' + ' of-service caused by carefully-chosen inputs that ' + 'exploit the\n' + ' worst case performance of a dict insertion, O(n^2) ' + 'complexity.\n' + ' See http://www.ocert.org/advisories/ocert-2011-003.html ' + 'for\n' + ' details.Changing hash values affects the iteration ' + 'order of\n' + ' dicts, sets and other mappings. Python has never made ' + 'guarantees\n' + ' about this ordering (and it typically varies between ' + '32-bit and\n' + ' 64-bit builds).See also "PYTHONHASHSEED".\n' + '\n' + ' Changed in version 3.3: Hash randomization is enabled by ' + 'default.\n' + '\n' + 'object.__bool__(self)\n' + '\n' + ' Called to implement truth value testing and the built-in ' + 'operation\n' + ' "bool()"; should return "False" or "True". When this ' + 'method is not\n' + ' defined, "__len__()" is called, if it is defined, and the ' + 'object is\n' + ' considered true if its result is nonzero. If a class ' + 'defines\n' + ' neither "__len__()" nor "__bool__()", all its instances ' + 'are\n' + ' considered true.\n' + '\n' + '\n' + 'Customizing attribute access\n' + '============================\n' + '\n' + 'The following methods can be defined to customize the ' + 'meaning of\n' + 'attribute access (use of, assignment to, or deletion of ' + '"x.name") for\n' + 'class instances.\n' + '\n' + 'object.__getattr__(self, name)\n' + '\n' + ' Called when an attribute lookup has not found the ' + 'attribute in the\n' + ' usual places (i.e. it is not an instance attribute nor is ' + 'it found\n' + ' in the class tree for "self"). "name" is the attribute ' + 'name. This\n' + ' method should return the (computed) attribute value or ' + 'raise an\n' + ' "AttributeError" exception.\n' + '\n' + ' Note that if the attribute is found through the normal ' + 'mechanism,\n' + ' "__getattr__()" is not called. (This is an intentional ' + 'asymmetry\n' + ' between "__getattr__()" and "__setattr__()".) This is ' + 'done both for\n' + ' efficiency reasons and because otherwise "__getattr__()" ' + 'would have\n' + ' no way to access other attributes of the instance. Note ' + 'that at\n' + ' least for instance variables, you can fake total control ' + 'by not\n' + ' inserting any values in the instance attribute dictionary ' + '(but\n' + ' instead inserting them in another object). See the\n' + ' "__getattribute__()" method below for a way to actually ' + 'get total\n' + ' control over attribute access.\n' + '\n' + 'object.__getattribute__(self, name)\n' + '\n' + ' Called unconditionally to implement attribute accesses ' + 'for\n' + ' instances of the class. If the class also defines ' + '"__getattr__()",\n' + ' the latter will not be called unless "__getattribute__()" ' + 'either\n' + ' calls it explicitly or raises an "AttributeError". This ' + 'method\n' + ' should return the (computed) attribute value or raise an\n' + ' "AttributeError" exception. In order to avoid infinite ' + 'recursion in\n' + ' this method, its implementation should always call the ' + 'base class\n' + ' method with the same name to access any attributes it ' + 'needs, for\n' + ' example, "object.__getattribute__(self, name)".\n' + '\n' + ' Note: This method may still be bypassed when looking up ' + 'special\n' + ' methods as the result of implicit invocation via ' + 'language syntax\n' + ' or built-in functions. See Special method lookup.\n' + '\n' + 'object.__setattr__(self, name, value)\n' + '\n' + ' Called when an attribute assignment is attempted. This ' + 'is called\n' + ' instead of the normal mechanism (i.e. store the value in ' + 'the\n' + ' instance dictionary). *name* is the attribute name, ' + '*value* is the\n' + ' value to be assigned to it.\n' + '\n' + ' If "__setattr__()" wants to assign to an instance ' + 'attribute, it\n' + ' should call the base class method with the same name, for ' + 'example,\n' + ' "object.__setattr__(self, name, value)".\n' + '\n' + 'object.__delattr__(self, name)\n' + '\n' + ' Like "__setattr__()" but for attribute deletion instead ' + 'of\n' + ' assignment. This should only be implemented if "del ' + 'obj.name" is\n' + ' meaningful for the object.\n' + '\n' + 'object.__dir__(self)\n' + '\n' + ' Called when "dir()" is called on the object. A sequence ' + 'must be\n' + ' returned. "dir()" converts the returned sequence to a ' + 'list and\n' + ' sorts it.\n' + '\n' + '\n' + 'Implementing Descriptors\n' + '------------------------\n' + '\n' + 'The following methods only apply when an instance of the ' + 'class\n' + 'containing the method (a so-called *descriptor* class) ' + 'appears in an\n' + "*owner* class (the descriptor must be in either the owner's " + 'class\n' + 'dictionary or in the class dictionary for one of its ' + 'parents). In the\n' + 'examples below, "the attribute" refers to the attribute ' + 'whose name is\n' + 'the key of the property in the owner class\' "__dict__".\n' + '\n' + 'object.__get__(self, instance, owner)\n' + '\n' + ' Called to get the attribute of the owner class (class ' + 'attribute\n' + ' access) or of an instance of that class (instance ' + 'attribute\n' + ' access). *owner* is always the owner class, while ' + '*instance* is the\n' + ' instance that the attribute was accessed through, or ' + '"None" when\n' + ' the attribute is accessed through the *owner*. This ' + 'method should\n' + ' return the (computed) attribute value or raise an ' + '"AttributeError"\n' + ' exception.\n' + '\n' + 'object.__set__(self, instance, value)\n' + '\n' + ' Called to set the attribute on an instance *instance* of ' + 'the owner\n' + ' class to a new value, *value*.\n' + '\n' + 'object.__delete__(self, instance)\n' + '\n' + ' Called to delete the attribute on an instance *instance* ' + 'of the\n' + ' owner class.\n' + '\n' + 'The attribute "__objclass__" is interpreted by the "inspect" ' + 'module as\n' + 'specifying the class where this object was defined (setting ' + 'this\n' + 'appropriately can assist in runtime introspection of dynamic ' + 'class\n' + 'attributes). For callables, it may indicate that an instance ' + 'of the\n' + 'given type (or a subclass) is expected or required as the ' + 'first\n' + 'positional argument (for example, CPython sets this ' + 'attribute for\n' + 'unbound methods that are implemented in C).\n' + '\n' + '\n' + 'Invoking Descriptors\n' + '--------------------\n' + '\n' + 'In general, a descriptor is an object attribute with ' + '"binding\n' + 'behavior", one whose attribute access has been overridden by ' + 'methods\n' + 'in the descriptor protocol: "__get__()", "__set__()", and\n' + '"__delete__()". If any of those methods are defined for an ' + 'object, it\n' + 'is said to be a descriptor.\n' + '\n' + 'The default behavior for attribute access is to get, set, or ' + 'delete\n' + "the attribute from an object's dictionary. For instance, " + '"a.x" has a\n' + 'lookup chain starting with "a.__dict__[\'x\']", then\n' + '"type(a).__dict__[\'x\']", and continuing through the base ' + 'classes of\n' + '"type(a)" excluding metaclasses.\n' + '\n' + 'However, if the looked-up value is an object defining one of ' + 'the\n' + 'descriptor methods, then Python may override the default ' + 'behavior and\n' + 'invoke the descriptor method instead. Where this occurs in ' + 'the\n' + 'precedence chain depends on which descriptor methods were ' + 'defined and\n' + 'how they were called.\n' + '\n' + 'The starting point for descriptor invocation is a binding, ' + '"a.x". How\n' + 'the arguments are assembled depends on "a":\n' + '\n' + 'Direct Call\n' + ' The simplest and least common call is when user code ' + 'directly\n' + ' invokes a descriptor method: "x.__get__(a)".\n' + '\n' + 'Instance Binding\n' + ' If binding to an object instance, "a.x" is transformed ' + 'into the\n' + ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' + '\n' + 'Class Binding\n' + ' If binding to a class, "A.x" is transformed into the ' + 'call:\n' + ' "A.__dict__[\'x\'].__get__(None, A)".\n' + '\n' + 'Super Binding\n' + ' If "a" is an instance of "super", then the binding ' + '"super(B,\n' + ' obj).m()" searches "obj.__class__.__mro__" for the base ' + 'class "A"\n' + ' immediately preceding "B" and then invokes the descriptor ' + 'with the\n' + ' call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n' + '\n' + 'For instance bindings, the precedence of descriptor ' + 'invocation depends\n' + 'on the which descriptor methods are defined. A descriptor ' + 'can define\n' + 'any combination of "__get__()", "__set__()" and ' + '"__delete__()". If it\n' + 'does not define "__get__()", then accessing the attribute ' + 'will return\n' + 'the descriptor object itself unless there is a value in the ' + "object's\n" + 'instance dictionary. If the descriptor defines "__set__()" ' + 'and/or\n' + '"__delete__()", it is a data descriptor; if it defines ' + 'neither, it is\n' + 'a non-data descriptor. Normally, data descriptors define ' + 'both\n' + '"__get__()" and "__set__()", while non-data descriptors have ' + 'just the\n' + '"__get__()" method. Data descriptors with "__set__()" and ' + '"__get__()"\n' + 'defined always override a redefinition in an instance ' + 'dictionary. In\n' + 'contrast, non-data descriptors can be overridden by ' + 'instances.\n' + '\n' + 'Python methods (including "staticmethod()" and ' + '"classmethod()") are\n' + 'implemented as non-data descriptors. Accordingly, instances ' + 'can\n' + 'redefine and override methods. This allows individual ' + 'instances to\n' + 'acquire behaviors that differ from other instances of the ' + 'same class.\n' + '\n' + 'The "property()" function is implemented as a data ' + 'descriptor.\n' + 'Accordingly, instances cannot override the behavior of a ' + 'property.\n' + '\n' + '\n' + '__slots__\n' + '---------\n' + '\n' + 'By default, instances of classes have a dictionary for ' + 'attribute\n' + 'storage. This wastes space for objects having very few ' + 'instance\n' + 'variables. The space consumption can become acute when ' + 'creating large\n' + 'numbers of instances.\n' + '\n' + 'The default can be overridden by defining *__slots__* in a ' + 'class\n' + 'definition. The *__slots__* declaration takes a sequence of ' + 'instance\n' + 'variables and reserves just enough space in each instance to ' + 'hold a\n' + 'value for each variable. Space is saved because *__dict__* ' + 'is not\n' + 'created for each instance.\n' + '\n' + 'object.__slots__\n' + '\n' + ' This class variable can be assigned a string, iterable, ' + 'or sequence\n' + ' of strings with variable names used by instances. ' + '*__slots__*\n' + ' reserves space for the declared variables and prevents ' + 'the\n' + ' automatic creation of *__dict__* and *__weakref__* for ' + 'each\n' + ' instance.\n' + '\n' + '\n' + 'Notes on using *__slots__*\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '* When inheriting from a class without *__slots__*, the ' + '*__dict__*\n' + ' attribute of that class will always be accessible, so a ' + '*__slots__*\n' + ' definition in the subclass is meaningless.\n' + '\n' + '* Without a *__dict__* variable, instances cannot be ' + 'assigned new\n' + ' variables not listed in the *__slots__* definition. ' + 'Attempts to\n' + ' assign to an unlisted variable name raises ' + '"AttributeError". If\n' + ' dynamic assignment of new variables is desired, then add\n' + ' "\'__dict__\'" to the sequence of strings in the ' + '*__slots__*\n' + ' declaration.\n' + '\n' + '* Without a *__weakref__* variable for each instance, ' + 'classes\n' + ' defining *__slots__* do not support weak references to ' + 'its\n' + ' instances. If weak reference support is needed, then add\n' + ' "\'__weakref__\'" to the sequence of strings in the ' + '*__slots__*\n' + ' declaration.\n' + '\n' + '* *__slots__* are implemented at the class level by ' + 'creating\n' + ' descriptors (Implementing Descriptors) for each variable ' + 'name. As a\n' + ' result, class attributes cannot be used to set default ' + 'values for\n' + ' instance variables defined by *__slots__*; otherwise, the ' + 'class\n' + ' attribute would overwrite the descriptor assignment.\n' + '\n' + '* The action of a *__slots__* declaration is limited to the ' + 'class\n' + ' where it is defined. As a result, subclasses will have a ' + '*__dict__*\n' + ' unless they also define *__slots__* (which must only ' + 'contain names\n' + ' of any *additional* slots).\n' + '\n' + '* If a class defines a slot also defined in a base class, ' + 'the\n' + ' instance variable defined by the base class slot is ' + 'inaccessible\n' + ' (except by retrieving its descriptor directly from the ' + 'base class).\n' + ' This renders the meaning of the program undefined. In the ' + 'future, a\n' + ' check may be added to prevent this.\n' + '\n' + '* Nonempty *__slots__* does not work for classes derived ' + 'from\n' + ' "variable-length" built-in types such as "int", "bytes" ' + 'and "tuple".\n' + '\n' + '* Any non-string iterable may be assigned to *__slots__*. ' + 'Mappings\n' + ' may also be used; however, in the future, special meaning ' + 'may be\n' + ' assigned to the values corresponding to each key.\n' + '\n' + '* *__class__* assignment works only if both classes have the ' + 'same\n' + ' *__slots__*.\n' + '\n' + '\n' + 'Customizing class creation\n' + '==========================\n' + '\n' + 'By default, classes are constructed using "type()". The ' + 'class body is\n' + 'executed in a new namespace and the class name is bound ' + 'locally to the\n' + 'result of "type(name, bases, namespace)".\n' + '\n' + 'The class creation process can be customised by passing the\n' + '"metaclass" keyword argument in the class definition line, ' + 'or by\n' + 'inheriting from an existing class that included such an ' + 'argument. In\n' + 'the following example, both "MyClass" and "MySubclass" are ' + 'instances\n' + 'of "Meta":\n' + '\n' + ' class Meta(type):\n' + ' pass\n' + '\n' + ' class MyClass(metaclass=Meta):\n' + ' pass\n' + '\n' + ' class MySubclass(MyClass):\n' + ' pass\n' + '\n' + 'Any other keyword arguments that are specified in the class ' + 'definition\n' + 'are passed through to all metaclass operations described ' + 'below.\n' + '\n' + 'When a class definition is executed, the following steps ' + 'occur:\n' + '\n' + '* the appropriate metaclass is determined\n' + '\n' + '* the class namespace is prepared\n' + '\n' + '* the class body is executed\n' + '\n' + '* the class object is created\n' + '\n' + '\n' + 'Determining the appropriate metaclass\n' + '-------------------------------------\n' + '\n' + 'The appropriate metaclass for a class definition is ' + 'determined as\n' + 'follows:\n' + '\n' + '* if no bases and no explicit metaclass are given, then ' + '"type()" is\n' + ' used\n' + '\n' + '* if an explicit metaclass is given and it is *not* an ' + 'instance of\n' + ' "type()", then it is used directly as the metaclass\n' + '\n' + '* if an instance of "type()" is given as the explicit ' + 'metaclass, or\n' + ' bases are defined, then the most derived metaclass is ' + 'used\n' + '\n' + 'The most derived metaclass is selected from the explicitly ' + 'specified\n' + 'metaclass (if any) and the metaclasses (i.e. "type(cls)") of ' + 'all\n' + 'specified base classes. The most derived metaclass is one ' + 'which is a\n' + 'subtype of *all* of these candidate metaclasses. If none of ' + 'the\n' + 'candidate metaclasses meets that criterion, then the class ' + 'definition\n' + 'will fail with "TypeError".\n' + '\n' + '\n' + 'Preparing the class namespace\n' + '-----------------------------\n' + '\n' + 'Once the appropriate metaclass has been identified, then the ' + 'class\n' + 'namespace is prepared. If the metaclass has a "__prepare__" ' + 'attribute,\n' + 'it is called as "namespace = metaclass.__prepare__(name, ' + 'bases,\n' + '**kwds)" (where the additional keyword arguments, if any, ' + 'come from\n' + 'the class definition).\n' + '\n' + 'If the metaclass has no "__prepare__" attribute, then the ' + 'class\n' + 'namespace is initialised as an empty "dict()" instance.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 3115** - Metaclasses in Python 3000\n' + ' Introduced the "__prepare__" namespace hook\n' + '\n' + '\n' + 'Executing the class body\n' + '------------------------\n' + '\n' + 'The class body is executed (approximately) as "exec(body, ' + 'globals(),\n' + 'namespace)". The key difference from a normal call to ' + '"exec()" is that\n' + 'lexical scoping allows the class body (including any ' + 'methods) to\n' + 'reference names from the current and outer scopes when the ' + 'class\n' + 'definition occurs inside a function.\n' + '\n' + 'However, even when the class definition occurs inside the ' + 'function,\n' + 'methods defined inside the class still cannot see names ' + 'defined at the\n' + 'class scope. Class variables must be accessed through the ' + 'first\n' + 'parameter of instance or class methods, and cannot be ' + 'accessed at all\n' + 'from static methods.\n' + '\n' + '\n' + 'Creating the class object\n' + '-------------------------\n' + '\n' + 'Once the class namespace has been populated by executing the ' + 'class\n' + 'body, the class object is created by calling ' + '"metaclass(name, bases,\n' + 'namespace, **kwds)" (the additional keywords passed here are ' + 'the same\n' + 'as those passed to "__prepare__").\n' + '\n' + 'This class object is the one that will be referenced by the ' + 'zero-\n' + 'argument form of "super()". "__class__" is an implicit ' + 'closure\n' + 'reference created by the compiler if any methods in a class ' + 'body refer\n' + 'to either "__class__" or "super". This allows the zero ' + 'argument form\n' + 'of "super()" to correctly identify the class being defined ' + 'based on\n' + 'lexical scoping, while the class or instance that was used ' + 'to make the\n' + 'current call is identified based on the first argument ' + 'passed to the\n' + 'method.\n' + '\n' + 'After the class object is created, it is passed to the ' + 'class\n' + 'decorators included in the class definition (if any) and the ' + 'resulting\n' + 'object is bound in the local namespace as the defined ' + 'class.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 3135** - New super\n' + ' Describes the implicit "__class__" closure reference\n' + '\n' + '\n' + 'Metaclass example\n' + '-----------------\n' + '\n' + 'The potential uses for metaclasses are boundless. Some ideas ' + 'that have\n' + 'been explored include logging, interface checking, ' + 'automatic\n' + 'delegation, automatic property creation, proxies, ' + 'frameworks, and\n' + 'automatic resource locking/synchronization.\n' + '\n' + 'Here is an example of a metaclass that uses an\n' + '"collections.OrderedDict" to remember the order that class ' + 'variables\n' + 'are defined:\n' + '\n' + ' class OrderedClass(type):\n' + '\n' + ' @classmethod\n' + ' def __prepare__(metacls, name, bases, **kwds):\n' + ' return collections.OrderedDict()\n' + '\n' + ' def __new__(cls, name, bases, namespace, **kwds):\n' + ' result = type.__new__(cls, name, bases, ' + 'dict(namespace))\n' + ' result.members = tuple(namespace)\n' + ' return result\n' + '\n' + ' class A(metaclass=OrderedClass):\n' + ' def one(self): pass\n' + ' def two(self): pass\n' + ' def three(self): pass\n' + ' def four(self): pass\n' + '\n' + ' >>> A.members\n' + " ('__module__', 'one', 'two', 'three', 'four')\n" + '\n' + 'When the class definition for *A* gets executed, the process ' + 'begins\n' + 'with calling the metaclass\'s "__prepare__()" method which ' + 'returns an\n' + 'empty "collections.OrderedDict". That mapping records the ' + 'methods and\n' + 'attributes of *A* as they are defined within the body of the ' + 'class\n' + 'statement. Once those definitions are executed, the ordered ' + 'dictionary\n' + 'is fully populated and the metaclass\'s "__new__()" method ' + 'gets\n' + 'invoked. That method builds the new type and it saves the ' + 'ordered\n' + 'dictionary keys in an attribute called "members".\n' + '\n' + '\n' + 'Customizing instance and subclass checks\n' + '========================================\n' + '\n' + 'The following methods are used to override the default ' + 'behavior of the\n' + '"isinstance()" and "issubclass()" built-in functions.\n' + '\n' + 'In particular, the metaclass "abc.ABCMeta" implements these ' + 'methods in\n' + 'order to allow the addition of Abstract Base Classes (ABCs) ' + 'as\n' + '"virtual base classes" to any class or type (including ' + 'built-in\n' + 'types), including other ABCs.\n' + '\n' + 'class.__instancecheck__(self, instance)\n' + '\n' + ' Return true if *instance* should be considered a (direct ' + 'or\n' + ' indirect) instance of *class*. If defined, called to ' + 'implement\n' + ' "isinstance(instance, class)".\n' + '\n' + 'class.__subclasscheck__(self, subclass)\n' + '\n' + ' Return true if *subclass* should be considered a (direct ' + 'or\n' + ' indirect) subclass of *class*. If defined, called to ' + 'implement\n' + ' "issubclass(subclass, class)".\n' + '\n' + 'Note that these methods are looked up on the type ' + '(metaclass) of a\n' + 'class. They cannot be defined as class methods in the ' + 'actual class.\n' + 'This is consistent with the lookup of special methods that ' + 'are called\n' + 'on instances, only in this case the instance is itself a ' + 'class.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 3119** - Introducing Abstract Base Classes\n' + ' Includes the specification for customizing ' + '"isinstance()" and\n' + ' "issubclass()" behavior through "__instancecheck__()" ' + 'and\n' + ' "__subclasscheck__()", with motivation for this ' + 'functionality in\n' + ' the context of adding Abstract Base Classes (see the ' + '"abc"\n' + ' module) to the language.\n' + '\n' + '\n' + 'Emulating callable objects\n' + '==========================\n' + '\n' + 'object.__call__(self[, args...])\n' + '\n' + ' Called when the instance is "called" as a function; if ' + 'this method\n' + ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' + ' "x.__call__(arg1, arg2, ...)".\n' + '\n' + '\n' + 'Emulating container types\n' + '=========================\n' + '\n' + 'The following methods can be defined to implement container ' + 'objects.\n' + 'Containers usually are sequences (such as lists or tuples) ' + 'or mappings\n' + '(like dictionaries), but can represent other containers as ' + 'well. The\n' + 'first set of methods is used either to emulate a sequence or ' + 'to\n' + 'emulate a mapping; the difference is that for a sequence, ' + 'the\n' + 'allowable keys should be the integers *k* for which "0 <= k ' + '< N" where\n' + '*N* is the length of the sequence, or slice objects, which ' + 'define a\n' + 'range of items. It is also recommended that mappings ' + 'provide the\n' + 'methods "keys()", "values()", "items()", "get()", ' + '"clear()",\n' + '"setdefault()", "pop()", "popitem()", "copy()", and ' + '"update()"\n' + "behaving similar to those for Python's standard dictionary " + 'objects.\n' + 'The "collections" module provides a "MutableMapping" ' + 'abstract base\n' + 'class to help create those methods from a base set of ' + '"__getitem__()",\n' + '"__setitem__()", "__delitem__()", and "keys()". Mutable ' + 'sequences\n' + 'should provide methods "append()", "count()", "index()", ' + '"extend()",\n' + '"insert()", "pop()", "remove()", "reverse()" and "sort()", ' + 'like Python\n' + 'standard list objects. Finally, sequence types should ' + 'implement\n' + 'addition (meaning concatenation) and multiplication ' + '(meaning\n' + 'repetition) by defining the methods "__add__()", ' + '"__radd__()",\n' + '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" ' + 'described\n' + 'below; they should not define other numerical operators. It ' + 'is\n' + 'recommended that both mappings and sequences implement the\n' + '"__contains__()" method to allow efficient use of the "in" ' + 'operator;\n' + 'for mappings, "in" should search the mapping\'s keys; for ' + 'sequences, it\n' + 'should search through the values. It is further recommended ' + 'that both\n' + 'mappings and sequences implement the "__iter__()" method to ' + 'allow\n' + 'efficient iteration through the container; for mappings, ' + '"__iter__()"\n' + 'should be the same as "keys()"; for sequences, it should ' + 'iterate\n' + 'through the values.\n' + '\n' + 'object.__len__(self)\n' + '\n' + ' Called to implement the built-in function "len()". ' + 'Should return\n' + ' the length of the object, an integer ">=" 0. Also, an ' + 'object that\n' + ' doesn\'t define a "__bool__()" method and whose ' + '"__len__()" method\n' + ' returns zero is considered to be false in a Boolean ' + 'context.\n' + '\n' + 'object.__length_hint__(self)\n' + '\n' + ' Called to implement "operator.length_hint()". Should ' + 'return an\n' + ' estimated length for the object (which may be greater or ' + 'less than\n' + ' the actual length). The length must be an integer ">=" 0. ' + 'This\n' + ' method is purely an optimization and is never required ' + 'for\n' + ' correctness.\n' + '\n' + ' New in version 3.4.\n' + '\n' + 'Note: Slicing is done exclusively with the following three ' + 'methods.\n' + ' A call like\n' + '\n' + ' a[1:2] = b\n' + '\n' + ' is translated to\n' + '\n' + ' a[slice(1, 2, None)] = b\n' + '\n' + ' and so forth. Missing slice items are always filled in ' + 'with "None".\n' + '\n' + 'object.__getitem__(self, key)\n' + '\n' + ' Called to implement evaluation of "self[key]". For ' + 'sequence types,\n' + ' the accepted keys should be integers and slice objects. ' + 'Note that\n' + ' the special interpretation of negative indexes (if the ' + 'class wishes\n' + ' to emulate a sequence type) is up to the "__getitem__()" ' + 'method. If\n' + ' *key* is of an inappropriate type, "TypeError" may be ' + 'raised; if of\n' + ' a value outside the set of indexes for the sequence ' + '(after any\n' + ' special interpretation of negative values), "IndexError" ' + 'should be\n' + ' raised. For mapping types, if *key* is missing (not in ' + 'the\n' + ' container), "KeyError" should be raised.\n' + '\n' + ' Note: "for" loops expect that an "IndexError" will be ' + 'raised for\n' + ' illegal indexes to allow proper detection of the end of ' + 'the\n' + ' sequence.\n' + '\n' + 'object.__missing__(self, key)\n' + '\n' + ' Called by "dict"."__getitem__()" to implement "self[key]" ' + 'for dict\n' + ' subclasses when key is not in the dictionary.\n' + '\n' + 'object.__setitem__(self, key, value)\n' + '\n' + ' Called to implement assignment to "self[key]". Same note ' + 'as for\n' + ' "__getitem__()". This should only be implemented for ' + 'mappings if\n' + ' the objects support changes to the values for keys, or if ' + 'new keys\n' + ' can be added, or for sequences if elements can be ' + 'replaced. The\n' + ' same exceptions should be raised for improper *key* ' + 'values as for\n' + ' the "__getitem__()" method.\n' + '\n' + 'object.__delitem__(self, key)\n' + '\n' + ' Called to implement deletion of "self[key]". Same note ' + 'as for\n' + ' "__getitem__()". This should only be implemented for ' + 'mappings if\n' + ' the objects support removal of keys, or for sequences if ' + 'elements\n' + ' can be removed from the sequence. The same exceptions ' + 'should be\n' + ' raised for improper *key* values as for the ' + '"__getitem__()" method.\n' + '\n' + 'object.__iter__(self)\n' + '\n' + ' This method is called when an iterator is required for a ' + 'container.\n' + ' This method should return a new iterator object that can ' + 'iterate\n' + ' over all the objects in the container. For mappings, it ' + 'should\n' + ' iterate over the keys of the container.\n' + '\n' + ' Iterator objects also need to implement this method; they ' + 'are\n' + ' required to return themselves. For more information on ' + 'iterator\n' + ' objects, see Iterator Types.\n' + '\n' + 'object.__reversed__(self)\n' + '\n' + ' Called (if present) by the "reversed()" built-in to ' + 'implement\n' + ' reverse iteration. It should return a new iterator ' + 'object that\n' + ' iterates over all the objects in the container in reverse ' + 'order.\n' + '\n' + ' If the "__reversed__()" method is not provided, the ' + '"reversed()"\n' + ' built-in will fall back to using the sequence protocol ' + '("__len__()"\n' + ' and "__getitem__()"). Objects that support the sequence ' + 'protocol\n' + ' should only provide "__reversed__()" if they can provide ' + 'an\n' + ' implementation that is more efficient than the one ' + 'provided by\n' + ' "reversed()".\n' + '\n' + 'The membership test operators ("in" and "not in") are ' + 'normally\n' + 'implemented as an iteration through a sequence. However, ' + 'container\n' + 'objects can supply the following special method with a more ' + 'efficient\n' + 'implementation, which also does not require the object be a ' + 'sequence.\n' + '\n' + 'object.__contains__(self, item)\n' + '\n' + ' Called to implement membership test operators. Should ' + 'return true\n' + ' if *item* is in *self*, false otherwise. For mapping ' + 'objects, this\n' + ' should consider the keys of the mapping rather than the ' + 'values or\n' + ' the key-item pairs.\n' + '\n' + ' For objects that don\'t define "__contains__()", the ' + 'membership test\n' + ' first tries iteration via "__iter__()", then the old ' + 'sequence\n' + ' iteration protocol via "__getitem__()", see this section ' + 'in the\n' + ' language reference.\n' + '\n' + '\n' + 'Emulating numeric types\n' + '=======================\n' + '\n' + 'The following methods can be defined to emulate numeric ' + 'objects.\n' + 'Methods corresponding to operations that are not supported ' + 'by the\n' + 'particular kind of number implemented (e.g., bitwise ' + 'operations for\n' + 'non-integral numbers) should be left undefined.\n' + '\n' + 'object.__add__(self, other)\n' + 'object.__sub__(self, other)\n' + 'object.__mul__(self, other)\n' + 'object.__matmul__(self, other)\n' + 'object.__truediv__(self, other)\n' + 'object.__floordiv__(self, other)\n' + 'object.__mod__(self, other)\n' + 'object.__divmod__(self, other)\n' + 'object.__pow__(self, other[, modulo])\n' + 'object.__lshift__(self, other)\n' + 'object.__rshift__(self, other)\n' + 'object.__and__(self, other)\n' + 'object.__xor__(self, other)\n' + 'object.__or__(self, other)\n' + '\n' + ' These methods are called to implement the binary ' + 'arithmetic\n' + ' operations ("+", "-", "*", "@", "/", "//", "%", ' + '"divmod()",\n' + ' "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, ' + 'to\n' + ' evaluate the expression "x + y", where *x* is an instance ' + 'of a\n' + ' class that has an "__add__()" method, "x.__add__(y)" is ' + 'called.\n' + ' The "__divmod__()" method should be the equivalent to ' + 'using\n' + ' "__floordiv__()" and "__mod__()"; it should not be ' + 'related to\n' + ' "__truediv__()". Note that "__pow__()" should be defined ' + 'to accept\n' + ' an optional third argument if the ternary version of the ' + 'built-in\n' + ' "pow()" function is to be supported.\n' + '\n' + ' If one of those methods does not support the operation ' + 'with the\n' + ' supplied arguments, it should return "NotImplemented".\n' + '\n' + 'object.__radd__(self, other)\n' + 'object.__rsub__(self, other)\n' + 'object.__rmul__(self, other)\n' + 'object.__rmatmul__(self, other)\n' + 'object.__rtruediv__(self, other)\n' + 'object.__rfloordiv__(self, other)\n' + 'object.__rmod__(self, other)\n' + 'object.__rdivmod__(self, other)\n' + 'object.__rpow__(self, other)\n' + 'object.__rlshift__(self, other)\n' + 'object.__rrshift__(self, other)\n' + 'object.__rand__(self, other)\n' + 'object.__rxor__(self, other)\n' + 'object.__ror__(self, other)\n' + '\n' + ' These methods are called to implement the binary ' + 'arithmetic\n' + ' operations ("+", "-", "*", "@", "/", "//", "%", ' + '"divmod()",\n' + ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' + '(swapped)\n' + ' operands. These functions are only called if the left ' + 'operand does\n' + ' not support the corresponding operation and the operands ' + 'are of\n' + ' different types. [2] For instance, to evaluate the ' + 'expression "x -\n' + ' y", where *y* is an instance of a class that has an ' + '"__rsub__()"\n' + ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' + 'returns\n' + ' *NotImplemented*.\n' + '\n' + ' Note that ternary "pow()" will not try calling ' + '"__rpow__()" (the\n' + ' coercion rules would become too complicated).\n' + '\n' + " Note: If the right operand's type is a subclass of the " + 'left\n' + " operand's type and that subclass provides the reflected " + 'method\n' + ' for the operation, this method will be called before ' + 'the left\n' + " operand's non-reflected method. This behavior allows " + 'subclasses\n' + " to override their ancestors' operations.\n" + '\n' + 'object.__iadd__(self, other)\n' + 'object.__isub__(self, other)\n' + 'object.__imul__(self, other)\n' + 'object.__imatmul__(self, other)\n' + 'object.__itruediv__(self, other)\n' + 'object.__ifloordiv__(self, other)\n' + 'object.__imod__(self, other)\n' + 'object.__ipow__(self, other[, modulo])\n' + 'object.__ilshift__(self, other)\n' + 'object.__irshift__(self, other)\n' + 'object.__iand__(self, other)\n' + 'object.__ixor__(self, other)\n' + 'object.__ior__(self, other)\n' + '\n' + ' These methods are called to implement the augmented ' + 'arithmetic\n' + ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' + '"**=",\n' + ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' + 'attempt to\n' + ' do the operation in-place (modifying *self*) and return ' + 'the result\n' + ' (which could be, but does not have to be, *self*). If a ' + 'specific\n' + ' method is not defined, the augmented assignment falls ' + 'back to the\n' + ' normal methods. For instance, if *x* is an instance of a ' + 'class\n' + ' with an "__iadd__()" method, "x += y" is equivalent to "x ' + '=\n' + ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' + '"y.__radd__(x)" are\n' + ' considered, as with the evaluation of "x + y". In ' + 'certain\n' + ' situations, augmented assignment can result in unexpected ' + 'errors\n' + " (see Why does a_tuple[i] += ['item'] raise an exception " + 'when the\n' + ' addition works?), but this behavior is in fact part of ' + 'the data\n' + ' model.\n' + '\n' + 'object.__neg__(self)\n' + 'object.__pos__(self)\n' + 'object.__abs__(self)\n' + 'object.__invert__(self)\n' + '\n' + ' Called to implement the unary arithmetic operations ("-", ' + '"+",\n' + ' "abs()" and "~").\n' + '\n' + 'object.__complex__(self)\n' + 'object.__int__(self)\n' + 'object.__float__(self)\n' + 'object.__round__(self[, n])\n' + '\n' + ' Called to implement the built-in functions "complex()", ' + '"int()",\n' + ' "float()" and "round()". Should return a value of the ' + 'appropriate\n' + ' type.\n' + '\n' + 'object.__index__(self)\n' + '\n' + ' Called to implement "operator.index()", and whenever ' + 'Python needs\n' + ' to losslessly convert the numeric object to an integer ' + 'object (such\n' + ' as in slicing, or in the built-in "bin()", "hex()" and ' + '"oct()"\n' + ' functions). Presence of this method indicates that the ' + 'numeric\n' + ' object is an integer type. Must return an integer.\n' + '\n' + ' Note: In order to have a coherent integer type class, ' + 'when\n' + ' "__index__()" is defined "__int__()" should also be ' + 'defined, and\n' + ' both should return the same value.\n' + '\n' + '\n' + 'With Statement Context Managers\n' + '===============================\n' + '\n' + 'A *context manager* is an object that defines the runtime ' + 'context to\n' + 'be established when executing a "with" statement. The ' + 'context manager\n' + 'handles the entry into, and the exit from, the desired ' + 'runtime context\n' + 'for the execution of the block of code. Context managers ' + 'are normally\n' + 'invoked using the "with" statement (described in section The ' + 'with\n' + 'statement), but can also be used by directly invoking their ' + 'methods.\n' + '\n' + 'Typical uses of context managers include saving and ' + 'restoring various\n' + 'kinds of global state, locking and unlocking resources, ' + 'closing opened\n' + 'files, etc.\n' + '\n' + 'For more information on context managers, see Context ' + 'Manager Types.\n' + '\n' + 'object.__enter__(self)\n' + '\n' + ' Enter the runtime context related to this object. The ' + '"with"\n' + " statement will bind this method's return value to the " + 'target(s)\n' + ' specified in the "as" clause of the statement, if any.\n' + '\n' + 'object.__exit__(self, exc_type, exc_value, traceback)\n' + '\n' + ' Exit the runtime context related to this object. The ' + 'parameters\n' + ' describe the exception that caused the context to be ' + 'exited. If the\n' + ' context was exited without an exception, all three ' + 'arguments will\n' + ' be "None".\n' + '\n' + ' If an exception is supplied, and the method wishes to ' + 'suppress the\n' + ' exception (i.e., prevent it from being propagated), it ' + 'should\n' + ' return a true value. Otherwise, the exception will be ' + 'processed\n' + ' normally upon exit from this method.\n' + '\n' + ' Note that "__exit__()" methods should not reraise the ' + 'passed-in\n' + " exception; this is the caller's responsibility.\n" + '\n' + 'See also:\n' + '\n' + ' **PEP 343** - The "with" statement\n' + ' The specification, background, and examples for the ' + 'Python "with"\n' + ' statement.\n' + '\n' + '\n' + 'Special method lookup\n' + '=====================\n' + '\n' + 'For custom classes, implicit invocations of special methods ' + 'are only\n' + "guaranteed to work correctly if defined on an object's type, " + 'not in\n' + "the object's instance dictionary. That behaviour is the " + 'reason why\n' + 'the following code raises an exception:\n' + '\n' + ' >>> class C:\n' + ' ... pass\n' + ' ...\n' + ' >>> c = C()\n' + ' >>> c.__len__ = lambda: 5\n' + ' >>> len(c)\n' + ' Traceback (most recent call last):\n' + ' File "", line 1, in \n' + " TypeError: object of type 'C' has no len()\n" + '\n' + 'The rationale behind this behaviour lies with a number of ' + 'special\n' + 'methods such as "__hash__()" and "__repr__()" that are ' + 'implemented by\n' + 'all objects, including type objects. If the implicit lookup ' + 'of these\n' + 'methods used the conventional lookup process, they would ' + 'fail when\n' + 'invoked on the type object itself:\n' + '\n' + ' >>> 1 .__hash__() == hash(1)\n' + ' True\n' + ' >>> int.__hash__() == hash(int)\n' + ' Traceback (most recent call last):\n' + ' File "", line 1, in \n' + " TypeError: descriptor '__hash__' of 'int' object needs an " + 'argument\n' + '\n' + 'Incorrectly attempting to invoke an unbound method of a ' + 'class in this\n' + "way is sometimes referred to as 'metaclass confusion', and " + 'is avoided\n' + 'by bypassing the instance when looking up special methods:\n' + '\n' + ' >>> type(1).__hash__(1) == hash(1)\n' + ' True\n' + ' >>> type(int).__hash__(int) == hash(int)\n' + ' True\n' + '\n' + 'In addition to bypassing any instance attributes in the ' + 'interest of\n' + 'correctness, implicit special method lookup generally also ' + 'bypasses\n' + 'the "__getattribute__()" method even of the object\'s ' + 'metaclass:\n' + '\n' + ' >>> class Meta(type):\n' + ' ... def __getattribute__(*args):\n' + ' ... print("Metaclass getattribute invoked")\n' + ' ... return type.__getattribute__(*args)\n' + ' ...\n' + ' >>> class C(object, metaclass=Meta):\n' + ' ... def __len__(self):\n' + ' ... return 10\n' + ' ... def __getattribute__(*args):\n' + ' ... print("Class getattribute invoked")\n' + ' ... return object.__getattribute__(*args)\n' + ' ...\n' + ' >>> c = C()\n' + ' >>> c.__len__() # Explicit lookup via ' + 'instance\n' + ' Class getattribute invoked\n' + ' 10\n' + ' >>> type(c).__len__(c) # Explicit lookup via ' + 'type\n' + ' Metaclass getattribute invoked\n' + ' 10\n' + ' >>> len(c) # Implicit lookup\n' + ' 10\n' + '\n' + 'Bypassing the "__getattribute__()" machinery in this fashion ' + 'provides\n' + 'significant scope for speed optimisations within the ' + 'interpreter, at\n' + 'the cost of some flexibility in the handling of special ' + 'methods (the\n' + 'special method *must* be set on the class object itself in ' + 'order to be\n' + 'consistently invoked by the interpreter).\n', + 'string-methods': '\n' + 'String Methods\n' + '**************\n' + '\n' + 'Strings implement all of the common sequence operations, ' + 'along with\n' + 'the additional methods described below.\n' + '\n' + 'Strings also support two styles of string formatting, one ' + 'providing a\n' + 'large degree of flexibility and customization (see ' + '"str.format()",\n' + 'Format String Syntax and Custom String Formatting) and the ' + 'other based\n' + 'on C "printf" style formatting that handles a narrower ' + 'range of types\n' + 'and is slightly harder to use correctly, but is often ' + 'faster for the\n' + 'cases it can handle (printf-style String Formatting).\n' + '\n' + 'The Text Processing Services section of the standard ' + 'library covers a\n' + 'number of other modules that provide various text related ' + 'utilities\n' + '(including regular expression support in the "re" ' + 'module).\n' + '\n' + 'str.capitalize()\n' + '\n' + ' Return a copy of the string with its first character ' + 'capitalized\n' + ' and the rest lowercased.\n' + '\n' + 'str.casefold()\n' + '\n' + ' Return a casefolded copy of the string. Casefolded ' + 'strings may be\n' + ' used for caseless matching.\n' + '\n' + ' Casefolding is similar to lowercasing but more ' + 'aggressive because\n' + ' it is intended to remove all case distinctions in a ' + 'string. For\n' + ' example, the German lowercase letter "\'ß\'" is ' + 'equivalent to ""ss"".\n' + ' Since it is already lowercase, "lower()" would do ' + 'nothing to "\'ß\'";\n' + ' "casefold()" converts it to ""ss"".\n' + '\n' + ' The casefolding algorithm is described in section 3.13 ' + 'of the\n' + ' Unicode Standard.\n' + '\n' + ' New in version 3.3.\n' + '\n' + 'str.center(width[, fillchar])\n' + '\n' + ' Return centered in a string of length *width*. Padding ' + 'is done\n' + ' using the specified *fillchar* (default is an ASCII ' + 'space). The\n' + ' original string is returned if *width* is less than or ' + 'equal to\n' + ' "len(s)".\n' + '\n' + 'str.count(sub[, start[, end]])\n' + '\n' + ' Return the number of non-overlapping occurrences of ' + 'substring *sub*\n' + ' in the range [*start*, *end*]. Optional arguments ' + '*start* and\n' + ' *end* are interpreted as in slice notation.\n' + '\n' + 'str.encode(encoding="utf-8", errors="strict")\n' + '\n' + ' Return an encoded version of the string as a bytes ' + 'object. Default\n' + ' encoding is "\'utf-8\'". *errors* may be given to set a ' + 'different\n' + ' error handling scheme. The default for *errors* is ' + '"\'strict\'",\n' + ' meaning that encoding errors raise a "UnicodeError". ' + 'Other possible\n' + ' values are "\'ignore\'", "\'replace\'", ' + '"\'xmlcharrefreplace\'",\n' + ' "\'backslashreplace\'" and any other name registered ' + 'via\n' + ' "codecs.register_error()", see section Error Handlers. ' + 'For a list\n' + ' of possible encodings, see section Standard Encodings.\n' + '\n' + ' Changed in version 3.1: Support for keyword arguments ' + 'added.\n' + '\n' + 'str.endswith(suffix[, start[, end]])\n' + '\n' + ' Return "True" if the string ends with the specified ' + '*suffix*,\n' + ' otherwise return "False". *suffix* can also be a tuple ' + 'of suffixes\n' + ' to look for. With optional *start*, test beginning at ' + 'that\n' + ' position. With optional *end*, stop comparing at that ' + 'position.\n' + '\n' + 'str.expandtabs(tabsize=8)\n' + '\n' + ' Return a copy of the string where all tab characters ' + 'are replaced\n' + ' by one or more spaces, depending on the current column ' + 'and the\n' + ' given tab size. Tab positions occur every *tabsize* ' + 'characters\n' + ' (default is 8, giving tab positions at columns 0, 8, 16 ' + 'and so on).\n' + ' To expand the string, the current column is set to zero ' + 'and the\n' + ' string is examined character by character. If the ' + 'character is a\n' + ' tab ("\\t"), one or more space characters are inserted ' + 'in the result\n' + ' until the current column is equal to the next tab ' + 'position. (The\n' + ' tab character itself is not copied.) If the character ' + 'is a newline\n' + ' ("\\n") or return ("\\r"), it is copied and the current ' + 'column is\n' + ' reset to zero. Any other character is copied unchanged ' + 'and the\n' + ' current column is incremented by one regardless of how ' + 'the\n' + ' character is represented when printed.\n' + '\n' + " >>> '01\\t012\\t0123\\t01234'.expandtabs()\n" + " '01 012 0123 01234'\n" + " >>> '01\\t012\\t0123\\t01234'.expandtabs(4)\n" + " '01 012 0123 01234'\n" + '\n' + 'str.find(sub[, start[, end]])\n' + '\n' + ' Return the lowest index in the string where substring ' + '*sub* is\n' + ' found within the slice "s[start:end]". Optional ' + 'arguments *start*\n' + ' and *end* are interpreted as in slice notation. Return ' + '"-1" if\n' + ' *sub* is not found.\n' + '\n' + ' Note: The "find()" method should be used only if you ' + 'need to know\n' + ' the position of *sub*. To check if *sub* is a ' + 'substring or not,\n' + ' use the "in" operator:\n' + '\n' + " >>> 'Py' in 'Python'\n" + ' True\n' + '\n' + 'str.format(*args, **kwargs)\n' + '\n' + ' Perform a string formatting operation. The string on ' + 'which this\n' + ' method is called can contain literal text or ' + 'replacement fields\n' + ' delimited by braces "{}". Each replacement field ' + 'contains either\n' + ' the numeric index of a positional argument, or the name ' + 'of a\n' + ' keyword argument. Returns a copy of the string where ' + 'each\n' + ' replacement field is replaced with the string value of ' + 'the\n' + ' corresponding argument.\n' + '\n' + ' >>> "The sum of 1 + 2 is {0}".format(1+2)\n' + " 'The sum of 1 + 2 is 3'\n" + '\n' + ' See Format String Syntax for a description of the ' + 'various\n' + ' formatting options that can be specified in format ' + 'strings.\n' + '\n' + 'str.format_map(mapping)\n' + '\n' + ' Similar to "str.format(**mapping)", except that ' + '"mapping" is used\n' + ' directly and not copied to a "dict". This is useful if ' + 'for example\n' + ' "mapping" is a dict subclass:\n' + '\n' + ' >>> class Default(dict):\n' + ' ... def __missing__(self, key):\n' + ' ... return key\n' + ' ...\n' + " >>> '{name} was born in " + "{country}'.format_map(Default(name='Guido'))\n" + " 'Guido was born in country'\n" + '\n' + ' New in version 3.2.\n' + '\n' + 'str.index(sub[, start[, end]])\n' + '\n' + ' Like "find()", but raise "ValueError" when the ' + 'substring is not\n' + ' found.\n' + '\n' + 'str.isalnum()\n' + '\n' + ' Return true if all characters in the string are ' + 'alphanumeric and\n' + ' there is at least one character, false otherwise. A ' + 'character "c"\n' + ' is alphanumeric if one of the following returns ' + '"True":\n' + ' "c.isalpha()", "c.isdecimal()", "c.isdigit()", or ' + '"c.isnumeric()".\n' + '\n' + 'str.isalpha()\n' + '\n' + ' Return true if all characters in the string are ' + 'alphabetic and\n' + ' there is at least one character, false otherwise. ' + 'Alphabetic\n' + ' characters are those characters defined in the Unicode ' + 'character\n' + ' database as "Letter", i.e., those with general category ' + 'property\n' + ' being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note ' + 'that this is\n' + ' different from the "Alphabetic" property defined in the ' + 'Unicode\n' + ' Standard.\n' + '\n' + 'str.isdecimal()\n' + '\n' + ' Return true if all characters in the string are decimal ' + 'characters\n' + ' and there is at least one character, false otherwise. ' + 'Decimal\n' + ' characters are those from general category "Nd". This ' + 'category\n' + ' includes digit characters, and all characters that can ' + 'be used to\n' + ' form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC ' + 'DIGIT ZERO.\n' + '\n' + 'str.isdigit()\n' + '\n' + ' Return true if all characters in the string are digits ' + 'and there is\n' + ' at least one character, false otherwise. Digits ' + 'include decimal\n' + ' characters and digits that need special handling, such ' + 'as the\n' + ' compatibility superscript digits. Formally, a digit is ' + 'a character\n' + ' that has the property value Numeric_Type=Digit or\n' + ' Numeric_Type=Decimal.\n' + '\n' + 'str.isidentifier()\n' + '\n' + ' Return true if the string is a valid identifier ' + 'according to the\n' + ' language definition, section Identifiers and keywords.\n' + '\n' + ' Use "keyword.iskeyword()" to test for reserved ' + 'identifiers such as\n' + ' "def" and "class".\n' + '\n' + 'str.islower()\n' + '\n' + ' Return true if all cased characters [4] in the string ' + 'are lowercase\n' + ' and there is at least one cased character, false ' + 'otherwise.\n' + '\n' + 'str.isnumeric()\n' + '\n' + ' Return true if all characters in the string are numeric ' + 'characters,\n' + ' and there is at least one character, false otherwise. ' + 'Numeric\n' + ' characters include digit characters, and all characters ' + 'that have\n' + ' the Unicode numeric value property, e.g. U+2155, VULGAR ' + 'FRACTION\n' + ' ONE FIFTH. Formally, numeric characters are those with ' + 'the\n' + ' property value Numeric_Type=Digit, Numeric_Type=Decimal ' + 'or\n' + ' Numeric_Type=Numeric.\n' + '\n' + 'str.isprintable()\n' + '\n' + ' Return true if all characters in the string are ' + 'printable or the\n' + ' string is empty, false otherwise. Nonprintable ' + 'characters are\n' + ' those characters defined in the Unicode character ' + 'database as\n' + ' "Other" or "Separator", excepting the ASCII space ' + '(0x20) which is\n' + ' considered printable. (Note that printable characters ' + 'in this\n' + ' context are those which should not be escaped when ' + '"repr()" is\n' + ' invoked on a string. It has no bearing on the handling ' + 'of strings\n' + ' written to "sys.stdout" or "sys.stderr".)\n' + '\n' + 'str.isspace()\n' + '\n' + ' Return true if there are only whitespace characters in ' + 'the string\n' + ' and there is at least one character, false otherwise. ' + 'Whitespace\n' + ' characters are those characters defined in the Unicode ' + 'character\n' + ' database as "Other" or "Separator" and those with ' + 'bidirectional\n' + ' property being one of "WS", "B", or "S".\n' + '\n' + 'str.istitle()\n' + '\n' + ' Return true if the string is a titlecased string and ' + 'there is at\n' + ' least one character, for example uppercase characters ' + 'may only\n' + ' follow uncased characters and lowercase characters only ' + 'cased ones.\n' + ' Return false otherwise.\n' + '\n' + 'str.isupper()\n' + '\n' + ' Return true if all cased characters [4] in the string ' + 'are uppercase\n' + ' and there is at least one cased character, false ' + 'otherwise.\n' + '\n' + 'str.join(iterable)\n' + '\n' + ' Return a string which is the concatenation of the ' + 'strings in the\n' + ' *iterable* *iterable*. A "TypeError" will be raised if ' + 'there are\n' + ' any non-string values in *iterable*, including "bytes" ' + 'objects.\n' + ' The separator between elements is the string providing ' + 'this method.\n' + '\n' + 'str.ljust(width[, fillchar])\n' + '\n' + ' Return the string left justified in a string of length ' + '*width*.\n' + ' Padding is done using the specified *fillchar* (default ' + 'is an ASCII\n' + ' space). The original string is returned if *width* is ' + 'less than or\n' + ' equal to "len(s)".\n' + '\n' + 'str.lower()\n' + '\n' + ' Return a copy of the string with all the cased ' + 'characters [4]\n' + ' converted to lowercase.\n' + '\n' + ' The lowercasing algorithm used is described in section ' + '3.13 of the\n' + ' Unicode Standard.\n' + '\n' + 'str.lstrip([chars])\n' + '\n' + ' Return a copy of the string with leading characters ' + 'removed. The\n' + ' *chars* argument is a string specifying the set of ' + 'characters to be\n' + ' removed. If omitted or "None", the *chars* argument ' + 'defaults to\n' + ' removing whitespace. The *chars* argument is not a ' + 'prefix; rather,\n' + ' all combinations of its values are stripped:\n' + '\n' + " >>> ' spacious '.lstrip()\n" + " 'spacious '\n" + " >>> 'www.example.com'.lstrip('cmowz.')\n" + " 'example.com'\n" + '\n' + 'static str.maketrans(x[, y[, z]])\n' + '\n' + ' This static method returns a translation table usable ' + 'for\n' + ' "str.translate()".\n' + '\n' + ' If there is only one argument, it must be a dictionary ' + 'mapping\n' + ' Unicode ordinals (integers) or characters (strings of ' + 'length 1) to\n' + ' Unicode ordinals, strings (of arbitrary lengths) or ' + 'None.\n' + ' Character keys will then be converted to ordinals.\n' + '\n' + ' If there are two arguments, they must be strings of ' + 'equal length,\n' + ' and in the resulting dictionary, each character in x ' + 'will be mapped\n' + ' to the character at the same position in y. If there ' + 'is a third\n' + ' argument, it must be a string, whose characters will be ' + 'mapped to\n' + ' None in the result.\n' + '\n' + 'str.partition(sep)\n' + '\n' + ' Split the string at the first occurrence of *sep*, and ' + 'return a\n' + ' 3-tuple containing the part before the separator, the ' + 'separator\n' + ' itself, and the part after the separator. If the ' + 'separator is not\n' + ' found, return a 3-tuple containing the string itself, ' + 'followed by\n' + ' two empty strings.\n' + '\n' + 'str.replace(old, new[, count])\n' + '\n' + ' Return a copy of the string with all occurrences of ' + 'substring *old*\n' + ' replaced by *new*. If the optional argument *count* is ' + 'given, only\n' + ' the first *count* occurrences are replaced.\n' + '\n' + 'str.rfind(sub[, start[, end]])\n' + '\n' + ' Return the highest index in the string where substring ' + '*sub* is\n' + ' found, such that *sub* is contained within ' + '"s[start:end]".\n' + ' Optional arguments *start* and *end* are interpreted as ' + 'in slice\n' + ' notation. Return "-1" on failure.\n' + '\n' + 'str.rindex(sub[, start[, end]])\n' + '\n' + ' Like "rfind()" but raises "ValueError" when the ' + 'substring *sub* is\n' + ' not found.\n' + '\n' + 'str.rjust(width[, fillchar])\n' + '\n' + ' Return the string right justified in a string of length ' + '*width*.\n' + ' Padding is done using the specified *fillchar* (default ' + 'is an ASCII\n' + ' space). The original string is returned if *width* is ' + 'less than or\n' + ' equal to "len(s)".\n' + '\n' + 'str.rpartition(sep)\n' + '\n' + ' Split the string at the last occurrence of *sep*, and ' + 'return a\n' + ' 3-tuple containing the part before the separator, the ' + 'separator\n' + ' itself, and the part after the separator. If the ' + 'separator is not\n' + ' found, return a 3-tuple containing two empty strings, ' + 'followed by\n' + ' the string itself.\n' + '\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' + '\n' + ' Return a list of the words in the string, using *sep* ' + 'as the\n' + ' delimiter string. If *maxsplit* is given, at most ' + '*maxsplit* splits\n' + ' are done, the *rightmost* ones. If *sep* is not ' + 'specified or\n' + ' "None", any whitespace string is a separator. Except ' + 'for splitting\n' + ' from the right, "rsplit()" behaves like "split()" which ' + 'is\n' + ' described in detail below.\n' + '\n' + 'str.rstrip([chars])\n' + '\n' + ' Return a copy of the string with trailing characters ' + 'removed. The\n' + ' *chars* argument is a string specifying the set of ' + 'characters to be\n' + ' removed. If omitted or "None", the *chars* argument ' + 'defaults to\n' + ' removing whitespace. The *chars* argument is not a ' + 'suffix; rather,\n' + ' all combinations of its values are stripped:\n' + '\n' + " >>> ' spacious '.rstrip()\n" + " ' spacious'\n" + " >>> 'mississippi'.rstrip('ipz')\n" + " 'mississ'\n" + '\n' + 'str.split(sep=None, maxsplit=-1)\n' + '\n' + ' Return a list of the words in the string, using *sep* ' + 'as the\n' + ' delimiter string. If *maxsplit* is given, at most ' + '*maxsplit*\n' + ' splits are done (thus, the list will have at most ' + '"maxsplit+1"\n' + ' elements). If *maxsplit* is not specified or "-1", ' + 'then there is\n' + ' no limit on the number of splits (all possible splits ' + 'are made).\n' + '\n' + ' If *sep* is given, consecutive delimiters are not ' + 'grouped together\n' + ' and are deemed to delimit empty strings (for example,\n' + ' "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', ' + '\'2\']"). The *sep* argument\n' + ' may consist of multiple characters (for example,\n' + ' "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', ' + '\'3\']"). Splitting an\n' + ' empty string with a specified separator returns ' + '"[\'\']".\n' + '\n' + ' For example:\n' + '\n' + " >>> '1,2,3'.split(',')\n" + " ['1', '2', '3']\n" + " >>> '1,2,3'.split(',', maxsplit=1)\n" + " ['1', '2,3']\n" + " >>> '1,2,,3,'.split(',')\n" + " ['1', '2', '', '3', '']\n" + '\n' + ' If *sep* is not specified or is "None", a different ' + 'splitting\n' + ' algorithm is applied: runs of consecutive whitespace ' + 'are regarded\n' + ' as a single separator, and the result will contain no ' + 'empty strings\n' + ' at the start or end if the string has leading or ' + 'trailing\n' + ' whitespace. Consequently, splitting an empty string or ' + 'a string\n' + ' consisting of just whitespace with a "None" separator ' + 'returns "[]".\n' + '\n' + ' For example:\n' + '\n' + " >>> '1 2 3'.split()\n" + " ['1', '2', '3']\n" + " >>> '1 2 3'.split(maxsplit=1)\n" + " ['1', '2 3']\n" + " >>> ' 1 2 3 '.split()\n" + " ['1', '2', '3']\n" + '\n' + 'str.splitlines([keepends])\n' + '\n' + ' Return a list of the lines in the string, breaking at ' + 'line\n' + ' boundaries. Line breaks are not included in the ' + 'resulting list\n' + ' unless *keepends* is given and true.\n' + '\n' + ' This method splits on the following line boundaries. ' + 'In\n' + ' particular, the boundaries are a superset of *universal ' + 'newlines*.\n' + '\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | Representation | ' + 'Description |\n' + ' ' + '+=========================+===============================+\n' + ' | "\\n" | Line ' + 'Feed |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\r" | Carriage ' + 'Return |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\r\\n" | Carriage Return + Line ' + 'Feed |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\v" or "\\x0b" | Line ' + 'Tabulation |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\f" or "\\x0c" | Form ' + 'Feed |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\x1c" | File ' + 'Separator |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\x1d" | Group ' + 'Separator |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\x1e" | Record ' + 'Separator |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\x85" | Next Line (C1 Control ' + 'Code) |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\u2028" | Line ' + 'Separator |\n' + ' ' + '+-------------------------+-------------------------------+\n' + ' | "\\u2029" | Paragraph ' + 'Separator |\n' + ' ' + '+-------------------------+-------------------------------+\n' + '\n' + ' Changed in version 3.2: "\\v" and "\\f" added to list ' + 'of line\n' + ' boundaries.\n' + '\n' + ' For example:\n' + '\n' + " >>> 'ab c\\n\\nde fg\\rkl\\r\\n'.splitlines()\n" + " ['ab c', '', 'de fg', 'kl']\n" + " >>> 'ab c\\n\\nde " + "fg\\rkl\\r\\n'.splitlines(keepends=True)\n" + " ['ab c\\n', '\\n', 'de fg\\r', 'kl\\r\\n']\n" + '\n' + ' Unlike "split()" when a delimiter string *sep* is ' + 'given, this\n' + ' method returns an empty list for the empty string, and ' + 'a terminal\n' + ' line break does not result in an extra line:\n' + '\n' + ' >>> "".splitlines()\n' + ' []\n' + ' >>> "One line\\n".splitlines()\n' + " ['One line']\n" + '\n' + ' For comparison, "split(\'\\n\')" gives:\n' + '\n' + " >>> ''.split('\\n')\n" + " ['']\n" + " >>> 'Two lines\\n'.split('\\n')\n" + " ['Two lines', '']\n" + '\n' + 'str.startswith(prefix[, start[, end]])\n' + '\n' + ' Return "True" if string starts with the *prefix*, ' + 'otherwise return\n' + ' "False". *prefix* can also be a tuple of prefixes to ' + 'look for.\n' + ' With optional *start*, test string beginning at that ' + 'position.\n' + ' With optional *end*, stop comparing string at that ' + 'position.\n' + '\n' + 'str.strip([chars])\n' + '\n' + ' Return a copy of the string with the leading and ' + 'trailing\n' + ' characters removed. The *chars* argument is a string ' + 'specifying the\n' + ' set of characters to be removed. If omitted or "None", ' + 'the *chars*\n' + ' argument defaults to removing whitespace. The *chars* ' + 'argument is\n' + ' not a prefix or suffix; rather, all combinations of its ' + 'values are\n' + ' stripped:\n' + '\n' + " >>> ' spacious '.strip()\n" + " 'spacious'\n" + " >>> 'www.example.com'.strip('cmowz.')\n" + " 'example'\n" + '\n' + ' The outermost leading and trailing *chars* argument ' + 'values are\n' + ' stripped from the string. Characters are removed from ' + 'the leading\n' + ' end until reaching a string character that is not ' + 'contained in the\n' + ' set of characters in *chars*. A similar action takes ' + 'place on the\n' + ' trailing end. For example:\n' + '\n' + " >>> comment_string = '#....... Section 3.2.1 Issue " + "#32 .......'\n" + " >>> comment_string.strip('.#! ')\n" + " 'Section 3.2.1 Issue #32'\n" + '\n' + 'str.swapcase()\n' + '\n' + ' Return a copy of the string with uppercase characters ' + 'converted to\n' + ' lowercase and vice versa. Note that it is not ' + 'necessarily true that\n' + ' "s.swapcase().swapcase() == s".\n' + '\n' + 'str.title()\n' + '\n' + ' Return a titlecased version of the string where words ' + 'start with an\n' + ' uppercase character and the remaining characters are ' + 'lowercase.\n' + '\n' + ' For example:\n' + '\n' + " >>> 'Hello world'.title()\n" + " 'Hello World'\n" + '\n' + ' The algorithm uses a simple language-independent ' + 'definition of a\n' + ' word as groups of consecutive letters. The definition ' + 'works in\n' + ' many contexts but it means that apostrophes in ' + 'contractions and\n' + ' possessives form word boundaries, which may not be the ' + 'desired\n' + ' result:\n' + '\n' + ' >>> "they\'re bill\'s friends from the UK".title()\n' + ' "They\'Re Bill\'S Friends From The Uk"\n' + '\n' + ' A workaround for apostrophes can be constructed using ' + 'regular\n' + ' expressions:\n' + '\n' + ' >>> import re\n' + ' >>> def titlecase(s):\n' + ' ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n' + ' ... lambda mo: ' + 'mo.group(0)[0].upper() +\n' + ' ... ' + 'mo.group(0)[1:].lower(),\n' + ' ... s)\n' + ' ...\n' + ' >>> titlecase("they\'re bill\'s friends.")\n' + ' "They\'re Bill\'s Friends."\n' + '\n' + 'str.translate(table)\n' + '\n' + ' Return a copy of the string in which each character has ' + 'been mapped\n' + ' through the given translation table. The table must be ' + 'an object\n' + ' that implements indexing via "__getitem__()", typically ' + 'a *mapping*\n' + ' or *sequence*. When indexed by a Unicode ordinal (an ' + 'integer), the\n' + ' table object can do any of the following: return a ' + 'Unicode ordinal\n' + ' or a string, to map the character to one or more other ' + 'characters;\n' + ' return "None", to delete the character from the return ' + 'string; or\n' + ' raise a "LookupError" exception, to map the character ' + 'to itself.\n' + '\n' + ' You can use "str.maketrans()" to create a translation ' + 'map from\n' + ' character-to-character mappings in different formats.\n' + '\n' + ' See also the "codecs" module for a more flexible ' + 'approach to custom\n' + ' character mappings.\n' + '\n' + 'str.upper()\n' + '\n' + ' Return a copy of the string with all the cased ' + 'characters [4]\n' + ' converted to uppercase. Note that ' + '"str.upper().isupper()" might be\n' + ' "False" if "s" contains uncased characters or if the ' + 'Unicode\n' + ' category of the resulting character(s) is not "Lu" ' + '(Letter,\n' + ' uppercase), but e.g. "Lt" (Letter, titlecase).\n' + '\n' + ' The uppercasing algorithm used is described in section ' + '3.13 of the\n' + ' Unicode Standard.\n' + '\n' + 'str.zfill(width)\n' + '\n' + ' Return a copy of the string left filled with ASCII ' + '"\'0\'" digits to\n' + ' make a string of length *width*. A leading sign prefix\n' + ' ("\'+\'"/"\'-\'") is handled by inserting the padding ' + '*after* the sign\n' + ' character rather than before. The original string is ' + 'returned if\n' + ' *width* is less than or equal to "len(s)".\n' + '\n' + ' For example:\n' + '\n' + ' >>> "42".zfill(5)\n' + " '00042'\n" + ' >>> "-42".zfill(5)\n' + " '-0042'\n", + 'strings': '\n' + 'String and Bytes literals\n' + '*************************\n' + '\n' + 'String literals are described by the following lexical ' + 'definitions:\n' + '\n' + ' stringliteral ::= [stringprefix](shortstring | longstring)\n' + ' stringprefix ::= "r" | "u" | "R" | "U" | "f" | "F"\n' + ' | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | ' + '"Rf" | "RF"\n' + ' shortstring ::= "\'" shortstringitem* "\'" | \'"\' ' + 'shortstringitem* \'"\'\n' + ' longstring ::= "\'\'\'" longstringitem* "\'\'\'" | ' + '\'"""\' longstringitem* \'"""\'\n' + ' shortstringitem ::= shortstringchar | stringescapeseq\n' + ' longstringitem ::= longstringchar | stringescapeseq\n' + ' shortstringchar ::= \n' + ' longstringchar ::= \n' + ' stringescapeseq ::= "\\" \n' + '\n' + ' bytesliteral ::= bytesprefix(shortbytes | longbytes)\n' + ' bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | ' + '"rb" | "rB" | "Rb" | "RB"\n' + ' shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' ' + 'shortbytesitem* \'"\'\n' + ' longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' ' + 'longbytesitem* \'"""\'\n' + ' shortbytesitem ::= shortbyteschar | bytesescapeseq\n' + ' longbytesitem ::= longbyteschar | bytesescapeseq\n' + ' shortbyteschar ::= \n' + ' longbyteschar ::= \n' + ' bytesescapeseq ::= "\\" \n' + '\n' + 'One syntactic restriction not indicated by these productions is ' + 'that\n' + 'whitespace is not allowed between the "stringprefix" or ' + '"bytesprefix"\n' + 'and the rest of the literal. The source character set is defined ' + 'by\n' + 'the encoding declaration; it is UTF-8 if no encoding declaration ' + 'is\n' + 'given in the source file; see section Encoding declarations.\n' + '\n' + 'In plain English: Both types of literals can be enclosed in ' + 'matching\n' + 'single quotes ("\'") or double quotes ("""). They can also be ' + 'enclosed\n' + 'in matching groups of three single or double quotes (these are\n' + 'generally referred to as *triple-quoted strings*). The ' + 'backslash\n' + '("\\") character is used to escape characters that otherwise have ' + 'a\n' + 'special meaning, such as newline, backslash itself, or the quote\n' + 'character.\n' + '\n' + 'Bytes literals are always prefixed with "\'b\'" or "\'B\'"; they ' + 'produce\n' + 'an instance of the "bytes" type instead of the "str" type. They ' + 'may\n' + 'only contain ASCII characters; bytes with a numeric value of 128 ' + 'or\n' + 'greater must be expressed with escapes.\n' + '\n' + 'As of Python 3.3 it is possible again to prefix string literals ' + 'with a\n' + '"u" prefix to simplify maintenance of dual 2.x and 3.x ' + 'codebases.\n' + '\n' + 'Both string and bytes literals may optionally be prefixed with a\n' + 'letter "\'r\'" or "\'R\'"; such strings are called *raw strings* ' + 'and treat\n' + 'backslashes as literal characters. As a result, in string ' + 'literals,\n' + '"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated ' + 'specially.\n' + "Given that Python 2.x's raw unicode literals behave differently " + 'than\n' + 'Python 3.x\'s the "\'ur\'" syntax is not supported.\n' + '\n' + 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' + 'been\n' + 'added as a synonym of "\'br\'".\n' + '\n' + 'New in version 3.3: Support for the unicode legacy literal\n' + '("u\'value\'") was reintroduced to simplify the maintenance of ' + 'dual\n' + 'Python 2.x and 3.x codebases. See **PEP 414** for more ' + 'information.\n' + '\n' + 'A string literal with "\'f\'" or "\'F\'" in its prefix is a ' + '*formatted\n' + 'string literal*; see Formatted string literals. The "\'f\'" may ' + 'be\n' + 'combined with "\'r\'", but not with "\'b\'" or "\'u\'", therefore ' + 'raw\n' + 'formatted strings are possible, but formatted bytes literals are ' + 'not.\n' + '\n' + 'In triple-quoted literals, unescaped newlines and quotes are ' + 'allowed\n' + '(and are retained), except that three unescaped quotes in a row\n' + 'terminate the literal. (A "quote" is the character used to open ' + 'the\n' + 'literal, i.e. either "\'" or """.)\n' + '\n' + 'Unless an "\'r\'" or "\'R\'" prefix is present, escape sequences ' + 'in string\n' + 'and bytes literals are interpreted according to rules similar to ' + 'those\n' + 'used by Standard C. The recognized escape sequences are:\n' + '\n' + '+-------------------+-----------------------------------+---------+\n' + '| Escape Sequence | Meaning | Notes ' + '|\n' + '+===================+===================================+=========+\n' + '| "\\newline" | Backslash and newline ignored ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\\\" | Backslash ("\\") ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\\'" | Single quote ("\'") ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\"" | Double quote (""") ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\a" | ASCII Bell (BEL) ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\b" | ASCII Backspace (BS) ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\f" | ASCII Formfeed (FF) ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\n" | ASCII Linefeed (LF) ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\r" | ASCII Carriage Return (CR) ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\t" | ASCII Horizontal Tab (TAB) ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\v" | ASCII Vertical Tab (VT) ' + '| |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\ooo" | Character with octal value *ooo* | ' + '(1,3) |\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\xhh" | Character with hex value *hh* | ' + '(2,3) |\n' + '+-------------------+-----------------------------------+---------+\n' + '\n' + 'Escape sequences only recognized in string literals are:\n' + '\n' + '+-------------------+-----------------------------------+---------+\n' + '| Escape Sequence | Meaning | Notes ' + '|\n' + '+===================+===================================+=========+\n' + '| "\\N{name}" | Character named *name* in the | ' + '(4) |\n' + '| | Unicode database | ' + '|\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\uxxxx" | Character with 16-bit hex value | ' + '(5) |\n' + '| | *xxxx* | ' + '|\n' + '+-------------------+-----------------------------------+---------+\n' + '| "\\Uxxxxxxxx" | Character with 32-bit hex value | ' + '(6) |\n' + '| | *xxxxxxxx* | ' + '|\n' + '+-------------------+-----------------------------------+---------+\n' + '\n' + 'Notes:\n' + '\n' + '1. As in Standard C, up to three octal digits are accepted.\n' + '\n' + '2. Unlike in Standard C, exactly two hex digits are required.\n' + '\n' + '3. In a bytes literal, hexadecimal and octal escapes denote the\n' + ' byte with the given value. In a string literal, these escapes\n' + ' denote a Unicode character with the given value.\n' + '\n' + '4. Changed in version 3.3: Support for name aliases [1] has been\n' + ' added.\n' + '\n' + '5. Exactly four hex digits are required.\n' + '\n' + '6. Any Unicode character can be encoded this way. Exactly eight\n' + ' hex digits are required.\n' + '\n' + 'Unlike Standard C, all unrecognized escape sequences are left in ' + 'the\n' + 'string unchanged, i.e., *the backslash is left in the result*. ' + '(This\n' + 'behavior is useful when debugging: if an escape sequence is ' + 'mistyped,\n' + 'the resulting output is more easily recognized as broken.) It is ' + 'also\n' + 'important to note that the escape sequences only recognized in ' + 'string\n' + 'literals fall into the category of unrecognized escapes for ' + 'bytes\n' + 'literals.\n' + '\n' + 'Even in a raw literal, quotes can be escaped with a backslash, ' + 'but the\n' + 'backslash remains in the result; for example, "r"\\""" is a ' + 'valid\n' + 'string literal consisting of two characters: a backslash and a ' + 'double\n' + 'quote; "r"\\"" is not a valid string literal (even a raw string ' + 'cannot\n' + 'end in an odd number of backslashes). Specifically, *a raw ' + 'literal\n' + 'cannot end in a single backslash* (since the backslash would ' + 'escape\n' + 'the following quote character). Note also that a single ' + 'backslash\n' + 'followed by a newline is interpreted as those two characters as ' + 'part\n' + 'of the literal, *not* as a line continuation.\n', + 'subscriptions': '\n' + 'Subscriptions\n' + '*************\n' + '\n' + 'A subscription selects an item of a sequence (string, tuple ' + 'or list)\n' + 'or mapping (dictionary) object:\n' + '\n' + ' subscription ::= primary "[" expression_list "]"\n' + '\n' + 'The primary must evaluate to an object that supports ' + 'subscription\n' + '(lists or dictionaries for example). User-defined objects ' + 'can support\n' + 'subscription by defining a "__getitem__()" method.\n' + '\n' + 'For built-in objects, there are two types of objects that ' + 'support\n' + 'subscription:\n' + '\n' + 'If the primary is a mapping, the expression list must ' + 'evaluate to an\n' + 'object whose value is one of the keys of the mapping, and ' + 'the\n' + 'subscription selects the value in the mapping that ' + 'corresponds to that\n' + 'key. (The expression list is a tuple except if it has ' + 'exactly one\n' + 'item.)\n' + '\n' + 'If the primary is a sequence, the expression (list) must ' + 'evaluate to\n' + 'an integer or a slice (as discussed in the following ' + 'section).\n' + '\n' + 'The formal syntax makes no special provision for negative ' + 'indices in\n' + 'sequences; however, built-in sequences all provide a ' + '"__getitem__()"\n' + 'method that interprets negative indices by adding the ' + 'length of the\n' + 'sequence to the index (so that "x[-1]" selects the last ' + 'item of "x").\n' + 'The resulting value must be a nonnegative integer less than ' + 'the number\n' + 'of items in the sequence, and the subscription selects the ' + 'item whose\n' + 'index is that value (counting from zero). Since the support ' + 'for\n' + "negative indices and slicing occurs in the object's " + '"__getitem__()"\n' + 'method, subclasses overriding this method will need to ' + 'explicitly add\n' + 'that support.\n' + '\n' + "A string's items are characters. A character is not a " + 'separate data\n' + 'type but a string of exactly one character.\n', + 'truth': '\n' + 'Truth Value Testing\n' + '*******************\n' + '\n' + 'Any object can be tested for truth value, for use in an "if" or\n' + '"while" condition or as operand of the Boolean operations below. ' + 'The\n' + 'following values are considered false:\n' + '\n' + '* "None"\n' + '\n' + '* "False"\n' + '\n' + '* zero of any numeric type, for example, "0", "0.0", "0j".\n' + '\n' + '* any empty sequence, for example, "\'\'", "()", "[]".\n' + '\n' + '* any empty mapping, for example, "{}".\n' + '\n' + '* instances of user-defined classes, if the class defines a\n' + ' "__bool__()" or "__len__()" method, when that method returns the\n' + ' integer zero or "bool" value "False". [1]\n' + '\n' + 'All other values are considered true --- so objects of many types ' + 'are\n' + 'always true.\n' + '\n' + 'Operations and built-in functions that have a Boolean result ' + 'always\n' + 'return "0" or "False" for false and "1" or "True" for true, unless\n' + 'otherwise stated. (Important exception: the Boolean operations ' + '"or"\n' + 'and "and" always return one of their operands.)\n', + 'try': '\n' + 'The "try" statement\n' + '*******************\n' + '\n' + 'The "try" statement specifies exception handlers and/or cleanup code\n' + 'for a group of statements:\n' + '\n' + ' try_stmt ::= try1_stmt | try2_stmt\n' + ' try1_stmt ::= "try" ":" suite\n' + ' ("except" [expression ["as" identifier]] ":" ' + 'suite)+\n' + ' ["else" ":" suite]\n' + ' ["finally" ":" suite]\n' + ' try2_stmt ::= "try" ":" suite\n' + ' "finally" ":" suite\n' + '\n' + 'The "except" clause(s) specify one or more exception handlers. When ' + 'no\n' + 'exception occurs in the "try" clause, no exception handler is\n' + 'executed. When an exception occurs in the "try" suite, a search for ' + 'an\n' + 'exception handler is started. This search inspects the except ' + 'clauses\n' + 'in turn until one is found that matches the exception. An ' + 'expression-\n' + 'less except clause, if present, must be last; it matches any\n' + 'exception. For an except clause with an expression, that expression\n' + 'is evaluated, and the clause matches the exception if the resulting\n' + 'object is "compatible" with the exception. An object is compatible\n' + 'with an exception if it is the class or a base class of the ' + 'exception\n' + 'object or a tuple containing an item compatible with the exception.\n' + '\n' + 'If no except clause matches the exception, the search for an ' + 'exception\n' + 'handler continues in the surrounding code and on the invocation ' + 'stack.\n' + '[1]\n' + '\n' + 'If the evaluation of an expression in the header of an except clause\n' + 'raises an exception, the original search for a handler is canceled ' + 'and\n' + 'a search starts for the new exception in the surrounding code and on\n' + 'the call stack (it is treated as if the entire "try" statement ' + 'raised\n' + 'the exception).\n' + '\n' + 'When a matching except clause is found, the exception is assigned to\n' + 'the target specified after the "as" keyword in that except clause, ' + 'if\n' + "present, and the except clause's suite is executed. All except\n" + 'clauses must have an executable block. When the end of this block ' + 'is\n' + 'reached, execution continues normally after the entire try ' + 'statement.\n' + '(This means that if two nested handlers exist for the same ' + 'exception,\n' + 'and the exception occurs in the try clause of the inner handler, the\n' + 'outer handler will not handle the exception.)\n' + '\n' + 'When an exception has been assigned using "as target", it is cleared\n' + 'at the end of the except clause. This is as if\n' + '\n' + ' except E as N:\n' + ' foo\n' + '\n' + 'was translated to\n' + '\n' + ' except E as N:\n' + ' try:\n' + ' foo\n' + ' finally:\n' + ' del N\n' + '\n' + 'This means the exception must be assigned to a different name to be\n' + 'able to refer to it after the except clause. Exceptions are cleared\n' + 'because with the traceback attached to them, they form a reference\n' + 'cycle with the stack frame, keeping all locals in that frame alive\n' + 'until the next garbage collection occurs.\n' + '\n' + "Before an except clause's suite is executed, details about the\n" + 'exception are stored in the "sys" module and can be accessed via\n' + '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of ' + 'the\n' + 'exception class, the exception instance and a traceback object (see\n' + 'section The standard type hierarchy) identifying the point in the\n' + 'program where the exception occurred. "sys.exc_info()" values are\n' + 'restored to their previous values (before the call) when returning\n' + 'from a function that handled an exception.\n' + '\n' + 'The optional "else" clause is executed if and when control flows off\n' + 'the end of the "try" clause. [2] Exceptions in the "else" clause are\n' + 'not handled by the preceding "except" clauses.\n' + '\n' + 'If "finally" is present, it specifies a \'cleanup\' handler. The ' + '"try"\n' + 'clause is executed, including any "except" and "else" clauses. If ' + 'an\n' + 'exception occurs in any of the clauses and is not handled, the\n' + 'exception is temporarily saved. The "finally" clause is executed. ' + 'If\n' + 'there is a saved exception it is re-raised at the end of the ' + '"finally"\n' + 'clause. If the "finally" clause raises another exception, the saved\n' + 'exception is set as the context of the new exception. If the ' + '"finally"\n' + 'clause executes a "return" or "break" statement, the saved exception\n' + 'is discarded:\n' + '\n' + ' >>> def f():\n' + ' ... try:\n' + ' ... 1/0\n' + ' ... finally:\n' + ' ... return 42\n' + ' ...\n' + ' >>> f()\n' + ' 42\n' + '\n' + 'The exception information is not available to the program during\n' + 'execution of the "finally" clause.\n' + '\n' + 'When a "return", "break" or "continue" statement is executed in the\n' + '"try" suite of a "try"..."finally" statement, the "finally" clause ' + 'is\n' + 'also executed \'on the way out.\' A "continue" statement is illegal ' + 'in\n' + 'the "finally" clause. (The reason is a problem with the current\n' + 'implementation --- this restriction may be lifted in the future).\n' + '\n' + 'The return value of a function is determined by the last "return"\n' + 'statement executed. Since the "finally" clause always executes, a\n' + '"return" statement executed in the "finally" clause will always be ' + 'the\n' + 'last one executed:\n' + '\n' + ' >>> def foo():\n' + ' ... try:\n' + " ... return 'try'\n" + ' ... finally:\n' + " ... return 'finally'\n" + ' ...\n' + ' >>> foo()\n' + " 'finally'\n" + '\n' + 'Additional information on exceptions can be found in section\n' + 'Exceptions, and information on using the "raise" statement to ' + 'generate\n' + 'exceptions may be found in section The raise statement.\n', + 'types': '\n' + 'The standard type hierarchy\n' + '***************************\n' + '\n' + 'Below is a list of the types that are built into Python. ' + 'Extension\n' + 'modules (written in C, Java, or other languages, depending on the\n' + 'implementation) can define additional types. Future versions of\n' + 'Python may add types to the type hierarchy (e.g., rational ' + 'numbers,\n' + 'efficiently stored arrays of integers, etc.), although such ' + 'additions\n' + 'will often be provided via the standard library instead.\n' + '\n' + 'Some of the type descriptions below contain a paragraph listing\n' + "'special attributes.' These are attributes that provide access to " + 'the\n' + 'implementation and are not intended for general use. Their ' + 'definition\n' + 'may change in the future.\n' + '\n' + 'None\n' + ' This type has a single value. There is a single object with ' + 'this\n' + ' value. This object is accessed through the built-in name "None". ' + 'It\n' + ' is used to signify the absence of a value in many situations, ' + 'e.g.,\n' + " it is returned from functions that don't explicitly return\n" + ' anything. Its truth value is false.\n' + '\n' + 'NotImplemented\n' + ' This type has a single value. There is a single object with ' + 'this\n' + ' value. This object is accessed through the built-in name\n' + ' "NotImplemented". Numeric methods and rich comparison methods\n' + ' should return this value if they do not implement the operation ' + 'for\n' + ' the operands provided. (The interpreter will then try the\n' + ' reflected operation, or some other fallback, depending on the\n' + ' operator.) Its truth value is true.\n' + '\n' + ' See Implementing the arithmetic operations for more details.\n' + '\n' + 'Ellipsis\n' + ' This type has a single value. There is a single object with ' + 'this\n' + ' value. This object is accessed through the literal "..." or the\n' + ' built-in name "Ellipsis". Its truth value is true.\n' + '\n' + '"numbers.Number"\n' + ' These are created by numeric literals and returned as results ' + 'by\n' + ' arithmetic operators and arithmetic built-in functions. ' + 'Numeric\n' + ' objects are immutable; once created their value never changes.\n' + ' Python numbers are of course strongly related to mathematical\n' + ' numbers, but subject to the limitations of numerical ' + 'representation\n' + ' in computers.\n' + '\n' + ' Python distinguishes between integers, floating point numbers, ' + 'and\n' + ' complex numbers:\n' + '\n' + ' "numbers.Integral"\n' + ' These represent elements from the mathematical set of ' + 'integers\n' + ' (positive and negative).\n' + '\n' + ' There are two types of integers:\n' + '\n' + ' Integers ("int")\n' + '\n' + ' These represent numbers in an unlimited range, subject to\n' + ' available (virtual) memory only. For the purpose of ' + 'shift\n' + ' and mask operations, a binary representation is assumed, ' + 'and\n' + " negative numbers are represented in a variant of 2's\n" + ' complement which gives the illusion of an infinite string ' + 'of\n' + ' sign bits extending to the left.\n' + '\n' + ' Booleans ("bool")\n' + ' These represent the truth values False and True. The two\n' + ' objects representing the values "False" and "True" are ' + 'the\n' + ' only Boolean objects. The Boolean type is a subtype of ' + 'the\n' + ' integer type, and Boolean values behave like the values 0 ' + 'and\n' + ' 1, respectively, in almost all contexts, the exception ' + 'being\n' + ' that when converted to a string, the strings ""False"" or\n' + ' ""True"" are returned, respectively.\n' + '\n' + ' The rules for integer representation are intended to give ' + 'the\n' + ' most meaningful interpretation of shift and mask operations\n' + ' involving negative integers.\n' + '\n' + ' "numbers.Real" ("float")\n' + ' These represent machine-level double precision floating ' + 'point\n' + ' numbers. You are at the mercy of the underlying machine\n' + ' architecture (and C or Java implementation) for the accepted\n' + ' range and handling of overflow. Python does not support ' + 'single-\n' + ' precision floating point numbers; the savings in processor ' + 'and\n' + ' memory usage that are usually the reason for using these are\n' + ' dwarfed by the overhead of using objects in Python, so there ' + 'is\n' + ' no reason to complicate the language with two kinds of ' + 'floating\n' + ' point numbers.\n' + '\n' + ' "numbers.Complex" ("complex")\n' + ' These represent complex numbers as a pair of machine-level\n' + ' double precision floating point numbers. The same caveats ' + 'apply\n' + ' as for floating point numbers. The real and imaginary parts ' + 'of a\n' + ' complex number "z" can be retrieved through the read-only\n' + ' attributes "z.real" and "z.imag".\n' + '\n' + 'Sequences\n' + ' These represent finite ordered sets indexed by non-negative\n' + ' numbers. The built-in function "len()" returns the number of ' + 'items\n' + ' of a sequence. When the length of a sequence is *n*, the index ' + 'set\n' + ' contains the numbers 0, 1, ..., *n*-1. Item *i* of sequence *a* ' + 'is\n' + ' selected by "a[i]".\n' + '\n' + ' Sequences also support slicing: "a[i:j]" selects all items with\n' + ' index *k* such that *i* "<=" *k* "<" *j*. When used as an\n' + ' expression, a slice is a sequence of the same type. This ' + 'implies\n' + ' that the index set is renumbered so that it starts at 0.\n' + '\n' + ' Some sequences also support "extended slicing" with a third ' + '"step"\n' + ' parameter: "a[i:j:k]" selects all items of *a* with index *x* ' + 'where\n' + ' "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n' + '\n' + ' Sequences are distinguished according to their mutability:\n' + '\n' + ' Immutable sequences\n' + ' An object of an immutable sequence type cannot change once it ' + 'is\n' + ' created. (If the object contains references to other ' + 'objects,\n' + ' these other objects may be mutable and may be changed; ' + 'however,\n' + ' the collection of objects directly referenced by an ' + 'immutable\n' + ' object cannot change.)\n' + '\n' + ' The following types are immutable sequences:\n' + '\n' + ' Strings\n' + ' A string is a sequence of values that represent Unicode ' + 'code\n' + ' points. All the code points in the range "U+0000 - ' + 'U+10FFFF"\n' + " can be represented in a string. Python doesn't have a " + '"char"\n' + ' type; instead, every code point in the string is ' + 'represented\n' + ' as a string object with length "1". The built-in ' + 'function\n' + ' "ord()" converts a code point from its string form to an\n' + ' integer in the range "0 - 10FFFF"; "chr()" converts an\n' + ' integer in the range "0 - 10FFFF" to the corresponding ' + 'length\n' + ' "1" string object. "str.encode()" can be used to convert ' + 'a\n' + ' "str" to "bytes" using the given text encoding, and\n' + ' "bytes.decode()" can be used to achieve the opposite.\n' + '\n' + ' Tuples\n' + ' The items of a tuple are arbitrary Python objects. Tuples ' + 'of\n' + ' two or more items are formed by comma-separated lists of\n' + " expressions. A tuple of one item (a 'singleton') can be\n" + ' formed by affixing a comma to an expression (an expression ' + 'by\n' + ' itself does not create a tuple, since parentheses must be\n' + ' usable for grouping of expressions). An empty tuple can ' + 'be\n' + ' formed by an empty pair of parentheses.\n' + '\n' + ' Bytes\n' + ' A bytes object is an immutable array. The items are ' + '8-bit\n' + ' bytes, represented by integers in the range 0 <= x < 256.\n' + ' Bytes literals (like "b\'abc\'") and the built-in ' + 'function\n' + ' "bytes()" can be used to construct bytes objects. Also,\n' + ' bytes objects can be decoded to strings via the ' + '"decode()"\n' + ' method.\n' + '\n' + ' Mutable sequences\n' + ' Mutable sequences can be changed after they are created. ' + 'The\n' + ' subscription and slicing notations can be used as the target ' + 'of\n' + ' assignment and "del" (delete) statements.\n' + '\n' + ' There are currently two intrinsic mutable sequence types:\n' + '\n' + ' Lists\n' + ' The items of a list are arbitrary Python objects. Lists ' + 'are\n' + ' formed by placing a comma-separated list of expressions ' + 'in\n' + ' square brackets. (Note that there are no special cases ' + 'needed\n' + ' to form lists of length 0 or 1.)\n' + '\n' + ' Byte Arrays\n' + ' A bytearray object is a mutable array. They are created ' + 'by\n' + ' the built-in "bytearray()" constructor. Aside from being\n' + ' mutable (and hence unhashable), byte arrays otherwise ' + 'provide\n' + ' the same interface and functionality as immutable bytes\n' + ' objects.\n' + '\n' + ' The extension module "array" provides an additional example ' + 'of a\n' + ' mutable sequence type, as does the "collections" module.\n' + '\n' + 'Set types\n' + ' These represent unordered, finite sets of unique, immutable\n' + ' objects. As such, they cannot be indexed by any subscript. ' + 'However,\n' + ' they can be iterated over, and the built-in function "len()"\n' + ' returns the number of items in a set. Common uses for sets are ' + 'fast\n' + ' membership testing, removing duplicates from a sequence, and\n' + ' computing mathematical operations such as intersection, union,\n' + ' difference, and symmetric difference.\n' + '\n' + ' For set elements, the same immutability rules apply as for\n' + ' dictionary keys. Note that numeric types obey the normal rules ' + 'for\n' + ' numeric comparison: if two numbers compare equal (e.g., "1" and\n' + ' "1.0"), only one of them can be contained in a set.\n' + '\n' + ' There are currently two intrinsic set types:\n' + '\n' + ' Sets\n' + ' These represent a mutable set. They are created by the ' + 'built-in\n' + ' "set()" constructor and can be modified afterwards by ' + 'several\n' + ' methods, such as "add()".\n' + '\n' + ' Frozen sets\n' + ' These represent an immutable set. They are created by the\n' + ' built-in "frozenset()" constructor. As a frozenset is ' + 'immutable\n' + ' and *hashable*, it can be used again as an element of ' + 'another\n' + ' set, or as a dictionary key.\n' + '\n' + 'Mappings\n' + ' These represent finite sets of objects indexed by arbitrary ' + 'index\n' + ' sets. The subscript notation "a[k]" selects the item indexed by ' + '"k"\n' + ' from the mapping "a"; this can be used in expressions and as ' + 'the\n' + ' target of assignments or "del" statements. The built-in ' + 'function\n' + ' "len()" returns the number of items in a mapping.\n' + '\n' + ' There is currently a single intrinsic mapping type:\n' + '\n' + ' Dictionaries\n' + ' These represent finite sets of objects indexed by nearly\n' + ' arbitrary values. The only types of values not acceptable ' + 'as\n' + ' keys are values containing lists or dictionaries or other\n' + ' mutable types that are compared by value rather than by ' + 'object\n' + ' identity, the reason being that the efficient implementation ' + 'of\n' + " dictionaries requires a key's hash value to remain constant.\n" + ' Numeric types used for keys obey the normal rules for ' + 'numeric\n' + ' comparison: if two numbers compare equal (e.g., "1" and ' + '"1.0")\n' + ' then they can be used interchangeably to index the same\n' + ' dictionary entry.\n' + '\n' + ' Dictionaries are mutable; they can be created by the "{...}"\n' + ' notation (see section Dictionary displays).\n' + '\n' + ' The extension modules "dbm.ndbm" and "dbm.gnu" provide\n' + ' additional examples of mapping types, as does the ' + '"collections"\n' + ' module.\n' + '\n' + 'Callable types\n' + ' These are the types to which the function call operation (see\n' + ' section Calls) can be applied:\n' + '\n' + ' User-defined functions\n' + ' A user-defined function object is created by a function\n' + ' definition (see section Function definitions). It should be\n' + ' called with an argument list containing the same number of ' + 'items\n' + " as the function's formal parameter list.\n" + '\n' + ' Special attributes:\n' + '\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | Attribute | Meaning ' + '| |\n' + ' ' + '+===========================+=================================+=============+\n' + ' | "__doc__" | The function\'s ' + 'documentation | Writable |\n' + ' | | string, or "None" if ' + '| |\n' + ' | | unavailable; not inherited by ' + '| |\n' + ' | | subclasses ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__name__" | The function\'s ' + 'name | Writable |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__qualname__" | The function\'s *qualified ' + 'name* | Writable |\n' + ' | | New in version 3.3. ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__module__" | The name of the module the ' + '| Writable |\n' + ' | | function was defined in, or ' + '| |\n' + ' | | "None" if unavailable. ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__defaults__" | A tuple containing default ' + '| Writable |\n' + ' | | argument values for those ' + '| |\n' + ' | | arguments that have defaults, ' + '| |\n' + ' | | or "None" if no arguments have ' + '| |\n' + ' | | a default value ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__code__" | The code object representing ' + '| Writable |\n' + ' | | the compiled function body. ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__globals__" | A reference to the dictionary ' + '| Read-only |\n' + " | | that holds the function's " + '| |\n' + ' | | global variables --- the global ' + '| |\n' + ' | | namespace of the module in ' + '| |\n' + ' | | which the function was defined. ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__dict__" | The namespace supporting ' + '| Writable |\n' + ' | | arbitrary function attributes. ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__closure__" | "None" or a tuple of cells that ' + '| Read-only |\n' + ' | | contain bindings for the ' + '| |\n' + " | | function's free variables. " + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__annotations__" | A dict containing annotations ' + '| Writable |\n' + ' | | of parameters. The keys of the ' + '| |\n' + ' | | dict are the parameter names, ' + '| |\n' + ' | | and "\'return\'" for the ' + 'return | |\n' + ' | | annotation, if provided. ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + ' | "__kwdefaults__" | A dict containing defaults for ' + '| Writable |\n' + ' | | keyword-only parameters. ' + '| |\n' + ' ' + '+---------------------------+---------------------------------+-------------+\n' + '\n' + ' Most of the attributes labelled "Writable" check the type of ' + 'the\n' + ' assigned value.\n' + '\n' + ' Function objects also support getting and setting arbitrary\n' + ' attributes, which can be used, for example, to attach ' + 'metadata\n' + ' to functions. Regular attribute dot-notation is used to get ' + 'and\n' + ' set such attributes. *Note that the current implementation ' + 'only\n' + ' supports function attributes on user-defined functions. ' + 'Function\n' + ' attributes on built-in functions may be supported in the\n' + ' future.*\n' + '\n' + " Additional information about a function's definition can be\n" + ' retrieved from its code object; see the description of ' + 'internal\n' + ' types below.\n' + '\n' + ' Instance methods\n' + ' An instance method object combines a class, a class instance ' + 'and\n' + ' any callable object (normally a user-defined function).\n' + '\n' + ' Special read-only attributes: "__self__" is the class ' + 'instance\n' + ' object, "__func__" is the function object; "__doc__" is the\n' + ' method\'s documentation (same as "__func__.__doc__"); ' + '"__name__"\n' + ' is the method name (same as "__func__.__name__"); ' + '"__module__"\n' + ' is the name of the module the method was defined in, or ' + '"None"\n' + ' if unavailable.\n' + '\n' + ' Methods also support accessing (but not setting) the ' + 'arbitrary\n' + ' function attributes on the underlying function object.\n' + '\n' + ' User-defined method objects may be created when getting an\n' + ' attribute of a class (perhaps via an instance of that class), ' + 'if\n' + ' that attribute is a user-defined function object or a class\n' + ' method object.\n' + '\n' + ' When an instance method object is created by retrieving a ' + 'user-\n' + ' defined function object from a class via one of its ' + 'instances,\n' + ' its "__self__" attribute is the instance, and the method ' + 'object\n' + ' is said to be bound. The new method\'s "__func__" attribute ' + 'is\n' + ' the original function object.\n' + '\n' + ' When a user-defined method object is created by retrieving\n' + ' another method object from a class or instance, the behaviour ' + 'is\n' + ' the same as for a function object, except that the ' + '"__func__"\n' + ' attribute of the new instance is not the original method ' + 'object\n' + ' but its "__func__" attribute.\n' + '\n' + ' When an instance method object is created by retrieving a ' + 'class\n' + ' method object from a class or instance, its "__self__" ' + 'attribute\n' + ' is the class itself, and its "__func__" attribute is the\n' + ' function object underlying the class method.\n' + '\n' + ' When an instance method object is called, the underlying\n' + ' function ("__func__") is called, inserting the class ' + 'instance\n' + ' ("__self__") in front of the argument list. For instance, ' + 'when\n' + ' "C" is a class which contains a definition for a function ' + '"f()",\n' + ' and "x" is an instance of "C", calling "x.f(1)" is equivalent ' + 'to\n' + ' calling "C.f(x, 1)".\n' + '\n' + ' When an instance method object is derived from a class ' + 'method\n' + ' object, the "class instance" stored in "__self__" will ' + 'actually\n' + ' be the class itself, so that calling either "x.f(1)" or ' + '"C.f(1)"\n' + ' is equivalent to calling "f(C,1)" where "f" is the ' + 'underlying\n' + ' function.\n' + '\n' + ' Note that the transformation from function object to ' + 'instance\n' + ' method object happens each time the attribute is retrieved ' + 'from\n' + ' the instance. In some cases, a fruitful optimization is to\n' + ' assign the attribute to a local variable and call that local\n' + ' variable. Also notice that this transformation only happens ' + 'for\n' + ' user-defined functions; other callable objects (and all non-\n' + ' callable objects) are retrieved without transformation. It ' + 'is\n' + ' also important to note that user-defined functions which are\n' + ' attributes of a class instance are not converted to bound\n' + ' methods; this *only* happens when the function is an ' + 'attribute\n' + ' of the class.\n' + '\n' + ' Generator functions\n' + ' A function or method which uses the "yield" statement (see\n' + ' section The yield statement) is called a *generator ' + 'function*.\n' + ' Such a function, when called, always returns an iterator ' + 'object\n' + ' which can be used to execute the body of the function: ' + 'calling\n' + ' the iterator\'s "iterator.__next__()" method will cause the\n' + ' function to execute until it provides a value using the ' + '"yield"\n' + ' statement. When the function executes a "return" statement ' + 'or\n' + ' falls off the end, a "StopIteration" exception is raised and ' + 'the\n' + ' iterator will have reached the end of the set of values to ' + 'be\n' + ' returned.\n' + '\n' + ' Coroutine functions\n' + ' A function or method which is defined using "async def" is\n' + ' called a *coroutine function*. Such a function, when ' + 'called,\n' + ' returns a *coroutine* object. It may contain "await"\n' + ' expressions, as well as "async with" and "async for" ' + 'statements.\n' + ' See also the Coroutine Objects section.\n' + '\n' + ' Built-in functions\n' + ' A built-in function object is a wrapper around a C function.\n' + ' Examples of built-in functions are "len()" and "math.sin()"\n' + ' ("math" is a standard built-in module). The number and type ' + 'of\n' + ' the arguments are determined by the C function. Special ' + 'read-\n' + ' only attributes: "__doc__" is the function\'s documentation\n' + ' string, or "None" if unavailable; "__name__" is the ' + "function's\n" + ' name; "__self__" is set to "None" (but see the next item);\n' + ' "__module__" is the name of the module the function was ' + 'defined\n' + ' in or "None" if unavailable.\n' + '\n' + ' Built-in methods\n' + ' This is really a different disguise of a built-in function, ' + 'this\n' + ' time containing an object passed to the C function as an\n' + ' implicit extra argument. An example of a built-in method is\n' + ' "alist.append()", assuming *alist* is a list object. In this\n' + ' case, the special read-only attribute "__self__" is set to ' + 'the\n' + ' object denoted by *alist*.\n' + '\n' + ' Classes\n' + ' Classes are callable. These objects normally act as ' + 'factories\n' + ' for new instances of themselves, but variations are possible ' + 'for\n' + ' class types that override "__new__()". The arguments of the\n' + ' call are passed to "__new__()" and, in the typical case, to\n' + ' "__init__()" to initialize the new instance.\n' + '\n' + ' Class Instances\n' + ' Instances of arbitrary classes can be made callable by ' + 'defining\n' + ' a "__call__()" method in their class.\n' + '\n' + 'Modules\n' + ' Modules are a basic organizational unit of Python code, and are\n' + ' created by the import system as invoked either by the "import"\n' + ' statement (see "import"), or by calling functions such as\n' + ' "importlib.import_module()" and built-in "__import__()". A ' + 'module\n' + ' object has a namespace implemented by a dictionary object (this ' + 'is\n' + ' the dictionary referenced by the "__globals__" attribute of\n' + ' functions defined in the module). Attribute references are\n' + ' translated to lookups in this dictionary, e.g., "m.x" is ' + 'equivalent\n' + ' to "m.__dict__["x"]". A module object does not contain the code\n' + " object used to initialize the module (since it isn't needed " + 'once\n' + ' the initialization is done).\n' + '\n' + " Attribute assignment updates the module's namespace dictionary,\n" + ' e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n' + '\n' + ' Special read-only attribute: "__dict__" is the module\'s ' + 'namespace\n' + ' as a dictionary object.\n' + '\n' + ' **CPython implementation detail:** Because of the way CPython\n' + ' clears module dictionaries, the module dictionary will be ' + 'cleared\n' + ' when the module falls out of scope even if the dictionary still ' + 'has\n' + ' live references. To avoid this, copy the dictionary or keep ' + 'the\n' + ' module around while using its dictionary directly.\n' + '\n' + ' Predefined (writable) attributes: "__name__" is the module\'s ' + 'name;\n' + ' "__doc__" is the module\'s documentation string, or "None" if\n' + ' unavailable; "__file__" is the pathname of the file from which ' + 'the\n' + ' module was loaded, if it was loaded from a file. The "__file__"\n' + ' attribute may be missing for certain types of modules, such as ' + 'C\n' + ' modules that are statically linked into the interpreter; for\n' + ' extension modules loaded dynamically from a shared library, it ' + 'is\n' + ' the pathname of the shared library file.\n' + '\n' + 'Custom classes\n' + ' Custom class types are typically created by class definitions ' + '(see\n' + ' section Class definitions). A class has a namespace implemented ' + 'by\n' + ' a dictionary object. Class attribute references are translated ' + 'to\n' + ' lookups in this dictionary, e.g., "C.x" is translated to\n' + ' "C.__dict__["x"]" (although there are a number of hooks which ' + 'allow\n' + ' for other means of locating attributes). When the attribute name ' + 'is\n' + ' not found there, the attribute search continues in the base\n' + ' classes. This search of the base classes uses the C3 method\n' + ' resolution order which behaves correctly even in the presence ' + 'of\n' + " 'diamond' inheritance structures where there are multiple\n" + ' inheritance paths leading back to a common ancestor. Additional\n' + ' details on the C3 MRO used by Python can be found in the\n' + ' documentation accompanying the 2.3 release at\n' + ' https://www.python.org/download/releases/2.3/mro/.\n' + '\n' + ' When a class attribute reference (for class "C", say) would ' + 'yield a\n' + ' class method object, it is transformed into an instance method\n' + ' object whose "__self__" attributes is "C". When it would yield ' + 'a\n' + ' static method object, it is transformed into the object wrapped ' + 'by\n' + ' the static method object. See section Implementing Descriptors ' + 'for\n' + ' another way in which attributes retrieved from a class may ' + 'differ\n' + ' from those actually contained in its "__dict__".\n' + '\n' + " Class attribute assignments update the class's dictionary, " + 'never\n' + ' the dictionary of a base class.\n' + '\n' + ' A class object can be called (see above) to yield a class ' + 'instance\n' + ' (see below).\n' + '\n' + ' Special attributes: "__name__" is the class name; "__module__" ' + 'is\n' + ' the module name in which the class was defined; "__dict__" is ' + 'the\n' + ' dictionary containing the class\'s namespace; "__bases__" is a ' + 'tuple\n' + ' (possibly empty or a singleton) containing the base classes, in ' + 'the\n' + ' order of their occurrence in the base class list; "__doc__" is ' + 'the\n' + " class's documentation string, or None if undefined.\n" + '\n' + 'Class instances\n' + ' A class instance is created by calling a class object (see ' + 'above).\n' + ' A class instance has a namespace implemented as a dictionary ' + 'which\n' + ' is the first place in which attribute references are searched.\n' + " When an attribute is not found there, and the instance's class " + 'has\n' + ' an attribute by that name, the search continues with the class\n' + ' attributes. If a class attribute is found that is a ' + 'user-defined\n' + ' function object, it is transformed into an instance method ' + 'object\n' + ' whose "__self__" attribute is the instance. Static method and\n' + ' class method objects are also transformed; see above under\n' + ' "Classes". See section Implementing Descriptors for another way ' + 'in\n' + ' which attributes of a class retrieved via its instances may ' + 'differ\n' + ' from the objects actually stored in the class\'s "__dict__". If ' + 'no\n' + " class attribute is found, and the object's class has a\n" + ' "__getattr__()" method, that is called to satisfy the lookup.\n' + '\n' + " Attribute assignments and deletions update the instance's\n" + " dictionary, never a class's dictionary. If the class has a\n" + ' "__setattr__()" or "__delattr__()" method, this is called ' + 'instead\n' + ' of updating the instance dictionary directly.\n' + '\n' + ' Class instances can pretend to be numbers, sequences, or ' + 'mappings\n' + ' if they have methods with certain special names. See section\n' + ' Special method names.\n' + '\n' + ' Special attributes: "__dict__" is the attribute dictionary;\n' + ' "__class__" is the instance\'s class.\n' + '\n' + 'I/O objects (also known as file objects)\n' + ' A *file object* represents an open file. Various shortcuts are\n' + ' available to create file objects: the "open()" built-in ' + 'function,\n' + ' and also "os.popen()", "os.fdopen()", and the "makefile()" ' + 'method\n' + ' of socket objects (and perhaps by other functions or methods\n' + ' provided by extension modules).\n' + '\n' + ' The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n' + " initialized to file objects corresponding to the interpreter's\n" + ' standard input, output and error streams; they are all open in ' + 'text\n' + ' mode and therefore follow the interface defined by the\n' + ' "io.TextIOBase" abstract class.\n' + '\n' + 'Internal types\n' + ' A few types used internally by the interpreter are exposed to ' + 'the\n' + ' user. Their definitions may change with future versions of the\n' + ' interpreter, but they are mentioned here for completeness.\n' + '\n' + ' Code objects\n' + ' Code objects represent *byte-compiled* executable Python ' + 'code,\n' + ' or *bytecode*. The difference between a code object and a\n' + ' function object is that the function object contains an ' + 'explicit\n' + " reference to the function's globals (the module in which it " + 'was\n' + ' defined), while a code object contains no context; also the\n' + ' default argument values are stored in the function object, ' + 'not\n' + ' in the code object (because they represent values calculated ' + 'at\n' + ' run-time). Unlike function objects, code objects are ' + 'immutable\n' + ' and contain no references (directly or indirectly) to ' + 'mutable\n' + ' objects.\n' + '\n' + ' Special read-only attributes: "co_name" gives the function ' + 'name;\n' + ' "co_argcount" is the number of positional arguments ' + '(including\n' + ' arguments with default values); "co_nlocals" is the number ' + 'of\n' + ' local variables used by the function (including arguments);\n' + ' "co_varnames" is a tuple containing the names of the local\n' + ' variables (starting with the argument names); "co_cellvars" ' + 'is a\n' + ' tuple containing the names of local variables that are\n' + ' referenced by nested functions; "co_freevars" is a tuple\n' + ' containing the names of free variables; "co_code" is a ' + 'string\n' + ' representing the sequence of bytecode instructions; ' + '"co_consts"\n' + ' is a tuple containing the literals used by the bytecode;\n' + ' "co_names" is a tuple containing the names used by the ' + 'bytecode;\n' + ' "co_filename" is the filename from which the code was ' + 'compiled;\n' + ' "co_firstlineno" is the first line number of the function;\n' + ' "co_lnotab" is a string encoding the mapping from bytecode\n' + ' offsets to line numbers (for details see the source code of ' + 'the\n' + ' interpreter); "co_stacksize" is the required stack size\n' + ' (including local variables); "co_flags" is an integer ' + 'encoding a\n' + ' number of flags for the interpreter.\n' + '\n' + ' The following flag bits are defined for "co_flags": bit ' + '"0x04"\n' + ' is set if the function uses the "*arguments" syntax to accept ' + 'an\n' + ' arbitrary number of positional arguments; bit "0x08" is set ' + 'if\n' + ' the function uses the "**keywords" syntax to accept ' + 'arbitrary\n' + ' keyword arguments; bit "0x20" is set if the function is a\n' + ' generator.\n' + '\n' + ' Future feature declarations ("from __future__ import ' + 'division")\n' + ' also use bits in "co_flags" to indicate whether a code ' + 'object\n' + ' was compiled with a particular feature enabled: bit "0x2000" ' + 'is\n' + ' set if the function was compiled with future division ' + 'enabled;\n' + ' bits "0x10" and "0x1000" were used in earlier versions of\n' + ' Python.\n' + '\n' + ' Other bits in "co_flags" are reserved for internal use.\n' + '\n' + ' If a code object represents a function, the first item in\n' + ' "co_consts" is the documentation string of the function, or\n' + ' "None" if undefined.\n' + '\n' + ' Frame objects\n' + ' Frame objects represent execution frames. They may occur in\n' + ' traceback objects (see below).\n' + '\n' + ' Special read-only attributes: "f_back" is to the previous ' + 'stack\n' + ' frame (towards the caller), or "None" if this is the bottom\n' + ' stack frame; "f_code" is the code object being executed in ' + 'this\n' + ' frame; "f_locals" is the dictionary used to look up local\n' + ' variables; "f_globals" is used for global variables;\n' + ' "f_builtins" is used for built-in (intrinsic) names; ' + '"f_lasti"\n' + ' gives the precise instruction (this is an index into the\n' + ' bytecode string of the code object).\n' + '\n' + ' Special writable attributes: "f_trace", if not "None", is a\n' + ' function called at the start of each source code line (this ' + 'is\n' + ' used by the debugger); "f_lineno" is the current line number ' + 'of\n' + ' the frame --- writing to this from within a trace function ' + 'jumps\n' + ' to the given line (only for the bottom-most frame). A ' + 'debugger\n' + ' can implement a Jump command (aka Set Next Statement) by ' + 'writing\n' + ' to f_lineno.\n' + '\n' + ' Frame objects support one method:\n' + '\n' + ' frame.clear()\n' + '\n' + ' This method clears all references to local variables held ' + 'by\n' + ' the frame. Also, if the frame belonged to a generator, ' + 'the\n' + ' generator is finalized. This helps break reference ' + 'cycles\n' + ' involving frame objects (for example when catching an\n' + ' exception and storing its traceback for later use).\n' + '\n' + ' "RuntimeError" is raised if the frame is currently ' + 'executing.\n' + '\n' + ' New in version 3.4.\n' + '\n' + ' Traceback objects\n' + ' Traceback objects represent a stack trace of an exception. ' + 'A\n' + ' traceback object is created when an exception occurs. When ' + 'the\n' + ' search for an exception handler unwinds the execution stack, ' + 'at\n' + ' each unwound level a traceback object is inserted in front ' + 'of\n' + ' the current traceback. When an exception handler is ' + 'entered,\n' + ' the stack trace is made available to the program. (See ' + 'section\n' + ' The try statement.) It is accessible as the third item of ' + 'the\n' + ' tuple returned by "sys.exc_info()". When the program contains ' + 'no\n' + ' suitable handler, the stack trace is written (nicely ' + 'formatted)\n' + ' to the standard error stream; if the interpreter is ' + 'interactive,\n' + ' it is also made available to the user as ' + '"sys.last_traceback".\n' + '\n' + ' Special read-only attributes: "tb_next" is the next level in ' + 'the\n' + ' stack trace (towards the frame where the exception occurred), ' + 'or\n' + ' "None" if there is no next level; "tb_frame" points to the\n' + ' execution frame of the current level; "tb_lineno" gives the ' + 'line\n' + ' number where the exception occurred; "tb_lasti" indicates ' + 'the\n' + ' precise instruction. The line number and last instruction ' + 'in\n' + ' the traceback may differ from the line number of its frame\n' + ' object if the exception occurred in a "try" statement with ' + 'no\n' + ' matching except clause or with a finally clause.\n' + '\n' + ' Slice objects\n' + ' Slice objects are used to represent slices for ' + '"__getitem__()"\n' + ' methods. They are also created by the built-in "slice()"\n' + ' function.\n' + '\n' + ' Special read-only attributes: "start" is the lower bound; ' + '"stop"\n' + ' is the upper bound; "step" is the step value; each is "None" ' + 'if\n' + ' omitted. These attributes can have any type.\n' + '\n' + ' Slice objects support one method:\n' + '\n' + ' slice.indices(self, length)\n' + '\n' + ' This method takes a single integer argument *length* and\n' + ' computes information about the slice that the slice ' + 'object\n' + ' would describe if applied to a sequence of *length* ' + 'items.\n' + ' It returns a tuple of three integers; respectively these ' + 'are\n' + ' the *start* and *stop* indices and the *step* or stride\n' + ' length of the slice. Missing or out-of-bounds indices are\n' + ' handled in a manner consistent with regular slices.\n' + '\n' + ' Static method objects\n' + ' Static method objects provide a way of defeating the\n' + ' transformation of function objects to method objects ' + 'described\n' + ' above. A static method object is a wrapper around any other\n' + ' object, usually a user-defined method object. When a static\n' + ' method object is retrieved from a class or a class instance, ' + 'the\n' + ' object actually returned is the wrapped object, which is not\n' + ' subject to any further transformation. Static method objects ' + 'are\n' + ' not themselves callable, although the objects they wrap ' + 'usually\n' + ' are. Static method objects are created by the built-in\n' + ' "staticmethod()" constructor.\n' + '\n' + ' Class method objects\n' + ' A class method object, like a static method object, is a ' + 'wrapper\n' + ' around another object that alters the way in which that ' + 'object\n' + ' is retrieved from classes and class instances. The behaviour ' + 'of\n' + ' class method objects upon such retrieval is described above,\n' + ' under "User-defined methods". Class method objects are ' + 'created\n' + ' by the built-in "classmethod()" constructor.\n', + 'typesfunctions': '\n' + 'Functions\n' + '*********\n' + '\n' + 'Function objects are created by function definitions. The ' + 'only\n' + 'operation on a function object is to call it: ' + '"func(argument-list)".\n' + '\n' + 'There are really two flavors of function objects: built-in ' + 'functions\n' + 'and user-defined functions. Both support the same ' + 'operation (to call\n' + 'the function), but the implementation is different, hence ' + 'the\n' + 'different object types.\n' + '\n' + 'See Function definitions for more information.\n', + 'typesmapping': '\n' + 'Mapping Types --- "dict"\n' + '************************\n' + '\n' + 'A *mapping* object maps *hashable* values to arbitrary ' + 'objects.\n' + 'Mappings are mutable objects. There is currently only one ' + 'standard\n' + 'mapping type, the *dictionary*. (For other containers see ' + 'the built-\n' + 'in "list", "set", and "tuple" classes, and the "collections" ' + 'module.)\n' + '\n' + "A dictionary's keys are *almost* arbitrary values. Values " + 'that are\n' + 'not *hashable*, that is, values containing lists, ' + 'dictionaries or\n' + 'other mutable types (that are compared by value rather than ' + 'by object\n' + 'identity) may not be used as keys. Numeric types used for ' + 'keys obey\n' + 'the normal rules for numeric comparison: if two numbers ' + 'compare equal\n' + '(such as "1" and "1.0") then they can be used ' + 'interchangeably to index\n' + 'the same dictionary entry. (Note however, that since ' + 'computers store\n' + 'floating-point numbers as approximations it is usually ' + 'unwise to use\n' + 'them as dictionary keys.)\n' + '\n' + 'Dictionaries can be created by placing a comma-separated ' + 'list of "key:\n' + 'value" pairs within braces, for example: "{\'jack\': 4098, ' + "'sjoerd':\n" + '4127}" or "{4098: \'jack\', 4127: \'sjoerd\'}", or by the ' + '"dict"\n' + 'constructor.\n' + '\n' + 'class dict(**kwarg)\n' + 'class dict(mapping, **kwarg)\n' + 'class dict(iterable, **kwarg)\n' + '\n' + ' Return a new dictionary initialized from an optional ' + 'positional\n' + ' argument and a possibly empty set of keyword arguments.\n' + '\n' + ' If no positional argument is given, an empty dictionary ' + 'is created.\n' + ' If a positional argument is given and it is a mapping ' + 'object, a\n' + ' dictionary is created with the same key-value pairs as ' + 'the mapping\n' + ' object. Otherwise, the positional argument must be an ' + '*iterable*\n' + ' object. Each item in the iterable must itself be an ' + 'iterable with\n' + ' exactly two objects. The first object of each item ' + 'becomes a key\n' + ' in the new dictionary, and the second object the ' + 'corresponding\n' + ' value. If a key occurs more than once, the last value ' + 'for that key\n' + ' becomes the corresponding value in the new dictionary.\n' + '\n' + ' If keyword arguments are given, the keyword arguments and ' + 'their\n' + ' values are added to the dictionary created from the ' + 'positional\n' + ' argument. If a key being added is already present, the ' + 'value from\n' + ' the keyword argument replaces the value from the ' + 'positional\n' + ' argument.\n' + '\n' + ' To illustrate, the following examples all return a ' + 'dictionary equal\n' + ' to "{"one": 1, "two": 2, "three": 3}":\n' + '\n' + ' >>> a = dict(one=1, two=2, three=3)\n' + " >>> b = {'one': 1, 'two': 2, 'three': 3}\n" + " >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3]))\n" + " >>> d = dict([('two', 2), ('one', 1), ('three', 3)])\n" + " >>> e = dict({'three': 3, 'one': 1, 'two': 2})\n" + ' >>> a == b == c == d == e\n' + ' True\n' + '\n' + ' Providing keyword arguments as in the first example only ' + 'works for\n' + ' keys that are valid Python identifiers. Otherwise, any ' + 'valid keys\n' + ' can be used.\n' + '\n' + ' These are the operations that dictionaries support (and ' + 'therefore,\n' + ' custom mapping types should support too):\n' + '\n' + ' len(d)\n' + '\n' + ' Return the number of items in the dictionary *d*.\n' + '\n' + ' d[key]\n' + '\n' + ' Return the item of *d* with key *key*. Raises a ' + '"KeyError" if\n' + ' *key* is not in the map.\n' + '\n' + ' If a subclass of dict defines a method "__missing__()" ' + 'and *key*\n' + ' is not present, the "d[key]" operation calls that ' + 'method with\n' + ' the key *key* as argument. The "d[key]" operation ' + 'then returns\n' + ' or raises whatever is returned or raised by the\n' + ' "__missing__(key)" call. No other operations or ' + 'methods invoke\n' + ' "__missing__()". If "__missing__()" is not defined, ' + '"KeyError"\n' + ' is raised. "__missing__()" must be a method; it cannot ' + 'be an\n' + ' instance variable:\n' + '\n' + ' >>> class Counter(dict):\n' + ' ... def __missing__(self, key):\n' + ' ... return 0\n' + ' >>> c = Counter()\n' + " >>> c['red']\n" + ' 0\n' + " >>> c['red'] += 1\n" + " >>> c['red']\n" + ' 1\n' + '\n' + ' The example above shows part of the implementation of\n' + ' "collections.Counter". A different "__missing__" ' + 'method is used\n' + ' by "collections.defaultdict".\n' + '\n' + ' d[key] = value\n' + '\n' + ' Set "d[key]" to *value*.\n' + '\n' + ' del d[key]\n' + '\n' + ' Remove "d[key]" from *d*. Raises a "KeyError" if ' + '*key* is not\n' + ' in the map.\n' + '\n' + ' key in d\n' + '\n' + ' Return "True" if *d* has a key *key*, else "False".\n' + '\n' + ' key not in d\n' + '\n' + ' Equivalent to "not key in d".\n' + '\n' + ' iter(d)\n' + '\n' + ' Return an iterator over the keys of the dictionary. ' + 'This is a\n' + ' shortcut for "iter(d.keys())".\n' + '\n' + ' clear()\n' + '\n' + ' Remove all items from the dictionary.\n' + '\n' + ' copy()\n' + '\n' + ' Return a shallow copy of the dictionary.\n' + '\n' + ' classmethod fromkeys(seq[, value])\n' + '\n' + ' Create a new dictionary with keys from *seq* and ' + 'values set to\n' + ' *value*.\n' + '\n' + ' "fromkeys()" is a class method that returns a new ' + 'dictionary.\n' + ' *value* defaults to "None".\n' + '\n' + ' get(key[, default])\n' + '\n' + ' Return the value for *key* if *key* is in the ' + 'dictionary, else\n' + ' *default*. If *default* is not given, it defaults to ' + '"None", so\n' + ' that this method never raises a "KeyError".\n' + '\n' + ' items()\n' + '\n' + ' Return a new view of the dictionary\'s items ("(key, ' + 'value)"\n' + ' pairs). See the documentation of view objects.\n' + '\n' + ' keys()\n' + '\n' + " Return a new view of the dictionary's keys. See the\n" + ' documentation of view objects.\n' + '\n' + ' pop(key[, default])\n' + '\n' + ' If *key* is in the dictionary, remove it and return ' + 'its value,\n' + ' else return *default*. If *default* is not given and ' + '*key* is\n' + ' not in the dictionary, a "KeyError" is raised.\n' + '\n' + ' popitem()\n' + '\n' + ' Remove and return an arbitrary "(key, value)" pair ' + 'from the\n' + ' dictionary.\n' + '\n' + ' "popitem()" is useful to destructively iterate over a\n' + ' dictionary, as often used in set algorithms. If the ' + 'dictionary\n' + ' is empty, calling "popitem()" raises a "KeyError".\n' + '\n' + ' setdefault(key[, default])\n' + '\n' + ' If *key* is in the dictionary, return its value. If ' + 'not, insert\n' + ' *key* with a value of *default* and return *default*. ' + '*default*\n' + ' defaults to "None".\n' + '\n' + ' update([other])\n' + '\n' + ' Update the dictionary with the key/value pairs from ' + '*other*,\n' + ' overwriting existing keys. Return "None".\n' + '\n' + ' "update()" accepts either another dictionary object or ' + 'an\n' + ' iterable of key/value pairs (as tuples or other ' + 'iterables of\n' + ' length two). If keyword arguments are specified, the ' + 'dictionary\n' + ' is then updated with those key/value pairs: ' + '"d.update(red=1,\n' + ' blue=2)".\n' + '\n' + ' values()\n' + '\n' + " Return a new view of the dictionary's values. See " + 'the\n' + ' documentation of view objects.\n' + '\n' + ' Dictionaries compare equal if and only if they have the ' + 'same "(key,\n' + ' value)" pairs. Order comparisons (\'<\', \'<=\', \'>=\', ' + "'>') raise\n" + ' "TypeError".\n' + '\n' + 'See also: "types.MappingProxyType" can be used to create a ' + 'read-only\n' + ' view of a "dict".\n' + '\n' + '\n' + 'Dictionary view objects\n' + '=======================\n' + '\n' + 'The objects returned by "dict.keys()", "dict.values()" and\n' + '"dict.items()" are *view objects*. They provide a dynamic ' + 'view on the\n' + "dictionary's entries, which means that when the dictionary " + 'changes,\n' + 'the view reflects these changes.\n' + '\n' + 'Dictionary views can be iterated over to yield their ' + 'respective data,\n' + 'and support membership tests:\n' + '\n' + 'len(dictview)\n' + '\n' + ' Return the number of entries in the dictionary.\n' + '\n' + 'iter(dictview)\n' + '\n' + ' Return an iterator over the keys, values or items ' + '(represented as\n' + ' tuples of "(key, value)") in the dictionary.\n' + '\n' + ' Keys and values are iterated over in an arbitrary order ' + 'which is\n' + ' non-random, varies across Python implementations, and ' + 'depends on\n' + " the dictionary's history of insertions and deletions. If " + 'keys,\n' + ' values and items views are iterated over with no ' + 'intervening\n' + ' modifications to the dictionary, the order of items will ' + 'directly\n' + ' correspond. This allows the creation of "(value, key)" ' + 'pairs using\n' + ' "zip()": "pairs = zip(d.values(), d.keys())". Another ' + 'way to\n' + ' create the same list is "pairs = [(v, k) for (k, v) in ' + 'd.items()]".\n' + '\n' + ' Iterating views while adding or deleting entries in the ' + 'dictionary\n' + ' may raise a "RuntimeError" or fail to iterate over all ' + 'entries.\n' + '\n' + 'x in dictview\n' + '\n' + ' Return "True" if *x* is in the underlying dictionary\'s ' + 'keys, values\n' + ' or items (in the latter case, *x* should be a "(key, ' + 'value)"\n' + ' tuple).\n' + '\n' + 'Keys views are set-like since their entries are unique and ' + 'hashable.\n' + 'If all values are hashable, so that "(key, value)" pairs are ' + 'unique\n' + 'and hashable, then the items view is also set-like. (Values ' + 'views are\n' + 'not treated as set-like since the entries are generally not ' + 'unique.)\n' + 'For set-like views, all of the operations defined for the ' + 'abstract\n' + 'base class "collections.abc.Set" are available (for example, ' + '"==",\n' + '"<", or "^").\n' + '\n' + 'An example of dictionary view usage:\n' + '\n' + " >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, " + "'spam': 500}\n" + ' >>> keys = dishes.keys()\n' + ' >>> values = dishes.values()\n' + '\n' + ' >>> # iteration\n' + ' >>> n = 0\n' + ' >>> for val in values:\n' + ' ... n += val\n' + ' >>> print(n)\n' + ' 504\n' + '\n' + ' >>> # keys and values are iterated over in the same ' + 'order\n' + ' >>> list(keys)\n' + " ['eggs', 'bacon', 'sausage', 'spam']\n" + ' >>> list(values)\n' + ' [2, 1, 1, 500]\n' + '\n' + ' >>> # view objects are dynamic and reflect dict changes\n' + " >>> del dishes['eggs']\n" + " >>> del dishes['sausage']\n" + ' >>> list(keys)\n' + " ['spam', 'bacon']\n" + '\n' + ' >>> # set operations\n' + " >>> keys & {'eggs', 'bacon', 'salad'}\n" + " {'bacon'}\n" + " >>> keys ^ {'sausage', 'juice'}\n" + " {'juice', 'sausage', 'bacon', 'spam'}\n", + 'typesmethods': '\n' + 'Methods\n' + '*******\n' + '\n' + 'Methods are functions that are called using the attribute ' + 'notation.\n' + 'There are two flavors: built-in methods (such as "append()" ' + 'on lists)\n' + 'and class instance methods. Built-in methods are described ' + 'with the\n' + 'types that support them.\n' + '\n' + 'If you access a method (a function defined in a class ' + 'namespace)\n' + 'through an instance, you get a special object: a *bound ' + 'method* (also\n' + 'called *instance method*) object. When called, it will add ' + 'the "self"\n' + 'argument to the argument list. Bound methods have two ' + 'special read-\n' + 'only attributes: "m.__self__" is the object on which the ' + 'method\n' + 'operates, and "m.__func__" is the function implementing the ' + 'method.\n' + 'Calling "m(arg-1, arg-2, ..., arg-n)" is completely ' + 'equivalent to\n' + 'calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n' + '\n' + 'Like function objects, bound method objects support getting ' + 'arbitrary\n' + 'attributes. However, since method attributes are actually ' + 'stored on\n' + 'the underlying function object ("meth.__func__"), setting ' + 'method\n' + 'attributes on bound methods is disallowed. Attempting to ' + 'set an\n' + 'attribute on a method results in an "AttributeError" being ' + 'raised. In\n' + 'order to set a method attribute, you need to explicitly set ' + 'it on the\n' + 'underlying function object:\n' + '\n' + ' >>> class C:\n' + ' ... def method(self):\n' + ' ... pass\n' + ' ...\n' + ' >>> c = C()\n' + " >>> c.method.whoami = 'my name is method' # can't set on " + 'the method\n' + ' Traceback (most recent call last):\n' + ' File "", line 1, in \n' + " AttributeError: 'method' object has no attribute " + "'whoami'\n" + " >>> c.method.__func__.whoami = 'my name is method'\n" + ' >>> c.method.whoami\n' + " 'my name is method'\n" + '\n' + 'See The standard type hierarchy for more information.\n', + 'typesmodules': '\n' + 'Modules\n' + '*******\n' + '\n' + 'The only special operation on a module is attribute access: ' + '"m.name",\n' + 'where *m* is a module and *name* accesses a name defined in ' + "*m*'s\n" + 'symbol table. Module attributes can be assigned to. (Note ' + 'that the\n' + '"import" statement is not, strictly speaking, an operation ' + 'on a module\n' + 'object; "import foo" does not require a module object named ' + '*foo* to\n' + 'exist, rather it requires an (external) *definition* for a ' + 'module\n' + 'named *foo* somewhere.)\n' + '\n' + 'A special attribute of every module is "__dict__". This is ' + 'the\n' + "dictionary containing the module's symbol table. Modifying " + 'this\n' + "dictionary will actually change the module's symbol table, " + 'but direct\n' + 'assignment to the "__dict__" attribute is not possible (you ' + 'can write\n' + '"m.__dict__[\'a\'] = 1", which defines "m.a" to be "1", but ' + "you can't\n" + 'write "m.__dict__ = {}"). Modifying "__dict__" directly is ' + 'not\n' + 'recommended.\n' + '\n' + 'Modules built into the interpreter are written like this: ' + '"". If loaded from a file, they are ' + 'written as\n' + '"".\n', + 'typesseq': '\n' + 'Sequence Types --- "list", "tuple", "range"\n' + '*******************************************\n' + '\n' + 'There are three basic sequence types: lists, tuples, and range\n' + 'objects. Additional sequence types tailored for processing of ' + 'binary\n' + 'data and text strings are described in dedicated sections.\n' + '\n' + '\n' + 'Common Sequence Operations\n' + '==========================\n' + '\n' + 'The operations in the following table are supported by most ' + 'sequence\n' + 'types, both mutable and immutable. The ' + '"collections.abc.Sequence" ABC\n' + 'is provided to make it easier to correctly implement these ' + 'operations\n' + 'on custom sequence types.\n' + '\n' + 'This table lists the sequence operations sorted in ascending ' + 'priority.\n' + 'In the table, *s* and *t* are sequences of the same type, *n*, ' + '*i*,\n' + '*j* and *k* are integers and *x* is an arbitrary object that ' + 'meets any\n' + 'type and value restrictions imposed by *s*.\n' + '\n' + 'The "in" and "not in" operations have the same priorities as ' + 'the\n' + 'comparison operations. The "+" (concatenation) and "*" ' + '(repetition)\n' + 'operations have the same priority as the corresponding numeric\n' + 'operations.\n' + '\n' + '+----------------------------+----------------------------------+------------+\n' + '| Operation | Result ' + '| Notes |\n' + '+============================+==================================+============+\n' + '| "x in s" | "True" if an item of *s* is ' + '| (1) |\n' + '| | equal to *x*, else "False" ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "x not in s" | "False" if an item of *s* is ' + '| (1) |\n' + '| | equal to *x*, else "True" ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "s + t" | the concatenation of *s* and *t* ' + '| (6)(7) |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "s * n" or "n * s" | equivalent to adding *s* to ' + '| (2)(7) |\n' + '| | itself *n* times ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "s[i]" | *i*th item of *s*, origin 0 ' + '| (3) |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "s[i:j]" | slice of *s* from *i* to *j* ' + '| (3)(4) |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "s[i:j:k]" | slice of *s* from *i* to *j* ' + '| (3)(5) |\n' + '| | with step *k* ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "len(s)" | length of *s* ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "min(s)" | smallest item of *s* ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "max(s)" | largest item of *s* ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "s.index(x[, i[, j]])" | index of the first occurrence of ' + '| (8) |\n' + '| | *x* in *s* (at or after index ' + '| |\n' + '| | *i* and before index *j*) ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '| "s.count(x)" | total number of occurrences of ' + '| |\n' + '| | *x* in *s* ' + '| |\n' + '+----------------------------+----------------------------------+------------+\n' + '\n' + 'Sequences of the same type also support comparisons. In ' + 'particular,\n' + 'tuples and lists are compared lexicographically by comparing\n' + 'corresponding elements. This means that to compare equal, every\n' + 'element must compare equal and the two sequences must be of the ' + 'same\n' + 'type and have the same length. (For full details see ' + 'Comparisons in\n' + 'the language reference.)\n' + '\n' + 'Notes:\n' + '\n' + '1. While the "in" and "not in" operations are used only for ' + 'simple\n' + ' containment testing in the general case, some specialised ' + 'sequences\n' + ' (such as "str", "bytes" and "bytearray") also use them for\n' + ' subsequence testing:\n' + '\n' + ' >>> "gg" in "eggs"\n' + ' True\n' + '\n' + '2. Values of *n* less than "0" are treated as "0" (which yields ' + 'an\n' + ' empty sequence of the same type as *s*). Note that items in ' + 'the\n' + ' sequence *s* are not copied; they are referenced multiple ' + 'times.\n' + ' This often haunts new Python programmers; consider:\n' + '\n' + ' >>> lists = [[]] * 3\n' + ' >>> lists\n' + ' [[], [], []]\n' + ' >>> lists[0].append(3)\n' + ' >>> lists\n' + ' [[3], [3], [3]]\n' + '\n' + ' What has happened is that "[[]]" is a one-element list ' + 'containing\n' + ' an empty list, so all three elements of "[[]] * 3" are ' + 'references\n' + ' to this single empty list. Modifying any of the elements of\n' + ' "lists" modifies this single list. You can create a list of\n' + ' different lists this way:\n' + '\n' + ' >>> lists = [[] for i in range(3)]\n' + ' >>> lists[0].append(3)\n' + ' >>> lists[1].append(5)\n' + ' >>> lists[2].append(7)\n' + ' >>> lists\n' + ' [[3], [5], [7]]\n' + '\n' + ' Further explanation is available in the FAQ entry How do I ' + 'create a\n' + ' multidimensional list?.\n' + '\n' + '3. If *i* or *j* is negative, the index is relative to the end ' + 'of\n' + ' the string: "len(s) + i" or "len(s) + j" is substituted. But ' + 'note\n' + ' that "-0" is still "0".\n' + '\n' + '4. The slice of *s* from *i* to *j* is defined as the sequence ' + 'of\n' + ' items with index *k* such that "i <= k < j". If *i* or *j* ' + 'is\n' + ' greater than "len(s)", use "len(s)". If *i* is omitted or ' + '"None",\n' + ' use "0". If *j* is omitted or "None", use "len(s)". If *i* ' + 'is\n' + ' greater than or equal to *j*, the slice is empty.\n' + '\n' + '5. The slice of *s* from *i* to *j* with step *k* is defined as ' + 'the\n' + ' sequence of items with index "x = i + n*k" such that "0 <= n ' + '<\n' + ' (j-i)/k". In other words, the indices are "i", "i+k", ' + '"i+2*k",\n' + ' "i+3*k" and so on, stopping when *j* is reached (but never\n' + ' including *j*). If *i* or *j* is greater than "len(s)", use\n' + ' "len(s)". If *i* or *j* are omitted or "None", they become ' + '"end"\n' + ' values (which end depends on the sign of *k*). Note, *k* ' + 'cannot be\n' + ' zero. If *k* is "None", it is treated like "1".\n' + '\n' + '6. Concatenating immutable sequences always results in a new\n' + ' object. This means that building up a sequence by repeated\n' + ' concatenation will have a quadratic runtime cost in the ' + 'total\n' + ' sequence length. To get a linear runtime cost, you must ' + 'switch to\n' + ' one of the alternatives below:\n' + '\n' + ' * if concatenating "str" objects, you can build a list and ' + 'use\n' + ' "str.join()" at the end or else write to an "io.StringIO"\n' + ' instance and retrieve its value when complete\n' + '\n' + ' * if concatenating "bytes" objects, you can similarly use\n' + ' "bytes.join()" or "io.BytesIO", or you can do in-place\n' + ' concatenation with a "bytearray" object. "bytearray" ' + 'objects are\n' + ' mutable and have an efficient overallocation mechanism\n' + '\n' + ' * if concatenating "tuple" objects, extend a "list" instead\n' + '\n' + ' * for other types, investigate the relevant class ' + 'documentation\n' + '\n' + '7. Some sequence types (such as "range") only support item\n' + " sequences that follow specific patterns, and hence don't " + 'support\n' + ' sequence concatenation or repetition.\n' + '\n' + '8. "index" raises "ValueError" when *x* is not found in *s*. ' + 'When\n' + ' supported, the additional arguments to the index method ' + 'allow\n' + ' efficient searching of subsections of the sequence. Passing ' + 'the\n' + ' extra arguments is roughly equivalent to using ' + '"s[i:j].index(x)",\n' + ' only without copying any data and with the returned index ' + 'being\n' + ' relative to the start of the sequence rather than the start ' + 'of the\n' + ' slice.\n' + '\n' + '\n' + 'Immutable Sequence Types\n' + '========================\n' + '\n' + 'The only operation that immutable sequence types generally ' + 'implement\n' + 'that is not also implemented by mutable sequence types is ' + 'support for\n' + 'the "hash()" built-in.\n' + '\n' + 'This support allows immutable sequences, such as "tuple" ' + 'instances, to\n' + 'be used as "dict" keys and stored in "set" and "frozenset" ' + 'instances.\n' + '\n' + 'Attempting to hash an immutable sequence that contains ' + 'unhashable\n' + 'values will result in "TypeError".\n' + '\n' + '\n' + 'Mutable Sequence Types\n' + '======================\n' + '\n' + 'The operations in the following table are defined on mutable ' + 'sequence\n' + 'types. The "collections.abc.MutableSequence" ABC is provided to ' + 'make\n' + 'it easier to correctly implement these operations on custom ' + 'sequence\n' + 'types.\n' + '\n' + 'In the table *s* is an instance of a mutable sequence type, *t* ' + 'is any\n' + 'iterable object and *x* is an arbitrary object that meets any ' + 'type and\n' + 'value restrictions imposed by *s* (for example, "bytearray" ' + 'only\n' + 'accepts integers that meet the value restriction "0 <= x <= ' + '255").\n' + '\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| Operation | ' + 'Result | Notes |\n' + '+================================+==================================+=======================+\n' + '| "s[i] = x" | item *i* of *s* is replaced ' + 'by | |\n' + '| | ' + '*x* | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s[i:j] = t" | slice of *s* from *i* to *j* ' + 'is | |\n' + '| | replaced by the contents of ' + 'the | |\n' + '| | iterable ' + '*t* | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "del s[i:j]" | same as "s[i:j] = ' + '[]" | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s[i:j:k] = t" | the elements of "s[i:j:k]" ' + 'are | (1) |\n' + '| | replaced by those of ' + '*t* | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "del s[i:j:k]" | removes the elements ' + 'of | |\n' + '| | "s[i:j:k]" from the ' + 'list | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.append(x)" | appends *x* to the end of ' + 'the | |\n' + '| | sequence (same ' + 'as | |\n' + '| | "s[len(s):len(s)] = ' + '[x]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.clear()" | removes all items from "s" ' + '(same | (5) |\n' + '| | as "del ' + 's[:]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.copy()" | creates a shallow copy of ' + '"s" | (5) |\n' + '| | (same as ' + '"s[:]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.extend(t)" or "s += t" | extends *s* with the contents ' + 'of | |\n' + '| | *t* (for the most part the ' + 'same | |\n' + '| | as "s[len(s):len(s)] = ' + 't") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s *= n" | updates *s* with its ' + 'contents | (6) |\n' + '| | repeated *n* ' + 'times | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.insert(i, x)" | inserts *x* into *s* at ' + 'the | |\n' + '| | index given by *i* (same ' + 'as | |\n' + '| | "s[i:i] = ' + '[x]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.pop([i])" | retrieves the item at *i* ' + 'and | (2) |\n' + '| | also removes it from ' + '*s* | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.remove(x)" | remove the first item from ' + '*s* | (3) |\n' + '| | where "s[i] == ' + 'x" | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.reverse()" | reverses the items of *s* ' + 'in | (4) |\n' + '| | ' + 'place | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '\n' + 'Notes:\n' + '\n' + '1. *t* must have the same length as the slice it is replacing.\n' + '\n' + '2. The optional argument *i* defaults to "-1", so that by ' + 'default\n' + ' the last item is removed and returned.\n' + '\n' + '3. "remove" raises "ValueError" when *x* is not found in *s*.\n' + '\n' + '4. The "reverse()" method modifies the sequence in place for\n' + ' economy of space when reversing a large sequence. To remind ' + 'users\n' + ' that it operates by side effect, it does not return the ' + 'reversed\n' + ' sequence.\n' + '\n' + '5. "clear()" and "copy()" are included for consistency with the\n' + " interfaces of mutable containers that don't support slicing\n" + ' operations (such as "dict" and "set")\n' + '\n' + ' New in version 3.3: "clear()" and "copy()" methods.\n' + '\n' + '6. The value *n* is an integer, or an object implementing\n' + ' "__index__()". Zero and negative values of *n* clear the ' + 'sequence.\n' + ' Items in the sequence are not copied; they are referenced ' + 'multiple\n' + ' times, as explained for "s * n" under Common Sequence ' + 'Operations.\n' + '\n' + '\n' + 'Lists\n' + '=====\n' + '\n' + 'Lists are mutable sequences, typically used to store collections ' + 'of\n' + 'homogeneous items (where the precise degree of similarity will ' + 'vary by\n' + 'application).\n' + '\n' + 'class list([iterable])\n' + '\n' + ' Lists may be constructed in several ways:\n' + '\n' + ' * Using a pair of square brackets to denote the empty list: ' + '"[]"\n' + '\n' + ' * Using square brackets, separating items with commas: ' + '"[a]",\n' + ' "[a, b, c]"\n' + '\n' + ' * Using a list comprehension: "[x for x in iterable]"\n' + '\n' + ' * Using the type constructor: "list()" or "list(iterable)"\n' + '\n' + ' The constructor builds a list whose items are the same and in ' + 'the\n' + " same order as *iterable*'s items. *iterable* may be either " + 'a\n' + ' sequence, a container that supports iteration, or an ' + 'iterator\n' + ' object. If *iterable* is already a list, a copy is made and\n' + ' returned, similar to "iterable[:]". For example, ' + '"list(\'abc\')"\n' + ' returns "[\'a\', \'b\', \'c\']" and "list( (1, 2, 3) )" ' + 'returns "[1, 2,\n' + ' 3]". If no argument is given, the constructor creates a new ' + 'empty\n' + ' list, "[]".\n' + '\n' + ' Many other operations also produce lists, including the ' + '"sorted()"\n' + ' built-in.\n' + '\n' + ' Lists implement all of the common and mutable sequence ' + 'operations.\n' + ' Lists also provide the following additional method:\n' + '\n' + ' sort(*, key=None, reverse=None)\n' + '\n' + ' This method sorts the list in place, using only "<" ' + 'comparisons\n' + ' between items. Exceptions are not suppressed - if any ' + 'comparison\n' + ' operations fail, the entire sort operation will fail (and ' + 'the\n' + ' list will likely be left in a partially modified state).\n' + '\n' + ' "sort()" accepts two arguments that can only be passed by\n' + ' keyword (keyword-only arguments):\n' + '\n' + ' *key* specifies a function of one argument that is used ' + 'to\n' + ' extract a comparison key from each list element (for ' + 'example,\n' + ' "key=str.lower"). The key corresponding to each item in ' + 'the list\n' + ' is calculated once and then used for the entire sorting ' + 'process.\n' + ' The default value of "None" means that list items are ' + 'sorted\n' + ' directly without calculating a separate key value.\n' + '\n' + ' The "functools.cmp_to_key()" utility is available to ' + 'convert a\n' + ' 2.x style *cmp* function to a *key* function.\n' + '\n' + ' *reverse* is a boolean value. If set to "True", then the ' + 'list\n' + ' elements are sorted as if each comparison were reversed.\n' + '\n' + ' This method modifies the sequence in place for economy of ' + 'space\n' + ' when sorting a large sequence. To remind users that it ' + 'operates\n' + ' by side effect, it does not return the sorted sequence ' + '(use\n' + ' "sorted()" to explicitly request a new sorted list ' + 'instance).\n' + '\n' + ' The "sort()" method is guaranteed to be stable. A sort ' + 'is\n' + ' stable if it guarantees not to change the relative order ' + 'of\n' + ' elements that compare equal --- this is helpful for ' + 'sorting in\n' + ' multiple passes (for example, sort by department, then by ' + 'salary\n' + ' grade).\n' + '\n' + ' **CPython implementation detail:** While a list is being ' + 'sorted,\n' + ' the effect of attempting to mutate, or even inspect, the ' + 'list is\n' + ' undefined. The C implementation of Python makes the list ' + 'appear\n' + ' empty for the duration, and raises "ValueError" if it can ' + 'detect\n' + ' that the list has been mutated during a sort.\n' + '\n' + '\n' + 'Tuples\n' + '======\n' + '\n' + 'Tuples are immutable sequences, typically used to store ' + 'collections of\n' + 'heterogeneous data (such as the 2-tuples produced by the ' + '"enumerate()"\n' + 'built-in). Tuples are also used for cases where an immutable ' + 'sequence\n' + 'of homogeneous data is needed (such as allowing storage in a ' + '"set" or\n' + '"dict" instance).\n' + '\n' + 'class tuple([iterable])\n' + '\n' + ' Tuples may be constructed in a number of ways:\n' + '\n' + ' * Using a pair of parentheses to denote the empty tuple: ' + '"()"\n' + '\n' + ' * Using a trailing comma for a singleton tuple: "a," or ' + '"(a,)"\n' + '\n' + ' * Separating items with commas: "a, b, c" or "(a, b, c)"\n' + '\n' + ' * Using the "tuple()" built-in: "tuple()" or ' + '"tuple(iterable)"\n' + '\n' + ' The constructor builds a tuple whose items are the same and ' + 'in the\n' + " same order as *iterable*'s items. *iterable* may be either " + 'a\n' + ' sequence, a container that supports iteration, or an ' + 'iterator\n' + ' object. If *iterable* is already a tuple, it is returned\n' + ' unchanged. For example, "tuple(\'abc\')" returns "(\'a\', ' + '\'b\', \'c\')"\n' + ' and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument ' + 'is\n' + ' given, the constructor creates a new empty tuple, "()".\n' + '\n' + ' Note that it is actually the comma which makes a tuple, not ' + 'the\n' + ' parentheses. The parentheses are optional, except in the ' + 'empty\n' + ' tuple case, or when they are needed to avoid syntactic ' + 'ambiguity.\n' + ' For example, "f(a, b, c)" is a function call with three ' + 'arguments,\n' + ' while "f((a, b, c))" is a function call with a 3-tuple as the ' + 'sole\n' + ' argument.\n' + '\n' + ' Tuples implement all of the common sequence operations.\n' + '\n' + 'For heterogeneous collections of data where access by name is ' + 'clearer\n' + 'than access by index, "collections.namedtuple()" may be a more\n' + 'appropriate choice than a simple tuple object.\n' + '\n' + '\n' + 'Ranges\n' + '======\n' + '\n' + 'The "range" type represents an immutable sequence of numbers and ' + 'is\n' + 'commonly used for looping a specific number of times in "for" ' + 'loops.\n' + '\n' + 'class range(stop)\n' + 'class range(start, stop[, step])\n' + '\n' + ' The arguments to the range constructor must be integers ' + '(either\n' + ' built-in "int" or any object that implements the "__index__"\n' + ' special method). If the *step* argument is omitted, it ' + 'defaults to\n' + ' "1". If the *start* argument is omitted, it defaults to "0". ' + 'If\n' + ' *step* is zero, "ValueError" is raised.\n' + '\n' + ' For a positive *step*, the contents of a range "r" are ' + 'determined\n' + ' by the formula "r[i] = start + step*i" where "i >= 0" and ' + '"r[i] <\n' + ' stop".\n' + '\n' + ' For a negative *step*, the contents of the range are still\n' + ' determined by the formula "r[i] = start + step*i", but the\n' + ' constraints are "i >= 0" and "r[i] > stop".\n' + '\n' + ' A range object will be empty if "r[0]" does not meet the ' + 'value\n' + ' constraint. Ranges do support negative indices, but these ' + 'are\n' + ' interpreted as indexing from the end of the sequence ' + 'determined by\n' + ' the positive indices.\n' + '\n' + ' Ranges containing absolute values larger than "sys.maxsize" ' + 'are\n' + ' permitted but some features (such as "len()") may raise\n' + ' "OverflowError".\n' + '\n' + ' Range examples:\n' + '\n' + ' >>> list(range(10))\n' + ' [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n' + ' >>> list(range(1, 11))\n' + ' [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n' + ' >>> list(range(0, 30, 5))\n' + ' [0, 5, 10, 15, 20, 25]\n' + ' >>> list(range(0, 10, 3))\n' + ' [0, 3, 6, 9]\n' + ' >>> list(range(0, -10, -1))\n' + ' [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n' + ' >>> list(range(0))\n' + ' []\n' + ' >>> list(range(1, 0))\n' + ' []\n' + '\n' + ' Ranges implement all of the common sequence operations ' + 'except\n' + ' concatenation and repetition (due to the fact that range ' + 'objects\n' + ' can only represent sequences that follow a strict pattern ' + 'and\n' + ' repetition and concatenation will usually violate that ' + 'pattern).\n' + '\n' + ' start\n' + '\n' + ' The value of the *start* parameter (or "0" if the ' + 'parameter was\n' + ' not supplied)\n' + '\n' + ' stop\n' + '\n' + ' The value of the *stop* parameter\n' + '\n' + ' step\n' + '\n' + ' The value of the *step* parameter (or "1" if the parameter ' + 'was\n' + ' not supplied)\n' + '\n' + 'The advantage of the "range" type over a regular "list" or ' + '"tuple" is\n' + 'that a "range" object will always take the same (small) amount ' + 'of\n' + 'memory, no matter the size of the range it represents (as it ' + 'only\n' + 'stores the "start", "stop" and "step" values, calculating ' + 'individual\n' + 'items and subranges as needed).\n' + '\n' + 'Range objects implement the "collections.abc.Sequence" ABC, and\n' + 'provide features such as containment tests, element index ' + 'lookup,\n' + 'slicing and support for negative indices (see Sequence Types --- ' + 'list,\n' + 'tuple, range):\n' + '\n' + '>>> r = range(0, 20, 2)\n' + '>>> r\n' + 'range(0, 20, 2)\n' + '>>> 11 in r\n' + 'False\n' + '>>> 10 in r\n' + 'True\n' + '>>> r.index(10)\n' + '5\n' + '>>> r[5]\n' + '10\n' + '>>> r[:5]\n' + 'range(0, 10, 2)\n' + '>>> r[-1]\n' + '18\n' + '\n' + 'Testing range objects for equality with "==" and "!=" compares ' + 'them as\n' + 'sequences. That is, two range objects are considered equal if ' + 'they\n' + 'represent the same sequence of values. (Note that two range ' + 'objects\n' + 'that compare equal might have different "start", "stop" and ' + '"step"\n' + 'attributes, for example "range(0) == range(2, 1, 3)" or ' + '"range(0, 3,\n' + '2) == range(0, 4, 2)".)\n' + '\n' + 'Changed in version 3.2: Implement the Sequence ABC. Support ' + 'slicing\n' + 'and negative indices. Test "int" objects for membership in ' + 'constant\n' + 'time instead of iterating through all items.\n' + '\n' + "Changed in version 3.3: Define '==' and '!=' to compare range " + 'objects\n' + 'based on the sequence of values they define (instead of ' + 'comparing\n' + 'based on object identity).\n' + '\n' + 'New in version 3.3: The "start", "stop" and "step" attributes.\n', + 'typesseq-mutable': '\n' + 'Mutable Sequence Types\n' + '**********************\n' + '\n' + 'The operations in the following table are defined on ' + 'mutable sequence\n' + 'types. The "collections.abc.MutableSequence" ABC is ' + 'provided to make\n' + 'it easier to correctly implement these operations on ' + 'custom sequence\n' + 'types.\n' + '\n' + 'In the table *s* is an instance of a mutable sequence ' + 'type, *t* is any\n' + 'iterable object and *x* is an arbitrary object that ' + 'meets any type and\n' + 'value restrictions imposed by *s* (for example, ' + '"bytearray" only\n' + 'accepts integers that meet the value restriction "0 <= x ' + '<= 255").\n' + '\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| Operation | ' + 'Result | Notes ' + '|\n' + '+================================+==================================+=======================+\n' + '| "s[i] = x" | item *i* of *s* is ' + 'replaced by | |\n' + '| | ' + '*x* | ' + '|\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s[i:j] = t" | slice of *s* from *i* ' + 'to *j* is | |\n' + '| | replaced by the ' + 'contents of the | |\n' + '| | iterable ' + '*t* | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "del s[i:j]" | same as "s[i:j] = ' + '[]" | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s[i:j:k] = t" | the elements of ' + '"s[i:j:k]" are | (1) |\n' + '| | replaced by those of ' + '*t* | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "del s[i:j:k]" | removes the elements ' + 'of | |\n' + '| | "s[i:j:k]" from the ' + 'list | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.append(x)" | appends *x* to the ' + 'end of the | |\n' + '| | sequence (same ' + 'as | |\n' + '| | "s[len(s):len(s)] = ' + '[x]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.clear()" | removes all items ' + 'from "s" (same | (5) |\n' + '| | as "del ' + 's[:]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.copy()" | creates a shallow ' + 'copy of "s" | (5) |\n' + '| | (same as ' + '"s[:]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.extend(t)" or "s += t" | extends *s* with the ' + 'contents of | |\n' + '| | *t* (for the most ' + 'part the same | |\n' + '| | as "s[len(s):len(s)] ' + '= t") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s *= n" | updates *s* with its ' + 'contents | (6) |\n' + '| | repeated *n* ' + 'times | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.insert(i, x)" | inserts *x* into *s* ' + 'at the | |\n' + '| | index given by *i* ' + '(same as | |\n' + '| | "s[i:i] = ' + '[x]") | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.pop([i])" | retrieves the item at ' + '*i* and | (2) |\n' + '| | also removes it from ' + '*s* | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.remove(x)" | remove the first item ' + 'from *s* | (3) |\n' + '| | where "s[i] == ' + 'x" | |\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '| "s.reverse()" | reverses the items of ' + '*s* in | (4) |\n' + '| | ' + 'place | ' + '|\n' + '+--------------------------------+----------------------------------+-----------------------+\n' + '\n' + 'Notes:\n' + '\n' + '1. *t* must have the same length as the slice it is ' + 'replacing.\n' + '\n' + '2. The optional argument *i* defaults to "-1", so that ' + 'by default\n' + ' the last item is removed and returned.\n' + '\n' + '3. "remove" raises "ValueError" when *x* is not found in ' + '*s*.\n' + '\n' + '4. The "reverse()" method modifies the sequence in place ' + 'for\n' + ' economy of space when reversing a large sequence. To ' + 'remind users\n' + ' that it operates by side effect, it does not return ' + 'the reversed\n' + ' sequence.\n' + '\n' + '5. "clear()" and "copy()" are included for consistency ' + 'with the\n' + " interfaces of mutable containers that don't support " + 'slicing\n' + ' operations (such as "dict" and "set")\n' + '\n' + ' New in version 3.3: "clear()" and "copy()" methods.\n' + '\n' + '6. The value *n* is an integer, or an object ' + 'implementing\n' + ' "__index__()". Zero and negative values of *n* clear ' + 'the sequence.\n' + ' Items in the sequence are not copied; they are ' + 'referenced multiple\n' + ' times, as explained for "s * n" under Common Sequence ' + 'Operations.\n', + 'unary': '\n' + 'Unary arithmetic and bitwise operations\n' + '***************************************\n' + '\n' + 'All unary arithmetic and bitwise operations have the same ' + 'priority:\n' + '\n' + ' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n' + '\n' + 'The unary "-" (minus) operator yields the negation of its numeric\n' + 'argument.\n' + '\n' + 'The unary "+" (plus) operator yields its numeric argument ' + 'unchanged.\n' + '\n' + 'The unary "~" (invert) operator yields the bitwise inversion of ' + 'its\n' + 'integer argument. The bitwise inversion of "x" is defined as\n' + '"-(x+1)". It only applies to integral numbers.\n' + '\n' + 'In all three cases, if the argument does not have the proper type, ' + 'a\n' + '"TypeError" exception is raised.\n', + 'while': '\n' + 'The "while" statement\n' + '*********************\n' + '\n' + 'The "while" statement is used for repeated execution as long as an\n' + 'expression is true:\n' + '\n' + ' while_stmt ::= "while" expression ":" suite\n' + ' ["else" ":" suite]\n' + '\n' + 'This repeatedly tests the expression and, if it is true, executes ' + 'the\n' + 'first suite; if the expression is false (which may be the first ' + 'time\n' + 'it is tested) the suite of the "else" clause, if present, is ' + 'executed\n' + 'and the loop terminates.\n' + '\n' + 'A "break" statement executed in the first suite terminates the ' + 'loop\n' + 'without executing the "else" clause\'s suite. A "continue" ' + 'statement\n' + 'executed in the first suite skips the rest of the suite and goes ' + 'back\n' + 'to testing the expression.\n', + 'with': '\n' + 'The "with" statement\n' + '********************\n' + '\n' + 'The "with" statement is used to wrap the execution of a block with\n' + 'methods defined by a context manager (see section With Statement\n' + 'Context Managers). This allows common "try"..."except"..."finally"\n' + 'usage patterns to be encapsulated for convenient reuse.\n' + '\n' + ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' + ' with_item ::= expression ["as" target]\n' + '\n' + 'The execution of the "with" statement with one "item" proceeds as\n' + 'follows:\n' + '\n' + '1. The context expression (the expression given in the "with_item")\n' + ' is evaluated to obtain a context manager.\n' + '\n' + '2. The context manager\'s "__exit__()" is loaded for later use.\n' + '\n' + '3. The context manager\'s "__enter__()" method is invoked.\n' + '\n' + '4. If a target was included in the "with" statement, the return\n' + ' value from "__enter__()" is assigned to it.\n' + '\n' + ' Note: The "with" statement guarantees that if the "__enter__()"\n' + ' method returns without an error, then "__exit__()" will always ' + 'be\n' + ' called. Thus, if an error occurs during the assignment to the\n' + ' target list, it will be treated the same as an error occurring\n' + ' within the suite would be. See step 6 below.\n' + '\n' + '5. The suite is executed.\n' + '\n' + '6. The context manager\'s "__exit__()" method is invoked. If an\n' + ' exception caused the suite to be exited, its type, value, and\n' + ' traceback are passed as arguments to "__exit__()". Otherwise, ' + 'three\n' + ' "None" arguments are supplied.\n' + '\n' + ' If the suite was exited due to an exception, and the return ' + 'value\n' + ' from the "__exit__()" method was false, the exception is ' + 'reraised.\n' + ' If the return value was true, the exception is suppressed, and\n' + ' execution continues with the statement following the "with"\n' + ' statement.\n' + '\n' + ' If the suite was exited for any reason other than an exception, ' + 'the\n' + ' return value from "__exit__()" is ignored, and execution ' + 'proceeds\n' + ' at the normal location for the kind of exit that was taken.\n' + '\n' + 'With more than one item, the context managers are processed as if\n' + 'multiple "with" statements were nested:\n' + '\n' + ' with A() as a, B() as b:\n' + ' suite\n' + '\n' + 'is equivalent to\n' + '\n' + ' with A() as a:\n' + ' with B() as b:\n' + ' suite\n' + '\n' + 'Changed in version 3.1: Support for multiple context expressions.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 343** - The "with" statement\n' + ' The specification, background, and examples for the Python ' + '"with"\n' + ' statement.\n', + 'yield': '\n' + 'The "yield" statement\n' + '*********************\n' + '\n' + ' yield_stmt ::= yield_expression\n' + '\n' + 'A "yield" statement is semantically equivalent to a yield ' + 'expression.\n' + 'The yield statement can be used to omit the parentheses that would\n' + 'otherwise be required in the equivalent yield expression ' + 'statement.\n' + 'For example, the yield statements\n' + '\n' + ' yield \n' + ' yield from \n' + '\n' + 'are equivalent to the yield expression statements\n' + '\n' + ' (yield )\n' + ' (yield from )\n' + '\n' + 'Yield expressions and statements are only used when defining a\n' + '*generator* function, and are only used in the body of the ' + 'generator\n' + 'function. Using yield in a function definition is sufficient to ' + 'cause\n' + 'that definition to create a generator function instead of a normal\n' + 'function.\n' + '\n' + 'For full details of "yield" semantics, refer to the Yield ' + 'expressions\n' + 'section.\n'} -- cgit v0.12 From 2e208b7d6280ef26483f7244006832fad8e4ec89 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 16 May 2016 22:35:46 +0300 Subject: Issue #27031: Removed dummy methods in Tkinter widget classes: tk_menuBar() and tk_bindForTraversal(). --- Doc/whatsnew/3.6.rst | 4 ++++ Lib/tkinter/__init__.py | 12 ------------ Misc/NEWS | 3 +++ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index bad0f9e..ce0ce82 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -489,6 +489,10 @@ API and Feature Removals :mod:`traceback` module. They were undocumented methods deprecated since Python 3.2 and equivalent functionality is available from private methods. +* The ``tk_menuBar()`` and ``tk_bindForTraversal()`` dummy methods in + :mod:`tkinter` widget classes were removed (corresponding Tk commands + were obsolete since Tk 4.0). + Porting to Python 3.6 ===================== diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 7fbe147..357385e 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -468,12 +468,6 @@ class Misc: disabledForeground, insertBackground, troughColor.""" self.tk.call(('tk_setPalette',) + _flatten(args) + _flatten(list(kw.items()))) - def tk_menuBar(self, *args): - """Do not use. Needed in Tk 3.6 and earlier.""" - # obsolete since Tk 4.0 - import warnings - warnings.warn('tk_menuBar() does nothing and will be removed in 3.6', - DeprecationWarning, stacklevel=2) def wait_variable(self, name='PY_VAR'): """Wait until the variable is modified. @@ -2705,12 +2699,6 @@ class Menu(Widget): def tk_popup(self, x, y, entry=""): """Post the menu at position X,Y with entry ENTRY.""" self.tk.call('tk_popup', self._w, x, y, entry) - def tk_bindForTraversal(self): - # obsolete since Tk 4.0 - import warnings - warnings.warn('tk_bindForTraversal() does nothing and ' - 'will be removed in 3.6', - DeprecationWarning, stacklevel=2) def activate(self, index): """Activate entry at INDEX.""" self.tk.call(self._w, 'activate', index) diff --git a/Misc/NEWS b/Misc/NEWS index 63299e3..5ba249b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -280,6 +280,9 @@ Core and Builtins Library ------- +- Issue #27031: Removed dummy methods in Tkinter widget classes: tk_menuBar() + and tk_bindForTraversal(). + - Issue #14132: Fix urllib.request redirect handling when the target only has a query string. Original fix by Ján Janech. -- cgit v0.12 From fc92e2c5fdfa691328e07c6a3bdd4d2ed4f7d3bd Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 16 May 2016 16:03:12 -0400 Subject: Update Mac installer ReadMe file for 3.6.0a1 --- Mac/BuildScript/resources/ReadMe.rtf | 55 +++++++++--------------------------- 1 file changed, 14 insertions(+), 41 deletions(-) diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index 65e3f14..1af2451 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,29 +1,25 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1348\cocoasubrtf170 +{\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf460 {\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fmodern\fcharset0 CourierNewPSMT;} {\colortbl;\red255\green255\blue255;} \margl1440\margr1440\vieww13380\viewh14600\viewkind0 -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \f0\fs24 \cf0 This package will install Python $FULL_VERSION for Mac OS X $MACOSX_DEPLOYMENT_TARGET for the following architecture(s): $ARCHITECTURES.\ \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \b \cf0 \ul \ulc0 Which installer variant should I use? \b0 \ulnone \ \ -Python.org provides two installer variants for download: one that installs a +For the initial alpha releases of Python 3.6, Python.org provides only one installer variant for download: one that installs a \i 64-bit/32-bit Intel \i0 Python capable of running on \i Mac OS X 10.6 (Snow Leopard) -\i0 or later; and one that installs a -\i 32-bit-only (Intel and PPC) -\i0 Python capable of running on -\i Mac OS X 10.5 (Leopard) -\i0 or later. This ReadMe was installed with the +\i0 or later. This will change prior to the beta releases of 3.6.0. This ReadMe was installed with the \i $MACOSX_DEPLOYMENT_TARGET -\i0 variant. Unless you are installing to an 10.5 system or you need to build applications that can run on 10.5 systems, use the 10.6 variant if possible. There are some additional operating system functions that are supported starting with 10.6 and you may see better performance using 64-bit mode. By default, Python will automatically run in 64-bit mode if your system supports it. Also see +\i0 variant. By default, Python will automatically run in 64-bit mode if your system supports it. Also see \i Certificate verification and OpenSSL -\i0 below. The Pythons installed by these installers are built with private copies of some third-party libraries not included with or newer than those in OS X itself. The list of these libraries varies by installer variant and is included at the end of the License.rtf file. +\i0 below. The Pythons installed by this installer is built with private copies of some third-party libraries not included with or newer than those in OS X itself. The list of these libraries varies by installer variant and is included at the end of the License.rtf file. \b \ul \ \ Update your version of Tcl/Tk to use IDLE or other Tk applications @@ -33,13 +29,13 @@ To use IDLE or other programs that use the Tkinter graphical user interface tool \i Tcl/Tk \i0 frameworks. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of \i Tcl/Tk -\i0 for this version of Python and of Mac OS X.\ +\i0 for this version of Python and of Mac OS X. For the initial alpha releases of Python 3.6, the installer is linked with Tcl/Tk 8.5; this will change prior to the beta releases of 3.6.0.\ \b \ul \ Certificate verification and OpenSSL\ \b0 \ulnone \ -Python 3.5 includes a number of network security enhancements that were released in Python 3.4.3 and Python 2.7.10. {\field{\*\fldinst{HYPERLINK "https://www.python.org/dev/peps/pep-0476/"}}{\fldrslt PEP 476}} changes several standard library modules, like +Python 3.6 includes a number of network security enhancements that were released in Python 3.4.3 and Python 2.7.10. {\field{\*\fldinst{HYPERLINK "https://www.python.org/dev/peps/pep-0476/"}}{\fldrslt PEP 476}} changes several standard library modules, like \i httplib \i0 , \i urllib @@ -51,56 +47,33 @@ Python 3.5 includes a number of network security enhancements that were released \i security \i0 command line utility.\ \ -For OS X 10.5, Apple provides -\i OpenSSL 0.9.7 -\i0 libraries. This version of Apple's OpenSSL -\b does not -\b0 use the certificates from the system security framework, even when used on newer versions of OS X. Instead it consults a traditional OpenSSL concatenated certificate file ( -\i cafile -\i0 ) or certificate directory ( -\i capath -\i0 ), located in -\f1 /System/Library/OpenSSL -\f0 . These directories are typically empty and not managed by OS X; you must manage them yourself or supply your own SSL contexts. OpenSSL 0.9.7 is obsolete by current security standards, lacking a number of important features found in later versions. Among the problems this causes is the inability to verify higher-security certificates now used by python.org services, including -\i t{\field{\*\fldinst{HYPERLINK "https://pypi.python.org/pypi"}}{\fldrslt he Python Package Index, PyPI}} -\i0 . To solve this problem, the -\i 10.5+ 32-bit-only python.org variant -\i0 is linked with a private copy of -\i OpenSSL 1.0.2 -\i0 ; it consults the same default certificate directory, -\f1 /System/Library/OpenSSL -\f0 . As before, it is still necessary to manage certificates yourself when you use this Python variant and, with certificate verification now enabled by default, you may now need to take additional steps to ensure your Python programs have access to CA certificates you trust. If you use this Python variant to build standalone applications with third-party tools like {\field{\*\fldinst{HYPERLINK "https://pypi.python.org/pypi/py2app/"}}{\fldrslt -\f1 py2app}}, you may now need to bundle CA certificates in them or otherwise supply non-default SSL contexts.\ -\ For OS X 10.6+, Apple also provides \i OpenSSL \i0 \i 0.9.8 libraries \i0 . Apple's 0.9.8 version includes an important additional feature: if a certificate cannot be verified using the manually administered certificates in \f1 /System/Library/OpenSSL -\f0 , the certificates managed by the system security framework In the user and system keychains are also consulted (using Apple private APIs). For this reason, the +\f0 , the certificates managed by the system security framework In the user and system keychains are also consulted (using Apple private APIs). For the initial alpha releases of Python 3.6, the \i 64-bit/32-bit 10.6+ python.org variant -\i0 continues to be dynamically linked with Apple's OpenSSL 0.9.8 since it was felt that the loss of the system-provided certificates and management tools outweighs the additional security features provided by newer versions of OpenSSL. This will likely change in future releases of the python.org installers as Apple has deprecated use of the system-supplied OpenSSL libraries. If you do need features from newer versions of OpenSSL, there are third-party OpenSSL wrapper packages available through +\i0 continues to be dynamically linked with Apple's OpenSSL 0.9.8 since it was felt that the loss of the system-provided certificates and management tools outweighs the additional security features provided by newer versions of OpenSSL. This will change prior to the beta releases of 3.6.0 as Apple has deprecated use of the system-supplied OpenSSL libraries. If you do need features from newer versions of OpenSSL, there are third-party OpenSSL wrapper packages available through \i PyPI \i0 .\ \ The bundled \f1 pip -\f0 included with the Python 3.5 installers has its own default certificate store for verifying download connections.\ +\f0 included with the Python 3.6 installers has its own default certificate store for verifying download connections.\ \ \b \ul Other changes\ \b0 \ulnone \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural -\cf0 For other changes in this release, see the +For other changes in this release, see the \i What's new \i0 section in the {\field{\*\fldinst{HYPERLINK "https://www.python.org/doc/"}}{\fldrslt Documentation Set}} for this release and its \i Release Notes \i0 link at {\field{\*\fldinst{HYPERLINK "https://www.python.org/downloads/"}}{\fldrslt https://www.python.org/downloads/}}.\ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural -\b \cf0 \ul \ +\b \ul \ Python 3 and Python 2 Co-existence\ \b0 \ulnone \ -- cgit v0.12 From cec00a7d87e7280b30eaef2e2567ebd0da5b8b9c Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 16 May 2016 16:03:51 -0400 Subject: Version bump for 3.6.0a1 --- Include/patchlevel.h | 4 ++-- Misc/NEWS | 2 +- README | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 246eba8..3574454 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 0 +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.0a0" +#define PY_VERSION "3.6.0a1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 33dedeb..0f79e53 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -5,7 +5,7 @@ Python News What's New in Python 3.6.0 alpha 1? =================================== -Release date: tba +Release date: 2016-05-16 Core and Builtins ----------------- diff --git a/README b/README index 0c95722..5e8e66b 100644 --- a/README +++ b/README @@ -81,7 +81,7 @@ What's New We have a comprehensive overview of the changes in the "What's New in Python 3.6" document, found at - http://docs.python.org/3.6/whatsnew/3.6.html + https://docs.python.org/3.6/whatsnew/3.6.html For a more detailed change log, read Misc/NEWS (though this file, too, is incomplete, and also doesn't list anything merged in from the 2.7 @@ -96,7 +96,7 @@ Documentation Documentation for Python 3.6 is online, updated daily: - http://docs.python.org/3.6/ + https://docs.python.org/3.6/ It can also be downloaded in many formats for faster access. The documentation is downloadable in HTML, PDF, and reStructuredText formats; the latter version @@ -106,7 +106,7 @@ formatting requirements. If you would like to contribute to the development of Python, relevant documentation is available at: - http://docs.python.org/devguide/ + https://docs.python.org/devguide/ For information about building Python's documentation, refer to Doc/README.txt. @@ -121,7 +121,7 @@ backported versions of certain key Python 3.x features. A source-to-source translation tool, "2to3", can take care of the mundane task of converting large amounts of source code. It is not a complete solution but is complemented by the deprecation warnings in 2.6. See -http://docs.python.org/3.6/library/2to3.html for more information. +https://docs.python.org/3.6/library/2to3.html for more information. Testing @@ -171,7 +171,7 @@ Issue Tracker and Mailing List We're soliciting bug reports about all aspects of the language. Fixes are also welcome, preferably in unified diff format. Please use the issue tracker: - http://bugs.python.org/ + https://bugs.python.org/ If you're not sure whether you're dealing with a bug or a feature, use the mailing list: @@ -180,7 +180,7 @@ mailing list: To subscribe to the list, use the mailman form: - http://mail.python.org/mailman/listinfo/python-dev/ + https://mail.python.org/mailman/listinfo/python-dev/ Proposals for enhancement @@ -190,13 +190,13 @@ If you have a proposal to change Python, you may want to send an email to the comp.lang.python or python-ideas mailing lists for initial feedback. A Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. All current PEPs, as well as guidelines for submitting a new PEP, are listed at -http://www.python.org/dev/peps/. +https://www.python.org/dev/peps/. Release Schedule ---------------- -See PEP 494 for release details: http://www.python.org/dev/peps/pep-0494/ +See PEP 494 for release details: https://www.python.org/dev/peps/pep-0494/ Copyright and License Information -- cgit v0.12 From 58ee0924cc4a7c1f533bb17ea90c7d80c316a1a5 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 16 May 2016 16:05:48 -0400 Subject: Added tag v3.6.0a1 for changeset 5896da372fb0 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index b12071d..d878f9f 100644 --- a/.hgtags +++ b/.hgtags @@ -161,3 +161,4 @@ cc15d736d860303b9da90d43cd32db39bab048df v3.5.0rc2 374f501f4567b7595f2ad7798aa09afa2456bb28 v3.5.0 948ef16a69513ba1ff15c9d7d0b012b949df4c80 v3.5.1rc1 37a07cee5969e6d3672583187a73cf636ff28e1b v3.5.1 +5896da372fb044e38595fb74495de1e1e7c8fb3c v3.6.0a1 -- cgit v0.12 From b4a0d52a3ffe4aa0f92174a0a38453d3c459b244 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 16 May 2016 16:25:16 -0400 Subject: docs: Update whatsnew/3.6 with asyncio changes --- Doc/whatsnew/3.6.rst | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index ce0ce82..52be134 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -190,6 +190,59 @@ New Modules Improved Modules ================ + +asyncio +------- + +Since the :mod:`asyncio` module is :term:`provisional `, +all changes introduced in Python 3.6 have also been backported to Python +3.5.x. + +Notable changes in the :mod:`asyncio` module since Python 3.5.0: + +* The :func:`~asyncio.ensure_future` function and all functions that + use it, such as :meth:`loop.run_until_complete() `, + now accept all kinds of :term:`awaitable objects `. + (Contributed by Yury Selivanov.) + +* New :func:`~asyncio.run_coroutine_threadsafe` function to submit + coroutines to event loops from other threads. + (Contributed by Vincent Michel.) + +* New :meth:`Transport.is_closing() ` + method to check if the transport is closing or closed. + (Contributed by Yury Selivanov.) + +* The :meth:`loop.create_server() ` + method can now accept a list of hosts. + (Contributed by Yann Sionneau.) + +* New :meth:`loop.create_future() ` + method to create Future objects. This allows alternative event + loop implementations, such as + `uvloop `_, to provide a faster + :class:`asyncio.Future` implementation. + (Contributed by Yury Selivanov.) + +* New :meth:`loop.get_exception_handler() ` + method to get the current exception handler. + (Contributed by Yury Selivanov.) + +* New :func:`~asyncio.timeout` context manager to simplify timeouts + handling code. + (Contributed by Andrew Svetlov.) + +* New :meth:`StreamReader.readuntil() ` + method to read data from the stream until a separator bytes + sequence appears. + (Contributed by Mark Korenberg.) + +* The :meth:`loop.getaddrinfo() ` + method is optimized to avoid calling the system ``getaddrinfo`` + function if the address is already resolved. + (Contributed by A. Jesse Jiryu Davis.) + + contextlib ---------- -- cgit v0.12 From 786ed5537c28dd11abd88a89ab020f5b88dbf736 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 17 May 2016 16:55:41 -0400 Subject: Post-release cleanup: 3.6.0a1 -> 3.6.0a2 --- Include/patchlevel.h | 2 +- Misc/NEWS | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 3574454..f0348bb 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.0a1" +#define PY_VERSION "3.6.0a1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 0f79e53..2043ae3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,18 @@ Python News +++++++++++ +What's New in Python 3.6.0 alpha 2 +================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + + What's New in Python 3.6.0 alpha 1? =================================== -- cgit v0.12 From bc7315068f188312e5c624ca57a3b010c7503eff Mon Sep 17 00:00:00 2001 From: "doko@ubuntu.com" Date: Wed, 18 May 2016 01:06:01 +0200 Subject: - make some internal symbols static --- Modules/_collectionsmodule.c | 2 +- Modules/_tracemalloc.c | 2 +- Modules/faulthandler.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index d57f1ba..3008879 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -264,7 +264,7 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); #define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) -int +static int deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->rightindex == BLOCKLEN - 1) { diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index beda2b6..e3329c7 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -262,7 +262,7 @@ hashtable_hash_pointer_t(_Py_hashtable_t *ht, const void *pkey) } -int +static int hashtable_compare_pointer_t(_Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry) { diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 570c875..d6322d0 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -414,7 +414,7 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) /* Install the handler for fatal signals, faulthandler_fatal_error(). */ -int +static int faulthandler_enable(void) { size_t i; -- cgit v0.12 From 094c9c921cc889f1aac99bbcc26b70b19f8ef1d9 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 18 May 2016 08:44:29 +0300 Subject: Issue #23275: Allow () = iterable assignment syntax Documentation updates by Martin Panter. --- Doc/reference/simple_stmts.rst | 31 +++++++++++++++---------------- Lib/test/test_codeop.py | 1 - Lib/test/test_syntax.py | 12 ------------ Lib/test/test_unpack.py | 21 +++++++++++++++++++++ Lib/test/test_with.py | 5 ----- Misc/NEWS | 3 +++ Python/ast.c | 9 ++------- 7 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index ff5b1ca..59493c3 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -84,8 +84,8 @@ attributes or items of mutable objects: assignment_stmt: (`target_list` "=")+ (`expression_list` | `yield_expression`) target_list: `target` ("," `target`)* [","] target: `identifier` - : | "(" `target_list` ")" - : | "[" `target_list` "]" + : | "(" [`target_list`] ")" + : | "[" [`target_list`] "]" : | `attributeref` : | `subscription` : | `slicing` @@ -115,21 +115,25 @@ given with the definition of the object types (see section :ref:`types`). Assignment of an object to a target list, optionally enclosed in parentheses or square brackets, is recursively defined as follows. -* If the target list is a single target: The object is assigned to that target. +* If the target list is empty: The object must also be an empty iterable. -* If the target list is a comma-separated list of targets: The object must be an - iterable with the same number of items as there are targets in the target list, - and the items are assigned, from left to right, to the corresponding targets. +* If the target list is a single target in parentheses: The object is assigned + to that target. + +* If the target list is a comma-separated list of targets, or a single target + in square brackets: The object must be an iterable with the same number of + items as there are targets in the target list, and the items are assigned, + from left to right, to the corresponding targets. * If the target list contains one target prefixed with an asterisk, called a - "starred" target: The object must be a sequence with at least as many items + "starred" target: The object must be an iterable with at least as many items as there are targets in the target list, minus one. The first items of the - sequence are assigned, from left to right, to the targets before the starred - target. The final items of the sequence are assigned to the targets after - the starred target. A list of the remaining items in the sequence is then + iterable are assigned, from left to right, to the targets before the starred + target. The final items of the iterable are assigned to the targets after + the starred target. A list of the remaining items in the iterable is then assigned to the starred target (the list can be empty). - * Else: The object must be a sequence with the same number of items as there + * Else: The object must be an iterable with the same number of items as there are targets in the target list, and the items are assigned, from left to right, to the corresponding targets. @@ -150,11 +154,6 @@ Assignment of an object to a single target is recursively defined as follows. count for the object previously bound to the name to reach zero, causing the object to be deallocated and its destructor (if it has one) to be called. -* If the target is a target list enclosed in parentheses or in square brackets: - The object must be an iterable with the same number of items as there are - targets in the target list, and its items are assigned, from left to right, - to the corresponding targets. - .. index:: pair: attribute; assignment * If the target is an attribute reference: The primary expression in the diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 509bf5d..98da26f 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -282,7 +282,6 @@ class CodeopTests(unittest.TestCase): ai("if (a == 1 and b = 2): pass") ai("del 1") - ai("del ()") ai("del (1,)") ai("del [1]") ai("del '1'") diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 057441c..38fc417 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -35,14 +35,6 @@ SyntaxError: invalid syntax Traceback (most recent call last): SyntaxError: can't assign to keyword -It's a syntax error to assign to the empty tuple. Why isn't it an -error to assign to the empty list? It will always raise some error at -runtime. - ->>> () = 1 -Traceback (most recent call last): -SyntaxError: can't assign to () - >>> f() = 1 Traceback (most recent call last): SyntaxError: can't assign to function call @@ -491,10 +483,6 @@ Traceback (most recent call last): ... SyntaxError: keyword argument repeated ->>> del () -Traceback (most recent call last): -SyntaxError: can't delete () - >>> {1, 2, 3} = 42 Traceback (most recent call last): SyntaxError: can't assign to literal diff --git a/Lib/test/test_unpack.py b/Lib/test/test_unpack.py index d1ccb38..3fcb18f 100644 --- a/Lib/test/test_unpack.py +++ b/Lib/test/test_unpack.py @@ -117,6 +117,27 @@ error) ... test.test_unpack.BozoError +Allow unpacking empty iterables + + >>> () = [] + >>> [] = () + >>> [] = [] + >>> () = () + +Unpacking non-iterables should raise TypeError + + >>> () = 42 + Traceback (most recent call last): + ... + TypeError: 'int' object is not iterable + +Unpacking to an empty iterable should raise ValueError + + >>> () = [42] + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 0) + """ __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index 3815062..e247ff6 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -140,11 +140,6 @@ class FailureTestCase(unittest.TestCase): 'with mock as (None):\n' ' pass') - def testAssignmentToEmptyTupleError(self): - self.assertRaisesSyntaxError( - 'with mock as ():\n' - ' pass') - def testAssignmentToTupleOnlyContainingNoneError(self): self.assertRaisesSyntaxError('with mock as None,:\n pass') self.assertRaisesSyntaxError( diff --git a/Misc/NEWS b/Misc/NEWS index cbea5fd..213a864 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,9 @@ Release date: 2016-05-16 Core and Builtins ----------------- +- Issue #23275: Allow assigning to an empty target list in round brackets: + () = iterable. + - Issue #26991: Fix possible refleak when creating a function with annotations. - Issue #27039: Fixed bytearray.remove() for values greater than 127. Based on diff --git a/Python/ast.c b/Python/ast.c index ad771cf..1efd0b7 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -990,13 +990,8 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) s = e->v.List.elts; break; case Tuple_kind: - if (asdl_seq_LEN(e->v.Tuple.elts)) { - e->v.Tuple.ctx = ctx; - s = e->v.Tuple.elts; - } - else { - expr_name = "()"; - } + e->v.Tuple.ctx = ctx; + s = e->v.Tuple.elts; break; case Lambda_kind: expr_name = "lambda"; -- cgit v0.12 From 9ef6b7fbde1fb09a0b4f548880e4160926c39c0f Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 18 May 2016 07:19:32 +0000 Subject: =?UTF-8?q?Issue=20#23275:=20Don=E2=80=99t=20think=20this=20made?= =?UTF-8?q?=20it=20into=20alpha=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Misc/NEWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 213a864..8cd2d2d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.0 alpha 2 Core and Builtins ----------------- +- Issue #23275: Allow assigning to an empty target list in round brackets: + () = iterable. + Library ------- @@ -22,9 +25,6 @@ Release date: 2016-05-16 Core and Builtins ----------------- -- Issue #23275: Allow assigning to an empty target list in round brackets: - () = iterable. - - Issue #26991: Fix possible refleak when creating a function with annotations. - Issue #27039: Fixed bytearray.remove() for values greater than 127. Based on -- cgit v0.12 From 744c34e2ea91ba8f9e945bbeba121c7e95063056 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 11:36:13 +0200 Subject: Cleanup import.c * Replace PyUnicode_RPartition() with PyUnicode_FindChar() and PyUnicode_Substring() to avoid the creation of a temporary tuple. * Use PyUnicode_FromFormat() to build a string and avoid the single_dot ('.') singleton Thanks Serhiy Storchaka for your review. --- Python/import.c | 67 ++++++++++++++++++++++++--------------------------------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/Python/import.c b/Python/import.c index 6d71f2a..d4c358f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1349,7 +1349,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, _Py_IDENTIFIER(_find_and_load); _Py_IDENTIFIER(_handle_fromlist); _Py_IDENTIFIER(_lock_unlock_module); - _Py_static_string(single_dot, "."); PyObject *abs_name = NULL; PyObject *builtins_import = NULL; PyObject *final_mod = NULL; @@ -1471,19 +1470,25 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { - PyObject *partition = NULL; - PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); - if (borrowed_dot == NULL) { + Py_ssize_t dot; + + if (PyUnicode_READY(package) < 0) { goto error; } - partition = PyUnicode_RPartition(package, borrowed_dot); - Py_DECREF(package); - if (partition == NULL) { + + dot = PyUnicode_FindChar(package, '.', + 0, PyUnicode_GET_LENGTH(package), -1); + if (dot == -2) { goto error; } - package = PyTuple_GET_ITEM(partition, 0); - Py_INCREF(package); - Py_DECREF(partition); + + if (dot >= 0) { + PyObject *substr = PyUnicode_Substring(package, 0, dot); + if (substr == NULL) { + goto error; + } + Py_SETREF(package, substr); + } } } @@ -1531,17 +1536,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, goto error; if (PyUnicode_GET_LENGTH(name) > 0) { - PyObject *borrowed_dot, *seq = NULL; - - borrowed_dot = _PyUnicode_FromId(&single_dot); - seq = PyTuple_Pack(2, base, name); + abs_name = PyUnicode_FromFormat("%U.%U", base, name); Py_DECREF(base); - if (borrowed_dot == NULL || seq == NULL) { - goto error; - } - - abs_name = PyUnicode_Join(borrowed_dot, seq); - Py_DECREF(seq); if (abs_name == NULL) { goto error; } @@ -1639,43 +1635,36 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, if (has_from < 0) goto error; if (!has_from) { - if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) { - PyObject *front = NULL; - PyObject *partition = NULL; - PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); + Py_ssize_t len = PyUnicode_GET_LENGTH(name); + if (level == 0 || len > 0) { + Py_ssize_t dot; - if (borrowed_dot == NULL) { + dot = PyUnicode_FindChar(name, '.', 0, len, 1); + if (dot == -2) { goto error; } - partition = PyUnicode_Partition(name, borrowed_dot); - if (partition == NULL) { - goto error; - } - - if (PyUnicode_GET_LENGTH(PyTuple_GET_ITEM(partition, 1)) == 0) { + if (dot == -1) { /* No dot in module name, simple exit */ - Py_DECREF(partition); final_mod = mod; Py_INCREF(mod); goto error; } - front = PyTuple_GET_ITEM(partition, 0); - Py_INCREF(front); - Py_DECREF(partition); - if (level == 0) { + PyObject *front = PyUnicode_Substring(name, 0, dot); + if (front == NULL) { + goto error; + } + final_mod = PyObject_CallFunctionObjArgs(builtins_import, front, NULL); Py_DECREF(front); } else { - Py_ssize_t cut_off = PyUnicode_GET_LENGTH(name) - - PyUnicode_GET_LENGTH(front); + Py_ssize_t cut_off = len - dot; Py_ssize_t abs_name_len = PyUnicode_GET_LENGTH(abs_name); PyObject *to_return = PyUnicode_Substring(abs_name, 0, abs_name_len - cut_off); - Py_DECREF(front); if (to_return == NULL) { goto error; } -- cgit v0.12 From 19ed27ec2b6d8871249e0dd8f56d40a0a78094f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 11:42:37 +0200 Subject: Optimize pickle.load() and pickle.loads() Issue #27056: Optimize pickle.load() and pickle.loads(), up to 10% faster to deserialize a lot of small objects. --- Doc/whatsnew/3.6.rst | 3 +++ Misc/NEWS | 5 ++++- Modules/_pickle.c | 45 ++++++++++++++++++++++++++------------------- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 52be134..67fd50f 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -467,6 +467,9 @@ Optimizations with a short lifetime, and use :c:func:`malloc` for larger memory blocks. (Contributed by Victor Stinner in :issue:`26249`). +* :func:`pickle.load` and :func:`pickle.loads` are now up to 10% faster when + deserializing many small objects (Contributed by Victor Stinner in + :issue:`27056`). Build and C API Changes ======================= diff --git a/Misc/NEWS b/Misc/NEWS index 94e508f..ba66c4e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- Issue #27056: Optimize pickle.load() and pickle.loads(), up to 10% faster + to deserialize a lot of small objects. + What's New in Python 3.6.0 alpha 1? =================================== @@ -341,7 +344,7 @@ Library - Issue #26977: Removed unnecessary, and ignored, call to sum of squares helper in statistics.pvariance. -- Issue #26002: Use bisect in statistics.median instead of a linear search. +- Issue #26002: Use bisect in statistics.median instead of a linear search. Patch by Upendra Kuma. - Issue #25974: Make use of new Decimal.as_integer_ratio() method in statistics diff --git a/Modules/_pickle.c b/Modules/_pickle.c index fdd60e0..e3aa7c5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1197,21 +1197,9 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) return read_size; } -/* Read `n` bytes from the unpickler's data source, storing the result in `*s`. - - This should be used for all data reads, rather than accessing the unpickler's - input buffer directly. This method deals correctly with reading from input - streams, which the input buffer doesn't deal with. - - Note that when reading from a file-like object, self->next_read_idx won't - be updated (it should remain at 0 for the entire unpickling process). You - should use this function's return value to know how many bytes you can - consume. - - Returns -1 (with an exception set) on failure. On success, return the - number of chars read. */ +/* Don't call it directly: use _Unpickler_Read() */ static Py_ssize_t -_Unpickler_Read(UnpicklerObject *self, char **s, Py_ssize_t n) +_Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) { Py_ssize_t num_read; @@ -1222,11 +1210,10 @@ _Unpickler_Read(UnpicklerObject *self, char **s, Py_ssize_t n) "read would overflow (invalid bytecode)"); return -1; } - if (self->next_read_idx + n <= self->input_len) { - *s = self->input_buffer + self->next_read_idx; - self->next_read_idx += n; - return n; - } + + /* This case is handled by the _Unpickler_Read() macro for efficiency */ + assert(self->next_read_idx + n > self->input_len); + if (!self->read) { PyErr_Format(PyExc_EOFError, "Ran out of input"); return -1; @@ -1243,6 +1230,26 @@ _Unpickler_Read(UnpicklerObject *self, char **s, Py_ssize_t n) return n; } +/* Read `n` bytes from the unpickler's data source, storing the result in `*s`. + + This should be used for all data reads, rather than accessing the unpickler's + input buffer directly. This method deals correctly with reading from input + streams, which the input buffer doesn't deal with. + + Note that when reading from a file-like object, self->next_read_idx won't + be updated (it should remain at 0 for the entire unpickling process). You + should use this function's return value to know how many bytes you can + consume. + + Returns -1 (with an exception set) on failure. On success, return the + number of chars read. */ +#define _Unpickler_Read(self, s, n) \ + (((self)->next_read_idx + (n) <= (self)->input_len) \ + ? (*(s) = (self)->input_buffer + (self)->next_read_idx, \ + (self)->next_read_idx += (n), \ + (n)) \ + : _Unpickler_ReadImpl(self, (s), (n))) + static Py_ssize_t _Unpickler_CopyLine(UnpicklerObject *self, char *line, Py_ssize_t len, char **result) -- cgit v0.12 From 7438c612ab02f9dd5c58fef687a5eb4671dfa18c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 12:43:15 +0200 Subject: Use "with popen:" in test_subprocess Issue #26741. --- Lib/test/test_subprocess.py | 113 +++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 012ff79..5239e5a 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -443,8 +443,8 @@ class ProcessTestCase(BaseTestCase): p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stdout.write("orange")'], stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read(), b"orange") + with p: + self.assertEqual(p.stdout.read(), b"orange") def test_stdout_filedes(self): # stdout is set to open file descriptor @@ -474,8 +474,8 @@ class ProcessTestCase(BaseTestCase): p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stderr.write("strawberry")'], stderr=subprocess.PIPE) - self.addCleanup(p.stderr.close) - self.assertStderrEqual(p.stderr.read(), b"strawberry") + with p: + self.assertStderrEqual(p.stderr.read(), b"strawberry") def test_stderr_filedes(self): # stderr is set to open file descriptor @@ -530,8 +530,8 @@ class ProcessTestCase(BaseTestCase): 'sys.stderr.write("orange")'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - self.addCleanup(p.stdout.close) - self.assertStderrEqual(p.stdout.read(), b"appleorange") + with p: + self.assertStderrEqual(p.stdout.read(), b"appleorange") def test_stdout_stderr_file(self): # capture stdout and stderr to the same open file @@ -789,18 +789,19 @@ class ProcessTestCase(BaseTestCase): stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=1) - p.stdin.write("line1\n") - p.stdin.flush() - self.assertEqual(p.stdout.readline(), "line1\n") - p.stdin.write("line3\n") - p.stdin.close() - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.readline(), - "line2\n") - self.assertEqual(p.stdout.read(6), - "line3\n") - self.assertEqual(p.stdout.read(), - "line4\nline5\nline6\nline7\nline8") + with p: + p.stdin.write("line1\n") + p.stdin.flush() + self.assertEqual(p.stdout.readline(), "line1\n") + p.stdin.write("line3\n") + p.stdin.close() + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.readline(), + "line2\n") + self.assertEqual(p.stdout.read(6), + "line3\n") + self.assertEqual(p.stdout.read(), + "line4\nline5\nline6\nline7\nline8") def test_universal_newlines_communicate(self): # universal newlines through communicate() @@ -1434,8 +1435,8 @@ class POSIXProcessTestCase(BaseTestCase): 'sys.stdout.write(os.getenv("FRUIT"))'], stdout=subprocess.PIPE, preexec_fn=lambda: os.putenv("FRUIT", "apple")) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read(), b"apple") + with p: + self.assertEqual(p.stdout.read(), b"apple") def test_preexec_exception(self): def raise_it(): @@ -1583,8 +1584,8 @@ class POSIXProcessTestCase(BaseTestCase): p = subprocess.Popen(["echo $FRUIT"], shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") def test_shell_string(self): # Run command through the shell (string) @@ -1593,8 +1594,8 @@ class POSIXProcessTestCase(BaseTestCase): p = subprocess.Popen("echo $FRUIT", shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") def test_call_string(self): # call() function with string argument on UNIX @@ -1626,8 +1627,8 @@ class POSIXProcessTestCase(BaseTestCase): for sh in shells: p = subprocess.Popen("echo $0", executable=sh, shell=True, stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) + with p: + self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) def _kill_process(self, method, *args): # Do not inherit file handles from the parent. @@ -2450,8 +2451,8 @@ class Win32ProcessTestCase(BaseTestCase): p = subprocess.Popen(["set"], shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertIn(b"physalis", p.stdout.read()) + with p: + self.assertIn(b"physalis", p.stdout.read()) def test_shell_string(self): # Run command through the shell (string) @@ -2460,8 +2461,8 @@ class Win32ProcessTestCase(BaseTestCase): p = subprocess.Popen("set", shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertIn(b"physalis", p.stdout.read()) + with p: + self.assertIn(b"physalis", p.stdout.read()) def test_call_string(self): # call() function with string argument on Windows @@ -2480,16 +2481,14 @@ class Win32ProcessTestCase(BaseTestCase): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - # Wait for the interpreter to be completely initialized before - # sending any signal. - p.stdout.read(1) - getattr(p, method)(*args) - _, stderr = p.communicate() - self.assertStderrEqual(stderr, b'') - returncode = p.wait() + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + returncode = p.wait() self.assertNotEqual(returncode, 0) def _kill_dead_process(self, method, *args): @@ -2502,19 +2501,17 @@ class Win32ProcessTestCase(BaseTestCase): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - # Wait for the interpreter to be completely initialized before - # sending any signal. - p.stdout.read(1) - # The process should end after this - time.sleep(1) - # This shouldn't raise even though the child is now dead - getattr(p, method)(*args) - _, stderr = p.communicate() - self.assertStderrEqual(stderr, b'') - rc = p.wait() + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() self.assertEqual(rc, 42) def test_send_signal(self): @@ -2602,11 +2599,11 @@ class CommandsWithSpaces (BaseTestCase): def with_spaces(self, *args, **kwargs): kwargs['stdout'] = subprocess.PIPE p = subprocess.Popen(*args, **kwargs) - self.addCleanup(p.stdout.close) - self.assertEqual( - p.stdout.read ().decode("mbcs"), - "2 [%r, 'ab cd']" % self.fname - ) + with p: + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) def test_shell_string_with_spaces(self): # call() function with string argument with spaces on Windows -- cgit v0.12 From a58e2c5c4928ae8031ee60a97f2ab4f863aff8cb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 12:08:12 +0200 Subject: Issue #26741: POSIX implementation of subprocess.Popen._execute_child() now sets the returncode attribute using the child process exit status when exec failed. --- Lib/subprocess.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 642c7f2..41a9de1 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1524,9 +1524,14 @@ class Popen(object): if errpipe_data: try: - os.waitpid(self.pid, 0) + pid, sts = os.waitpid(self.pid, 0) + if pid == self.pid: + self._handle_exitstatus(sts) + else: + self.returncode = sys.maxsize except ChildProcessError: pass + try: exception_name, hex_errno, err_msg = ( errpipe_data.split(b':', 2)) -- cgit v0.12 From 5a48e21ff163107a6f1788050b2f1ffc8bce2b6d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 12:11:15 +0200 Subject: subprocess now emits a ResourceWarning warning Issue #26741: subprocess.Popen destructor now emits a ResourceWarning warning if the child process is still running. --- Doc/library/subprocess.rst | 4 ++++ Doc/whatsnew/3.6.rst | 10 ++++++++++ Lib/subprocess.py | 3 +++ Lib/test/test_subprocess.py | 8 ++++++-- Misc/NEWS | 3 +++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 2ec94bf..0545267 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -497,6 +497,10 @@ functions. .. versionchanged:: 3.2 Added context manager support. + .. versionchanged:: 3.6 + Popen destructor now emits a :exc:`ResourceWarning` warning if the child + process is still running. + Exceptions ^^^^^^^^^^ diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 67fd50f..2ec8f00 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -330,6 +330,16 @@ protocol. (Contributed by Aviv Palivoda in :issue:`26404`.) +subprocess +---------- + +:class:`subprocess.Popen` destructor now emits a :exc:`ResourceWarning` warning +if the child process is still running. Use the context manager protocol (``with +proc: ...``) or call explicitly the :meth:`~subprocess.Popen.wait` method to +read the exit status of the child process (Contributed by Victor Stinner in +:issue:`26741`). + + telnetlib --------- diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 41a9de1..b853f4d 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1006,6 +1006,9 @@ class Popen(object): if not self._child_created: # We didn't get to successfully create a child process. return + if self.returncode is None: + warnings.warn("running subprocess %r" % self, ResourceWarning, + source=self) # In case the child hasn't been waited on, check if it's done. self._internal_poll(_deadstate=_maxsize) if self.returncode is None and _active is not None: diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 5239e5a..a8f0a64 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2286,7 +2286,9 @@ class POSIXProcessTestCase(BaseTestCase): self.addCleanup(p.stderr.close) ident = id(p) pid = p.pid - del p + with support.check_warnings(('', ResourceWarning)): + p = None + # check that p is in the active processes list self.assertIn(ident, [id(o) for o in subprocess._active]) @@ -2305,7 +2307,9 @@ class POSIXProcessTestCase(BaseTestCase): self.addCleanup(p.stderr.close) ident = id(p) pid = p.pid - del p + with support.check_warnings(('', ResourceWarning)): + p = None + os.kill(pid, signal.SIGKILL) # check that p is in the active processes list self.assertIn(ident, [id(o) for o in subprocess._active]) diff --git a/Misc/NEWS b/Misc/NEWS index ba66c4e..5dccc5c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- Issue #26741: subprocess.Popen destructor now emits a ResourceWarning warning + if the child process is still running. + - Issue #27056: Optimize pickle.load() and pickle.loads(), up to 10% faster to deserialize a lot of small objects. -- cgit v0.12 From 6d81a2136d61cb787e7bd7fc26a7ba2a363d8c40 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 13:15:55 +0200 Subject: regrtest doesn't ignore -j1 anymore * regrtest now uses subprocesses when the -j1 command line option is used: each test file runs in a fresh child process. Before, the -j1 option was ignored. * Tools/buildbot/test.bat script now uses -j1 by default to run each test file in fresh child process. --- Lib/test/libregrtest/cmdline.py | 2 -- Misc/NEWS | 10 ++++++++++ Tools/buildbot/test.bat | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index c7e990d..de09a01 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -318,8 +318,6 @@ def _parse_args(args, **kwargs): if ns.use_mp <= 0: # Use all cores + extras for tests that like to sleep ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None if ns.use: for a in ns.use: for r in a: diff --git a/Misc/NEWS b/Misc/NEWS index 5dccc5c..628426b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,16 @@ Library - Issue #27056: Optimize pickle.load() and pickle.loads(), up to 10% faster to deserialize a lot of small objects. +Tests +----- + +- Issue #25285: regrtest now uses subprocesses when the -j1 command line option + is used: each test file runs in a fresh child process. Before, the -j1 option + was ignored. + +- Issue #25285: Tools/buildbot/test.bat script now uses -j1 by default to run + each test file in fresh child process. + What's New in Python 3.6.0 alpha 1? =================================== diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index c01400c..5972d5e 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -4,7 +4,7 @@ setlocal set here=%~dp0 set rt_opts=-q -d -set regrtest_args= +set regrtest_args=-j1 :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -- cgit v0.12 From 1b8b42344ed3f6a982b26fcc4255b8490c059527 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 13:37:40 +0200 Subject: regrtest: display test result (passed, failed, ...) * in multiprocessing mode: always display the result * sequential mode: only display the result if the test did not pass --- Lib/test/libregrtest/main.py | 12 ++++++++---- Lib/test/libregrtest/runtest.py | 17 +++++++++++++++-- Lib/test/libregrtest/runtest_mp.py | 5 +++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 447d99f..e503c13 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -15,7 +15,7 @@ from test.libregrtest.runtest import ( findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR, - PROGRESS_MIN_TIME) + PROGRESS_MIN_TIME, format_test_result) from test.libregrtest.setup import setup_tests from test import support try: @@ -326,7 +326,9 @@ class Regrtest: # if on a false return value from main. cmd = ('result = runtest(self.ns, test); ' 'self.accumulate_result(test, result)') - self.tracer.runctx(cmd, globals=globals(), locals=vars()) + ns = dict(locals()) + self.tracer.runctx(cmd, globals=globals(), locals=ns) + result = ns['result'] else: try: result = runtest(self.ns, test) @@ -337,10 +339,12 @@ class Regrtest: else: self.accumulate_result(test, result) + previous_test = format_test_result(test, result[0]) test_time = time.monotonic() - start_time if test_time >= PROGRESS_MIN_TIME: - previous_test = '%s took %.0f sec' % (test, test_time) - else: + previous_test = "%s in %.0f sec" % (previous_test, test_time) + elif result[0] == PASSED: + # be quiet: say nothing if the test passed shortly previous_test = None if self.ns.findleaks: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 601f2b2..ef1feb7 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -20,12 +20,20 @@ RESOURCE_DENIED = -3 INTERRUPTED = -4 CHILD_ERROR = -5 # error in a child process +_FORMAT_TEST_RESULT = { + PASSED: '%s passed', + FAILED: '%s failed', + ENV_CHANGED: '%s failed (env changed)', + SKIPPED: '%s skipped', + RESOURCE_DENIED: '%s skipped (resource denied)', + INTERRUPTED: '%s interrupted', + CHILD_ERROR: '%s crashed', +} + # Minimum duration of a test to display its duration or to mention that # the test is running in background PROGRESS_MIN_TIME = 30.0 # seconds - - # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ @@ -45,6 +53,11 @@ STDTESTS = [ NOTTESTS = set() +def format_test_result(test_name, result): + fmt = _FORMAT_TEST_RESULT.get(result, "%s") + return fmt % test_name + + def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 33f632d..9604c16 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -14,7 +14,8 @@ except ImportError: sys.exit(2) from test.libregrtest.runtest import ( - runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME) + runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, + format_test_result) from test.libregrtest.setup import setup_tests @@ -196,8 +197,8 @@ def run_tests_multiprocess(regrtest): regrtest.accumulate_result(test, result) # Display progress - text = test ok, test_time = result + text = format_test_result(test, ok) if (ok not in (CHILD_ERROR, INTERRUPTED) and test_time >= PROGRESS_MIN_TIME and not regrtest.ns.pgo): -- cgit v0.12 From da23056a3ed33d2ae69752f7d113059333176297 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 May 2016 21:16:59 +0200 Subject: Issue #27056: Fix _Unpickler_Read() to avoid integer overflow --- Modules/_pickle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index e3aa7c5..1c9b9eb 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1244,7 +1244,7 @@ _Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) Returns -1 (with an exception set) on failure. On success, return the number of chars read. */ #define _Unpickler_Read(self, s, n) \ - (((self)->next_read_idx + (n) <= (self)->input_len) \ + (((n) <= (self)->input_len - (self)->next_read_idx) \ ? (*(s) = (self)->input_buffer + (self)->next_read_idx, \ (self)->next_read_idx += (n), \ (n)) \ -- cgit v0.12 From 7f3d16f8f582de1c6d19b40f7d1b8966af06cce0 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 22 May 2016 03:10:11 +0000 Subject: Extra NEWS formatting fix for 3.6 --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index ce20fe7..363454d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1025,7 +1025,7 @@ Library IDLE ---- -- Issue 15348: Stop the debugger engine (normally in a user process) +- Issue #15348: Stop the debugger engine (normally in a user process) before closing the debugger window (running in the IDLE process). This prevents the RuntimeErrors that were being caught and ignored. -- cgit v0.12 From 835416cf7caff0d397c10b98eb146372d6d6dfd0 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 22 May 2016 12:28:41 +0100 Subject: Issue #27064: The py.exe launcher now defaults to Python 3. The Windows launcher ``py.exe`` no longer prefers an installed Python 2 version over Python 3 by default when used interactively. --- Doc/using/windows.rst | 22 +++++++++++++++------- Doc/whatsnew/3.6.rst | 9 +++++++++ Misc/NEWS | 7 +++++++ PC/launcher.c | 21 ++++++++++++++------- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 7520d60..2399278 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -418,6 +418,8 @@ Getting started From the command-line ^^^^^^^^^^^^^^^^^^^^^ +.. versionchanged:: 3.6 + System-wide installations of Python 3.3 and later will put the launcher on your :envvar:`PATH`. The launcher is compatible with all available versions of Python, so it does not matter which version is installed. To check that the @@ -427,25 +429,26 @@ launcher is available, execute the following command in Command Prompt: py -You should find that the latest version of Python 2.x you have installed is +You should find that the latest version of Python you have installed is started - it can be exited as normal, and any additional command-line arguments specified will be sent directly to Python. -If you have multiple versions of Python 2.x installed (e.g., 2.6 and 2.7) you -will have noticed that Python 2.7 was started - to launch Python 2.6, try the +If you have multiple versions of Python installed (e.g., 2.7 and 3.6) you +will have noticed that Python 3.6 was started - to launch Python 2.7, try the command: :: - py -2.6 + py -2.7 -If you have a Python 3.x installed, try the command: +If you want the latest version of Python 2.x you have installed, try the +command: :: - py -3 + py -2 -You should find the latest version of Python 3.x starts. +You should find the latest version of Python 2.x starts. If you see the following error, you do not have the launcher installed: @@ -500,6 +503,11 @@ version qualifier. Assuming you have Python 2.6 installed, try changing the first line to ``#! python2.6`` and you should find the 2.6 version information printed. +Note that unlike interactive use, a bare "python" will use the latest +version of Python 2.x that you have installed. This is for backward +compatibility and for compatibility with Unix, where the command ``python`` +typically refers to Python 2. + From file associations ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 2ec8f00..f829e4a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -62,8 +62,17 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.6. Brevity is key. +New syntax features: + * PEP 498: :ref:`Formatted string literals ` +Windows improvements: + +* The ``py.exe`` launcher, when used interactively, no longer prefers + Python 2 over Python 3 when the user doesn't specify a version (via + command line arguments or a config file). Handling of shebang lines + remains unchanged - "python" refers to Python 2 in that case. + .. PEP-sized items next. .. _pep-4XX: diff --git a/Misc/NEWS b/Misc/NEWS index 363454d..b7d9607 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,13 @@ Tests - Issue #25285: Tools/buildbot/test.bat script now uses -j1 by default to run each test file in fresh child process. +Windows +------- + +- Issue #27064: The py.exe launcher now defaults to Python 3. + The Windows launcher ``py.exe`` no longer prefers an installed + Python 2 version over Python 3 by default when used interactively. + What's New in Python 3.6.0 alpha 1? =================================== diff --git a/PC/launcher.c b/PC/launcher.c index e5f2cea..e4d3e8e 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -465,7 +465,7 @@ get_configured_value(wchar_t * key) } static INSTALLED_PYTHON * -locate_python(wchar_t * wanted_ver) +locate_python(wchar_t * wanted_ver, BOOL from_shebang) { static wchar_t config_key [] = { L"pythonX" }; static wchar_t * last_char = &config_key[sizeof(config_key) / @@ -497,10 +497,17 @@ locate_python(wchar_t * wanted_ver) configured_value = get_configured_value(config_key); if (configured_value) result = find_python_by_version(configured_value); + /* Not found a value yet - try by major version. + * If we're looking for an interpreter specified in a shebang line, + * we want to try Python 2 first, then Python 3 (for Unix and backward + * compatibility). If we're being called interactively, assume the user + * wants the latest version available, so try Python 3 first, then + * Python 2. + */ if (result == NULL) - result = find_python_by_version(L"2"); + result = find_python_by_version(from_shebang ? L"2" : L"3"); if (result == NULL) - result = find_python_by_version(L"3"); + result = find_python_by_version(from_shebang ? L"3" : L"2"); debug(L"search for default Python found "); if (result) { debug(L"version %ls at '%ls'\n", @@ -1094,7 +1101,7 @@ find_by_magic(unsigned short magic) for (mp = magic_values; mp->min; mp++) { if ((magic >= mp->min) && (magic <= mp->max)) { - result = locate_python(mp->version); + result = locate_python(mp->version, FALSE); if (result != NULL) break; } @@ -1279,7 +1286,7 @@ specification: '%ls'.\nIn the first line of the script, 'python' needs to be \ followed by a valid version specifier.\nPlease check the documentation.", command); /* TODO could call validate_version(command) */ - ip = locate_python(command); + ip = locate_python(command, TRUE); if (ip == NULL) { error(RC_NO_PYTHON, L"Requested Python version \ (%ls) is not installed", command); @@ -1485,7 +1492,7 @@ process(int argc, wchar_t ** argv) plen = wcslen(p); valid = (*p == L'-') && validate_version(&p[1]); if (valid) { - ip = locate_python(&p[1]); + ip = locate_python(&p[1], FALSE); if (ip == NULL) error(RC_NO_PYTHON, L"Requested Python version (%ls) not \ installed", &p[1]); @@ -1512,7 +1519,7 @@ installed", &p[1]); /* If we didn't find one, look for the default Python */ if (executable == NULL) { - ip = locate_python(L""); + ip = locate_python(L"", FALSE); if (ip == NULL) error(RC_NO_PYTHON, L"Can't find a default Python."); executable = ip->executable; -- cgit v0.12 From 1f9eb879d573454cd3ea321993e72ad2cca65c9d Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Sun, 22 May 2016 17:35:34 +0200 Subject: Issue #26857: The gethostbyaddr_r() workaround is no longer needed with api-level >= 23. Patch by Xavier de Gaye. --- Include/pyport.h | 4 ++++ Modules/socketmodule.c | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Include/pyport.h b/Include/pyport.h index 66e00d4..8b0a89f 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -897,4 +897,8 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif /* _MSC_VER >= 1900 */ #endif /* Py_BUILD_CORE */ +#ifdef __ANDROID__ +#include +#endif + #endif /* Py_PYPORT_H */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 46eeed1..dc57810 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -163,7 +163,11 @@ if_indextoname(index) -- return the corresponding interface name\n\ # include #endif -#if !defined(WITH_THREAD) || defined(__ANDROID__) +#if !defined(WITH_THREAD) +# undef HAVE_GETHOSTBYNAME_R +#endif + +#if defined(__ANDROID__) && __ANDROID_API__ < 23 # undef HAVE_GETHOSTBYNAME_R #endif -- cgit v0.12 From 0d9220e162f1e5f8caa3d7ebaa54665776d361a1 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 22 May 2016 19:10:31 -0400 Subject: Issue #24225: Rename many idlelib/*.py and idlelib/idle_test/test_*.py files. --- Lib/idlelib/AutoComplete.py | 233 ---- Lib/idlelib/AutoCompleteWindow.py | 416 ------ Lib/idlelib/AutoExpand.py | 104 -- Lib/idlelib/Bindings.py | 94 -- Lib/idlelib/CallTipWindow.py | 161 --- Lib/idlelib/CallTips.py | 175 --- Lib/idlelib/ClassBrowser.py | 236 ---- Lib/idlelib/CodeContext.py | 176 --- Lib/idlelib/ColorDelegator.py | 256 ---- Lib/idlelib/Debugger.py | 539 -------- Lib/idlelib/Delegator.py | 33 - Lib/idlelib/EditorWindow.py | 1703 ------------------------ Lib/idlelib/FileList.py | 129 -- Lib/idlelib/FormatParagraph.py | 195 --- Lib/idlelib/GrepDialog.py | 158 --- Lib/idlelib/HyperParser.py | 313 ----- Lib/idlelib/IOBinding.py | 565 -------- Lib/idlelib/IdleHistory.py | 104 -- Lib/idlelib/MultiCall.py | 446 ------- Lib/idlelib/MultiStatusBar.py | 47 - Lib/idlelib/ObjectBrowser.py | 143 -- Lib/idlelib/OutputWindow.py | 144 -- Lib/idlelib/ParenMatch.py | 178 --- Lib/idlelib/PathBrowser.py | 108 -- Lib/idlelib/Percolator.py | 105 -- Lib/idlelib/PyParse.py | 617 --------- Lib/idlelib/PyShell.py | 1619 ---------------------- Lib/idlelib/RemoteDebugger.py | 388 ------ Lib/idlelib/RemoteObjectBrowser.py | 36 - Lib/idlelib/ReplaceDialog.py | 241 ---- Lib/idlelib/RstripExtension.py | 33 - Lib/idlelib/ScriptBinding.py | 206 --- Lib/idlelib/ScrolledList.py | 145 -- Lib/idlelib/SearchDialog.py | 97 -- Lib/idlelib/SearchDialogBase.py | 184 --- Lib/idlelib/SearchEngine.py | 233 ---- Lib/idlelib/StackViewer.py | 151 --- Lib/idlelib/ToolTip.py | 97 -- Lib/idlelib/TreeWidget.py | 466 ------- Lib/idlelib/UndoDelegator.py | 368 ----- Lib/idlelib/WidgetRedirector.py | 176 --- Lib/idlelib/WindowList.py | 90 -- Lib/idlelib/ZoomHeight.py | 51 - Lib/idlelib/aboutDialog.py | 149 --- Lib/idlelib/autocomplete.py | 233 ++++ Lib/idlelib/autocomplete_w.py | 416 ++++++ Lib/idlelib/autoexpand.py | 104 ++ Lib/idlelib/browser.py | 236 ++++ Lib/idlelib/calltip_w.py | 161 +++ Lib/idlelib/calltips.py | 175 +++ Lib/idlelib/codecontext.py | 176 +++ Lib/idlelib/colorizer.py | 256 ++++ Lib/idlelib/config.py | 760 +++++++++++ Lib/idlelib/configDialog.py | 1434 -------------------- Lib/idlelib/configHandler.py | 760 ----------- Lib/idlelib/configHelpSourceEdit.py | 170 --- Lib/idlelib/configSectionNameDialog.py | 98 -- Lib/idlelib/config_help.py | 170 +++ Lib/idlelib/config_key.py | 266 ++++ Lib/idlelib/config_sec.py | 98 ++ Lib/idlelib/configdialog.py | 1434 ++++++++++++++++++++ Lib/idlelib/debugger.py | 539 ++++++++ Lib/idlelib/debugger_r.py | 388 ++++++ Lib/idlelib/debugobj.py | 143 ++ Lib/idlelib/debugobj_r.py | 36 + Lib/idlelib/delegator.py | 33 + Lib/idlelib/dynOptionMenuWidget.py | 57 - Lib/idlelib/dynoption.py | 57 + Lib/idlelib/editor.py | 1703 ++++++++++++++++++++++++ Lib/idlelib/filelist.py | 129 ++ Lib/idlelib/grep.py | 158 +++ Lib/idlelib/help_about.py | 149 +++ Lib/idlelib/history.py | 104 ++ Lib/idlelib/hyperparser.py | 313 +++++ Lib/idlelib/idle_test/test_formatparagraph.py | 377 ------ Lib/idlelib/idle_test/test_history.py | 167 +++ Lib/idlelib/idle_test/test_idlehistory.py | 167 --- Lib/idlelib/idle_test/test_io.py | 233 ---- Lib/idlelib/idle_test/test_iomenu.py | 233 ++++ Lib/idlelib/idle_test/test_paragraph.py | 377 ++++++ Lib/idlelib/idle_test/test_redirector.py | 122 ++ Lib/idlelib/idle_test/test_replace.py | 292 ++++ Lib/idlelib/idle_test/test_replacedialog.py | 292 ---- Lib/idlelib/idle_test/test_search.py | 80 ++ Lib/idlelib/idle_test/test_searchbase.py | 165 +++ Lib/idlelib/idle_test/test_searchdialog.py | 80 -- Lib/idlelib/idle_test/test_searchdialogbase.py | 165 --- Lib/idlelib/idle_test/test_undo.py | 134 ++ Lib/idlelib/idle_test/test_undodelegator.py | 134 -- Lib/idlelib/idle_test/test_widgetredir.py | 122 -- Lib/idlelib/iomenu.py | 565 ++++++++ Lib/idlelib/keybindingDialog.py | 266 ---- Lib/idlelib/macosx.py | 239 ++++ Lib/idlelib/macosxSupport.py | 239 ---- Lib/idlelib/mainmenu.py | 94 ++ Lib/idlelib/multicall.py | 446 +++++++ Lib/idlelib/outwin.py | 144 ++ Lib/idlelib/paragraph.py | 195 +++ Lib/idlelib/parenmatch.py | 178 +++ Lib/idlelib/pathbrowser.py | 108 ++ Lib/idlelib/percolator.py | 105 ++ Lib/idlelib/pyparse.py | 617 +++++++++ Lib/idlelib/pyshell.py | 1619 ++++++++++++++++++++++ Lib/idlelib/redirector.py | 176 +++ Lib/idlelib/replace.py | 241 ++++ Lib/idlelib/rstrip.py | 33 + Lib/idlelib/runscript.py | 206 +++ Lib/idlelib/scrolledlist.py | 145 ++ Lib/idlelib/search.py | 97 ++ Lib/idlelib/searchbase.py | 184 +++ Lib/idlelib/searchengine.py | 233 ++++ Lib/idlelib/stackviewer.py | 151 +++ Lib/idlelib/statusbar.py | 47 + Lib/idlelib/textView.py | 86 -- Lib/idlelib/textview.py | 86 ++ Lib/idlelib/tooltip.py | 97 ++ Lib/idlelib/tree.py | 466 +++++++ Lib/idlelib/undo.py | 368 +++++ Lib/idlelib/windows.py | 90 ++ Lib/idlelib/zoomheight.py | 51 + 120 files changed, 16788 insertions(+), 16788 deletions(-) delete mode 100644 Lib/idlelib/AutoComplete.py delete mode 100644 Lib/idlelib/AutoCompleteWindow.py delete mode 100644 Lib/idlelib/AutoExpand.py delete mode 100644 Lib/idlelib/Bindings.py delete mode 100644 Lib/idlelib/CallTipWindow.py delete mode 100644 Lib/idlelib/CallTips.py delete mode 100644 Lib/idlelib/ClassBrowser.py delete mode 100644 Lib/idlelib/CodeContext.py delete mode 100644 Lib/idlelib/ColorDelegator.py delete mode 100644 Lib/idlelib/Debugger.py delete mode 100644 Lib/idlelib/Delegator.py delete mode 100644 Lib/idlelib/EditorWindow.py delete mode 100644 Lib/idlelib/FileList.py delete mode 100644 Lib/idlelib/FormatParagraph.py delete mode 100644 Lib/idlelib/GrepDialog.py delete mode 100644 Lib/idlelib/HyperParser.py delete mode 100644 Lib/idlelib/IOBinding.py delete mode 100644 Lib/idlelib/IdleHistory.py delete mode 100644 Lib/idlelib/MultiCall.py delete mode 100644 Lib/idlelib/MultiStatusBar.py delete mode 100644 Lib/idlelib/ObjectBrowser.py delete mode 100644 Lib/idlelib/OutputWindow.py delete mode 100644 Lib/idlelib/ParenMatch.py delete mode 100644 Lib/idlelib/PathBrowser.py delete mode 100644 Lib/idlelib/Percolator.py delete mode 100644 Lib/idlelib/PyParse.py delete mode 100755 Lib/idlelib/PyShell.py delete mode 100644 Lib/idlelib/RemoteDebugger.py delete mode 100644 Lib/idlelib/RemoteObjectBrowser.py delete mode 100644 Lib/idlelib/ReplaceDialog.py delete mode 100644 Lib/idlelib/RstripExtension.py delete mode 100644 Lib/idlelib/ScriptBinding.py delete mode 100644 Lib/idlelib/ScrolledList.py delete mode 100644 Lib/idlelib/SearchDialog.py delete mode 100644 Lib/idlelib/SearchDialogBase.py delete mode 100644 Lib/idlelib/SearchEngine.py delete mode 100644 Lib/idlelib/StackViewer.py delete mode 100644 Lib/idlelib/ToolTip.py delete mode 100644 Lib/idlelib/TreeWidget.py delete mode 100644 Lib/idlelib/UndoDelegator.py delete mode 100644 Lib/idlelib/WidgetRedirector.py delete mode 100644 Lib/idlelib/WindowList.py delete mode 100644 Lib/idlelib/ZoomHeight.py delete mode 100644 Lib/idlelib/aboutDialog.py create mode 100644 Lib/idlelib/autocomplete.py create mode 100644 Lib/idlelib/autocomplete_w.py create mode 100644 Lib/idlelib/autoexpand.py create mode 100644 Lib/idlelib/browser.py create mode 100644 Lib/idlelib/calltip_w.py create mode 100644 Lib/idlelib/calltips.py create mode 100644 Lib/idlelib/codecontext.py create mode 100644 Lib/idlelib/colorizer.py create mode 100644 Lib/idlelib/config.py delete mode 100644 Lib/idlelib/configDialog.py delete mode 100644 Lib/idlelib/configHandler.py delete mode 100644 Lib/idlelib/configHelpSourceEdit.py delete mode 100644 Lib/idlelib/configSectionNameDialog.py create mode 100644 Lib/idlelib/config_help.py create mode 100644 Lib/idlelib/config_key.py create mode 100644 Lib/idlelib/config_sec.py create mode 100644 Lib/idlelib/configdialog.py create mode 100644 Lib/idlelib/debugger.py create mode 100644 Lib/idlelib/debugger_r.py create mode 100644 Lib/idlelib/debugobj.py create mode 100644 Lib/idlelib/debugobj_r.py create mode 100644 Lib/idlelib/delegator.py delete mode 100644 Lib/idlelib/dynOptionMenuWidget.py create mode 100644 Lib/idlelib/dynoption.py create mode 100644 Lib/idlelib/editor.py create mode 100644 Lib/idlelib/filelist.py create mode 100644 Lib/idlelib/grep.py create mode 100644 Lib/idlelib/help_about.py create mode 100644 Lib/idlelib/history.py create mode 100644 Lib/idlelib/hyperparser.py delete mode 100644 Lib/idlelib/idle_test/test_formatparagraph.py create mode 100644 Lib/idlelib/idle_test/test_history.py delete mode 100644 Lib/idlelib/idle_test/test_idlehistory.py delete mode 100644 Lib/idlelib/idle_test/test_io.py create mode 100644 Lib/idlelib/idle_test/test_iomenu.py create mode 100644 Lib/idlelib/idle_test/test_paragraph.py create mode 100644 Lib/idlelib/idle_test/test_redirector.py create mode 100644 Lib/idlelib/idle_test/test_replace.py delete mode 100644 Lib/idlelib/idle_test/test_replacedialog.py create mode 100644 Lib/idlelib/idle_test/test_search.py create mode 100644 Lib/idlelib/idle_test/test_searchbase.py delete mode 100644 Lib/idlelib/idle_test/test_searchdialog.py delete mode 100644 Lib/idlelib/idle_test/test_searchdialogbase.py create mode 100644 Lib/idlelib/idle_test/test_undo.py delete mode 100644 Lib/idlelib/idle_test/test_undodelegator.py delete mode 100644 Lib/idlelib/idle_test/test_widgetredir.py create mode 100644 Lib/idlelib/iomenu.py delete mode 100644 Lib/idlelib/keybindingDialog.py create mode 100644 Lib/idlelib/macosx.py delete mode 100644 Lib/idlelib/macosxSupport.py create mode 100644 Lib/idlelib/mainmenu.py create mode 100644 Lib/idlelib/multicall.py create mode 100644 Lib/idlelib/outwin.py create mode 100644 Lib/idlelib/paragraph.py create mode 100644 Lib/idlelib/parenmatch.py create mode 100644 Lib/idlelib/pathbrowser.py create mode 100644 Lib/idlelib/percolator.py create mode 100644 Lib/idlelib/pyparse.py create mode 100755 Lib/idlelib/pyshell.py create mode 100644 Lib/idlelib/redirector.py create mode 100644 Lib/idlelib/replace.py create mode 100644 Lib/idlelib/rstrip.py create mode 100644 Lib/idlelib/runscript.py create mode 100644 Lib/idlelib/scrolledlist.py create mode 100644 Lib/idlelib/search.py create mode 100644 Lib/idlelib/searchbase.py create mode 100644 Lib/idlelib/searchengine.py create mode 100644 Lib/idlelib/stackviewer.py create mode 100644 Lib/idlelib/statusbar.py delete mode 100644 Lib/idlelib/textView.py create mode 100644 Lib/idlelib/textview.py create mode 100644 Lib/idlelib/tooltip.py create mode 100644 Lib/idlelib/tree.py create mode 100644 Lib/idlelib/undo.py create mode 100644 Lib/idlelib/windows.py create mode 100644 Lib/idlelib/zoomheight.py diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py deleted file mode 100644 index b9ec539..0000000 --- a/Lib/idlelib/AutoComplete.py +++ /dev/null @@ -1,233 +0,0 @@ -"""AutoComplete.py - An IDLE extension for automatically completing names. - -This extension can complete either attribute names of file names. It can pop -a window with all available names, for the user to select from. -""" -import os -import sys -import string - -from idlelib.configHandler import idleConf - -# This string includes all chars that may be in an identifier -ID_CHARS = string.ascii_letters + string.digits + "_" - -# These constants represent the two different types of completions -COMPLETE_ATTRIBUTES, COMPLETE_FILES = range(1, 2+1) - -from idlelib import AutoCompleteWindow -from idlelib.HyperParser import HyperParser - -import __main__ - -SEPS = os.sep -if os.altsep: # e.g. '/' on Windows... - SEPS += os.altsep - -class AutoComplete: - - menudefs = [ - ('edit', [ - ("Show Completions", "<>"), - ]) - ] - - popupwait = idleConf.GetOption("extensions", "AutoComplete", - "popupwait", type="int", default=0) - - def __init__(self, editwin=None): - self.editwin = editwin - if editwin is None: # subprocess and test - return - self.text = editwin.text - self.autocompletewindow = None - - # id of delayed call, and the index of the text insert when the delayed - # call was issued. If _delayed_completion_id is None, there is no - # delayed call. - self._delayed_completion_id = None - self._delayed_completion_index = None - - def _make_autocomplete_window(self): - return AutoCompleteWindow.AutoCompleteWindow(self.text) - - def _remove_autocomplete_window(self, event=None): - if self.autocompletewindow: - self.autocompletewindow.hide_window() - self.autocompletewindow = None - - def force_open_completions_event(self, event): - """Happens when the user really wants to open a completion list, even - if a function call is needed. - """ - self.open_completions(True, False, True) - - def try_open_completions_event(self, event): - """Happens when it would be nice to open a completion list, but not - really necessary, for example after a dot, so function - calls won't be made. - """ - lastchar = self.text.get("insert-1c") - if lastchar == ".": - self._open_completions_later(False, False, False, - COMPLETE_ATTRIBUTES) - elif lastchar in SEPS: - self._open_completions_later(False, False, False, - COMPLETE_FILES) - - def autocomplete_event(self, event): - """Happens when the user wants to complete his word, and if necessary, - open a completion list after that (if there is more than one - completion) - """ - if hasattr(event, "mc_state") and event.mc_state: - # A modifier was pressed along with the tab, continue as usual. - return - if self.autocompletewindow and self.autocompletewindow.is_active(): - self.autocompletewindow.complete() - return "break" - else: - opened = self.open_completions(False, True, True) - if opened: - return "break" - - def _open_completions_later(self, *args): - self._delayed_completion_index = self.text.index("insert") - if self._delayed_completion_id is not None: - self.text.after_cancel(self._delayed_completion_id) - self._delayed_completion_id = \ - self.text.after(self.popupwait, self._delayed_open_completions, - *args) - - def _delayed_open_completions(self, *args): - self._delayed_completion_id = None - if self.text.index("insert") != self._delayed_completion_index: - return - self.open_completions(*args) - - def open_completions(self, evalfuncs, complete, userWantsWin, mode=None): - """Find the completions and create the AutoCompleteWindow. - Return True if successful (no syntax error or so found). - if complete is True, then if there's nothing to complete and no - start of completion, won't open completions and return False. - If mode is given, will open a completion list only in this mode. - """ - # Cancel another delayed call, if it exists. - if self._delayed_completion_id is not None: - self.text.after_cancel(self._delayed_completion_id) - self._delayed_completion_id = None - - hp = HyperParser(self.editwin, "insert") - curline = self.text.get("insert linestart", "insert") - i = j = len(curline) - if hp.is_in_string() and (not mode or mode==COMPLETE_FILES): - # Find the beginning of the string - # fetch_completions will look at the file system to determine whether the - # string value constitutes an actual file name - # XXX could consider raw strings here and unescape the string value if it's - # not raw. - self._remove_autocomplete_window() - mode = COMPLETE_FILES - # Find last separator or string start - while i and curline[i-1] not in "'\"" + SEPS: - i -= 1 - comp_start = curline[i:j] - j = i - # Find string start - while i and curline[i-1] not in "'\"": - i -= 1 - comp_what = curline[i:j] - elif hp.is_in_code() and (not mode or mode==COMPLETE_ATTRIBUTES): - self._remove_autocomplete_window() - mode = COMPLETE_ATTRIBUTES - while i and (curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127): - i -= 1 - comp_start = curline[i:j] - if i and curline[i-1] == '.': - hp.set_index("insert-%dc" % (len(curline)-(i-1))) - comp_what = hp.get_expression() - if not comp_what or \ - (not evalfuncs and comp_what.find('(') != -1): - return - else: - comp_what = "" - else: - return - - if complete and not comp_what and not comp_start: - return - comp_lists = self.fetch_completions(comp_what, mode) - if not comp_lists[0]: - return - self.autocompletewindow = self._make_autocomplete_window() - return not self.autocompletewindow.show_window( - comp_lists, "insert-%dc" % len(comp_start), - complete, mode, userWantsWin) - - def fetch_completions(self, what, mode): - """Return a pair of lists of completions for something. The first list - is a sublist of the second. Both are sorted. - - If there is a Python subprocess, get the comp. list there. Otherwise, - either fetch_completions() is running in the subprocess itself or it - was called in an IDLE EditorWindow before any script had been run. - - The subprocess environment is that of the most recently run script. If - two unrelated modules are being edited some calltips in the current - module may be inoperative if the module was not the last to run. - """ - try: - rpcclt = self.editwin.flist.pyshell.interp.rpcclt - except: - rpcclt = None - if rpcclt: - return rpcclt.remotecall("exec", "get_the_completion_list", - (what, mode), {}) - else: - if mode == COMPLETE_ATTRIBUTES: - if what == "": - namespace = __main__.__dict__.copy() - namespace.update(__main__.__builtins__.__dict__) - bigl = eval("dir()", namespace) - bigl.sort() - if "__all__" in bigl: - smalll = sorted(eval("__all__", namespace)) - else: - smalll = [s for s in bigl if s[:1] != '_'] - else: - try: - entity = self.get_entity(what) - bigl = dir(entity) - bigl.sort() - if "__all__" in bigl: - smalll = sorted(entity.__all__) - else: - smalll = [s for s in bigl if s[:1] != '_'] - except: - return [], [] - - elif mode == COMPLETE_FILES: - if what == "": - what = "." - try: - expandedpath = os.path.expanduser(what) - bigl = os.listdir(expandedpath) - bigl.sort() - smalll = [s for s in bigl if s[:1] != '.'] - except OSError: - return [], [] - - if not smalll: - smalll = bigl - return smalll, bigl - - def get_entity(self, name): - """Lookup name in a namespace spanning sys.modules and __main.dict__""" - namespace = sys.modules.copy() - namespace.update(__main__.__dict__) - return eval(name, namespace) - - -if __name__ == '__main__': - from unittest import main - main('idlelib.idle_test.test_autocomplete', verbosity=2) diff --git a/Lib/idlelib/AutoCompleteWindow.py b/Lib/idlelib/AutoCompleteWindow.py deleted file mode 100644 index 2ee6878..0000000 --- a/Lib/idlelib/AutoCompleteWindow.py +++ /dev/null @@ -1,416 +0,0 @@ -""" -An auto-completion window for IDLE, used by the AutoComplete extension -""" -from tkinter import * -from idlelib.MultiCall import MC_SHIFT -from idlelib.AutoComplete import COMPLETE_FILES, COMPLETE_ATTRIBUTES - -HIDE_VIRTUAL_EVENT_NAME = "<>" -HIDE_SEQUENCES = ("", "") -KEYPRESS_VIRTUAL_EVENT_NAME = "<>" -# We need to bind event beyond so that the function will be called -# before the default specific IDLE function -KEYPRESS_SEQUENCES = ("", "", "", "", - "", "", "", "", - "", "") -KEYRELEASE_VIRTUAL_EVENT_NAME = "<>" -KEYRELEASE_SEQUENCE = "" -LISTUPDATE_SEQUENCE = "" -WINCONFIG_SEQUENCE = "" -DOUBLECLICK_SEQUENCE = "" - -class AutoCompleteWindow: - - def __init__(self, widget): - # The widget (Text) on which we place the AutoCompleteWindow - self.widget = widget - # The widgets we create - self.autocompletewindow = self.listbox = self.scrollbar = None - # The default foreground and background of a selection. Saved because - # they are changed to the regular colors of list items when the - # completion start is not a prefix of the selected completion - self.origselforeground = self.origselbackground = None - # The list of completions - self.completions = None - # A list with more completions, or None - self.morecompletions = None - # The completion mode. Either AutoComplete.COMPLETE_ATTRIBUTES or - # AutoComplete.COMPLETE_FILES - self.mode = None - # The current completion start, on the text box (a string) - self.start = None - # The index of the start of the completion - self.startindex = None - # The last typed start, used so that when the selection changes, - # the new start will be as close as possible to the last typed one. - self.lasttypedstart = None - # Do we have an indication that the user wants the completion window - # (for example, he clicked the list) - self.userwantswindow = None - # event ids - self.hideid = self.keypressid = self.listupdateid = self.winconfigid \ - = self.keyreleaseid = self.doubleclickid = None - # Flag set if last keypress was a tab - self.lastkey_was_tab = False - - def _change_start(self, newstart): - min_len = min(len(self.start), len(newstart)) - i = 0 - while i < min_len and self.start[i] == newstart[i]: - i += 1 - if i < len(self.start): - self.widget.delete("%s+%dc" % (self.startindex, i), - "%s+%dc" % (self.startindex, len(self.start))) - if i < len(newstart): - self.widget.insert("%s+%dc" % (self.startindex, i), - newstart[i:]) - self.start = newstart - - def _binary_search(self, s): - """Find the first index in self.completions where completions[i] is - greater or equal to s, or the last index if there is no such - one.""" - i = 0; j = len(self.completions) - while j > i: - m = (i + j) // 2 - if self.completions[m] >= s: - j = m - else: - i = m + 1 - return min(i, len(self.completions)-1) - - def _complete_string(self, s): - """Assuming that s is the prefix of a string in self.completions, - return the longest string which is a prefix of all the strings which - s is a prefix of them. If s is not a prefix of a string, return s.""" - first = self._binary_search(s) - if self.completions[first][:len(s)] != s: - # There is not even one completion which s is a prefix of. - return s - # Find the end of the range of completions where s is a prefix of. - i = first + 1 - j = len(self.completions) - while j > i: - m = (i + j) // 2 - if self.completions[m][:len(s)] != s: - j = m - else: - i = m + 1 - last = i-1 - - if first == last: # only one possible completion - return self.completions[first] - - # We should return the maximum prefix of first and last - first_comp = self.completions[first] - last_comp = self.completions[last] - min_len = min(len(first_comp), len(last_comp)) - i = len(s) - while i < min_len and first_comp[i] == last_comp[i]: - i += 1 - return first_comp[:i] - - def _selection_changed(self): - """Should be called when the selection of the Listbox has changed. - Updates the Listbox display and calls _change_start.""" - cursel = int(self.listbox.curselection()[0]) - - self.listbox.see(cursel) - - lts = self.lasttypedstart - selstart = self.completions[cursel] - if self._binary_search(lts) == cursel: - newstart = lts - else: - min_len = min(len(lts), len(selstart)) - i = 0 - while i < min_len and lts[i] == selstart[i]: - i += 1 - newstart = selstart[:i] - self._change_start(newstart) - - if self.completions[cursel][:len(self.start)] == self.start: - # start is a prefix of the selected completion - self.listbox.configure(selectbackground=self.origselbackground, - selectforeground=self.origselforeground) - else: - self.listbox.configure(selectbackground=self.listbox.cget("bg"), - selectforeground=self.listbox.cget("fg")) - # If there are more completions, show them, and call me again. - if self.morecompletions: - self.completions = self.morecompletions - self.morecompletions = None - self.listbox.delete(0, END) - for item in self.completions: - self.listbox.insert(END, item) - self.listbox.select_set(self._binary_search(self.start)) - self._selection_changed() - - def show_window(self, comp_lists, index, complete, mode, userWantsWin): - """Show the autocomplete list, bind events. - If complete is True, complete the text, and if there is exactly one - matching completion, don't open a list.""" - # Handle the start we already have - self.completions, self.morecompletions = comp_lists - self.mode = mode - self.startindex = self.widget.index(index) - self.start = self.widget.get(self.startindex, "insert") - if complete: - completed = self._complete_string(self.start) - start = self.start - self._change_start(completed) - i = self._binary_search(completed) - if self.completions[i] == completed and \ - (i == len(self.completions)-1 or - self.completions[i+1][:len(completed)] != completed): - # There is exactly one matching completion - return completed == start - self.userwantswindow = userWantsWin - self.lasttypedstart = self.start - - # Put widgets in place - self.autocompletewindow = acw = Toplevel(self.widget) - # Put it in a position so that it is not seen. - acw.wm_geometry("+10000+10000") - # Make it float - acw.wm_overrideredirect(1) - try: - # This command is only needed and available on Tk >= 8.4.0 for OSX - # Without it, call tips intrude on the typing process by grabbing - # the focus. - acw.tk.call("::tk::unsupported::MacWindowStyle", "style", acw._w, - "help", "noActivates") - except TclError: - pass - self.scrollbar = scrollbar = Scrollbar(acw, orient=VERTICAL) - self.listbox = listbox = Listbox(acw, yscrollcommand=scrollbar.set, - exportselection=False, bg="white") - for item in self.completions: - listbox.insert(END, item) - self.origselforeground = listbox.cget("selectforeground") - self.origselbackground = listbox.cget("selectbackground") - scrollbar.config(command=listbox.yview) - scrollbar.pack(side=RIGHT, fill=Y) - listbox.pack(side=LEFT, fill=BOTH, expand=True) - acw.lift() # work around bug in Tk 8.5.18+ (issue #24570) - - # Initialize the listbox selection - self.listbox.select_set(self._binary_search(self.start)) - self._selection_changed() - - # bind events - self.hideid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, - self.hide_event) - for seq in HIDE_SEQUENCES: - self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq) - self.keypressid = self.widget.bind(KEYPRESS_VIRTUAL_EVENT_NAME, - self.keypress_event) - for seq in KEYPRESS_SEQUENCES: - self.widget.event_add(KEYPRESS_VIRTUAL_EVENT_NAME, seq) - self.keyreleaseid = self.widget.bind(KEYRELEASE_VIRTUAL_EVENT_NAME, - self.keyrelease_event) - self.widget.event_add(KEYRELEASE_VIRTUAL_EVENT_NAME,KEYRELEASE_SEQUENCE) - self.listupdateid = listbox.bind(LISTUPDATE_SEQUENCE, - self.listselect_event) - self.winconfigid = acw.bind(WINCONFIG_SEQUENCE, self.winconfig_event) - self.doubleclickid = listbox.bind(DOUBLECLICK_SEQUENCE, - self.doubleclick_event) - - def winconfig_event(self, event): - if not self.is_active(): - return - # Position the completion list window - text = self.widget - text.see(self.startindex) - x, y, cx, cy = text.bbox(self.startindex) - acw = self.autocompletewindow - acw_width, acw_height = acw.winfo_width(), acw.winfo_height() - text_width, text_height = text.winfo_width(), text.winfo_height() - new_x = text.winfo_rootx() + min(x, max(0, text_width - acw_width)) - new_y = text.winfo_rooty() + y - if (text_height - (y + cy) >= acw_height # enough height below - or y < acw_height): # not enough height above - # place acw below current line - new_y += cy - else: - # place acw above current line - new_y -= acw_height - acw.wm_geometry("+%d+%d" % (new_x, new_y)) - - def hide_event(self, event): - if not self.is_active(): - return - self.hide_window() - - def listselect_event(self, event): - if not self.is_active(): - return - self.userwantswindow = True - cursel = int(self.listbox.curselection()[0]) - self._change_start(self.completions[cursel]) - - def doubleclick_event(self, event): - # Put the selected completion in the text, and close the list - cursel = int(self.listbox.curselection()[0]) - self._change_start(self.completions[cursel]) - self.hide_window() - - def keypress_event(self, event): - if not self.is_active(): - return - keysym = event.keysym - if hasattr(event, "mc_state"): - state = event.mc_state - else: - state = 0 - if keysym != "Tab": - self.lastkey_was_tab = False - if (len(keysym) == 1 or keysym in ("underscore", "BackSpace") - or (self.mode == COMPLETE_FILES and keysym in - ("period", "minus"))) \ - and not (state & ~MC_SHIFT): - # Normal editing of text - if len(keysym) == 1: - self._change_start(self.start + keysym) - elif keysym == "underscore": - self._change_start(self.start + '_') - elif keysym == "period": - self._change_start(self.start + '.') - elif keysym == "minus": - self._change_start(self.start + '-') - else: - # keysym == "BackSpace" - if len(self.start) == 0: - self.hide_window() - return - self._change_start(self.start[:-1]) - self.lasttypedstart = self.start - self.listbox.select_clear(0, int(self.listbox.curselection()[0])) - self.listbox.select_set(self._binary_search(self.start)) - self._selection_changed() - return "break" - - elif keysym == "Return": - self.hide_window() - return - - elif (self.mode == COMPLETE_ATTRIBUTES and keysym in - ("period", "space", "parenleft", "parenright", "bracketleft", - "bracketright")) or \ - (self.mode == COMPLETE_FILES and keysym in - ("slash", "backslash", "quotedbl", "apostrophe")) \ - and not (state & ~MC_SHIFT): - # If start is a prefix of the selection, but is not '' when - # completing file names, put the whole - # selected completion. Anyway, close the list. - cursel = int(self.listbox.curselection()[0]) - if self.completions[cursel][:len(self.start)] == self.start \ - and (self.mode == COMPLETE_ATTRIBUTES or self.start): - self._change_start(self.completions[cursel]) - self.hide_window() - return - - elif keysym in ("Home", "End", "Prior", "Next", "Up", "Down") and \ - not state: - # Move the selection in the listbox - self.userwantswindow = True - cursel = int(self.listbox.curselection()[0]) - if keysym == "Home": - newsel = 0 - elif keysym == "End": - newsel = len(self.completions)-1 - elif keysym in ("Prior", "Next"): - jump = self.listbox.nearest(self.listbox.winfo_height()) - \ - self.listbox.nearest(0) - if keysym == "Prior": - newsel = max(0, cursel-jump) - else: - assert keysym == "Next" - newsel = min(len(self.completions)-1, cursel+jump) - elif keysym == "Up": - newsel = max(0, cursel-1) - else: - assert keysym == "Down" - newsel = min(len(self.completions)-1, cursel+1) - self.listbox.select_clear(cursel) - self.listbox.select_set(newsel) - self._selection_changed() - self._change_start(self.completions[newsel]) - return "break" - - elif (keysym == "Tab" and not state): - if self.lastkey_was_tab: - # two tabs in a row; insert current selection and close acw - cursel = int(self.listbox.curselection()[0]) - self._change_start(self.completions[cursel]) - self.hide_window() - return "break" - else: - # first tab; let AutoComplete handle the completion - self.userwantswindow = True - self.lastkey_was_tab = True - return - - elif any(s in keysym for s in ("Shift", "Control", "Alt", - "Meta", "Command", "Option")): - # A modifier key, so ignore - return - - elif event.char and event.char >= ' ': - # Regular character with a non-length-1 keycode - self._change_start(self.start + event.char) - self.lasttypedstart = self.start - self.listbox.select_clear(0, int(self.listbox.curselection()[0])) - self.listbox.select_set(self._binary_search(self.start)) - self._selection_changed() - return "break" - - else: - # Unknown event, close the window and let it through. - self.hide_window() - return - - def keyrelease_event(self, event): - if not self.is_active(): - return - if self.widget.index("insert") != \ - self.widget.index("%s+%dc" % (self.startindex, len(self.start))): - # If we didn't catch an event which moved the insert, close window - self.hide_window() - - def is_active(self): - return self.autocompletewindow is not None - - def complete(self): - self._change_start(self._complete_string(self.start)) - # The selection doesn't change. - - def hide_window(self): - if not self.is_active(): - return - - # unbind events - for seq in HIDE_SEQUENCES: - self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideid) - self.hideid = None - for seq in KEYPRESS_SEQUENCES: - self.widget.event_delete(KEYPRESS_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(KEYPRESS_VIRTUAL_EVENT_NAME, self.keypressid) - self.keypressid = None - self.widget.event_delete(KEYRELEASE_VIRTUAL_EVENT_NAME, - KEYRELEASE_SEQUENCE) - self.widget.unbind(KEYRELEASE_VIRTUAL_EVENT_NAME, self.keyreleaseid) - self.keyreleaseid = None - self.listbox.unbind(LISTUPDATE_SEQUENCE, self.listupdateid) - self.listupdateid = None - self.autocompletewindow.unbind(WINCONFIG_SEQUENCE, self.winconfigid) - self.winconfigid = None - - # destroy widgets - self.scrollbar.destroy() - self.scrollbar = None - self.listbox.destroy() - self.listbox = None - self.autocompletewindow.destroy() - self.autocompletewindow = None diff --git a/Lib/idlelib/AutoExpand.py b/Lib/idlelib/AutoExpand.py deleted file mode 100644 index 7059054..0000000 --- a/Lib/idlelib/AutoExpand.py +++ /dev/null @@ -1,104 +0,0 @@ -'''Complete the current word before the cursor with words in the editor. - -Each menu selection or shortcut key selection replaces the word with a -different word with the same prefix. The search for matches begins -before the target and moves toward the top of the editor. It then starts -after the cursor and moves down. It then returns to the original word and -the cycle starts again. - -Changing the current text line or leaving the cursor in a different -place before requesting the next selection causes AutoExpand to reset -its state. - -This is an extension file and there is only one instance of AutoExpand. -''' -import string -import re - -###$ event <> -###$ win -###$ unix - -class AutoExpand: - - menudefs = [ - ('edit', [ - ('E_xpand Word', '<>'), - ]), - ] - - wordchars = string.ascii_letters + string.digits + "_" - - def __init__(self, editwin): - self.text = editwin.text - self.state = None - - def expand_word_event(self, event): - "Replace the current word with the next expansion." - curinsert = self.text.index("insert") - curline = self.text.get("insert linestart", "insert lineend") - if not self.state: - words = self.getwords() - index = 0 - else: - words, index, insert, line = self.state - if insert != curinsert or line != curline: - words = self.getwords() - index = 0 - if not words: - self.text.bell() - return "break" - word = self.getprevword() - self.text.delete("insert - %d chars" % len(word), "insert") - newword = words[index] - index = (index + 1) % len(words) - if index == 0: - self.text.bell() # Warn we cycled around - self.text.insert("insert", newword) - curinsert = self.text.index("insert") - curline = self.text.get("insert linestart", "insert lineend") - self.state = words, index, curinsert, curline - return "break" - - def getwords(self): - "Return a list of words that match the prefix before the cursor." - word = self.getprevword() - if not word: - return [] - before = self.text.get("1.0", "insert wordstart") - wbefore = re.findall(r"\b" + word + r"\w+\b", before) - del before - after = self.text.get("insert wordend", "end") - wafter = re.findall(r"\b" + word + r"\w+\b", after) - del after - if not wbefore and not wafter: - return [] - words = [] - dict = {} - # search backwards through words before - wbefore.reverse() - for w in wbefore: - if dict.get(w): - continue - words.append(w) - dict[w] = w - # search onwards through words after - for w in wafter: - if dict.get(w): - continue - words.append(w) - dict[w] = w - words.append(word) - return words - - def getprevword(self): - "Return the word prefix before the cursor." - line = self.text.get("insert linestart", "insert") - i = len(line) - while i > 0 and line[i-1] in self.wordchars: - i = i-1 - return line[i:] - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_autoexpand', verbosity=2) diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py deleted file mode 100644 index ab25ff1..0000000 --- a/Lib/idlelib/Bindings.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Define the menu contents, hotkeys, and event bindings. - -There is additional configuration information in the EditorWindow class (and -subclasses): the menus are created there based on the menu_specs (class) -variable, and menus not created are silently skipped in the code here. This -makes it possible, for example, to define a Debug menu which is only present in -the PythonShell window, and a Format menu which is only present in the Editor -windows. - -""" -from importlib.util import find_spec - -from idlelib.configHandler import idleConf - -# Warning: menudefs is altered in macosxSupport.overrideRootMenu() -# after it is determined that an OS X Aqua Tk is in use, -# which cannot be done until after Tk() is first called. -# Do not alter the 'file', 'options', or 'help' cascades here -# without altering overrideRootMenu() as well. -# TODO: Make this more robust - -menudefs = [ - # underscore prefixes character to underscore - ('file', [ - ('_New File', '<>'), - ('_Open...', '<>'), - ('Open _Module...', '<>'), - ('Class _Browser', '<>'), - ('_Path Browser', '<>'), - None, - ('_Save', '<>'), - ('Save _As...', '<>'), - ('Save Cop_y As...', '<>'), - None, - ('Prin_t Window', '<>'), - None, - ('_Close', '<>'), - ('E_xit', '<>'), - ]), - ('edit', [ - ('_Undo', '<>'), - ('_Redo', '<>'), - None, - ('Cu_t', '<>'), - ('_Copy', '<>'), - ('_Paste', '<>'), - ('Select _All', '<>'), - None, - ('_Find...', '<>'), - ('Find A_gain', '<>'), - ('Find _Selection', '<>'), - ('Find in Files...', '<>'), - ('R_eplace...', '<>'), - ('Go to _Line', '<>'), - ]), -('format', [ - ('_Indent Region', '<>'), - ('_Dedent Region', '<>'), - ('Comment _Out Region', '<>'), - ('U_ncomment Region', '<>'), - ('Tabify Region', '<>'), - ('Untabify Region', '<>'), - ('Toggle Tabs', '<>'), - ('New Indent Width', '<>'), - ]), - ('run', [ - ('Python Shell', '<>'), - ]), - ('shell', [ - ('_View Last Restart', '<>'), - ('_Restart Shell', '<>'), - ]), - ('debug', [ - ('_Go to File/Line', '<>'), - ('!_Debugger', '<>'), - ('_Stack Viewer', '<>'), - ('!_Auto-open Stack Viewer', '<>'), - ]), - ('options', [ - ('Configure _IDLE', '<>'), - None, - ]), - ('help', [ - ('_About IDLE', '<>'), - None, - ('_IDLE Help', '<>'), - ('Python _Docs', '<>'), - ]), -] - -if find_spec('turtledemo'): - menudefs[-1][1].append(('Turtle Demo', '<>')) - -default_keydefs = idleConf.GetCurrentKeySet() diff --git a/Lib/idlelib/CallTipWindow.py b/Lib/idlelib/CallTipWindow.py deleted file mode 100644 index 8e68a76..0000000 --- a/Lib/idlelib/CallTipWindow.py +++ /dev/null @@ -1,161 +0,0 @@ -"""A CallTip window class for Tkinter/IDLE. - -After ToolTip.py, which uses ideas gleaned from PySol -Used by the CallTips IDLE extension. -""" -from tkinter import Toplevel, Label, LEFT, SOLID, TclError - -HIDE_VIRTUAL_EVENT_NAME = "<>" -HIDE_SEQUENCES = ("", "") -CHECKHIDE_VIRTUAL_EVENT_NAME = "<>" -CHECKHIDE_SEQUENCES = ("", "") -CHECKHIDE_TIME = 100 # miliseconds - -MARK_RIGHT = "calltipwindowregion_right" - -class CallTip: - - def __init__(self, widget): - self.widget = widget - self.tipwindow = self.label = None - self.parenline = self.parencol = None - self.lastline = None - self.hideid = self.checkhideid = None - self.checkhide_after_id = None - - def position_window(self): - """Check if needs to reposition the window, and if so - do it.""" - curline = int(self.widget.index("insert").split('.')[0]) - if curline == self.lastline: - return - self.lastline = curline - self.widget.see("insert") - if curline == self.parenline: - box = self.widget.bbox("%d.%d" % (self.parenline, - self.parencol)) - else: - box = self.widget.bbox("%d.0" % curline) - if not box: - box = list(self.widget.bbox("insert")) - # align to left of window - box[0] = 0 - box[2] = 0 - x = box[0] + self.widget.winfo_rootx() + 2 - y = box[1] + box[3] + self.widget.winfo_rooty() - self.tipwindow.wm_geometry("+%d+%d" % (x, y)) - - def showtip(self, text, parenleft, parenright): - """Show the calltip, bind events which will close it and reposition it. - """ - # Only called in CallTips, where lines are truncated - self.text = text - if self.tipwindow or not self.text: - return - - self.widget.mark_set(MARK_RIGHT, parenright) - self.parenline, self.parencol = map( - int, self.widget.index(parenleft).split(".")) - - self.tipwindow = tw = Toplevel(self.widget) - self.position_window() - # remove border on calltip window - tw.wm_overrideredirect(1) - try: - # This command is only needed and available on Tk >= 8.4.0 for OSX - # Without it, call tips intrude on the typing process by grabbing - # the focus. - tw.tk.call("::tk::unsupported::MacWindowStyle", "style", tw._w, - "help", "noActivates") - except TclError: - pass - self.label = Label(tw, text=self.text, justify=LEFT, - background="#ffffe0", relief=SOLID, borderwidth=1, - font = self.widget['font']) - self.label.pack() - tw.lift() # work around bug in Tk 8.5.18+ (issue #24570) - - self.checkhideid = self.widget.bind(CHECKHIDE_VIRTUAL_EVENT_NAME, - self.checkhide_event) - for seq in CHECKHIDE_SEQUENCES: - self.widget.event_add(CHECKHIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.after(CHECKHIDE_TIME, self.checkhide_event) - self.hideid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, - self.hide_event) - for seq in HIDE_SEQUENCES: - self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq) - - def checkhide_event(self, event=None): - if not self.tipwindow: - # If the event was triggered by the same event that unbinded - # this function, the function will be called nevertheless, - # so do nothing in this case. - return - curline, curcol = map(int, self.widget.index("insert").split('.')) - if curline < self.parenline or \ - (curline == self.parenline and curcol <= self.parencol) or \ - self.widget.compare("insert", ">", MARK_RIGHT): - self.hidetip() - else: - self.position_window() - if self.checkhide_after_id is not None: - self.widget.after_cancel(self.checkhide_after_id) - self.checkhide_after_id = \ - self.widget.after(CHECKHIDE_TIME, self.checkhide_event) - - def hide_event(self, event): - if not self.tipwindow: - # See the explanation in checkhide_event. - return - self.hidetip() - - def hidetip(self): - if not self.tipwindow: - return - - for seq in CHECKHIDE_SEQUENCES: - self.widget.event_delete(CHECKHIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(CHECKHIDE_VIRTUAL_EVENT_NAME, self.checkhideid) - self.checkhideid = None - for seq in HIDE_SEQUENCES: - self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideid) - self.hideid = None - - self.label.destroy() - self.label = None - self.tipwindow.destroy() - self.tipwindow = None - - self.widget.mark_unset(MARK_RIGHT) - self.parenline = self.parencol = self.lastline = None - - def is_active(self): - return bool(self.tipwindow) - - -def _calltip_window(parent): # htest # - from tkinter import Toplevel, Text, LEFT, BOTH - - top = Toplevel(parent) - top.title("Test calltips") - top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, - parent.winfo_rooty() + 150)) - text = Text(top) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - top.update() - calltip = CallTip(text) - - def calltip_show(event): - calltip.showtip("(s=Hello world)", "insert", "end") - def calltip_hide(event): - calltip.hidetip() - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", calltip_show) - text.bind("<>", calltip_hide) - text.focus_set() - -if __name__=='__main__': - from idlelib.idle_test.htest import run - run(_calltip_window) diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py deleted file mode 100644 index 81bd5f1..0000000 --- a/Lib/idlelib/CallTips.py +++ /dev/null @@ -1,175 +0,0 @@ -"""CallTips.py - An IDLE Extension to Jog Your Memory - -Call Tips are floating windows which display function, class, and method -parameter and docstring information when you type an opening parenthesis, and -which disappear when you type a closing parenthesis. - -""" -import __main__ -import inspect -import re -import sys -import textwrap -import types - -from idlelib import CallTipWindow -from idlelib.HyperParser import HyperParser - -class CallTips: - - menudefs = [ - ('edit', [ - ("Show call tip", "<>"), - ]) - ] - - def __init__(self, editwin=None): - if editwin is None: # subprocess and test - self.editwin = None - else: - self.editwin = editwin - self.text = editwin.text - self.active_calltip = None - self._calltip_window = self._make_tk_calltip_window - - def close(self): - self._calltip_window = None - - def _make_tk_calltip_window(self): - # See __init__ for usage - return CallTipWindow.CallTip(self.text) - - def _remove_calltip_window(self, event=None): - if self.active_calltip: - self.active_calltip.hidetip() - self.active_calltip = None - - def force_open_calltip_event(self, event): - "The user selected the menu entry or hotkey, open the tip." - self.open_calltip(True) - - def try_open_calltip_event(self, event): - """Happens when it would be nice to open a CallTip, but not really - necessary, for example after an opening bracket, so function calls - won't be made. - """ - self.open_calltip(False) - - def refresh_calltip_event(self, event): - if self.active_calltip and self.active_calltip.is_active(): - self.open_calltip(False) - - def open_calltip(self, evalfuncs): - self._remove_calltip_window() - - hp = HyperParser(self.editwin, "insert") - sur_paren = hp.get_surrounding_brackets('(') - if not sur_paren: - return - hp.set_index(sur_paren[0]) - expression = hp.get_expression() - if not expression: - return - if not evalfuncs and (expression.find('(') != -1): - return - argspec = self.fetch_tip(expression) - if not argspec: - return - self.active_calltip = self._calltip_window() - self.active_calltip.showtip(argspec, sur_paren[0], sur_paren[1]) - - def fetch_tip(self, expression): - """Return the argument list and docstring of a function or class. - - If there is a Python subprocess, get the calltip there. Otherwise, - either this fetch_tip() is running in the subprocess or it was - called in an IDLE running without the subprocess. - - The subprocess environment is that of the most recently run script. If - two unrelated modules are being edited some calltips in the current - module may be inoperative if the module was not the last to run. - - To find methods, fetch_tip must be fed a fully qualified name. - - """ - try: - rpcclt = self.editwin.flist.pyshell.interp.rpcclt - except AttributeError: - rpcclt = None - if rpcclt: - return rpcclt.remotecall("exec", "get_the_calltip", - (expression,), {}) - else: - return get_argspec(get_entity(expression)) - -def get_entity(expression): - """Return the object corresponding to expression evaluated - in a namespace spanning sys.modules and __main.dict__. - """ - if expression: - namespace = sys.modules.copy() - namespace.update(__main__.__dict__) - try: - return eval(expression, namespace) - except BaseException: - # An uncaught exception closes idle, and eval can raise any - # exception, especially if user classes are involved. - return None - -# The following are used in get_argspec and some in tests -_MAX_COLS = 85 -_MAX_LINES = 5 # enough for bytes -_INDENT = ' '*4 # for wrapped signatures -_first_param = re.compile('(?<=\()\w*\,?\s*') -_default_callable_argspec = "See source or doc" - - -def get_argspec(ob): - '''Return a string describing the signature of a callable object, or ''. - - For Python-coded functions and methods, the first line is introspected. - Delete 'self' parameter for classes (.__init__) and bound methods. - The next lines are the first lines of the doc string up to the first - empty line or _MAX_LINES. For builtins, this typically includes - the arguments in addition to the return value. - ''' - argspec = "" - try: - ob_call = ob.__call__ - except BaseException: - return argspec - if isinstance(ob, type): - fob = ob.__init__ - elif isinstance(ob_call, types.MethodType): - fob = ob_call - else: - fob = ob - if isinstance(fob, (types.FunctionType, types.MethodType)): - argspec = inspect.formatargspec(*inspect.getfullargspec(fob)) - if (isinstance(ob, (type, types.MethodType)) or - isinstance(ob_call, types.MethodType)): - argspec = _first_param.sub("", argspec) - - lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) - if len(argspec) > _MAX_COLS else [argspec] if argspec else []) - - if isinstance(ob_call, types.MethodType): - doc = ob_call.__doc__ - else: - doc = getattr(ob, "__doc__", "") - if doc: - for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: - line = line.strip() - if not line: - break - if len(line) > _MAX_COLS: - line = line[: _MAX_COLS - 3] + '...' - lines.append(line) - argspec = '\n'.join(lines) - if not argspec: - argspec = _default_callable_argspec - return argspec - -if __name__ == '__main__': - from unittest import main - main('idlelib.idle_test.test_calltips', verbosity=2) diff --git a/Lib/idlelib/ClassBrowser.py b/Lib/idlelib/ClassBrowser.py deleted file mode 100644 index d09c52f..0000000 --- a/Lib/idlelib/ClassBrowser.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Class browser. - -XXX TO DO: - -- reparse when source changed (maybe just a button would be OK?) - (or recheck on window popup) -- add popup menu with more options (e.g. doc strings, base classes, imports) -- show function argument list? (have to do pattern matching on source) -- should the classes and methods lists also be in the module's menu bar? -- add base classes to class browser tree -""" - -import os -import sys -import pyclbr - -from idlelib import PyShell -from idlelib.WindowList import ListedToplevel -from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas -from idlelib.configHandler import idleConf - -file_open = None # Method...Item and Class...Item use this. -# Normally PyShell.flist.open, but there is no PyShell.flist for htest. - -class ClassBrowser: - - def __init__(self, flist, name, path, _htest=False): - # XXX This API should change, if the file doesn't end in ".py" - # XXX the code here is bogus! - """ - _htest - bool, change box when location running htest. - """ - global file_open - if not _htest: - file_open = PyShell.flist.open - self.name = name - self.file = os.path.join(path[0], self.name + ".py") - self._htest = _htest - self.init(flist) - - def close(self, event=None): - self.top.destroy() - self.node.destroy() - - def init(self, flist): - self.flist = flist - # reset pyclbr - pyclbr._modules.clear() - # create top - self.top = top = ListedToplevel(flist.root) - top.protocol("WM_DELETE_WINDOW", self.close) - top.bind("", self.close) - if self._htest: # place dialog below parent if running htest - top.geometry("+%d+%d" % - (flist.root.winfo_rootx(), flist.root.winfo_rooty() + 200)) - self.settitle() - top.focus_set() - # create scrolled canvas - theme = idleConf.CurrentTheme() - background = idleConf.GetHighlight(theme, 'normal')['background'] - sc = ScrolledCanvas(top, bg=background, highlightthickness=0, takefocus=1) - sc.frame.pack(expand=1, fill="both") - item = self.rootnode() - self.node = node = TreeNode(sc.canvas, None, item) - node.update() - node.expand() - - def settitle(self): - self.top.wm_title("Class Browser - " + self.name) - self.top.wm_iconname("Class Browser") - - def rootnode(self): - return ModuleBrowserTreeItem(self.file) - -class ModuleBrowserTreeItem(TreeItem): - - def __init__(self, file): - self.file = file - - def GetText(self): - return os.path.basename(self.file) - - def GetIconName(self): - return "python" - - def GetSubList(self): - sublist = [] - for name in self.listclasses(): - item = ClassBrowserTreeItem(name, self.classes, self.file) - sublist.append(item) - return sublist - - def OnDoubleClick(self): - if os.path.normcase(self.file[-3:]) != ".py": - return - if not os.path.exists(self.file): - return - PyShell.flist.open(self.file) - - def IsExpandable(self): - return os.path.normcase(self.file[-3:]) == ".py" - - def listclasses(self): - dir, file = os.path.split(self.file) - name, ext = os.path.splitext(file) - if os.path.normcase(ext) != ".py": - return [] - try: - dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError: - return [] - items = [] - self.classes = {} - for key, cl in dict.items(): - if cl.module == name: - s = key - if hasattr(cl, 'super') and cl.super: - supers = [] - for sup in cl.super: - if type(sup) is type(''): - sname = sup - else: - sname = sup.name - if sup.module != cl.module: - sname = "%s.%s" % (sup.module, sname) - supers.append(sname) - s = s + "(%s)" % ", ".join(supers) - items.append((cl.lineno, s)) - self.classes[s] = cl - items.sort() - list = [] - for item, s in items: - list.append(s) - return list - -class ClassBrowserTreeItem(TreeItem): - - def __init__(self, name, classes, file): - self.name = name - self.classes = classes - self.file = file - try: - self.cl = self.classes[self.name] - except (IndexError, KeyError): - self.cl = None - self.isfunction = isinstance(self.cl, pyclbr.Function) - - def GetText(self): - if self.isfunction: - return "def " + self.name + "(...)" - else: - return "class " + self.name - - def GetIconName(self): - if self.isfunction: - return "python" - else: - return "folder" - - def IsExpandable(self): - if self.cl: - try: - return not not self.cl.methods - except AttributeError: - return False - - def GetSubList(self): - if not self.cl: - return [] - sublist = [] - for name in self.listmethods(): - item = MethodBrowserTreeItem(name, self.cl, self.file) - sublist.append(item) - return sublist - - def OnDoubleClick(self): - if not os.path.exists(self.file): - return - edit = file_open(self.file) - if hasattr(self.cl, 'lineno'): - lineno = self.cl.lineno - edit.gotoline(lineno) - - def listmethods(self): - if not self.cl: - return [] - items = [] - for name, lineno in self.cl.methods.items(): - items.append((lineno, name)) - items.sort() - list = [] - for item, name in items: - list.append(name) - return list - -class MethodBrowserTreeItem(TreeItem): - - def __init__(self, name, cl, file): - self.name = name - self.cl = cl - self.file = file - - def GetText(self): - return "def " + self.name + "(...)" - - def GetIconName(self): - return "python" # XXX - - def IsExpandable(self): - return 0 - - def OnDoubleClick(self): - if not os.path.exists(self.file): - return - edit = file_open(self.file) - edit.gotoline(self.cl.methods[self.name]) - -def _class_browser(parent): #Wrapper for htest - try: - file = __file__ - except NameError: - file = sys.argv[0] - if sys.argv[1:]: - file = sys.argv[1] - else: - file = sys.argv[0] - dir, file = os.path.split(file) - name = os.path.splitext(file)[0] - flist = PyShell.PyShellFileList(parent) - global file_open - file_open = flist.open - ClassBrowser(flist, name, [dir], _htest=True) - -if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_class_browser) diff --git a/Lib/idlelib/CodeContext.py b/Lib/idlelib/CodeContext.py deleted file mode 100644 index 7d25ada..0000000 --- a/Lib/idlelib/CodeContext.py +++ /dev/null @@ -1,176 +0,0 @@ -"""CodeContext - Extension to display the block context above the edit window - -Once code has scrolled off the top of a window, it can be difficult to -determine which block you are in. This extension implements a pane at the top -of each IDLE edit window which provides block structure hints. These hints are -the lines which contain the block opening keywords, e.g. 'if', for the -enclosing block. The number of hint lines is determined by the numlines -variable in the CodeContext section of config-extensions.def. Lines which do -not open blocks are not shown in the context hints pane. - -""" -import tkinter -from tkinter.constants import TOP, LEFT, X, W, SUNKEN -import re -from sys import maxsize as INFINITY -from idlelib.configHandler import idleConf - -BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", - "if", "try", "while", "with"} -UPDATEINTERVAL = 100 # millisec -FONTUPDATEINTERVAL = 1000 # millisec - -getspacesfirstword =\ - lambda s, c=re.compile(r"^(\s*)(\w*)"): c.match(s).groups() - -class CodeContext: - menudefs = [('options', [('!Code Conte_xt', '<>')])] - context_depth = idleConf.GetOption("extensions", "CodeContext", - "numlines", type="int", default=3) - bgcolor = idleConf.GetOption("extensions", "CodeContext", - "bgcolor", type="str", default="LightGray") - fgcolor = idleConf.GetOption("extensions", "CodeContext", - "fgcolor", type="str", default="Black") - def __init__(self, editwin): - self.editwin = editwin - self.text = editwin.text - self.textfont = self.text["font"] - self.label = None - # self.info is a list of (line number, indent level, line text, block - # keyword) tuples providing the block structure associated with - # self.topvisible (the linenumber of the line displayed at the top of - # the edit window). self.info[0] is initialized as a 'dummy' line which - # starts the toplevel 'block' of the module. - self.info = [(0, -1, "", False)] - self.topvisible = 1 - visible = idleConf.GetOption("extensions", "CodeContext", - "visible", type="bool", default=False) - if visible: - self.toggle_code_context_event() - self.editwin.setvar('<>', True) - # Start two update cycles, one for context lines, one for font changes. - self.text.after(UPDATEINTERVAL, self.timer_event) - self.text.after(FONTUPDATEINTERVAL, self.font_timer_event) - - def toggle_code_context_event(self, event=None): - if not self.label: - # Calculate the border width and horizontal padding required to - # align the context with the text in the main Text widget. - # - # All values are passed through getint(), since some - # values may be pixel objects, which can't simply be added to ints. - widgets = self.editwin.text, self.editwin.text_frame - # Calculate the required vertical padding - padx = 0 - for widget in widgets: - padx += widget.tk.getint(widget.pack_info()['padx']) - padx += widget.tk.getint(widget.cget('padx')) - # Calculate the required border width - border = 0 - for widget in widgets: - border += widget.tk.getint(widget.cget('border')) - self.label = tkinter.Label(self.editwin.top, - text="\n" * (self.context_depth - 1), - anchor=W, justify=LEFT, - font=self.textfont, - bg=self.bgcolor, fg=self.fgcolor, - width=1, #don't request more than we get - padx=padx, border=border, - relief=SUNKEN) - # Pack the label widget before and above the text_frame widget, - # thus ensuring that it will appear directly above text_frame - self.label.pack(side=TOP, fill=X, expand=False, - before=self.editwin.text_frame) - else: - self.label.destroy() - self.label = None - idleConf.SetOption("extensions", "CodeContext", "visible", - str(self.label is not None)) - idleConf.SaveUserCfgFiles() - - def get_line_info(self, linenum): - """Get the line indent value, text, and any block start keyword - - If the line does not start a block, the keyword value is False. - The indentation of empty lines (or comment lines) is INFINITY. - - """ - text = self.text.get("%d.0" % linenum, "%d.end" % linenum) - spaces, firstword = getspacesfirstword(text) - opener = firstword in BLOCKOPENERS and firstword - if len(text) == len(spaces) or text[len(spaces)] == '#': - indent = INFINITY - else: - indent = len(spaces) - return indent, text, opener - - def get_context(self, new_topvisible, stopline=1, stopindent=0): - """Get context lines, starting at new_topvisible and working backwards. - - Stop when stopline or stopindent is reached. Return a tuple of context - data and the indent level at the top of the region inspected. - - """ - assert stopline > 0 - lines = [] - # The indentation level we are currently in: - lastindent = INFINITY - # For a line to be interesting, it must begin with a block opening - # keyword, and have less indentation than lastindent. - for linenum in range(new_topvisible, stopline-1, -1): - indent, text, opener = self.get_line_info(linenum) - if indent < lastindent: - lastindent = indent - if opener in ("else", "elif"): - # We also show the if statement - lastindent += 1 - if opener and linenum < new_topvisible and indent >= stopindent: - lines.append((linenum, indent, text, opener)) - if lastindent <= stopindent: - break - lines.reverse() - return lines, lastindent - - def update_code_context(self): - """Update context information and lines visible in the context pane. - - """ - new_topvisible = int(self.text.index("@0,0").split('.')[0]) - if self.topvisible == new_topvisible: # haven't scrolled - return - if self.topvisible < new_topvisible: # scroll down - lines, lastindent = self.get_context(new_topvisible, - self.topvisible) - # retain only context info applicable to the region - # between topvisible and new_topvisible: - while self.info[-1][1] >= lastindent: - del self.info[-1] - elif self.topvisible > new_topvisible: # scroll up - stopindent = self.info[-1][1] + 1 - # retain only context info associated - # with lines above new_topvisible: - while self.info[-1][0] >= new_topvisible: - stopindent = self.info[-1][1] - del self.info[-1] - lines, lastindent = self.get_context(new_topvisible, - self.info[-1][0]+1, - stopindent) - self.info.extend(lines) - self.topvisible = new_topvisible - # empty lines in context pane: - context_strings = [""] * max(0, self.context_depth - len(self.info)) - # followed by the context hint lines: - context_strings += [x[2] for x in self.info[-self.context_depth:]] - self.label["text"] = '\n'.join(context_strings) - - def timer_event(self): - if self.label: - self.update_code_context() - self.text.after(UPDATEINTERVAL, self.timer_event) - - def font_timer_event(self): - newtextfont = self.text["font"] - if self.label and newtextfont != self.textfont: - self.textfont = newtextfont - self.label["font"] = self.textfont - self.text.after(FONTUPDATEINTERVAL, self.font_timer_event) diff --git a/Lib/idlelib/ColorDelegator.py b/Lib/idlelib/ColorDelegator.py deleted file mode 100644 index 9f31349..0000000 --- a/Lib/idlelib/ColorDelegator.py +++ /dev/null @@ -1,256 +0,0 @@ -import time -import re -import keyword -import builtins -from idlelib.Delegator import Delegator -from idlelib.configHandler import idleConf - -DEBUG = False - -def any(name, alternates): - "Return a named group pattern matching list of alternates." - return "(?P<%s>" % name + "|".join(alternates) + ")" - -def make_pat(): - kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" - builtinlist = [str(name) for name in dir(builtins) - if not name.startswith('_') and \ - name not in keyword.kwlist] - # self.file = open("file") : - # 1st 'file' colorized normal, 2nd as builtin, 3rd as string - builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b" - comment = any("COMMENT", [r"#[^\n]*"]) - stringprefix = r"(\br|u|ur|R|U|UR|Ur|uR|b|B|br|Br|bR|BR|rb|rB|Rb|RB)?" - sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?" - dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?' - sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" - dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' - string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) - return kw + "|" + builtin + "|" + comment + "|" + string +\ - "|" + any("SYNC", [r"\n"]) - -prog = re.compile(make_pat(), re.S) -idprog = re.compile(r"\s+(\w+)", re.S) - -class ColorDelegator(Delegator): - - def __init__(self): - Delegator.__init__(self) - self.prog = prog - self.idprog = idprog - self.LoadTagDefs() - - def setdelegate(self, delegate): - if self.delegate is not None: - self.unbind("<>") - Delegator.setdelegate(self, delegate) - if delegate is not None: - self.config_colors() - self.bind("<>", self.toggle_colorize_event) - self.notify_range("1.0", "end") - else: - # No delegate - stop any colorizing - self.stop_colorizing = True - self.allow_colorizing = False - - def config_colors(self): - for tag, cnf in self.tagdefs.items(): - if cnf: - self.tag_configure(tag, **cnf) - self.tag_raise('sel') - - def LoadTagDefs(self): - theme = idleConf.CurrentTheme() - self.tagdefs = { - "COMMENT": idleConf.GetHighlight(theme, "comment"), - "KEYWORD": idleConf.GetHighlight(theme, "keyword"), - "BUILTIN": idleConf.GetHighlight(theme, "builtin"), - "STRING": idleConf.GetHighlight(theme, "string"), - "DEFINITION": idleConf.GetHighlight(theme, "definition"), - "SYNC": {'background':None,'foreground':None}, - "TODO": {'background':None,'foreground':None}, - "ERROR": idleConf.GetHighlight(theme, "error"), - # The following is used by ReplaceDialog: - "hit": idleConf.GetHighlight(theme, "hit"), - } - - if DEBUG: print('tagdefs',self.tagdefs) - - def insert(self, index, chars, tags=None): - index = self.index(index) - self.delegate.insert(index, chars, tags) - self.notify_range(index, index + "+%dc" % len(chars)) - - def delete(self, index1, index2=None): - index1 = self.index(index1) - self.delegate.delete(index1, index2) - self.notify_range(index1) - - after_id = None - allow_colorizing = True - colorizing = False - - def notify_range(self, index1, index2=None): - self.tag_add("TODO", index1, index2) - if self.after_id: - if DEBUG: print("colorizing already scheduled") - return - if self.colorizing: - self.stop_colorizing = True - if DEBUG: print("stop colorizing") - if self.allow_colorizing: - if DEBUG: print("schedule colorizing") - self.after_id = self.after(1, self.recolorize) - - close_when_done = None # Window to be closed when done colorizing - - def close(self, close_when_done=None): - if self.after_id: - after_id = self.after_id - self.after_id = None - if DEBUG: print("cancel scheduled recolorizer") - self.after_cancel(after_id) - self.allow_colorizing = False - self.stop_colorizing = True - if close_when_done: - if not self.colorizing: - close_when_done.destroy() - else: - self.close_when_done = close_when_done - - def toggle_colorize_event(self, event): - if self.after_id: - after_id = self.after_id - self.after_id = None - if DEBUG: print("cancel scheduled recolorizer") - self.after_cancel(after_id) - if self.allow_colorizing and self.colorizing: - if DEBUG: print("stop colorizing") - self.stop_colorizing = True - self.allow_colorizing = not self.allow_colorizing - if self.allow_colorizing and not self.colorizing: - self.after_id = self.after(1, self.recolorize) - if DEBUG: - print("auto colorizing turned",\ - self.allow_colorizing and "on" or "off") - return "break" - - def recolorize(self): - self.after_id = None - if not self.delegate: - if DEBUG: print("no delegate") - return - if not self.allow_colorizing: - if DEBUG: print("auto colorizing is off") - return - if self.colorizing: - if DEBUG: print("already colorizing") - return - try: - self.stop_colorizing = False - self.colorizing = True - if DEBUG: print("colorizing...") - t0 = time.perf_counter() - self.recolorize_main() - t1 = time.perf_counter() - if DEBUG: print("%.3f seconds" % (t1-t0)) - finally: - self.colorizing = False - if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"): - if DEBUG: print("reschedule colorizing") - self.after_id = self.after(1, self.recolorize) - if self.close_when_done: - top = self.close_when_done - self.close_when_done = None - top.destroy() - - def recolorize_main(self): - next = "1.0" - while True: - item = self.tag_nextrange("TODO", next) - if not item: - break - head, tail = item - self.tag_remove("SYNC", head, tail) - item = self.tag_prevrange("SYNC", head) - if item: - head = item[1] - else: - head = "1.0" - - chars = "" - next = head - lines_to_get = 1 - ok = False - while not ok: - mark = next - next = self.index(mark + "+%d lines linestart" % - lines_to_get) - lines_to_get = min(lines_to_get * 2, 100) - ok = "SYNC" in self.tag_names(next + "-1c") - line = self.get(mark, next) - ##print head, "get", mark, next, "->", repr(line) - if not line: - return - for tag in self.tagdefs: - self.tag_remove(tag, mark, next) - chars = chars + line - m = self.prog.search(chars) - while m: - for key, value in m.groupdict().items(): - if value: - a, b = m.span(key) - self.tag_add(key, - head + "+%dc" % a, - head + "+%dc" % b) - if value in ("def", "class"): - m1 = self.idprog.match(chars, b) - if m1: - a, b = m1.span(1) - self.tag_add("DEFINITION", - head + "+%dc" % a, - head + "+%dc" % b) - m = self.prog.search(chars, m.end()) - if "SYNC" in self.tag_names(next + "-1c"): - head = next - chars = "" - else: - ok = False - if not ok: - # We're in an inconsistent state, and the call to - # update may tell us to stop. It may also change - # the correct value for "next" (since this is a - # line.col string, not a true mark). So leave a - # crumb telling the next invocation to resume here - # in case update tells us to leave. - self.tag_add("TODO", next) - self.update() - if self.stop_colorizing: - if DEBUG: print("colorizing stopped") - return - - def removecolors(self): - for tag in self.tagdefs: - self.tag_remove(tag, "1.0", "end") - -def _color_delegator(parent): # htest # - from tkinter import Toplevel, Text - from idlelib.Percolator import Percolator - - top = Toplevel(parent) - top.title("Test ColorDelegator") - top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, - parent.winfo_rooty() + 150)) - source = "if somename: x = 'abc' # comment\nprint\n" - text = Text(top, background="white") - text.pack(expand=1, fill="both") - text.insert("insert", source) - text.focus_set() - - p = Percolator(text) - d = ColorDelegator() - p.insertfilter(d) - -if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_color_delegator) diff --git a/Lib/idlelib/Debugger.py b/Lib/idlelib/Debugger.py deleted file mode 100644 index d5e217d..0000000 --- a/Lib/idlelib/Debugger.py +++ /dev/null @@ -1,539 +0,0 @@ -import os -import bdb -from tkinter import * -from idlelib.WindowList import ListedToplevel -from idlelib.ScrolledList import ScrolledList -from idlelib import macosxSupport - - -class Idb(bdb.Bdb): - - def __init__(self, gui): - self.gui = gui - bdb.Bdb.__init__(self) - - def user_line(self, frame): - if self.in_rpc_code(frame): - self.set_step() - return - message = self.__frame2message(frame) - try: - self.gui.interaction(message, frame) - except TclError: # When closing debugger window with [x] in 3.x - pass - - def user_exception(self, frame, info): - if self.in_rpc_code(frame): - self.set_step() - return - message = self.__frame2message(frame) - self.gui.interaction(message, frame, info) - - def in_rpc_code(self, frame): - if frame.f_code.co_filename.count('rpc.py'): - return True - else: - prev_frame = frame.f_back - if prev_frame.f_code.co_filename.count('Debugger.py'): - # (that test will catch both Debugger.py and RemoteDebugger.py) - return False - return self.in_rpc_code(prev_frame) - - def __frame2message(self, frame): - code = frame.f_code - filename = code.co_filename - lineno = frame.f_lineno - basename = os.path.basename(filename) - message = "%s:%s" % (basename, lineno) - if code.co_name != "?": - message = "%s: %s()" % (message, code.co_name) - return message - - -class Debugger: - - vstack = vsource = vlocals = vglobals = None - - def __init__(self, pyshell, idb=None): - if idb is None: - idb = Idb(self) - self.pyshell = pyshell - self.idb = idb - self.frame = None - self.make_gui() - self.interacting = 0 - self.nesting_level = 0 - - def run(self, *args): - # Deal with the scenario where we've already got a program running - # in the debugger and we want to start another. If that is the case, - # our second 'run' was invoked from an event dispatched not from - # the main event loop, but from the nested event loop in 'interaction' - # below. So our stack looks something like this: - # outer main event loop - # run() - # - # callback to debugger's interaction() - # nested event loop - # run() for second command - # - # This kind of nesting of event loops causes all kinds of problems - # (see e.g. issue #24455) especially when dealing with running as a - # subprocess, where there's all kinds of extra stuff happening in - # there - insert a traceback.print_stack() to check it out. - # - # By this point, we've already called restart_subprocess() in - # ScriptBinding. However, we also need to unwind the stack back to - # that outer event loop. To accomplish this, we: - # - return immediately from the nested run() - # - abort_loop ensures the nested event loop will terminate - # - the debugger's interaction routine completes normally - # - the restart_subprocess() will have taken care of stopping - # the running program, which will also let the outer run complete - # - # That leaves us back at the outer main event loop, at which point our - # after event can fire, and we'll come back to this routine with a - # clean stack. - if self.nesting_level > 0: - self.abort_loop() - self.root.after(100, lambda: self.run(*args)) - return - try: - self.interacting = 1 - return self.idb.run(*args) - finally: - self.interacting = 0 - - def close(self, event=None): - try: - self.quit() - except Exception: - pass - if self.interacting: - self.top.bell() - return - if self.stackviewer: - self.stackviewer.close(); self.stackviewer = None - # Clean up pyshell if user clicked debugger control close widget. - # (Causes a harmless extra cycle through close_debugger() if user - # toggled debugger from pyshell Debug menu) - self.pyshell.close_debugger() - # Now close the debugger control window.... - self.top.destroy() - - def make_gui(self): - pyshell = self.pyshell - self.flist = pyshell.flist - self.root = root = pyshell.root - self.top = top = ListedToplevel(root) - self.top.wm_title("Debug Control") - self.top.wm_iconname("Debug") - top.wm_protocol("WM_DELETE_WINDOW", self.close) - self.top.bind("", self.close) - # - self.bframe = bframe = Frame(top) - self.bframe.pack(anchor="w") - self.buttons = bl = [] - # - self.bcont = b = Button(bframe, text="Go", command=self.cont) - bl.append(b) - self.bstep = b = Button(bframe, text="Step", command=self.step) - bl.append(b) - self.bnext = b = Button(bframe, text="Over", command=self.next) - bl.append(b) - self.bret = b = Button(bframe, text="Out", command=self.ret) - bl.append(b) - self.bret = b = Button(bframe, text="Quit", command=self.quit) - bl.append(b) - # - for b in bl: - b.configure(state="disabled") - b.pack(side="left") - # - self.cframe = cframe = Frame(bframe) - self.cframe.pack(side="left") - # - if not self.vstack: - self.__class__.vstack = BooleanVar(top) - self.vstack.set(1) - self.bstack = Checkbutton(cframe, - text="Stack", command=self.show_stack, variable=self.vstack) - self.bstack.grid(row=0, column=0) - if not self.vsource: - self.__class__.vsource = BooleanVar(top) - self.bsource = Checkbutton(cframe, - text="Source", command=self.show_source, variable=self.vsource) - self.bsource.grid(row=0, column=1) - if not self.vlocals: - self.__class__.vlocals = BooleanVar(top) - self.vlocals.set(1) - self.blocals = Checkbutton(cframe, - text="Locals", command=self.show_locals, variable=self.vlocals) - self.blocals.grid(row=1, column=0) - if not self.vglobals: - self.__class__.vglobals = BooleanVar(top) - self.bglobals = Checkbutton(cframe, - text="Globals", command=self.show_globals, variable=self.vglobals) - self.bglobals.grid(row=1, column=1) - # - self.status = Label(top, anchor="w") - self.status.pack(anchor="w") - self.error = Label(top, anchor="w") - self.error.pack(anchor="w", fill="x") - self.errorbg = self.error.cget("background") - # - self.fstack = Frame(top, height=1) - self.fstack.pack(expand=1, fill="both") - self.flocals = Frame(top) - self.flocals.pack(expand=1, fill="both") - self.fglobals = Frame(top, height=1) - self.fglobals.pack(expand=1, fill="both") - # - if self.vstack.get(): - self.show_stack() - if self.vlocals.get(): - self.show_locals() - if self.vglobals.get(): - self.show_globals() - - def interaction(self, message, frame, info=None): - self.frame = frame - self.status.configure(text=message) - # - if info: - type, value, tb = info - try: - m1 = type.__name__ - except AttributeError: - m1 = "%s" % str(type) - if value is not None: - try: - m1 = "%s: %s" % (m1, str(value)) - except: - pass - bg = "yellow" - else: - m1 = "" - tb = None - bg = self.errorbg - self.error.configure(text=m1, background=bg) - # - sv = self.stackviewer - if sv: - stack, i = self.idb.get_stack(self.frame, tb) - sv.load_stack(stack, i) - # - self.show_variables(1) - # - if self.vsource.get(): - self.sync_source_line() - # - for b in self.buttons: - b.configure(state="normal") - # - self.top.wakeup() - # Nested main loop: Tkinter's main loop is not reentrant, so use - # Tcl's vwait facility, which reenters the event loop until an - # event handler sets the variable we're waiting on - self.nesting_level += 1 - self.root.tk.call('vwait', '::idledebugwait') - self.nesting_level -= 1 - # - for b in self.buttons: - b.configure(state="disabled") - self.status.configure(text="") - self.error.configure(text="", background=self.errorbg) - self.frame = None - - def sync_source_line(self): - frame = self.frame - if not frame: - return - filename, lineno = self.__frame2fileline(frame) - if filename[:1] + filename[-1:] != "<>" and os.path.exists(filename): - self.flist.gotofileline(filename, lineno) - - def __frame2fileline(self, frame): - code = frame.f_code - filename = code.co_filename - lineno = frame.f_lineno - return filename, lineno - - def cont(self): - self.idb.set_continue() - self.abort_loop() - - def step(self): - self.idb.set_step() - self.abort_loop() - - def next(self): - self.idb.set_next(self.frame) - self.abort_loop() - - def ret(self): - self.idb.set_return(self.frame) - self.abort_loop() - - def quit(self): - self.idb.set_quit() - self.abort_loop() - - def abort_loop(self): - self.root.tk.call('set', '::idledebugwait', '1') - - stackviewer = None - - def show_stack(self): - if not self.stackviewer and self.vstack.get(): - self.stackviewer = sv = StackViewer(self.fstack, self.flist, self) - if self.frame: - stack, i = self.idb.get_stack(self.frame, None) - sv.load_stack(stack, i) - else: - sv = self.stackviewer - if sv and not self.vstack.get(): - self.stackviewer = None - sv.close() - self.fstack['height'] = 1 - - def show_source(self): - if self.vsource.get(): - self.sync_source_line() - - def show_frame(self, stackitem): - self.frame = stackitem[0] # lineno is stackitem[1] - self.show_variables() - - localsviewer = None - globalsviewer = None - - def show_locals(self): - lv = self.localsviewer - if self.vlocals.get(): - if not lv: - self.localsviewer = NamespaceViewer(self.flocals, "Locals") - else: - if lv: - self.localsviewer = None - lv.close() - self.flocals['height'] = 1 - self.show_variables() - - def show_globals(self): - gv = self.globalsviewer - if self.vglobals.get(): - if not gv: - self.globalsviewer = NamespaceViewer(self.fglobals, "Globals") - else: - if gv: - self.globalsviewer = None - gv.close() - self.fglobals['height'] = 1 - self.show_variables() - - def show_variables(self, force=0): - lv = self.localsviewer - gv = self.globalsviewer - frame = self.frame - if not frame: - ldict = gdict = None - else: - ldict = frame.f_locals - gdict = frame.f_globals - if lv and gv and ldict is gdict: - ldict = None - if lv: - lv.load_dict(ldict, force, self.pyshell.interp.rpcclt) - if gv: - gv.load_dict(gdict, force, self.pyshell.interp.rpcclt) - - def set_breakpoint_here(self, filename, lineno): - self.idb.set_break(filename, lineno) - - def clear_breakpoint_here(self, filename, lineno): - self.idb.clear_break(filename, lineno) - - def clear_file_breaks(self, filename): - self.idb.clear_all_file_breaks(filename) - - def load_breakpoints(self): - "Load PyShellEditorWindow breakpoints into subprocess debugger" - for editwin in self.pyshell.flist.inversedict: - filename = editwin.io.filename - try: - for lineno in editwin.breakpoints: - self.set_breakpoint_here(filename, lineno) - except AttributeError: - continue - -class StackViewer(ScrolledList): - - def __init__(self, master, flist, gui): - if macosxSupport.isAquaTk(): - # At least on with the stock AquaTk version on OSX 10.4 you'll - # get a shaking GUI that eventually kills IDLE if the width - # argument is specified. - ScrolledList.__init__(self, master) - else: - ScrolledList.__init__(self, master, width=80) - self.flist = flist - self.gui = gui - self.stack = [] - - def load_stack(self, stack, index=None): - self.stack = stack - self.clear() - for i in range(len(stack)): - frame, lineno = stack[i] - try: - modname = frame.f_globals["__name__"] - except: - modname = "?" - code = frame.f_code - filename = code.co_filename - funcname = code.co_name - import linecache - sourceline = linecache.getline(filename, lineno) - sourceline = sourceline.strip() - if funcname in ("?", "", None): - item = "%s, line %d: %s" % (modname, lineno, sourceline) - else: - item = "%s.%s(), line %d: %s" % (modname, funcname, - lineno, sourceline) - if i == index: - item = "> " + item - self.append(item) - if index is not None: - self.select(index) - - def popup_event(self, event): - "override base method" - if self.stack: - return ScrolledList.popup_event(self, event) - - def fill_menu(self): - "override base method" - menu = self.menu - menu.add_command(label="Go to source line", - command=self.goto_source_line) - menu.add_command(label="Show stack frame", - command=self.show_stack_frame) - - def on_select(self, index): - "override base method" - if 0 <= index < len(self.stack): - self.gui.show_frame(self.stack[index]) - - def on_double(self, index): - "override base method" - self.show_source(index) - - def goto_source_line(self): - index = self.listbox.index("active") - self.show_source(index) - - def show_stack_frame(self): - index = self.listbox.index("active") - if 0 <= index < len(self.stack): - self.gui.show_frame(self.stack[index]) - - def show_source(self, index): - if not (0 <= index < len(self.stack)): - return - frame, lineno = self.stack[index] - code = frame.f_code - filename = code.co_filename - if os.path.isfile(filename): - edit = self.flist.open(filename) - if edit: - edit.gotoline(lineno) - - -class NamespaceViewer: - - def __init__(self, master, title, dict=None): - width = 0 - height = 40 - if dict: - height = 20*len(dict) # XXX 20 == observed height of Entry widget - self.master = master - self.title = title - import reprlib - self.repr = reprlib.Repr() - self.repr.maxstring = 60 - self.repr.maxother = 60 - self.frame = frame = Frame(master) - self.frame.pack(expand=1, fill="both") - self.label = Label(frame, text=title, borderwidth=2, relief="groove") - self.label.pack(fill="x") - self.vbar = vbar = Scrollbar(frame, name="vbar") - vbar.pack(side="right", fill="y") - self.canvas = canvas = Canvas(frame, - height=min(300, max(40, height)), - scrollregion=(0, 0, width, height)) - canvas.pack(side="left", fill="both", expand=1) - vbar["command"] = canvas.yview - canvas["yscrollcommand"] = vbar.set - self.subframe = subframe = Frame(canvas) - self.sfid = canvas.create_window(0, 0, window=subframe, anchor="nw") - self.load_dict(dict) - - dict = -1 - - def load_dict(self, dict, force=0, rpc_client=None): - if dict is self.dict and not force: - return - subframe = self.subframe - frame = self.frame - for c in list(subframe.children.values()): - c.destroy() - self.dict = None - if not dict: - l = Label(subframe, text="None") - l.grid(row=0, column=0) - else: - #names = sorted(dict) - ### - # Because of (temporary) limitations on the dict_keys type (not yet - # public or pickleable), have the subprocess to send a list of - # keys, not a dict_keys object. sorted() will take a dict_keys - # (no subprocess) or a list. - # - # There is also an obscure bug in sorted(dict) where the - # interpreter gets into a loop requesting non-existing dict[0], - # dict[1], dict[2], etc from the RemoteDebugger.DictProxy. - ### - keys_list = dict.keys() - names = sorted(keys_list) - ### - row = 0 - for name in names: - value = dict[name] - svalue = self.repr.repr(value) # repr(value) - # Strip extra quotes caused by calling repr on the (already) - # repr'd value sent across the RPC interface: - if rpc_client: - svalue = svalue[1:-1] - l = Label(subframe, text=name) - l.grid(row=row, column=0, sticky="nw") - l = Entry(subframe, width=0, borderwidth=0) - l.insert(0, svalue) - l.grid(row=row, column=1, sticky="nw") - row = row+1 - self.dict = dict - # XXX Could we use a callback for the following? - subframe.update_idletasks() # Alas! - width = subframe.winfo_reqwidth() - height = subframe.winfo_reqheight() - canvas = self.canvas - self.canvas["scrollregion"] = (0, 0, width, height) - if height > 300: - canvas["height"] = 300 - frame.pack(expand=1) - else: - canvas["height"] = height - frame.pack(expand=0) - - def close(self): - self.frame.destroy() diff --git a/Lib/idlelib/Delegator.py b/Lib/idlelib/Delegator.py deleted file mode 100644 index dc2a1aa..0000000 --- a/Lib/idlelib/Delegator.py +++ /dev/null @@ -1,33 +0,0 @@ -class Delegator: - - def __init__(self, delegate=None): - self.delegate = delegate - self.__cache = set() - # Cache is used to only remove added attributes - # when changing the delegate. - - def __getattr__(self, name): - attr = getattr(self.delegate, name) # May raise AttributeError - setattr(self, name, attr) - self.__cache.add(name) - return attr - - def resetcache(self): - "Removes added attributes while leaving original attributes." - # Function is really about resetting delagator dict - # to original state. Cache is just a means - for key in self.__cache: - try: - delattr(self, key) - except AttributeError: - pass - self.__cache.clear() - - def setdelegate(self, delegate): - "Reset attributes and change delegate." - self.resetcache() - self.delegate = delegate - -if __name__ == '__main__': - from unittest import main - main('idlelib.idle_test.test_delegator', verbosity=2) diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py deleted file mode 100644 index b5868be..0000000 --- a/Lib/idlelib/EditorWindow.py +++ /dev/null @@ -1,1703 +0,0 @@ -import importlib -import importlib.abc -import importlib.util -import os -import platform -import re -import string -import sys -from tkinter import * -import tkinter.simpledialog as tkSimpleDialog -import tkinter.messagebox as tkMessageBox -import traceback -import webbrowser - -from idlelib.MultiCall import MultiCallCreator -from idlelib import WindowList -from idlelib import SearchDialog -from idlelib import GrepDialog -from idlelib import ReplaceDialog -from idlelib import PyParse -from idlelib.configHandler import idleConf -from idlelib import aboutDialog, textView, configDialog -from idlelib import macosxSupport -from idlelib import help - -# The default tab setting for a Text widget, in average-width characters. -TK_TABWIDTH_DEFAULT = 8 - -_py_version = ' (%s)' % platform.python_version() - -def _sphinx_version(): - "Format sys.version_info to produce the Sphinx version string used to install the chm docs" - major, minor, micro, level, serial = sys.version_info - release = '%s%s' % (major, minor) - release += '%s' % (micro,) - if level == 'candidate': - release += 'rc%s' % (serial,) - elif level != 'final': - release += '%s%s' % (level[0], serial) - return release - - -class HelpDialog(object): - - def __init__(self): - self.parent = None # parent of help window - self.dlg = None # the help window iteself - - def display(self, parent, near=None): - """ Display the help dialog. - - parent - parent widget for the help window - - near - a Toplevel widget (e.g. EditorWindow or PyShell) - to use as a reference for placing the help window - """ - import warnings as w - w.warn("EditorWindow.HelpDialog is no longer used by Idle.\n" - "It will be removed in 3.6 or later.\n" - "It has been replaced by private help.HelpWindow\n", - DeprecationWarning, stacklevel=2) - if self.dlg is None: - self.show_dialog(parent) - if near: - self.nearwindow(near) - - def show_dialog(self, parent): - self.parent = parent - fn=os.path.join(os.path.abspath(os.path.dirname(__file__)),'help.txt') - self.dlg = dlg = textView.view_file(parent,'Help',fn, modal=False) - dlg.bind('', self.destroy, '+') - - def nearwindow(self, near): - # Place the help dialog near the window specified by parent. - # Note - this may not reposition the window in Metacity - # if "/apps/metacity/general/disable_workarounds" is enabled - dlg = self.dlg - geom = (near.winfo_rootx() + 10, near.winfo_rooty() + 10) - dlg.withdraw() - dlg.geometry("=+%d+%d" % geom) - dlg.deiconify() - dlg.lift() - - def destroy(self, ev=None): - self.dlg = None - self.parent = None - -helpDialog = HelpDialog() # singleton instance, no longer used - - -class EditorWindow(object): - from idlelib.Percolator import Percolator - from idlelib.ColorDelegator import ColorDelegator - from idlelib.UndoDelegator import UndoDelegator - from idlelib.IOBinding import IOBinding, filesystemencoding, encoding - from idlelib import Bindings - from tkinter import Toplevel - from idlelib.MultiStatusBar import MultiStatusBar - - help_url = None - - def __init__(self, flist=None, filename=None, key=None, root=None): - if EditorWindow.help_url is None: - dochome = os.path.join(sys.base_prefix, 'Doc', 'index.html') - if sys.platform.count('linux'): - # look for html docs in a couple of standard places - pyver = 'python-docs-' + '%s.%s.%s' % sys.version_info[:3] - if os.path.isdir('/var/www/html/python/'): # "python2" rpm - dochome = '/var/www/html/python/index.html' - else: - basepath = '/usr/share/doc/' # standard location - dochome = os.path.join(basepath, pyver, - 'Doc', 'index.html') - elif sys.platform[:3] == 'win': - chmfile = os.path.join(sys.base_prefix, 'Doc', - 'Python%s.chm' % _sphinx_version()) - if os.path.isfile(chmfile): - dochome = chmfile - elif sys.platform == 'darwin': - # documentation may be stored inside a python framework - dochome = os.path.join(sys.base_prefix, - 'Resources/English.lproj/Documentation/index.html') - dochome = os.path.normpath(dochome) - if os.path.isfile(dochome): - EditorWindow.help_url = dochome - if sys.platform == 'darwin': - # Safari requires real file:-URLs - EditorWindow.help_url = 'file://' + EditorWindow.help_url - else: - EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] - self.flist = flist - root = root or flist.root - self.root = root - try: - sys.ps1 - except AttributeError: - sys.ps1 = '>>> ' - self.menubar = Menu(root) - self.top = top = WindowList.ListedToplevel(root, menu=self.menubar) - if flist: - self.tkinter_vars = flist.vars - #self.top.instance_dict makes flist.inversedict available to - #configDialog.py so it can access all EditorWindow instances - self.top.instance_dict = flist.inversedict - else: - self.tkinter_vars = {} # keys: Tkinter event names - # values: Tkinter variable instances - self.top.instance_dict = {} - self.recent_files_path = os.path.join(idleConf.GetUserCfgDir(), - 'recent-files.lst') - self.text_frame = text_frame = Frame(top) - self.vbar = vbar = Scrollbar(text_frame, name='vbar') - self.width = idleConf.GetOption('main', 'EditorWindow', - 'width', type='int') - text_options = { - 'name': 'text', - 'padx': 5, - 'wrap': 'none', - 'highlightthickness': 0, - 'width': self.width, - 'height': idleConf.GetOption('main', 'EditorWindow', - 'height', type='int')} - if TkVersion >= 8.5: - # Starting with tk 8.5 we have to set the new tabstyle option - # to 'wordprocessor' to achieve the same display of tabs as in - # older tk versions. - text_options['tabstyle'] = 'wordprocessor' - self.text = text = MultiCallCreator(Text)(text_frame, **text_options) - self.top.focused_widget = self.text - - self.createmenubar() - self.apply_bindings() - - self.top.protocol("WM_DELETE_WINDOW", self.close) - self.top.bind("<>", self.close_event) - if macosxSupport.isAquaTk(): - # Command-W on editorwindows doesn't work without this. - text.bind('<>', self.close_event) - # Some OS X systems have only one mouse button, so use - # control-click for popup context menus there. For two - # buttons, AquaTk defines <2> as the right button, not <3>. - text.bind("",self.right_menu_event) - text.bind("<2>", self.right_menu_event) - else: - # Elsewhere, use right-click for popup menus. - text.bind("<3>",self.right_menu_event) - text.bind("<>", self.cut) - text.bind("<>", self.copy) - text.bind("<>", self.paste) - text.bind("<>", self.center_insert_event) - text.bind("<>", self.help_dialog) - text.bind("<>", self.python_docs) - text.bind("<>", self.about_dialog) - text.bind("<>", self.config_dialog) - text.bind("<>", self.open_module) - text.bind("<>", lambda event: "break") - text.bind("<>", self.select_all) - text.bind("<>", self.remove_selection) - text.bind("<>", self.find_event) - text.bind("<>", self.find_again_event) - text.bind("<>", self.find_in_files_event) - text.bind("<>", self.find_selection_event) - text.bind("<>", self.replace_event) - text.bind("<>", self.goto_line_event) - text.bind("<>",self.smart_backspace_event) - text.bind("<>",self.newline_and_indent_event) - text.bind("<>",self.smart_indent_event) - text.bind("<>",self.indent_region_event) - text.bind("<>",self.dedent_region_event) - text.bind("<>",self.comment_region_event) - text.bind("<>",self.uncomment_region_event) - text.bind("<>",self.tabify_region_event) - text.bind("<>",self.untabify_region_event) - text.bind("<>",self.toggle_tabs_event) - text.bind("<>",self.change_indentwidth_event) - text.bind("", self.move_at_edge_if_selection(0)) - text.bind("", self.move_at_edge_if_selection(1)) - text.bind("<>", self.del_word_left) - text.bind("<>", self.del_word_right) - text.bind("<>", self.home_callback) - - if flist: - flist.inversedict[self] = key - if key: - flist.dict[key] = self - text.bind("<>", self.new_callback) - text.bind("<>", self.flist.close_all_callback) - text.bind("<>", self.open_class_browser) - text.bind("<>", self.open_path_browser) - text.bind("<>", self.open_turtle_demo) - - self.set_status_bar() - vbar['command'] = text.yview - vbar.pack(side=RIGHT, fill=Y) - text['yscrollcommand'] = vbar.set - text['font'] = idleConf.GetFont(self.root, 'main', 'EditorWindow') - text_frame.pack(side=LEFT, fill=BOTH, expand=1) - text.pack(side=TOP, fill=BOTH, expand=1) - text.focus_set() - - # usetabs true -> literal tab characters are used by indent and - # dedent cmds, possibly mixed with spaces if - # indentwidth is not a multiple of tabwidth, - # which will cause Tabnanny to nag! - # false -> tab characters are converted to spaces by indent - # and dedent cmds, and ditto TAB keystrokes - # Although use-spaces=0 can be configured manually in config-main.def, - # configuration of tabs v. spaces is not supported in the configuration - # dialog. IDLE promotes the preferred Python indentation: use spaces! - usespaces = idleConf.GetOption('main', 'Indent', - 'use-spaces', type='bool') - self.usetabs = not usespaces - - # tabwidth is the display width of a literal tab character. - # CAUTION: telling Tk to use anything other than its default - # tab setting causes it to use an entirely different tabbing algorithm, - # treating tab stops as fixed distances from the left margin. - # Nobody expects this, so for now tabwidth should never be changed. - self.tabwidth = 8 # must remain 8 until Tk is fixed. - - # indentwidth is the number of screen characters per indent level. - # The recommended Python indentation is four spaces. - self.indentwidth = self.tabwidth - self.set_notabs_indentwidth() - - # If context_use_ps1 is true, parsing searches back for a ps1 line; - # else searches for a popular (if, def, ...) Python stmt. - self.context_use_ps1 = False - - # When searching backwards for a reliable place to begin parsing, - # first start num_context_lines[0] lines back, then - # num_context_lines[1] lines back if that didn't work, and so on. - # The last value should be huge (larger than the # of lines in a - # conceivable file). - # Making the initial values larger slows things down more often. - self.num_context_lines = 50, 500, 5000000 - self.per = per = self.Percolator(text) - self.undo = undo = self.UndoDelegator() - per.insertfilter(undo) - text.undo_block_start = undo.undo_block_start - text.undo_block_stop = undo.undo_block_stop - undo.set_saved_change_hook(self.saved_change_hook) - # IOBinding implements file I/O and printing functionality - self.io = io = self.IOBinding(self) - io.set_filename_change_hook(self.filename_change_hook) - self.good_load = False - self.set_indentation_params(False) - self.color = None # initialized below in self.ResetColorizer - if filename: - if os.path.exists(filename) and not os.path.isdir(filename): - if io.loadfile(filename): - self.good_load = True - is_py_src = self.ispythonsource(filename) - self.set_indentation_params(is_py_src) - else: - io.set_filename(filename) - self.good_load = True - - self.ResetColorizer() - self.saved_change_hook() - self.update_recent_files_list() - self.load_extensions() - menu = self.menudict.get('windows') - if menu: - end = menu.index("end") - if end is None: - end = -1 - if end >= 0: - menu.add_separator() - end = end + 1 - self.wmenu_end = end - WindowList.register_callback(self.postwindowsmenu) - - # Some abstractions so IDLE extensions are cross-IDE - self.askyesno = tkMessageBox.askyesno - self.askinteger = tkSimpleDialog.askinteger - self.showerror = tkMessageBox.showerror - - def _filename_to_unicode(self, filename): - """Return filename as BMP unicode so diplayable in Tk.""" - # Decode bytes to unicode. - if isinstance(filename, bytes): - try: - filename = filename.decode(self.filesystemencoding) - except UnicodeDecodeError: - try: - filename = filename.decode(self.encoding) - except UnicodeDecodeError: - # byte-to-byte conversion - filename = filename.decode('iso8859-1') - # Replace non-BMP char with diamond questionmark. - return re.sub('[\U00010000-\U0010FFFF]', '\ufffd', filename) - - def new_callback(self, event): - dirname, basename = self.io.defaultfilename() - self.flist.new(dirname) - return "break" - - def home_callback(self, event): - if (event.state & 4) != 0 and event.keysym == "Home": - # state&4==Control. If , use the Tk binding. - return - if self.text.index("iomark") and \ - self.text.compare("iomark", "<=", "insert lineend") and \ - self.text.compare("insert linestart", "<=", "iomark"): - # In Shell on input line, go to just after prompt - insertpt = int(self.text.index("iomark").split(".")[1]) - else: - line = self.text.get("insert linestart", "insert lineend") - for insertpt in range(len(line)): - if line[insertpt] not in (' ','\t'): - break - else: - insertpt=len(line) - lineat = int(self.text.index("insert").split('.')[1]) - if insertpt == lineat: - insertpt = 0 - dest = "insert linestart+"+str(insertpt)+"c" - if (event.state&1) == 0: - # shift was not pressed - self.text.tag_remove("sel", "1.0", "end") - else: - if not self.text.index("sel.first"): - # there was no previous selection - self.text.mark_set("my_anchor", "insert") - else: - if self.text.compare(self.text.index("sel.first"), "<", - self.text.index("insert")): - self.text.mark_set("my_anchor", "sel.first") # extend back - else: - self.text.mark_set("my_anchor", "sel.last") # extend forward - first = self.text.index(dest) - last = self.text.index("my_anchor") - if self.text.compare(first,">",last): - first,last = last,first - self.text.tag_remove("sel", "1.0", "end") - self.text.tag_add("sel", first, last) - self.text.mark_set("insert", dest) - self.text.see("insert") - return "break" - - def set_status_bar(self): - self.status_bar = self.MultiStatusBar(self.top) - sep = Frame(self.top, height=1, borderwidth=1, background='grey75') - if sys.platform == "darwin": - # Insert some padding to avoid obscuring some of the statusbar - # by the resize widget. - self.status_bar.set_label('_padding1', ' ', side=RIGHT) - self.status_bar.set_label('column', 'Col: ?', side=RIGHT) - self.status_bar.set_label('line', 'Ln: ?', side=RIGHT) - self.status_bar.pack(side=BOTTOM, fill=X) - sep.pack(side=BOTTOM, fill=X) - self.text.bind("<>", self.set_line_and_column) - self.text.event_add("<>", - "", "") - self.text.after_idle(self.set_line_and_column) - - def set_line_and_column(self, event=None): - line, column = self.text.index(INSERT).split('.') - self.status_bar.set_label('column', 'Col: %s' % column) - self.status_bar.set_label('line', 'Ln: %s' % line) - - menu_specs = [ - ("file", "_File"), - ("edit", "_Edit"), - ("format", "F_ormat"), - ("run", "_Run"), - ("options", "_Options"), - ("windows", "_Window"), - ("help", "_Help"), - ] - - - def createmenubar(self): - mbar = self.menubar - self.menudict = menudict = {} - for name, label in self.menu_specs: - underline, label = prepstr(label) - menudict[name] = menu = Menu(mbar, name=name, tearoff=0) - mbar.add_cascade(label=label, menu=menu, underline=underline) - if macosxSupport.isCarbonTk(): - # Insert the application menu - menudict['application'] = menu = Menu(mbar, name='apple', - tearoff=0) - mbar.add_cascade(label='IDLE', menu=menu) - self.fill_menus() - self.recent_files_menu = Menu(self.menubar, tearoff=0) - self.menudict['file'].insert_cascade(3, label='Recent Files', - underline=0, - menu=self.recent_files_menu) - self.base_helpmenu_length = self.menudict['help'].index(END) - self.reset_help_menu_entries() - - def postwindowsmenu(self): - # Only called when Windows menu exists - menu = self.menudict['windows'] - end = menu.index("end") - if end is None: - end = -1 - if end > self.wmenu_end: - menu.delete(self.wmenu_end+1, end) - WindowList.add_windows_to_menu(menu) - - rmenu = None - - def right_menu_event(self, event): - self.text.mark_set("insert", "@%d,%d" % (event.x, event.y)) - if not self.rmenu: - self.make_rmenu() - rmenu = self.rmenu - self.event = event - iswin = sys.platform[:3] == 'win' - if iswin: - self.text.config(cursor="arrow") - - for item in self.rmenu_specs: - try: - label, eventname, verify_state = item - except ValueError: # see issue1207589 - continue - - if verify_state is None: - continue - state = getattr(self, verify_state)() - rmenu.entryconfigure(label, state=state) - - - rmenu.tk_popup(event.x_root, event.y_root) - if iswin: - self.text.config(cursor="ibeam") - - rmenu_specs = [ - # ("Label", "<>", "statefuncname"), ... - ("Close", "<>", None), # Example - ] - - def make_rmenu(self): - rmenu = Menu(self.text, tearoff=0) - for item in self.rmenu_specs: - label, eventname = item[0], item[1] - if label is not None: - def command(text=self.text, eventname=eventname): - text.event_generate(eventname) - rmenu.add_command(label=label, command=command) - else: - rmenu.add_separator() - self.rmenu = rmenu - - def rmenu_check_cut(self): - return self.rmenu_check_copy() - - def rmenu_check_copy(self): - try: - indx = self.text.index('sel.first') - except TclError: - return 'disabled' - else: - return 'normal' if indx else 'disabled' - - def rmenu_check_paste(self): - try: - self.text.tk.call('tk::GetSelection', self.text, 'CLIPBOARD') - except TclError: - return 'disabled' - else: - return 'normal' - - def about_dialog(self, event=None): - "Handle Help 'About IDLE' event." - # Synchronize with macosxSupport.overrideRootMenu.about_dialog. - aboutDialog.AboutDialog(self.top,'About IDLE') - - def config_dialog(self, event=None): - "Handle Options 'Configure IDLE' event." - # Synchronize with macosxSupport.overrideRootMenu.config_dialog. - configDialog.ConfigDialog(self.top,'Settings') - - def help_dialog(self, event=None): - "Handle Help 'IDLE Help' event." - # Synchronize with macosxSupport.overrideRootMenu.help_dialog. - if self.root: - parent = self.root - else: - parent = self.top - help.show_idlehelp(parent) - - def python_docs(self, event=None): - if sys.platform[:3] == 'win': - try: - os.startfile(self.help_url) - except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', - message=str(why), parent=self.text) - else: - webbrowser.open(self.help_url) - return "break" - - def cut(self,event): - self.text.event_generate("<>") - return "break" - - def copy(self,event): - if not self.text.tag_ranges("sel"): - # There is no selection, so do nothing and maybe interrupt. - return - self.text.event_generate("<>") - return "break" - - def paste(self,event): - self.text.event_generate("<>") - self.text.see("insert") - return "break" - - def select_all(self, event=None): - self.text.tag_add("sel", "1.0", "end-1c") - self.text.mark_set("insert", "1.0") - self.text.see("insert") - return "break" - - def remove_selection(self, event=None): - self.text.tag_remove("sel", "1.0", "end") - self.text.see("insert") - - def move_at_edge_if_selection(self, edge_index): - """Cursor move begins at start or end of selection - - When a left/right cursor key is pressed create and return to Tkinter a - function which causes a cursor move from the associated edge of the - selection. - - """ - self_text_index = self.text.index - self_text_mark_set = self.text.mark_set - edges_table = ("sel.first+1c", "sel.last-1c") - def move_at_edge(event): - if (event.state & 5) == 0: # no shift(==1) or control(==4) pressed - try: - self_text_index("sel.first") - self_text_mark_set("insert", edges_table[edge_index]) - except TclError: - pass - return move_at_edge - - def del_word_left(self, event): - self.text.event_generate('') - return "break" - - def del_word_right(self, event): - self.text.event_generate('') - return "break" - - def find_event(self, event): - SearchDialog.find(self.text) - return "break" - - def find_again_event(self, event): - SearchDialog.find_again(self.text) - return "break" - - def find_selection_event(self, event): - SearchDialog.find_selection(self.text) - return "break" - - def find_in_files_event(self, event): - GrepDialog.grep(self.text, self.io, self.flist) - return "break" - - def replace_event(self, event): - ReplaceDialog.replace(self.text) - return "break" - - def goto_line_event(self, event): - text = self.text - lineno = tkSimpleDialog.askinteger("Goto", - "Go to line number:",parent=text) - if lineno is None: - return "break" - if lineno <= 0: - text.bell() - return "break" - text.mark_set("insert", "%d.0" % lineno) - text.see("insert") - - def open_module(self, event=None): - # XXX Shouldn't this be in IOBinding? - try: - name = self.text.get("sel.first", "sel.last") - except TclError: - name = "" - else: - name = name.strip() - name = tkSimpleDialog.askstring("Module", - "Enter the name of a Python module\n" - "to search on sys.path and open:", - parent=self.text, initialvalue=name) - if name: - name = name.strip() - if not name: - return - # XXX Ought to insert current file's directory in front of path - try: - spec = importlib.util.find_spec(name) - except (ValueError, ImportError) as msg: - tkMessageBox.showerror("Import error", str(msg), parent=self.text) - return - if spec is None: - tkMessageBox.showerror("Import error", "module not found", - parent=self.text) - return - if not isinstance(spec.loader, importlib.abc.SourceLoader): - tkMessageBox.showerror("Import error", "not a source-based module", - parent=self.text) - return - try: - file_path = spec.loader.get_filename(name) - except AttributeError: - tkMessageBox.showerror("Import error", - "loader does not support get_filename", - parent=self.text) - return - if self.flist: - self.flist.open(file_path) - else: - self.io.loadfile(file_path) - return file_path - - def open_class_browser(self, event=None): - filename = self.io.filename - if not (self.__class__.__name__ == 'PyShellEditorWindow' - and filename): - filename = self.open_module() - if filename is None: - return - head, tail = os.path.split(filename) - base, ext = os.path.splitext(tail) - from idlelib import ClassBrowser - ClassBrowser.ClassBrowser(self.flist, base, [head]) - - def open_path_browser(self, event=None): - from idlelib import PathBrowser - PathBrowser.PathBrowser(self.flist) - - def open_turtle_demo(self, event = None): - import subprocess - - cmd = [sys.executable, - '-c', - 'from turtledemo.__main__ import main; main()'] - subprocess.Popen(cmd, shell=False) - - def gotoline(self, lineno): - if lineno is not None and lineno > 0: - self.text.mark_set("insert", "%d.0" % lineno) - self.text.tag_remove("sel", "1.0", "end") - self.text.tag_add("sel", "insert", "insert +1l") - self.center() - - def ispythonsource(self, filename): - if not filename or os.path.isdir(filename): - return True - base, ext = os.path.splitext(os.path.basename(filename)) - if os.path.normcase(ext) in (".py", ".pyw"): - return True - line = self.text.get('1.0', '1.0 lineend') - return line.startswith('#!') and 'python' in line - - def close_hook(self): - if self.flist: - self.flist.unregister_maybe_terminate(self) - self.flist = None - - def set_close_hook(self, close_hook): - self.close_hook = close_hook - - def filename_change_hook(self): - if self.flist: - self.flist.filename_changed_edit(self) - self.saved_change_hook() - self.top.update_windowlist_registry(self) - self.ResetColorizer() - - def _addcolorizer(self): - if self.color: - return - if self.ispythonsource(self.io.filename): - self.color = self.ColorDelegator() - # can add more colorizers here... - if self.color: - self.per.removefilter(self.undo) - self.per.insertfilter(self.color) - self.per.insertfilter(self.undo) - - def _rmcolorizer(self): - if not self.color: - return - self.color.removecolors() - self.per.removefilter(self.color) - self.color = None - - def ResetColorizer(self): - "Update the color theme" - # Called from self.filename_change_hook and from configDialog.py - self._rmcolorizer() - self._addcolorizer() - theme = idleConf.CurrentTheme() - normal_colors = idleConf.GetHighlight(theme, 'normal') - cursor_color = idleConf.GetHighlight(theme, 'cursor', fgBg='fg') - select_colors = idleConf.GetHighlight(theme, 'hilite') - self.text.config( - foreground=normal_colors['foreground'], - background=normal_colors['background'], - insertbackground=cursor_color, - selectforeground=select_colors['foreground'], - selectbackground=select_colors['background'], - ) - if TkVersion >= 8.5: - self.text.config( - inactiveselectbackground=select_colors['background']) - - IDENTCHARS = string.ascii_letters + string.digits + "_" - - def colorize_syntax_error(self, text, pos): - text.tag_add("ERROR", pos) - char = text.get(pos) - if char and char in self.IDENTCHARS: - text.tag_add("ERROR", pos + " wordstart", pos) - if '\n' == text.get(pos): # error at line end - text.mark_set("insert", pos) - else: - text.mark_set("insert", pos + "+1c") - text.see(pos) - - def ResetFont(self): - "Update the text widgets' font if it is changed" - # Called from configDialog.py - - self.text['font'] = idleConf.GetFont(self.root, 'main','EditorWindow') - - def RemoveKeybindings(self): - "Remove the keybindings before they are changed." - # Called from configDialog.py - self.Bindings.default_keydefs = keydefs = idleConf.GetCurrentKeySet() - for event, keylist in keydefs.items(): - self.text.event_delete(event, *keylist) - for extensionName in self.get_standard_extension_names(): - xkeydefs = idleConf.GetExtensionBindings(extensionName) - if xkeydefs: - for event, keylist in xkeydefs.items(): - self.text.event_delete(event, *keylist) - - def ApplyKeybindings(self): - "Update the keybindings after they are changed" - # Called from configDialog.py - self.Bindings.default_keydefs = keydefs = idleConf.GetCurrentKeySet() - self.apply_bindings() - for extensionName in self.get_standard_extension_names(): - xkeydefs = idleConf.GetExtensionBindings(extensionName) - if xkeydefs: - self.apply_bindings(xkeydefs) - #update menu accelerators - menuEventDict = {} - for menu in self.Bindings.menudefs: - menuEventDict[menu[0]] = {} - for item in menu[1]: - if item: - menuEventDict[menu[0]][prepstr(item[0])[1]] = item[1] - for menubarItem in self.menudict: - menu = self.menudict[menubarItem] - end = menu.index(END) - if end is None: - # Skip empty menus - continue - end += 1 - for index in range(0, end): - if menu.type(index) == 'command': - accel = menu.entrycget(index, 'accelerator') - if accel: - itemName = menu.entrycget(index, 'label') - event = '' - if menubarItem in menuEventDict: - if itemName in menuEventDict[menubarItem]: - event = menuEventDict[menubarItem][itemName] - if event: - accel = get_accelerator(keydefs, event) - menu.entryconfig(index, accelerator=accel) - - def set_notabs_indentwidth(self): - "Update the indentwidth if changed and not using tabs in this window" - # Called from configDialog.py - if not self.usetabs: - self.indentwidth = idleConf.GetOption('main', 'Indent','num-spaces', - type='int') - - def reset_help_menu_entries(self): - "Update the additional help entries on the Help menu" - help_list = idleConf.GetAllExtraHelpSourcesList() - helpmenu = self.menudict['help'] - # first delete the extra help entries, if any - helpmenu_length = helpmenu.index(END) - if helpmenu_length > self.base_helpmenu_length: - helpmenu.delete((self.base_helpmenu_length + 1), helpmenu_length) - # then rebuild them - if help_list: - helpmenu.add_separator() - for entry in help_list: - cmd = self.__extra_help_callback(entry[1]) - helpmenu.add_command(label=entry[0], command=cmd) - # and update the menu dictionary - self.menudict['help'] = helpmenu - - def __extra_help_callback(self, helpfile): - "Create a callback with the helpfile value frozen at definition time" - def display_extra_help(helpfile=helpfile): - if not helpfile.startswith(('www', 'http')): - helpfile = os.path.normpath(helpfile) - if sys.platform[:3] == 'win': - try: - os.startfile(helpfile) - except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', - message=str(why), parent=self.text) - else: - webbrowser.open(helpfile) - return display_extra_help - - def update_recent_files_list(self, new_file=None): - "Load and update the recent files list and menus" - rf_list = [] - if os.path.exists(self.recent_files_path): - with open(self.recent_files_path, 'r', - encoding='utf_8', errors='replace') as rf_list_file: - rf_list = rf_list_file.readlines() - if new_file: - new_file = os.path.abspath(new_file) + '\n' - if new_file in rf_list: - rf_list.remove(new_file) # move to top - rf_list.insert(0, new_file) - # clean and save the recent files list - bad_paths = [] - for path in rf_list: - if '\0' in path or not os.path.exists(path[0:-1]): - bad_paths.append(path) - rf_list = [path for path in rf_list if path not in bad_paths] - ulchars = "1234567890ABCDEFGHIJK" - rf_list = rf_list[0:len(ulchars)] - try: - with open(self.recent_files_path, 'w', - encoding='utf_8', errors='replace') as rf_file: - rf_file.writelines(rf_list) - except OSError as err: - if not getattr(self.root, "recentfilelist_error_displayed", False): - self.root.recentfilelist_error_displayed = True - tkMessageBox.showwarning(title='IDLE Warning', - message="Cannot update File menu Recent Files list. " - "Your operating system says:\n%s\n" - "Select OK and IDLE will continue without updating." - % self._filename_to_unicode(str(err)), - parent=self.text) - # for each edit window instance, construct the recent files menu - for instance in self.top.instance_dict: - menu = instance.recent_files_menu - menu.delete(0, END) # clear, and rebuild: - for i, file_name in enumerate(rf_list): - file_name = file_name.rstrip() # zap \n - # make unicode string to display non-ASCII chars correctly - ufile_name = self._filename_to_unicode(file_name) - callback = instance.__recent_file_callback(file_name) - menu.add_command(label=ulchars[i] + " " + ufile_name, - command=callback, - underline=0) - - def __recent_file_callback(self, file_name): - def open_recent_file(fn_closure=file_name): - self.io.open(editFile=fn_closure) - return open_recent_file - - def saved_change_hook(self): - short = self.short_title() - long = self.long_title() - if short and long: - title = short + " - " + long + _py_version - elif short: - title = short - elif long: - title = long - else: - title = "Untitled" - icon = short or long or title - if not self.get_saved(): - title = "*%s*" % title - icon = "*%s" % icon - self.top.wm_title(title) - self.top.wm_iconname(icon) - - def get_saved(self): - return self.undo.get_saved() - - def set_saved(self, flag): - self.undo.set_saved(flag) - - def reset_undo(self): - self.undo.reset_undo() - - def short_title(self): - filename = self.io.filename - if filename: - filename = os.path.basename(filename) - else: - filename = "Untitled" - # return unicode string to display non-ASCII chars correctly - return self._filename_to_unicode(filename) - - def long_title(self): - # return unicode string to display non-ASCII chars correctly - return self._filename_to_unicode(self.io.filename or "") - - def center_insert_event(self, event): - self.center() - - def center(self, mark="insert"): - text = self.text - top, bot = self.getwindowlines() - lineno = self.getlineno(mark) - height = bot - top - newtop = max(1, lineno - height//2) - text.yview(float(newtop)) - - def getwindowlines(self): - text = self.text - top = self.getlineno("@0,0") - bot = self.getlineno("@0,65535") - if top == bot and text.winfo_height() == 1: - # Geometry manager hasn't run yet - height = int(text['height']) - bot = top + height - 1 - return top, bot - - def getlineno(self, mark="insert"): - text = self.text - return int(float(text.index(mark))) - - def get_geometry(self): - "Return (width, height, x, y)" - geom = self.top.wm_geometry() - m = re.match(r"(\d+)x(\d+)\+(-?\d+)\+(-?\d+)", geom) - return list(map(int, m.groups())) - - def close_event(self, event): - self.close() - - def maybesave(self): - if self.io: - if not self.get_saved(): - if self.top.state()!='normal': - self.top.deiconify() - self.top.lower() - self.top.lift() - return self.io.maybesave() - - def close(self): - reply = self.maybesave() - if str(reply) != "cancel": - self._close() - return reply - - def _close(self): - if self.io.filename: - self.update_recent_files_list(new_file=self.io.filename) - WindowList.unregister_callback(self.postwindowsmenu) - self.unload_extensions() - self.io.close() - self.io = None - self.undo = None - if self.color: - self.color.close(False) - self.color = None - self.text = None - self.tkinter_vars = None - self.per.close() - self.per = None - self.top.destroy() - if self.close_hook: - # unless override: unregister from flist, terminate if last window - self.close_hook() - - def load_extensions(self): - self.extensions = {} - self.load_standard_extensions() - - def unload_extensions(self): - for ins in list(self.extensions.values()): - if hasattr(ins, "close"): - ins.close() - self.extensions = {} - - def load_standard_extensions(self): - for name in self.get_standard_extension_names(): - try: - self.load_extension(name) - except: - print("Failed to load extension", repr(name)) - traceback.print_exc() - - def get_standard_extension_names(self): - return idleConf.GetExtensions(editor_only=True) - - def load_extension(self, name): - try: - try: - mod = importlib.import_module('.' + name, package=__package__) - except (ImportError, TypeError): - mod = importlib.import_module(name) - except ImportError: - print("\nFailed to import extension: ", name) - raise - cls = getattr(mod, name) - keydefs = idleConf.GetExtensionBindings(name) - if hasattr(cls, "menudefs"): - self.fill_menus(cls.menudefs, keydefs) - ins = cls(self) - self.extensions[name] = ins - if keydefs: - self.apply_bindings(keydefs) - for vevent in keydefs: - methodname = vevent.replace("-", "_") - while methodname[:1] == '<': - methodname = methodname[1:] - while methodname[-1:] == '>': - methodname = methodname[:-1] - methodname = methodname + "_event" - if hasattr(ins, methodname): - self.text.bind(vevent, getattr(ins, methodname)) - - def apply_bindings(self, keydefs=None): - if keydefs is None: - keydefs = self.Bindings.default_keydefs - text = self.text - text.keydefs = keydefs - for event, keylist in keydefs.items(): - if keylist: - text.event_add(event, *keylist) - - def fill_menus(self, menudefs=None, keydefs=None): - """Add appropriate entries to the menus and submenus - - Menus that are absent or None in self.menudict are ignored. - """ - if menudefs is None: - menudefs = self.Bindings.menudefs - if keydefs is None: - keydefs = self.Bindings.default_keydefs - menudict = self.menudict - text = self.text - for mname, entrylist in menudefs: - menu = menudict.get(mname) - if not menu: - continue - for entry in entrylist: - if not entry: - menu.add_separator() - else: - label, eventname = entry - checkbutton = (label[:1] == '!') - if checkbutton: - label = label[1:] - underline, label = prepstr(label) - accelerator = get_accelerator(keydefs, eventname) - def command(text=text, eventname=eventname): - text.event_generate(eventname) - if checkbutton: - var = self.get_var_obj(eventname, BooleanVar) - menu.add_checkbutton(label=label, underline=underline, - command=command, accelerator=accelerator, - variable=var) - else: - menu.add_command(label=label, underline=underline, - command=command, - accelerator=accelerator) - - def getvar(self, name): - var = self.get_var_obj(name) - if var: - value = var.get() - return value - else: - raise NameError(name) - - def setvar(self, name, value, vartype=None): - var = self.get_var_obj(name, vartype) - if var: - var.set(value) - else: - raise NameError(name) - - def get_var_obj(self, name, vartype=None): - var = self.tkinter_vars.get(name) - if not var and vartype: - # create a Tkinter variable object with self.text as master: - self.tkinter_vars[name] = var = vartype(self.text) - return var - - # Tk implementations of "virtual text methods" -- each platform - # reusing IDLE's support code needs to define these for its GUI's - # flavor of widget. - - # Is character at text_index in a Python string? Return 0 for - # "guaranteed no", true for anything else. This info is expensive - # to compute ab initio, but is probably already known by the - # platform's colorizer. - - def is_char_in_string(self, text_index): - if self.color: - # Return true iff colorizer hasn't (re)gotten this far - # yet, or the character is tagged as being in a string - return self.text.tag_prevrange("TODO", text_index) or \ - "STRING" in self.text.tag_names(text_index) - else: - # The colorizer is missing: assume the worst - return 1 - - # If a selection is defined in the text widget, return (start, - # end) as Tkinter text indices, otherwise return (None, None) - def get_selection_indices(self): - try: - first = self.text.index("sel.first") - last = self.text.index("sel.last") - return first, last - except TclError: - return None, None - - # Return the text widget's current view of what a tab stop means - # (equivalent width in spaces). - - def get_tk_tabwidth(self): - current = self.text['tabs'] or TK_TABWIDTH_DEFAULT - return int(current) - - # Set the text widget's current view of what a tab stop means. - - def set_tk_tabwidth(self, newtabwidth): - text = self.text - if self.get_tk_tabwidth() != newtabwidth: - # Set text widget tab width - pixels = text.tk.call("font", "measure", text["font"], - "-displayof", text.master, - "n" * newtabwidth) - text.configure(tabs=pixels) - -### begin autoindent code ### (configuration was moved to beginning of class) - - def set_indentation_params(self, is_py_src, guess=True): - if is_py_src and guess: - i = self.guess_indent() - if 2 <= i <= 8: - self.indentwidth = i - if self.indentwidth != self.tabwidth: - self.usetabs = False - self.set_tk_tabwidth(self.tabwidth) - - def smart_backspace_event(self, event): - text = self.text - first, last = self.get_selection_indices() - if first and last: - text.delete(first, last) - text.mark_set("insert", first) - return "break" - # Delete whitespace left, until hitting a real char or closest - # preceding virtual tab stop. - chars = text.get("insert linestart", "insert") - if chars == '': - if text.compare("insert", ">", "1.0"): - # easy: delete preceding newline - text.delete("insert-1c") - else: - text.bell() # at start of buffer - return "break" - if chars[-1] not in " \t": - # easy: delete preceding real char - text.delete("insert-1c") - return "break" - # Ick. It may require *inserting* spaces if we back up over a - # tab character! This is written to be clear, not fast. - tabwidth = self.tabwidth - have = len(chars.expandtabs(tabwidth)) - assert have > 0 - want = ((have - 1) // self.indentwidth) * self.indentwidth - # Debug prompt is multilined.... - if self.context_use_ps1: - last_line_of_prompt = sys.ps1.split('\n')[-1] - else: - last_line_of_prompt = '' - ncharsdeleted = 0 - while 1: - if chars == last_line_of_prompt: - break - chars = chars[:-1] - ncharsdeleted = ncharsdeleted + 1 - have = len(chars.expandtabs(tabwidth)) - if have <= want or chars[-1] not in " \t": - break - text.undo_block_start() - text.delete("insert-%dc" % ncharsdeleted, "insert") - if have < want: - text.insert("insert", ' ' * (want - have)) - text.undo_block_stop() - return "break" - - def smart_indent_event(self, event): - # if intraline selection: - # delete it - # elif multiline selection: - # do indent-region - # else: - # indent one level - text = self.text - first, last = self.get_selection_indices() - text.undo_block_start() - try: - if first and last: - if index2line(first) != index2line(last): - return self.indent_region_event(event) - text.delete(first, last) - text.mark_set("insert", first) - prefix = text.get("insert linestart", "insert") - raw, effective = classifyws(prefix, self.tabwidth) - if raw == len(prefix): - # only whitespace to the left - self.reindent_to(effective + self.indentwidth) - else: - # tab to the next 'stop' within or to right of line's text: - if self.usetabs: - pad = '\t' - else: - effective = len(prefix.expandtabs(self.tabwidth)) - n = self.indentwidth - pad = ' ' * (n - effective % n) - text.insert("insert", pad) - text.see("insert") - return "break" - finally: - text.undo_block_stop() - - def newline_and_indent_event(self, event): - text = self.text - first, last = self.get_selection_indices() - text.undo_block_start() - try: - if first and last: - text.delete(first, last) - text.mark_set("insert", first) - line = text.get("insert linestart", "insert") - i, n = 0, len(line) - while i < n and line[i] in " \t": - i = i+1 - if i == n: - # the cursor is in or at leading indentation in a continuation - # line; just inject an empty line at the start - text.insert("insert linestart", '\n') - return "break" - indent = line[:i] - # strip whitespace before insert point unless it's in the prompt - i = 0 - last_line_of_prompt = sys.ps1.split('\n')[-1] - while line and line[-1] in " \t" and line != last_line_of_prompt: - line = line[:-1] - i = i+1 - if i: - text.delete("insert - %d chars" % i, "insert") - # strip whitespace after insert point - while text.get("insert") in " \t": - text.delete("insert") - # start new line - text.insert("insert", '\n') - - # adjust indentation for continuations and block - # open/close first need to find the last stmt - lno = index2line(text.index('insert')) - y = PyParse.Parser(self.indentwidth, self.tabwidth) - if not self.context_use_ps1: - for context in self.num_context_lines: - startat = max(lno - context, 1) - startatindex = repr(startat) + ".0" - rawtext = text.get(startatindex, "insert") - y.set_str(rawtext) - bod = y.find_good_parse_start( - self.context_use_ps1, - self._build_char_in_string_func(startatindex)) - if bod is not None or startat == 1: - break - y.set_lo(bod or 0) - else: - r = text.tag_prevrange("console", "insert") - if r: - startatindex = r[1] - else: - startatindex = "1.0" - rawtext = text.get(startatindex, "insert") - y.set_str(rawtext) - y.set_lo(0) - - c = y.get_continuation_type() - if c != PyParse.C_NONE: - # The current stmt hasn't ended yet. - if c == PyParse.C_STRING_FIRST_LINE: - # after the first line of a string; do not indent at all - pass - elif c == PyParse.C_STRING_NEXT_LINES: - # inside a string which started before this line; - # just mimic the current indent - text.insert("insert", indent) - elif c == PyParse.C_BRACKET: - # line up with the first (if any) element of the - # last open bracket structure; else indent one - # level beyond the indent of the line with the - # last open bracket - self.reindent_to(y.compute_bracket_indent()) - elif c == PyParse.C_BACKSLASH: - # if more than one line in this stmt already, just - # mimic the current indent; else if initial line - # has a start on an assignment stmt, indent to - # beyond leftmost =; else to beyond first chunk of - # non-whitespace on initial line - if y.get_num_lines_in_stmt() > 1: - text.insert("insert", indent) - else: - self.reindent_to(y.compute_backslash_indent()) - else: - assert 0, "bogus continuation type %r" % (c,) - return "break" - - # This line starts a brand new stmt; indent relative to - # indentation of initial line of closest preceding - # interesting stmt. - indent = y.get_base_indent_string() - text.insert("insert", indent) - if y.is_block_opener(): - self.smart_indent_event(event) - elif indent and y.is_block_closer(): - self.smart_backspace_event(event) - return "break" - finally: - text.see("insert") - text.undo_block_stop() - - # Our editwin provides an is_char_in_string function that works - # with a Tk text index, but PyParse only knows about offsets into - # a string. This builds a function for PyParse that accepts an - # offset. - - def _build_char_in_string_func(self, startindex): - def inner(offset, _startindex=startindex, - _icis=self.is_char_in_string): - return _icis(_startindex + "+%dc" % offset) - return inner - - def indent_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines)): - line = lines[pos] - if line: - raw, effective = classifyws(line, self.tabwidth) - effective = effective + self.indentwidth - lines[pos] = self._make_blanks(effective) + line[raw:] - self.set_region(head, tail, chars, lines) - return "break" - - def dedent_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines)): - line = lines[pos] - if line: - raw, effective = classifyws(line, self.tabwidth) - effective = max(effective - self.indentwidth, 0) - lines[pos] = self._make_blanks(effective) + line[raw:] - self.set_region(head, tail, chars, lines) - return "break" - - def comment_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines) - 1): - line = lines[pos] - lines[pos] = '##' + line - self.set_region(head, tail, chars, lines) - - def uncomment_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines)): - line = lines[pos] - if not line: - continue - if line[:2] == '##': - line = line[2:] - elif line[:1] == '#': - line = line[1:] - lines[pos] = line - self.set_region(head, tail, chars, lines) - - def tabify_region_event(self, event): - head, tail, chars, lines = self.get_region() - tabwidth = self._asktabwidth() - if tabwidth is None: return - for pos in range(len(lines)): - line = lines[pos] - if line: - raw, effective = classifyws(line, tabwidth) - ntabs, nspaces = divmod(effective, tabwidth) - lines[pos] = '\t' * ntabs + ' ' * nspaces + line[raw:] - self.set_region(head, tail, chars, lines) - - def untabify_region_event(self, event): - head, tail, chars, lines = self.get_region() - tabwidth = self._asktabwidth() - if tabwidth is None: return - for pos in range(len(lines)): - lines[pos] = lines[pos].expandtabs(tabwidth) - self.set_region(head, tail, chars, lines) - - def toggle_tabs_event(self, event): - if self.askyesno( - "Toggle tabs", - "Turn tabs " + ("on", "off")[self.usetabs] + - "?\nIndent width " + - ("will be", "remains at")[self.usetabs] + " 8." + - "\n Note: a tab is always 8 columns", - parent=self.text): - self.usetabs = not self.usetabs - # Try to prevent inconsistent indentation. - # User must change indent width manually after using tabs. - self.indentwidth = 8 - return "break" - - # XXX this isn't bound to anything -- see tabwidth comments -## def change_tabwidth_event(self, event): -## new = self._asktabwidth() -## if new != self.tabwidth: -## self.tabwidth = new -## self.set_indentation_params(0, guess=0) -## return "break" - - def change_indentwidth_event(self, event): - new = self.askinteger( - "Indent width", - "New indent width (2-16)\n(Always use 8 when using tabs)", - parent=self.text, - initialvalue=self.indentwidth, - minvalue=2, - maxvalue=16) - if new and new != self.indentwidth and not self.usetabs: - self.indentwidth = new - return "break" - - def get_region(self): - text = self.text - first, last = self.get_selection_indices() - if first and last: - head = text.index(first + " linestart") - tail = text.index(last + "-1c lineend +1c") - else: - head = text.index("insert linestart") - tail = text.index("insert lineend +1c") - chars = text.get(head, tail) - lines = chars.split("\n") - return head, tail, chars, lines - - def set_region(self, head, tail, chars, lines): - text = self.text - newchars = "\n".join(lines) - if newchars == chars: - text.bell() - return - text.tag_remove("sel", "1.0", "end") - text.mark_set("insert", head) - text.undo_block_start() - text.delete(head, tail) - text.insert(head, newchars) - text.undo_block_stop() - text.tag_add("sel", head, "insert") - - # Make string that displays as n leading blanks. - - def _make_blanks(self, n): - if self.usetabs: - ntabs, nspaces = divmod(n, self.tabwidth) - return '\t' * ntabs + ' ' * nspaces - else: - return ' ' * n - - # Delete from beginning of line to insert point, then reinsert - # column logical (meaning use tabs if appropriate) spaces. - - def reindent_to(self, column): - text = self.text - text.undo_block_start() - if text.compare("insert linestart", "!=", "insert"): - text.delete("insert linestart", "insert") - if column: - text.insert("insert", self._make_blanks(column)) - text.undo_block_stop() - - def _asktabwidth(self): - return self.askinteger( - "Tab width", - "Columns per tab? (2-16)", - parent=self.text, - initialvalue=self.indentwidth, - minvalue=2, - maxvalue=16) - - # Guess indentwidth from text content. - # Return guessed indentwidth. This should not be believed unless - # it's in a reasonable range (e.g., it will be 0 if no indented - # blocks are found). - - def guess_indent(self): - opener, indented = IndentSearcher(self.text, self.tabwidth).run() - if opener and indented: - raw, indentsmall = classifyws(opener, self.tabwidth) - raw, indentlarge = classifyws(indented, self.tabwidth) - else: - indentsmall = indentlarge = 0 - return indentlarge - indentsmall - -# "line.col" -> line, as an int -def index2line(index): - return int(float(index)) - -# Look at the leading whitespace in s. -# Return pair (# of leading ws characters, -# effective # of leading blanks after expanding -# tabs to width tabwidth) - -def classifyws(s, tabwidth): - raw = effective = 0 - for ch in s: - if ch == ' ': - raw = raw + 1 - effective = effective + 1 - elif ch == '\t': - raw = raw + 1 - effective = (effective // tabwidth + 1) * tabwidth - else: - break - return raw, effective - -import tokenize -_tokenize = tokenize -del tokenize - -class IndentSearcher(object): - - # .run() chews over the Text widget, looking for a block opener - # and the stmt following it. Returns a pair, - # (line containing block opener, line containing stmt) - # Either or both may be None. - - def __init__(self, text, tabwidth): - self.text = text - self.tabwidth = tabwidth - self.i = self.finished = 0 - self.blkopenline = self.indentedline = None - - def readline(self): - if self.finished: - return "" - i = self.i = self.i + 1 - mark = repr(i) + ".0" - if self.text.compare(mark, ">=", "end"): - return "" - return self.text.get(mark, mark + " lineend+1c") - - def tokeneater(self, type, token, start, end, line, - INDENT=_tokenize.INDENT, - NAME=_tokenize.NAME, - OPENERS=('class', 'def', 'for', 'if', 'try', 'while')): - if self.finished: - pass - elif type == NAME and token in OPENERS: - self.blkopenline = line - elif type == INDENT and self.blkopenline: - self.indentedline = line - self.finished = 1 - - def run(self): - save_tabsize = _tokenize.tabsize - _tokenize.tabsize = self.tabwidth - try: - try: - tokens = _tokenize.generate_tokens(self.readline) - for token in tokens: - self.tokeneater(*token) - except (_tokenize.TokenError, SyntaxError): - # since we cut off the tokenizer early, we can trigger - # spurious errors - pass - finally: - _tokenize.tabsize = save_tabsize - return self.blkopenline, self.indentedline - -### end autoindent code ### - -def prepstr(s): - # Helper to extract the underscore from a string, e.g. - # prepstr("Co_py") returns (2, "Copy"). - i = s.find('_') - if i >= 0: - s = s[:i] + s[i+1:] - return i, s - - -keynames = { - 'bracketleft': '[', - 'bracketright': ']', - 'slash': '/', -} - -def get_accelerator(keydefs, eventname): - keylist = keydefs.get(eventname) - # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5 - # if not keylist: - if (not keylist) or (macosxSupport.isCocoaTk() and eventname in { - "<>", - "<>", - "<>"}): - return "" - s = keylist[0] - s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s) - s = re.sub(r"\b\w+\b", lambda m: keynames.get(m.group(), m.group()), s) - s = re.sub("Key-", "", s) - s = re.sub("Cancel","Ctrl-Break",s) # dscherer@cmu.edu - s = re.sub("Control-", "Ctrl-", s) - s = re.sub("-", "+", s) - s = re.sub("><", " ", s) - s = re.sub("<", "", s) - s = re.sub(">", "", s) - return s - - -def fixwordbreaks(root): - # Make sure that Tk's double-click and next/previous word - # operations use our definition of a word (i.e. an identifier) - tk = root.tk - tk.call('tcl_wordBreakAfter', 'a b', 0) # make sure word.tcl is loaded - tk.call('set', 'tcl_wordchars', '[a-zA-Z0-9_]') - tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') - - -def _editor_window(parent): # htest # - # error if close master window first - timer event, after script - root = parent - fixwordbreaks(root) - if sys.argv[1:]: - filename = sys.argv[1] - else: - filename = None - macosxSupport.setupApp(root, None) - edit = EditorWindow(root=root, filename=filename) - edit.text.bind("<>", edit.close_event) - # Does not stop error, neither does following - # edit.text.bind("<>", edit.close_event) - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_editor_window) diff --git a/Lib/idlelib/FileList.py b/Lib/idlelib/FileList.py deleted file mode 100644 index a9989a8..0000000 --- a/Lib/idlelib/FileList.py +++ /dev/null @@ -1,129 +0,0 @@ -import os -from tkinter import * -import tkinter.messagebox as tkMessageBox - - -class FileList: - - # N.B. this import overridden in PyShellFileList. - from idlelib.EditorWindow import EditorWindow - - def __init__(self, root): - self.root = root - self.dict = {} - self.inversedict = {} - self.vars = {} # For EditorWindow.getrawvar (shared Tcl variables) - - def open(self, filename, action=None): - assert filename - filename = self.canonize(filename) - if os.path.isdir(filename): - # This can happen when bad filename is passed on command line: - tkMessageBox.showerror( - "File Error", - "%r is a directory." % (filename,), - master=self.root) - return None - key = os.path.normcase(filename) - if key in self.dict: - edit = self.dict[key] - edit.top.wakeup() - return edit - if action: - # Don't create window, perform 'action', e.g. open in same window - return action(filename) - else: - edit = self.EditorWindow(self, filename, key) - if edit.good_load: - return edit - else: - edit._close() - return None - - def gotofileline(self, filename, lineno=None): - edit = self.open(filename) - if edit is not None and lineno is not None: - edit.gotoline(lineno) - - def new(self, filename=None): - return self.EditorWindow(self, filename) - - def close_all_callback(self, *args, **kwds): - for edit in list(self.inversedict): - reply = edit.close() - if reply == "cancel": - break - return "break" - - def unregister_maybe_terminate(self, edit): - try: - key = self.inversedict[edit] - except KeyError: - print("Don't know this EditorWindow object. (close)") - return - if key: - del self.dict[key] - del self.inversedict[edit] - if not self.inversedict: - self.root.quit() - - def filename_changed_edit(self, edit): - edit.saved_change_hook() - try: - key = self.inversedict[edit] - except KeyError: - print("Don't know this EditorWindow object. (rename)") - return - filename = edit.io.filename - if not filename: - if key: - del self.dict[key] - self.inversedict[edit] = None - return - filename = self.canonize(filename) - newkey = os.path.normcase(filename) - if newkey == key: - return - if newkey in self.dict: - conflict = self.dict[newkey] - self.inversedict[conflict] = None - tkMessageBox.showerror( - "Name Conflict", - "You now have multiple edit windows open for %r" % (filename,), - master=self.root) - self.dict[newkey] = edit - self.inversedict[edit] = newkey - if key: - try: - del self.dict[key] - except KeyError: - pass - - def canonize(self, filename): - if not os.path.isabs(filename): - try: - pwd = os.getcwd() - except OSError: - pass - else: - filename = os.path.join(pwd, filename) - return os.path.normpath(filename) - - -def _test(): - from idlelib.EditorWindow import fixwordbreaks - import sys - root = Tk() - fixwordbreaks(root) - root.withdraw() - flist = FileList(root) - if sys.argv[1:]: - for filename in sys.argv[1:]: - flist.open(filename) - else: - flist.new() - if flist.inversedict: - root.mainloop() - -if __name__ == '__main__': - _test() diff --git a/Lib/idlelib/FormatParagraph.py b/Lib/idlelib/FormatParagraph.py deleted file mode 100644 index 7a9d185..0000000 --- a/Lib/idlelib/FormatParagraph.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Extension to format a paragraph or selection to a max width. - -Does basic, standard text formatting, and also understands Python -comment blocks. Thus, for editing Python source code, this -extension is really only suitable for reformatting these comment -blocks or triple-quoted strings. - -Known problems with comment reformatting: -* If there is a selection marked, and the first line of the - selection is not complete, the block will probably not be detected - as comments, and will have the normal "text formatting" rules - applied. -* If a comment block has leading whitespace that mixes tabs and - spaces, they will not be considered part of the same block. -* Fancy comments, like this bulleted list, aren't handled :-) -""" - -import re -from idlelib.configHandler import idleConf - -class FormatParagraph: - - menudefs = [ - ('format', [ # /s/edit/format dscherer@cmu.edu - ('Format Paragraph', '<>'), - ]) - ] - - def __init__(self, editwin): - self.editwin = editwin - - def close(self): - self.editwin = None - - def format_paragraph_event(self, event, limit=None): - """Formats paragraph to a max width specified in idleConf. - - If text is selected, format_paragraph_event will start breaking lines - at the max width, starting from the beginning selection. - - If no text is selected, format_paragraph_event uses the current - cursor location to determine the paragraph (lines of text surrounded - by blank lines) and formats it. - - The length limit parameter is for testing with a known value. - """ - if limit is None: - # The default length limit is that defined by pep8 - limit = idleConf.GetOption( - 'extensions', 'FormatParagraph', 'max-width', - type='int', default=72) - text = self.editwin.text - first, last = self.editwin.get_selection_indices() - if first and last: - data = text.get(first, last) - comment_header = get_comment_header(data) - else: - first, last, comment_header, data = \ - find_paragraph(text, text.index("insert")) - if comment_header: - newdata = reformat_comment(data, limit, comment_header) - else: - newdata = reformat_paragraph(data, limit) - text.tag_remove("sel", "1.0", "end") - - if newdata != data: - text.mark_set("insert", first) - text.undo_block_start() - text.delete(first, last) - text.insert(first, newdata) - text.undo_block_stop() - else: - text.mark_set("insert", last) - text.see("insert") - return "break" - -def find_paragraph(text, mark): - """Returns the start/stop indices enclosing the paragraph that mark is in. - - Also returns the comment format string, if any, and paragraph of text - between the start/stop indices. - """ - lineno, col = map(int, mark.split(".")) - line = text.get("%d.0" % lineno, "%d.end" % lineno) - - # Look for start of next paragraph if the index passed in is a blank line - while text.compare("%d.0" % lineno, "<", "end") and is_all_white(line): - lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - first_lineno = lineno - comment_header = get_comment_header(line) - comment_header_len = len(comment_header) - - # Once start line found, search for end of paragraph (a blank line) - while get_comment_header(line)==comment_header and \ - not is_all_white(line[comment_header_len:]): - lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - last = "%d.0" % lineno - - # Search back to beginning of paragraph (first blank line before) - lineno = first_lineno - 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - while lineno > 0 and \ - get_comment_header(line)==comment_header and \ - not is_all_white(line[comment_header_len:]): - lineno = lineno - 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - first = "%d.0" % (lineno+1) - - return first, last, comment_header, text.get(first, last) - -# This should perhaps be replaced with textwrap.wrap -def reformat_paragraph(data, limit): - """Return data reformatted to specified width (limit).""" - lines = data.split("\n") - i = 0 - n = len(lines) - while i < n and is_all_white(lines[i]): - i = i+1 - if i >= n: - return data - indent1 = get_indent(lines[i]) - if i+1 < n and not is_all_white(lines[i+1]): - indent2 = get_indent(lines[i+1]) - else: - indent2 = indent1 - new = lines[:i] - partial = indent1 - while i < n and not is_all_white(lines[i]): - # XXX Should take double space after period (etc.) into account - words = re.split("(\s+)", lines[i]) - for j in range(0, len(words), 2): - word = words[j] - if not word: - continue # Can happen when line ends in whitespace - if len((partial + word).expandtabs()) > limit and \ - partial != indent1: - new.append(partial.rstrip()) - partial = indent2 - partial = partial + word + " " - if j+1 < len(words) and words[j+1] != " ": - partial = partial + " " - i = i+1 - new.append(partial.rstrip()) - # XXX Should reformat remaining paragraphs as well - new.extend(lines[i:]) - return "\n".join(new) - -def reformat_comment(data, limit, comment_header): - """Return data reformatted to specified width with comment header.""" - - # Remove header from the comment lines - lc = len(comment_header) - data = "\n".join(line[lc:] for line in data.split("\n")) - # Reformat to maxformatwidth chars or a 20 char width, - # whichever is greater. - format_width = max(limit - len(comment_header), 20) - newdata = reformat_paragraph(data, format_width) - # re-split and re-insert the comment header. - newdata = newdata.split("\n") - # If the block ends in a \n, we dont want the comment prefix - # inserted after it. (Im not sure it makes sense to reformat a - # comment block that is not made of complete lines, but whatever!) - # Can't think of a clean solution, so we hack away - block_suffix = "" - if not newdata[-1]: - block_suffix = "\n" - newdata = newdata[:-1] - return '\n'.join(comment_header+line for line in newdata) + block_suffix - -def is_all_white(line): - """Return True if line is empty or all whitespace.""" - - return re.match(r"^\s*$", line) is not None - -def get_indent(line): - """Return the initial space or tab indent of line.""" - return re.match(r"^([ \t]*)", line).group() - -def get_comment_header(line): - """Return string with leading whitespace and '#' from line or ''. - - A null return indicates that the line is not a comment line. A non- - null return, such as ' #', will be used to find the other lines of - a comment block with the same indent. - """ - m = re.match(r"^([ \t]*#*)", line) - if m is None: return "" - return m.group(1) - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_formatparagraph', - verbosity=2, exit=False) diff --git a/Lib/idlelib/GrepDialog.py b/Lib/idlelib/GrepDialog.py deleted file mode 100644 index 721b231..0000000 --- a/Lib/idlelib/GrepDialog.py +++ /dev/null @@ -1,158 +0,0 @@ -import os -import fnmatch -import re # for htest -import sys -from tkinter import StringVar, BooleanVar, Checkbutton # for GrepDialog -from tkinter import Tk, Text, Button, SEL, END # for htest -from idlelib import SearchEngine -from idlelib.SearchDialogBase import SearchDialogBase -# Importing OutputWindow fails due to import loop -# EditorWindow -> GrepDialop -> OutputWindow -> EditorWindow - -def grep(text, io=None, flist=None): - root = text._root() - engine = SearchEngine.get(root) - if not hasattr(engine, "_grepdialog"): - engine._grepdialog = GrepDialog(root, engine, flist) - dialog = engine._grepdialog - searchphrase = text.get("sel.first", "sel.last") - dialog.open(text, searchphrase, io) - -class GrepDialog(SearchDialogBase): - - title = "Find in Files Dialog" - icon = "Grep" - needwrapbutton = 0 - - def __init__(self, root, engine, flist): - SearchDialogBase.__init__(self, root, engine) - self.flist = flist - self.globvar = StringVar(root) - self.recvar = BooleanVar(root) - - def open(self, text, searchphrase, io=None): - SearchDialogBase.open(self, text, searchphrase) - if io: - path = io.filename or "" - else: - path = "" - dir, base = os.path.split(path) - head, tail = os.path.splitext(base) - if not tail: - tail = ".py" - self.globvar.set(os.path.join(dir, "*" + tail)) - - def create_entries(self): - SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar)[0] - - def create_other_buttons(self): - f = self.make_frame()[0] - - btn = Checkbutton(f, anchor="w", - variable=self.recvar, - text="Recurse down subdirectories") - btn.pack(side="top", fill="both") - btn.select() - - def create_command_buttons(self): - SearchDialogBase.create_command_buttons(self) - self.make_button("Search Files", self.default_command, 1) - - def default_command(self, event=None): - prog = self.engine.getprog() - if not prog: - return - path = self.globvar.get() - if not path: - self.top.bell() - return - from idlelib.OutputWindow import OutputWindow # leave here! - save = sys.stdout - try: - sys.stdout = OutputWindow(self.flist) - self.grep_it(prog, path) - finally: - sys.stdout = save - - def grep_it(self, prog, path): - dir, base = os.path.split(path) - list = self.findfiles(dir, base, self.recvar.get()) - list.sort() - self.close() - pat = self.engine.getpat() - print("Searching %r in %s ..." % (pat, path)) - hits = 0 - try: - for fn in list: - try: - with open(fn, errors='replace') as f: - for lineno, line in enumerate(f, 1): - if line[-1:] == '\n': - line = line[:-1] - if prog.search(line): - sys.stdout.write("%s: %s: %s\n" % - (fn, lineno, line)) - hits += 1 - except OSError as msg: - print(msg) - print(("Hits found: %s\n" - "(Hint: right-click to open locations.)" - % hits) if hits else "No hits.") - except AttributeError: - # Tk window has been closed, OutputWindow.text = None, - # so in OW.write, OW.text.insert fails. - pass - - def findfiles(self, dir, base, rec): - try: - names = os.listdir(dir or os.curdir) - except OSError as msg: - print(msg) - return [] - list = [] - subdirs = [] - for name in names: - fn = os.path.join(dir, name) - if os.path.isdir(fn): - subdirs.append(fn) - else: - if fnmatch.fnmatch(name, base): - list.append(fn) - if rec: - for subdir in subdirs: - list.extend(self.findfiles(subdir, base, rec)) - return list - - def close(self, event=None): - if self.top: - self.top.grab_release() - self.top.withdraw() - - -def _grep_dialog(parent): # htest # - from idlelib.PyShell import PyShellFileList - root = Tk() - root.title("Test GrepDialog") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - - flist = PyShellFileList(root) - text = Text(root, height=5) - text.pack() - - def show_grep_dialog(): - text.tag_add(SEL, "1.0", END) - grep(text, flist=flist) - text.tag_remove(SEL, "1.0", END) - - button = Button(root, text="Show GrepDialog", command=show_grep_dialog) - button.pack() - root.mainloop() - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_grep', verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(_grep_dialog) diff --git a/Lib/idlelib/HyperParser.py b/Lib/idlelib/HyperParser.py deleted file mode 100644 index 77cb057..0000000 --- a/Lib/idlelib/HyperParser.py +++ /dev/null @@ -1,313 +0,0 @@ -"""Provide advanced parsing abilities for ParenMatch and other extensions. - -HyperParser uses PyParser. PyParser mostly gives information on the -proper indentation of code. HyperParser gives additional information on -the structure of code. -""" - -import string -from keyword import iskeyword -from idlelib import PyParse - - -# all ASCII chars that may be in an identifier -_ASCII_ID_CHARS = frozenset(string.ascii_letters + string.digits + "_") -# all ASCII chars that may be the first char of an identifier -_ASCII_ID_FIRST_CHARS = frozenset(string.ascii_letters + "_") - -# lookup table for whether 7-bit ASCII chars are valid in a Python identifier -_IS_ASCII_ID_CHAR = [(chr(x) in _ASCII_ID_CHARS) for x in range(128)] -# lookup table for whether 7-bit ASCII chars are valid as the first -# char in a Python identifier -_IS_ASCII_ID_FIRST_CHAR = \ - [(chr(x) in _ASCII_ID_FIRST_CHARS) for x in range(128)] - - -class HyperParser: - def __init__(self, editwin, index): - "To initialize, analyze the surroundings of the given index." - - self.editwin = editwin - self.text = text = editwin.text - - parser = PyParse.Parser(editwin.indentwidth, editwin.tabwidth) - - def index2line(index): - return int(float(index)) - lno = index2line(text.index(index)) - - if not editwin.context_use_ps1: - for context in editwin.num_context_lines: - startat = max(lno - context, 1) - startatindex = repr(startat) + ".0" - stopatindex = "%d.end" % lno - # We add the newline because PyParse requires a newline - # at end. We add a space so that index won't be at end - # of line, so that its status will be the same as the - # char before it, if should. - parser.set_str(text.get(startatindex, stopatindex)+' \n') - bod = parser.find_good_parse_start( - editwin._build_char_in_string_func(startatindex)) - if bod is not None or startat == 1: - break - parser.set_lo(bod or 0) - else: - r = text.tag_prevrange("console", index) - if r: - startatindex = r[1] - else: - startatindex = "1.0" - stopatindex = "%d.end" % lno - # We add the newline because PyParse requires it. We add a - # space so that index won't be at end of line, so that its - # status will be the same as the char before it, if should. - parser.set_str(text.get(startatindex, stopatindex)+' \n') - parser.set_lo(0) - - # We want what the parser has, minus the last newline and space. - self.rawtext = parser.str[:-2] - # Parser.str apparently preserves the statement we are in, so - # that stopatindex can be used to synchronize the string with - # the text box indices. - self.stopatindex = stopatindex - self.bracketing = parser.get_last_stmt_bracketing() - # find which pairs of bracketing are openers. These always - # correspond to a character of rawtext. - self.isopener = [i>0 and self.bracketing[i][1] > - self.bracketing[i-1][1] - for i in range(len(self.bracketing))] - - self.set_index(index) - - def set_index(self, index): - """Set the index to which the functions relate. - - The index must be in the same statement. - """ - indexinrawtext = (len(self.rawtext) - - len(self.text.get(index, self.stopatindex))) - if indexinrawtext < 0: - raise ValueError("Index %s precedes the analyzed statement" - % index) - self.indexinrawtext = indexinrawtext - # find the rightmost bracket to which index belongs - self.indexbracket = 0 - while (self.indexbracket < len(self.bracketing)-1 and - self.bracketing[self.indexbracket+1][0] < self.indexinrawtext): - self.indexbracket += 1 - if (self.indexbracket < len(self.bracketing)-1 and - self.bracketing[self.indexbracket+1][0] == self.indexinrawtext and - not self.isopener[self.indexbracket+1]): - self.indexbracket += 1 - - def is_in_string(self): - """Is the index given to the HyperParser in a string?""" - # The bracket to which we belong should be an opener. - # If it's an opener, it has to have a character. - return (self.isopener[self.indexbracket] and - self.rawtext[self.bracketing[self.indexbracket][0]] - in ('"', "'")) - - def is_in_code(self): - """Is the index given to the HyperParser in normal code?""" - return (not self.isopener[self.indexbracket] or - self.rawtext[self.bracketing[self.indexbracket][0]] - not in ('#', '"', "'")) - - def get_surrounding_brackets(self, openers='([{', mustclose=False): - """Return bracket indexes or None. - - If the index given to the HyperParser is surrounded by a - bracket defined in openers (or at least has one before it), - return the indices of the opening bracket and the closing - bracket (or the end of line, whichever comes first). - - If it is not surrounded by brackets, or the end of line comes - before the closing bracket and mustclose is True, returns None. - """ - - bracketinglevel = self.bracketing[self.indexbracket][1] - before = self.indexbracket - while (not self.isopener[before] or - self.rawtext[self.bracketing[before][0]] not in openers or - self.bracketing[before][1] > bracketinglevel): - before -= 1 - if before < 0: - return None - bracketinglevel = min(bracketinglevel, self.bracketing[before][1]) - after = self.indexbracket + 1 - while (after < len(self.bracketing) and - self.bracketing[after][1] >= bracketinglevel): - after += 1 - - beforeindex = self.text.index("%s-%dc" % - (self.stopatindex, len(self.rawtext)-self.bracketing[before][0])) - if (after >= len(self.bracketing) or - self.bracketing[after][0] > len(self.rawtext)): - if mustclose: - return None - afterindex = self.stopatindex - else: - # We are after a real char, so it is a ')' and we give the - # index before it. - afterindex = self.text.index( - "%s-%dc" % (self.stopatindex, - len(self.rawtext)-(self.bracketing[after][0]-1))) - - return beforeindex, afterindex - - # the set of built-in identifiers which are also keywords, - # i.e. keyword.iskeyword() returns True for them - _ID_KEYWORDS = frozenset({"True", "False", "None"}) - - @classmethod - def _eat_identifier(cls, str, limit, pos): - """Given a string and pos, return the number of chars in the - identifier which ends at pos, or 0 if there is no such one. - - This ignores non-identifier eywords are not identifiers. - """ - is_ascii_id_char = _IS_ASCII_ID_CHAR - - # Start at the end (pos) and work backwards. - i = pos - - # Go backwards as long as the characters are valid ASCII - # identifier characters. This is an optimization, since it - # is faster in the common case where most of the characters - # are ASCII. - while i > limit and ( - ord(str[i - 1]) < 128 and - is_ascii_id_char[ord(str[i - 1])] - ): - i -= 1 - - # If the above loop ended due to reaching a non-ASCII - # character, continue going backwards using the most generic - # test for whether a string contains only valid identifier - # characters. - if i > limit and ord(str[i - 1]) >= 128: - while i - 4 >= limit and ('a' + str[i - 4:pos]).isidentifier(): - i -= 4 - if i - 2 >= limit and ('a' + str[i - 2:pos]).isidentifier(): - i -= 2 - if i - 1 >= limit and ('a' + str[i - 1:pos]).isidentifier(): - i -= 1 - - # The identifier candidate starts here. If it isn't a valid - # identifier, don't eat anything. At this point that is only - # possible if the first character isn't a valid first - # character for an identifier. - if not str[i:pos].isidentifier(): - return 0 - elif i < pos: - # All characters in str[i:pos] are valid ASCII identifier - # characters, so it is enough to check that the first is - # valid as the first character of an identifier. - if not _IS_ASCII_ID_FIRST_CHAR[ord(str[i])]: - return 0 - - # All keywords are valid identifiers, but should not be - # considered identifiers here, except for True, False and None. - if i < pos and ( - iskeyword(str[i:pos]) and - str[i:pos] not in cls._ID_KEYWORDS - ): - return 0 - - return pos - i - - # This string includes all chars that may be in a white space - _whitespace_chars = " \t\n\\" - - def get_expression(self): - """Return a string with the Python expression which ends at the - given index, which is empty if there is no real one. - """ - if not self.is_in_code(): - raise ValueError("get_expression should only be called" - "if index is inside a code.") - - rawtext = self.rawtext - bracketing = self.bracketing - - brck_index = self.indexbracket - brck_limit = bracketing[brck_index][0] - pos = self.indexinrawtext - - last_identifier_pos = pos - postdot_phase = True - - while 1: - # Eat whitespaces, comments, and if postdot_phase is False - a dot - while 1: - if pos>brck_limit and rawtext[pos-1] in self._whitespace_chars: - # Eat a whitespace - pos -= 1 - elif (not postdot_phase and - pos > brck_limit and rawtext[pos-1] == '.'): - # Eat a dot - pos -= 1 - postdot_phase = True - # The next line will fail if we are *inside* a comment, - # but we shouldn't be. - elif (pos == brck_limit and brck_index > 0 and - rawtext[bracketing[brck_index-1][0]] == '#'): - # Eat a comment - brck_index -= 2 - brck_limit = bracketing[brck_index][0] - pos = bracketing[brck_index+1][0] - else: - # If we didn't eat anything, quit. - break - - if not postdot_phase: - # We didn't find a dot, so the expression end at the - # last identifier pos. - break - - ret = self._eat_identifier(rawtext, brck_limit, pos) - if ret: - # There is an identifier to eat - pos = pos - ret - last_identifier_pos = pos - # Now, to continue the search, we must find a dot. - postdot_phase = False - # (the loop continues now) - - elif pos == brck_limit: - # We are at a bracketing limit. If it is a closing - # bracket, eat the bracket, otherwise, stop the search. - level = bracketing[brck_index][1] - while brck_index > 0 and bracketing[brck_index-1][1] > level: - brck_index -= 1 - if bracketing[brck_index][0] == brck_limit: - # We were not at the end of a closing bracket - break - pos = bracketing[brck_index][0] - brck_index -= 1 - brck_limit = bracketing[brck_index][0] - last_identifier_pos = pos - if rawtext[pos] in "([": - # [] and () may be used after an identifier, so we - # continue. postdot_phase is True, so we don't allow a dot. - pass - else: - # We can't continue after other types of brackets - if rawtext[pos] in "'\"": - # Scan a string prefix - while pos > 0 and rawtext[pos - 1] in "rRbBuU": - pos -= 1 - last_identifier_pos = pos - break - - else: - # We've found an operator or something. - break - - return rawtext[last_identifier_pos:self.indexinrawtext] - - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_hyperparser', verbosity=2) diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py deleted file mode 100644 index 84f39a2..0000000 --- a/Lib/idlelib/IOBinding.py +++ /dev/null @@ -1,565 +0,0 @@ -import codecs -from codecs import BOM_UTF8 -import os -import re -import shlex -import sys -import tempfile - -import tkinter.filedialog as tkFileDialog -import tkinter.messagebox as tkMessageBox -from tkinter.simpledialog import askstring - -from idlelib.configHandler import idleConf - - -# Try setting the locale, so that we can find out -# what encoding to use -try: - import locale - locale.setlocale(locale.LC_CTYPE, "") -except (ImportError, locale.Error): - pass - -# Encoding for file names -filesystemencoding = sys.getfilesystemencoding() ### currently unused - -locale_encoding = 'ascii' -if sys.platform == 'win32': - # On Windows, we could use "mbcs". However, to give the user - # a portable encoding name, we need to find the code page - try: - locale_encoding = locale.getdefaultlocale()[1] - codecs.lookup(locale_encoding) - except LookupError: - pass -else: - try: - # Different things can fail here: the locale module may not be - # loaded, it may not offer nl_langinfo, or CODESET, or the - # resulting codeset may be unknown to Python. We ignore all - # these problems, falling back to ASCII - locale_encoding = locale.nl_langinfo(locale.CODESET) - if locale_encoding is None or locale_encoding is '': - # situation occurs on Mac OS X - locale_encoding = 'ascii' - codecs.lookup(locale_encoding) - except (NameError, AttributeError, LookupError): - # Try getdefaultlocale: it parses environment variables, - # which may give a clue. Unfortunately, getdefaultlocale has - # bugs that can cause ValueError. - try: - locale_encoding = locale.getdefaultlocale()[1] - if locale_encoding is None or locale_encoding is '': - # situation occurs on Mac OS X - locale_encoding = 'ascii' - codecs.lookup(locale_encoding) - except (ValueError, LookupError): - pass - -locale_encoding = locale_encoding.lower() - -encoding = locale_encoding ### KBK 07Sep07 This is used all over IDLE, check! - ### 'encoding' is used below in encode(), check! - -coding_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) - -def coding_spec(data): - """Return the encoding declaration according to PEP 263. - - When checking encoded data, only the first two lines should be passed - in to avoid a UnicodeDecodeError if the rest of the data is not unicode. - The first two lines would contain the encoding specification. - - Raise a LookupError if the encoding is declared but unknown. - """ - if isinstance(data, bytes): - # This encoding might be wrong. However, the coding - # spec must be ASCII-only, so any non-ASCII characters - # around here will be ignored. Decoding to Latin-1 should - # never fail (except for memory outage) - lines = data.decode('iso-8859-1') - else: - lines = data - # consider only the first two lines - if '\n' in lines: - lst = lines.split('\n', 2)[:2] - elif '\r' in lines: - lst = lines.split('\r', 2)[:2] - else: - lst = [lines] - for line in lst: - match = coding_re.match(line) - if match is not None: - break - if not blank_re.match(line): - return None - else: - return None - name = match.group(1) - try: - codecs.lookup(name) - except LookupError: - # The standard encoding error does not indicate the encoding - raise LookupError("Unknown encoding: "+name) - return name - - -class IOBinding: - - def __init__(self, editwin): - self.editwin = editwin - self.text = editwin.text - self.__id_open = self.text.bind("<>", self.open) - self.__id_save = self.text.bind("<>", self.save) - self.__id_saveas = self.text.bind("<>", - self.save_as) - self.__id_savecopy = self.text.bind("<>", - self.save_a_copy) - self.fileencoding = None - self.__id_print = self.text.bind("<>", self.print_window) - - def close(self): - # Undo command bindings - self.text.unbind("<>", self.__id_open) - self.text.unbind("<>", self.__id_save) - self.text.unbind("<>",self.__id_saveas) - self.text.unbind("<>", self.__id_savecopy) - self.text.unbind("<>", self.__id_print) - # Break cycles - self.editwin = None - self.text = None - self.filename_change_hook = None - - def get_saved(self): - return self.editwin.get_saved() - - def set_saved(self, flag): - self.editwin.set_saved(flag) - - def reset_undo(self): - self.editwin.reset_undo() - - filename_change_hook = None - - def set_filename_change_hook(self, hook): - self.filename_change_hook = hook - - filename = None - dirname = None - - def set_filename(self, filename): - if filename and os.path.isdir(filename): - self.filename = None - self.dirname = filename - else: - self.filename = filename - self.dirname = None - self.set_saved(1) - if self.filename_change_hook: - self.filename_change_hook() - - def open(self, event=None, editFile=None): - flist = self.editwin.flist - # Save in case parent window is closed (ie, during askopenfile()). - if flist: - if not editFile: - filename = self.askopenfile() - else: - filename=editFile - if filename: - # If editFile is valid and already open, flist.open will - # shift focus to its existing window. - # If the current window exists and is a fresh unnamed, - # unmodified editor window (not an interpreter shell), - # pass self.loadfile to flist.open so it will load the file - # in the current window (if the file is not already open) - # instead of a new window. - if (self.editwin and - not getattr(self.editwin, 'interp', None) and - not self.filename and - self.get_saved()): - flist.open(filename, self.loadfile) - else: - flist.open(filename) - else: - if self.text: - self.text.focus_set() - return "break" - - # Code for use outside IDLE: - if self.get_saved(): - reply = self.maybesave() - if reply == "cancel": - self.text.focus_set() - return "break" - if not editFile: - filename = self.askopenfile() - else: - filename=editFile - if filename: - self.loadfile(filename) - else: - self.text.focus_set() - return "break" - - eol = r"(\r\n)|\n|\r" # \r\n (Windows), \n (UNIX), or \r (Mac) - eol_re = re.compile(eol) - eol_convention = os.linesep # default - - def loadfile(self, filename): - try: - # open the file in binary mode so that we can handle - # end-of-line convention ourselves. - with open(filename, 'rb') as f: - two_lines = f.readline() + f.readline() - f.seek(0) - bytes = f.read() - except OSError as msg: - tkMessageBox.showerror("I/O Error", str(msg), parent=self.text) - return False - chars, converted = self._decode(two_lines, bytes) - if chars is None: - tkMessageBox.showerror("Decoding Error", - "File %s\nFailed to Decode" % filename, - parent=self.text) - return False - # We now convert all end-of-lines to '\n's - firsteol = self.eol_re.search(chars) - if firsteol: - self.eol_convention = firsteol.group(0) - chars = self.eol_re.sub(r"\n", chars) - self.text.delete("1.0", "end") - self.set_filename(None) - self.text.insert("1.0", chars) - self.reset_undo() - self.set_filename(filename) - if converted: - # We need to save the conversion results first - # before being able to execute the code - self.set_saved(False) - self.text.mark_set("insert", "1.0") - self.text.yview("insert") - self.updaterecentfileslist(filename) - return True - - def _decode(self, two_lines, bytes): - "Create a Unicode string." - chars = None - # Check presence of a UTF-8 signature first - if bytes.startswith(BOM_UTF8): - try: - chars = bytes[3:].decode("utf-8") - except UnicodeDecodeError: - # has UTF-8 signature, but fails to decode... - return None, False - else: - # Indicates that this file originally had a BOM - self.fileencoding = 'BOM' - return chars, False - # Next look for coding specification - try: - enc = coding_spec(two_lines) - except LookupError as name: - tkMessageBox.showerror( - title="Error loading the file", - message="The encoding '%s' is not known to this Python "\ - "installation. The file may not display correctly" % name, - parent = self.text) - enc = None - except UnicodeDecodeError: - return None, False - if enc: - try: - chars = str(bytes, enc) - self.fileencoding = enc - return chars, False - except UnicodeDecodeError: - pass - # Try ascii: - try: - chars = str(bytes, 'ascii') - self.fileencoding = None - return chars, False - except UnicodeDecodeError: - pass - # Try utf-8: - try: - chars = str(bytes, 'utf-8') - self.fileencoding = 'utf-8' - return chars, False - except UnicodeDecodeError: - pass - # Finally, try the locale's encoding. This is deprecated; - # the user should declare a non-ASCII encoding - try: - # Wait for the editor window to appear - self.editwin.text.update() - enc = askstring( - "Specify file encoding", - "The file's encoding is invalid for Python 3.x.\n" - "IDLE will convert it to UTF-8.\n" - "What is the current encoding of the file?", - initialvalue = locale_encoding, - parent = self.editwin.text) - - if enc: - chars = str(bytes, enc) - self.fileencoding = None - return chars, True - except (UnicodeDecodeError, LookupError): - pass - return None, False # None on failure - - def maybesave(self): - if self.get_saved(): - return "yes" - message = "Do you want to save %s before closing?" % ( - self.filename or "this untitled document") - confirm = tkMessageBox.askyesnocancel( - title="Save On Close", - message=message, - default=tkMessageBox.YES, - parent=self.text) - if confirm: - reply = "yes" - self.save(None) - if not self.get_saved(): - reply = "cancel" - elif confirm is None: - reply = "cancel" - else: - reply = "no" - self.text.focus_set() - return reply - - def save(self, event): - if not self.filename: - self.save_as(event) - else: - if self.writefile(self.filename): - self.set_saved(True) - try: - self.editwin.store_file_breaks() - except AttributeError: # may be a PyShell - pass - self.text.focus_set() - return "break" - - def save_as(self, event): - filename = self.asksavefile() - if filename: - if self.writefile(filename): - self.set_filename(filename) - self.set_saved(1) - try: - self.editwin.store_file_breaks() - except AttributeError: - pass - self.text.focus_set() - self.updaterecentfileslist(filename) - return "break" - - def save_a_copy(self, event): - filename = self.asksavefile() - if filename: - self.writefile(filename) - self.text.focus_set() - self.updaterecentfileslist(filename) - return "break" - - def writefile(self, filename): - self.fixlastline() - text = self.text.get("1.0", "end-1c") - if self.eol_convention != "\n": - text = text.replace("\n", self.eol_convention) - chars = self.encode(text) - try: - with open(filename, "wb") as f: - f.write(chars) - return True - except OSError as msg: - tkMessageBox.showerror("I/O Error", str(msg), - parent=self.text) - return False - - def encode(self, chars): - if isinstance(chars, bytes): - # This is either plain ASCII, or Tk was returning mixed-encoding - # text to us. Don't try to guess further. - return chars - # Preserve a BOM that might have been present on opening - if self.fileencoding == 'BOM': - return BOM_UTF8 + chars.encode("utf-8") - # See whether there is anything non-ASCII in it. - # If not, no need to figure out the encoding. - try: - return chars.encode('ascii') - except UnicodeError: - pass - # Check if there is an encoding declared - try: - # a string, let coding_spec slice it to the first two lines - enc = coding_spec(chars) - failed = None - except LookupError as msg: - failed = msg - enc = None - else: - if not enc: - # PEP 3120: default source encoding is UTF-8 - enc = 'utf-8' - if enc: - try: - return chars.encode(enc) - except UnicodeError: - failed = "Invalid encoding '%s'" % enc - tkMessageBox.showerror( - "I/O Error", - "%s.\nSaving as UTF-8" % failed, - parent = self.text) - # Fallback: save as UTF-8, with BOM - ignoring the incorrect - # declared encoding - return BOM_UTF8 + chars.encode("utf-8") - - def fixlastline(self): - c = self.text.get("end-2c") - if c != '\n': - self.text.insert("end-1c", "\n") - - def print_window(self, event): - confirm = tkMessageBox.askokcancel( - title="Print", - message="Print to Default Printer", - default=tkMessageBox.OK, - parent=self.text) - if not confirm: - self.text.focus_set() - return "break" - tempfilename = None - saved = self.get_saved() - if saved: - filename = self.filename - # shell undo is reset after every prompt, looks saved, probably isn't - if not saved or filename is None: - (tfd, tempfilename) = tempfile.mkstemp(prefix='IDLE_tmp_') - filename = tempfilename - os.close(tfd) - if not self.writefile(tempfilename): - os.unlink(tempfilename) - return "break" - platform = os.name - printPlatform = True - if platform == 'posix': #posix platform - command = idleConf.GetOption('main','General', - 'print-command-posix') - command = command + " 2>&1" - elif platform == 'nt': #win32 platform - command = idleConf.GetOption('main','General','print-command-win') - else: #no printing for this platform - printPlatform = False - if printPlatform: #we can try to print for this platform - command = command % shlex.quote(filename) - pipe = os.popen(command, "r") - # things can get ugly on NT if there is no printer available. - output = pipe.read().strip() - status = pipe.close() - if status: - output = "Printing failed (exit status 0x%x)\n" % \ - status + output - if output: - output = "Printing command: %s\n" % repr(command) + output - tkMessageBox.showerror("Print status", output, parent=self.text) - else: #no printing for this platform - message = "Printing is not enabled for this platform: %s" % platform - tkMessageBox.showinfo("Print status", message, parent=self.text) - if tempfilename: - os.unlink(tempfilename) - return "break" - - opendialog = None - savedialog = None - - filetypes = [ - ("Python files", "*.py *.pyw", "TEXT"), - ("Text files", "*.txt", "TEXT"), - ("All files", "*"), - ] - - defaultextension = '.py' if sys.platform == 'darwin' else '' - - def askopenfile(self): - dir, base = self.defaultfilename("open") - if not self.opendialog: - self.opendialog = tkFileDialog.Open(parent=self.text, - filetypes=self.filetypes) - filename = self.opendialog.show(initialdir=dir, initialfile=base) - return filename - - def defaultfilename(self, mode="open"): - if self.filename: - return os.path.split(self.filename) - elif self.dirname: - return self.dirname, "" - else: - try: - pwd = os.getcwd() - except OSError: - pwd = "" - return pwd, "" - - def asksavefile(self): - dir, base = self.defaultfilename("save") - if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs( - parent=self.text, - filetypes=self.filetypes, - defaultextension=self.defaultextension) - filename = self.savedialog.show(initialdir=dir, initialfile=base) - return filename - - def updaterecentfileslist(self,filename): - "Update recent file list on all editor windows" - if self.editwin.flist: - self.editwin.update_recent_files_list(filename) - -def _io_binding(parent): # htest # - from tkinter import Toplevel, Text - - root = Toplevel(parent) - root.title("Test IOBinding") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - class MyEditWin: - def __init__(self, text): - self.text = text - self.flist = None - self.text.bind("", self.open) - self.text.bind('', self.print) - self.text.bind("", self.save) - self.text.bind("", self.saveas) - self.text.bind('', self.savecopy) - def get_saved(self): return 0 - def set_saved(self, flag): pass - def reset_undo(self): pass - def open(self, event): - self.text.event_generate("<>") - def print(self, event): - self.text.event_generate("<>") - def save(self, event): - self.text.event_generate("<>") - def saveas(self, event): - self.text.event_generate("<>") - def savecopy(self, event): - self.text.event_generate("<>") - - text = Text(root) - text.pack() - text.focus_set() - editwin = MyEditWin(text) - IOBinding(editwin) - -if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_io_binding) diff --git a/Lib/idlelib/IdleHistory.py b/Lib/idlelib/IdleHistory.py deleted file mode 100644 index 078af29..0000000 --- a/Lib/idlelib/IdleHistory.py +++ /dev/null @@ -1,104 +0,0 @@ -"Implement Idle Shell history mechanism with History class" - -from idlelib.configHandler import idleConf - -class History: - ''' Implement Idle Shell history mechanism. - - store - Store source statement (called from PyShell.resetoutput). - fetch - Fetch stored statement matching prefix already entered. - history_next - Bound to <> event (default Alt-N). - history_prev - Bound to <> event (default Alt-P). - ''' - def __init__(self, text): - '''Initialize data attributes and bind event methods. - - .text - Idle wrapper of tk Text widget, with .bell(). - .history - source statements, possibly with multiple lines. - .prefix - source already entered at prompt; filters history list. - .pointer - index into history. - .cyclic - wrap around history list (or not). - ''' - self.text = text - self.history = [] - self.prefix = None - self.pointer = None - self.cyclic = idleConf.GetOption("main", "History", "cyclic", 1, "bool") - text.bind("<>", self.history_prev) - text.bind("<>", self.history_next) - - def history_next(self, event): - "Fetch later statement; start with ealiest if cyclic." - self.fetch(reverse=False) - return "break" - - def history_prev(self, event): - "Fetch earlier statement; start with most recent." - self.fetch(reverse=True) - return "break" - - def fetch(self, reverse): - '''Fetch statememt and replace current line in text widget. - - Set prefix and pointer as needed for successive fetches. - Reset them to None, None when returning to the start line. - Sound bell when return to start line or cannot leave a line - because cyclic is False. - ''' - nhist = len(self.history) - pointer = self.pointer - prefix = self.prefix - if pointer is not None and prefix is not None: - if self.text.compare("insert", "!=", "end-1c") or \ - self.text.get("iomark", "end-1c") != self.history[pointer]: - pointer = prefix = None - self.text.mark_set("insert", "end-1c") # != after cursor move - if pointer is None or prefix is None: - prefix = self.text.get("iomark", "end-1c") - if reverse: - pointer = nhist # will be decremented - else: - if self.cyclic: - pointer = -1 # will be incremented - else: # abort history_next - self.text.bell() - return - nprefix = len(prefix) - while 1: - pointer += -1 if reverse else 1 - if pointer < 0 or pointer >= nhist: - self.text.bell() - if not self.cyclic and pointer < 0: # abort history_prev - return - else: - if self.text.get("iomark", "end-1c") != prefix: - self.text.delete("iomark", "end-1c") - self.text.insert("iomark", prefix) - pointer = prefix = None - break - item = self.history[pointer] - if item[:nprefix] == prefix and len(item) > nprefix: - self.text.delete("iomark", "end-1c") - self.text.insert("iomark", item) - break - self.text.see("insert") - self.text.tag_remove("sel", "1.0", "end") - self.pointer = pointer - self.prefix = prefix - - def store(self, source): - "Store Shell input statement into history list." - source = source.strip() - if len(source) > 2: - # avoid duplicates - try: - self.history.remove(source) - except ValueError: - pass - self.history.append(source) - self.pointer = None - self.prefix = None - -if __name__ == "__main__": - from unittest import main - main('idlelib.idle_test.test_idlehistory', verbosity=2, exit=False) diff --git a/Lib/idlelib/MultiCall.py b/Lib/idlelib/MultiCall.py deleted file mode 100644 index 251a84d..0000000 --- a/Lib/idlelib/MultiCall.py +++ /dev/null @@ -1,446 +0,0 @@ -""" -MultiCall - a class which inherits its methods from a Tkinter widget (Text, for -example), but enables multiple calls of functions per virtual event - all -matching events will be called, not only the most specific one. This is done -by wrapping the event functions - event_add, event_delete and event_info. -MultiCall recognizes only a subset of legal event sequences. Sequences which -are not recognized are treated by the original Tk handling mechanism. A -more-specific event will be called before a less-specific event. - -The recognized sequences are complete one-event sequences (no emacs-style -Ctrl-X Ctrl-C, no shortcuts like <3>), for all types of events. -Key/Button Press/Release events can have modifiers. -The recognized modifiers are Shift, Control, Option and Command for Mac, and -Control, Alt, Shift, Meta/M for other platforms. - -For all events which were handled by MultiCall, a new member is added to the -event instance passed to the binded functions - mc_type. This is one of the -event type constants defined in this module (such as MC_KEYPRESS). -For Key/Button events (which are handled by MultiCall and may receive -modifiers), another member is added - mc_state. This member gives the state -of the recognized modifiers, as a combination of the modifier constants -also defined in this module (for example, MC_SHIFT). -Using these members is absolutely portable. - -The order by which events are called is defined by these rules: -1. A more-specific event will be called before a less-specific event. -2. A recently-binded event will be called before a previously-binded event, - unless this conflicts with the first rule. -Each function will be called at most once for each event. -""" - -import sys -import re -import tkinter - -# the event type constants, which define the meaning of mc_type -MC_KEYPRESS=0; MC_KEYRELEASE=1; MC_BUTTONPRESS=2; MC_BUTTONRELEASE=3; -MC_ACTIVATE=4; MC_CIRCULATE=5; MC_COLORMAP=6; MC_CONFIGURE=7; -MC_DEACTIVATE=8; MC_DESTROY=9; MC_ENTER=10; MC_EXPOSE=11; MC_FOCUSIN=12; -MC_FOCUSOUT=13; MC_GRAVITY=14; MC_LEAVE=15; MC_MAP=16; MC_MOTION=17; -MC_MOUSEWHEEL=18; MC_PROPERTY=19; MC_REPARENT=20; MC_UNMAP=21; MC_VISIBILITY=22; -# the modifier state constants, which define the meaning of mc_state -MC_SHIFT = 1<<0; MC_CONTROL = 1<<2; MC_ALT = 1<<3; MC_META = 1<<5 -MC_OPTION = 1<<6; MC_COMMAND = 1<<7 - -# define the list of modifiers, to be used in complex event types. -if sys.platform == "darwin": - _modifiers = (("Shift",), ("Control",), ("Option",), ("Command",)) - _modifier_masks = (MC_SHIFT, MC_CONTROL, MC_OPTION, MC_COMMAND) -else: - _modifiers = (("Control",), ("Alt",), ("Shift",), ("Meta", "M")) - _modifier_masks = (MC_CONTROL, MC_ALT, MC_SHIFT, MC_META) - -# a dictionary to map a modifier name into its number -_modifier_names = dict([(name, number) - for number in range(len(_modifiers)) - for name in _modifiers[number]]) - -# In 3.4, if no shell window is ever open, the underlying Tk widget is -# destroyed before .__del__ methods here are called. The following -# is used to selectively ignore shutdown exceptions to avoid -# 'Exception ignored' messages. See http://bugs.python.org/issue20167 -APPLICATION_GONE = "application has been destroyed" - -# A binder is a class which binds functions to one type of event. It has two -# methods: bind and unbind, which get a function and a parsed sequence, as -# returned by _parse_sequence(). There are two types of binders: -# _SimpleBinder handles event types with no modifiers and no detail. -# No Python functions are called when no events are binded. -# _ComplexBinder handles event types with modifiers and a detail. -# A Python function is called each time an event is generated. - -class _SimpleBinder: - def __init__(self, type, widget, widgetinst): - self.type = type - self.sequence = '<'+_types[type][0]+'>' - self.widget = widget - self.widgetinst = widgetinst - self.bindedfuncs = [] - self.handlerid = None - - def bind(self, triplet, func): - if not self.handlerid: - def handler(event, l = self.bindedfuncs, mc_type = self.type): - event.mc_type = mc_type - wascalled = {} - for i in range(len(l)-1, -1, -1): - func = l[i] - if func not in wascalled: - wascalled[func] = True - r = func(event) - if r: - return r - self.handlerid = self.widget.bind(self.widgetinst, - self.sequence, handler) - self.bindedfuncs.append(func) - - def unbind(self, triplet, func): - self.bindedfuncs.remove(func) - if not self.bindedfuncs: - self.widget.unbind(self.widgetinst, self.sequence, self.handlerid) - self.handlerid = None - - def __del__(self): - if self.handlerid: - try: - self.widget.unbind(self.widgetinst, self.sequence, - self.handlerid) - except tkinter.TclError as e: - if not APPLICATION_GONE in e.args[0]: - raise - -# An int in range(1 << len(_modifiers)) represents a combination of modifiers -# (if the least significent bit is on, _modifiers[0] is on, and so on). -# _state_subsets gives for each combination of modifiers, or *state*, -# a list of the states which are a subset of it. This list is ordered by the -# number of modifiers is the state - the most specific state comes first. -_states = range(1 << len(_modifiers)) -_state_names = [''.join(m[0]+'-' - for i, m in enumerate(_modifiers) - if (1 << i) & s) - for s in _states] - -def expand_substates(states): - '''For each item of states return a list containing all combinations of - that item with individual bits reset, sorted by the number of set bits. - ''' - def nbits(n): - "number of bits set in n base 2" - nb = 0 - while n: - n, rem = divmod(n, 2) - nb += rem - return nb - statelist = [] - for state in states: - substates = list(set(state & x for x in states)) - substates.sort(key=nbits, reverse=True) - statelist.append(substates) - return statelist - -_state_subsets = expand_substates(_states) - -# _state_codes gives for each state, the portable code to be passed as mc_state -_state_codes = [] -for s in _states: - r = 0 - for i in range(len(_modifiers)): - if (1 << i) & s: - r |= _modifier_masks[i] - _state_codes.append(r) - -class _ComplexBinder: - # This class binds many functions, and only unbinds them when it is deleted. - # self.handlerids is the list of seqs and ids of binded handler functions. - # The binded functions sit in a dictionary of lists of lists, which maps - # a detail (or None) and a state into a list of functions. - # When a new detail is discovered, handlers for all the possible states - # are binded. - - def __create_handler(self, lists, mc_type, mc_state): - def handler(event, lists = lists, - mc_type = mc_type, mc_state = mc_state, - ishandlerrunning = self.ishandlerrunning, - doafterhandler = self.doafterhandler): - ishandlerrunning[:] = [True] - event.mc_type = mc_type - event.mc_state = mc_state - wascalled = {} - r = None - for l in lists: - for i in range(len(l)-1, -1, -1): - func = l[i] - if func not in wascalled: - wascalled[func] = True - r = l[i](event) - if r: - break - if r: - break - ishandlerrunning[:] = [] - # Call all functions in doafterhandler and remove them from list - for f in doafterhandler: - f() - doafterhandler[:] = [] - if r: - return r - return handler - - def __init__(self, type, widget, widgetinst): - self.type = type - self.typename = _types[type][0] - self.widget = widget - self.widgetinst = widgetinst - self.bindedfuncs = {None: [[] for s in _states]} - self.handlerids = [] - # we don't want to change the lists of functions while a handler is - # running - it will mess up the loop and anyway, we usually want the - # change to happen from the next event. So we have a list of functions - # for the handler to run after it finishes calling the binded functions. - # It calls them only once. - # ishandlerrunning is a list. An empty one means no, otherwise - yes. - # this is done so that it would be mutable. - self.ishandlerrunning = [] - self.doafterhandler = [] - for s in _states: - lists = [self.bindedfuncs[None][i] for i in _state_subsets[s]] - handler = self.__create_handler(lists, type, _state_codes[s]) - seq = '<'+_state_names[s]+self.typename+'>' - self.handlerids.append((seq, self.widget.bind(self.widgetinst, - seq, handler))) - - def bind(self, triplet, func): - if triplet[2] not in self.bindedfuncs: - self.bindedfuncs[triplet[2]] = [[] for s in _states] - for s in _states: - lists = [ self.bindedfuncs[detail][i] - for detail in (triplet[2], None) - for i in _state_subsets[s] ] - handler = self.__create_handler(lists, self.type, - _state_codes[s]) - seq = "<%s%s-%s>"% (_state_names[s], self.typename, triplet[2]) - self.handlerids.append((seq, self.widget.bind(self.widgetinst, - seq, handler))) - doit = lambda: self.bindedfuncs[triplet[2]][triplet[0]].append(func) - if not self.ishandlerrunning: - doit() - else: - self.doafterhandler.append(doit) - - def unbind(self, triplet, func): - doit = lambda: self.bindedfuncs[triplet[2]][triplet[0]].remove(func) - if not self.ishandlerrunning: - doit() - else: - self.doafterhandler.append(doit) - - def __del__(self): - for seq, id in self.handlerids: - try: - self.widget.unbind(self.widgetinst, seq, id) - except tkinter.TclError as e: - if not APPLICATION_GONE in e.args[0]: - raise - -# define the list of event types to be handled by MultiEvent. the order is -# compatible with the definition of event type constants. -_types = ( - ("KeyPress", "Key"), ("KeyRelease",), ("ButtonPress", "Button"), - ("ButtonRelease",), ("Activate",), ("Circulate",), ("Colormap",), - ("Configure",), ("Deactivate",), ("Destroy",), ("Enter",), ("Expose",), - ("FocusIn",), ("FocusOut",), ("Gravity",), ("Leave",), ("Map",), - ("Motion",), ("MouseWheel",), ("Property",), ("Reparent",), ("Unmap",), - ("Visibility",), -) - -# which binder should be used for every event type? -_binder_classes = (_ComplexBinder,) * 4 + (_SimpleBinder,) * (len(_types)-4) - -# A dictionary to map a type name into its number -_type_names = dict([(name, number) - for number in range(len(_types)) - for name in _types[number]]) - -_keysym_re = re.compile(r"^\w+$") -_button_re = re.compile(r"^[1-5]$") -def _parse_sequence(sequence): - """Get a string which should describe an event sequence. If it is - successfully parsed as one, return a tuple containing the state (as an int), - the event type (as an index of _types), and the detail - None if none, or a - string if there is one. If the parsing is unsuccessful, return None. - """ - if not sequence or sequence[0] != '<' or sequence[-1] != '>': - return None - words = sequence[1:-1].split('-') - modifiers = 0 - while words and words[0] in _modifier_names: - modifiers |= 1 << _modifier_names[words[0]] - del words[0] - if words and words[0] in _type_names: - type = _type_names[words[0]] - del words[0] - else: - return None - if _binder_classes[type] is _SimpleBinder: - if modifiers or words: - return None - else: - detail = None - else: - # _ComplexBinder - if type in [_type_names[s] for s in ("KeyPress", "KeyRelease")]: - type_re = _keysym_re - else: - type_re = _button_re - - if not words: - detail = None - elif len(words) == 1 and type_re.match(words[0]): - detail = words[0] - else: - return None - - return modifiers, type, detail - -def _triplet_to_sequence(triplet): - if triplet[2]: - return '<'+_state_names[triplet[0]]+_types[triplet[1]][0]+'-'+ \ - triplet[2]+'>' - else: - return '<'+_state_names[triplet[0]]+_types[triplet[1]][0]+'>' - -_multicall_dict = {} -def MultiCallCreator(widget): - """Return a MultiCall class which inherits its methods from the - given widget class (for example, Tkinter.Text). This is used - instead of a templating mechanism. - """ - if widget in _multicall_dict: - return _multicall_dict[widget] - - class MultiCall (widget): - assert issubclass(widget, tkinter.Misc) - - def __init__(self, *args, **kwargs): - widget.__init__(self, *args, **kwargs) - # a dictionary which maps a virtual event to a tuple with: - # 0. the function binded - # 1. a list of triplets - the sequences it is binded to - self.__eventinfo = {} - self.__binders = [_binder_classes[i](i, widget, self) - for i in range(len(_types))] - - def bind(self, sequence=None, func=None, add=None): - #print("bind(%s, %s, %s)" % (sequence, func, add), - # file=sys.__stderr__) - if type(sequence) is str and len(sequence) > 2 and \ - sequence[:2] == "<<" and sequence[-2:] == ">>": - if sequence in self.__eventinfo: - ei = self.__eventinfo[sequence] - if ei[0] is not None: - for triplet in ei[1]: - self.__binders[triplet[1]].unbind(triplet, ei[0]) - ei[0] = func - if ei[0] is not None: - for triplet in ei[1]: - self.__binders[triplet[1]].bind(triplet, func) - else: - self.__eventinfo[sequence] = [func, []] - return widget.bind(self, sequence, func, add) - - def unbind(self, sequence, funcid=None): - if type(sequence) is str and len(sequence) > 2 and \ - sequence[:2] == "<<" and sequence[-2:] == ">>" and \ - sequence in self.__eventinfo: - func, triplets = self.__eventinfo[sequence] - if func is not None: - for triplet in triplets: - self.__binders[triplet[1]].unbind(triplet, func) - self.__eventinfo[sequence][0] = None - return widget.unbind(self, sequence, funcid) - - def event_add(self, virtual, *sequences): - #print("event_add(%s, %s)" % (repr(virtual), repr(sequences)), - # file=sys.__stderr__) - if virtual not in self.__eventinfo: - self.__eventinfo[virtual] = [None, []] - - func, triplets = self.__eventinfo[virtual] - for seq in sequences: - triplet = _parse_sequence(seq) - if triplet is None: - #print("Tkinter event_add(%s)" % seq, file=sys.__stderr__) - widget.event_add(self, virtual, seq) - else: - if func is not None: - self.__binders[triplet[1]].bind(triplet, func) - triplets.append(triplet) - - def event_delete(self, virtual, *sequences): - if virtual not in self.__eventinfo: - return - func, triplets = self.__eventinfo[virtual] - for seq in sequences: - triplet = _parse_sequence(seq) - if triplet is None: - #print("Tkinter event_delete: %s" % seq, file=sys.__stderr__) - widget.event_delete(self, virtual, seq) - else: - if func is not None: - self.__binders[triplet[1]].unbind(triplet, func) - triplets.remove(triplet) - - def event_info(self, virtual=None): - if virtual is None or virtual not in self.__eventinfo: - return widget.event_info(self, virtual) - else: - return tuple(map(_triplet_to_sequence, - self.__eventinfo[virtual][1])) + \ - widget.event_info(self, virtual) - - def __del__(self): - for virtual in self.__eventinfo: - func, triplets = self.__eventinfo[virtual] - if func: - for triplet in triplets: - try: - self.__binders[triplet[1]].unbind(triplet, func) - except tkinter.TclError as e: - if not APPLICATION_GONE in e.args[0]: - raise - - _multicall_dict[widget] = MultiCall - return MultiCall - - -def _multi_call(parent): - root = tkinter.Tk() - root.title("Test MultiCall") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - text = MultiCallCreator(tkinter.Text)(root) - text.pack() - def bindseq(seq, n=[0]): - def handler(event): - print(seq) - text.bind("<>"%n[0], handler) - text.event_add("<>"%n[0], seq) - n[0] += 1 - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - bindseq("") - root.mainloop() - -if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_multi_call) diff --git a/Lib/idlelib/MultiStatusBar.py b/Lib/idlelib/MultiStatusBar.py deleted file mode 100644 index e82ba9a..0000000 --- a/Lib/idlelib/MultiStatusBar.py +++ /dev/null @@ -1,47 +0,0 @@ -from tkinter import * - -class MultiStatusBar(Frame): - - def __init__(self, master=None, **kw): - if master is None: - master = Tk() - Frame.__init__(self, master, **kw) - self.labels = {} - - def set_label(self, name, text='', side=LEFT, width=0): - if name not in self.labels: - label = Label(self, borderwidth=0, anchor=W) - label.pack(side=side, pady=0, padx=4) - self.labels[name] = label - else: - label = self.labels[name] - if width != 0: - label.config(width=width) - label.config(text=text) - -def _multistatus_bar(parent): - root = Tk() - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d" %(x, y + 150)) - root.title("Test multistatus bar") - frame = Frame(root) - text = Text(frame) - text.pack() - msb = MultiStatusBar(frame) - msb.set_label("one", "hello") - msb.set_label("two", "world") - msb.pack(side=BOTTOM, fill=X) - - def change(): - msb.set_label("one", "foo") - msb.set_label("two", "bar") - - button = Button(root, text="Update status", command=change) - button.pack(side=BOTTOM) - frame.pack() - frame.mainloop() - root.mainloop() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_multistatus_bar) diff --git a/Lib/idlelib/ObjectBrowser.py b/Lib/idlelib/ObjectBrowser.py deleted file mode 100644 index 7b57aa4..0000000 --- a/Lib/idlelib/ObjectBrowser.py +++ /dev/null @@ -1,143 +0,0 @@ -# XXX TO DO: -# - popup menu -# - support partial or total redisplay -# - more doc strings -# - tooltips - -# object browser - -# XXX TO DO: -# - for classes/modules, add "open source" to object browser - -import re - -from idlelib.TreeWidget import TreeItem, TreeNode, ScrolledCanvas - -from reprlib import Repr - -myrepr = Repr() -myrepr.maxstring = 100 -myrepr.maxother = 100 - -class ObjectTreeItem(TreeItem): - def __init__(self, labeltext, object, setfunction=None): - self.labeltext = labeltext - self.object = object - self.setfunction = setfunction - def GetLabelText(self): - return self.labeltext - def GetText(self): - return myrepr.repr(self.object) - def GetIconName(self): - if not self.IsExpandable(): - return "python" - def IsEditable(self): - return self.setfunction is not None - def SetText(self, text): - try: - value = eval(text) - self.setfunction(value) - except: - pass - else: - self.object = value - def IsExpandable(self): - return not not dir(self.object) - def GetSubList(self): - keys = dir(self.object) - sublist = [] - for key in keys: - try: - value = getattr(self.object, key) - except AttributeError: - continue - item = make_objecttreeitem( - str(key) + " =", - value, - lambda value, key=key, object=self.object: - setattr(object, key, value)) - sublist.append(item) - return sublist - -class ClassTreeItem(ObjectTreeItem): - def IsExpandable(self): - return True - def GetSubList(self): - sublist = ObjectTreeItem.GetSubList(self) - if len(self.object.__bases__) == 1: - item = make_objecttreeitem("__bases__[0] =", - self.object.__bases__[0]) - else: - item = make_objecttreeitem("__bases__ =", self.object.__bases__) - sublist.insert(0, item) - return sublist - -class AtomicObjectTreeItem(ObjectTreeItem): - def IsExpandable(self): - return 0 - -class SequenceTreeItem(ObjectTreeItem): - def IsExpandable(self): - return len(self.object) > 0 - def keys(self): - return range(len(self.object)) - def GetSubList(self): - sublist = [] - for key in self.keys(): - try: - value = self.object[key] - except KeyError: - continue - def setfunction(value, key=key, object=self.object): - object[key] = value - item = make_objecttreeitem("%r:" % (key,), value, setfunction) - sublist.append(item) - return sublist - -class DictTreeItem(SequenceTreeItem): - def keys(self): - keys = list(self.object.keys()) - try: - keys.sort() - except: - pass - return keys - -dispatch = { - int: AtomicObjectTreeItem, - float: AtomicObjectTreeItem, - str: AtomicObjectTreeItem, - tuple: SequenceTreeItem, - list: SequenceTreeItem, - dict: DictTreeItem, - type: ClassTreeItem, -} - -def make_objecttreeitem(labeltext, object, setfunction=None): - t = type(object) - if t in dispatch: - c = dispatch[t] - else: - c = ObjectTreeItem - return c(labeltext, object, setfunction) - - -def _object_browser(parent): - import sys - from tkinter import Tk - root = Tk() - root.title("Test ObjectBrowser") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - root.configure(bd=0, bg="yellow") - root.focus_set() - sc = ScrolledCanvas(root, bg="white", highlightthickness=0, takefocus=1) - sc.frame.pack(expand=1, fill="both") - item = make_objecttreeitem("sys", sys) - node = TreeNode(sc.canvas, None, item) - node.update() - root.mainloop() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_object_browser) diff --git a/Lib/idlelib/OutputWindow.py b/Lib/idlelib/OutputWindow.py deleted file mode 100644 index e614f9b..0000000 --- a/Lib/idlelib/OutputWindow.py +++ /dev/null @@ -1,144 +0,0 @@ -from tkinter import * -from idlelib.EditorWindow import EditorWindow -import re -import tkinter.messagebox as tkMessageBox -from idlelib import IOBinding - -class OutputWindow(EditorWindow): - - """An editor window that can serve as an output file. - - Also the future base class for the Python shell window. - This class has no input facilities. - """ - - def __init__(self, *args): - EditorWindow.__init__(self, *args) - self.text.bind("<>", self.goto_file_line) - - # Customize EditorWindow - - def ispythonsource(self, filename): - # No colorization needed - return 0 - - def short_title(self): - return "Output" - - def maybesave(self): - # Override base class method -- don't ask any questions - if self.get_saved(): - return "yes" - else: - return "no" - - # Act as output file - - def write(self, s, tags=(), mark="insert"): - if isinstance(s, (bytes, bytes)): - s = s.decode(IOBinding.encoding, "replace") - self.text.insert(mark, s, tags) - self.text.see(mark) - self.text.update() - return len(s) - - def writelines(self, lines): - for line in lines: - self.write(line) - - def flush(self): - pass - - # Our own right-button menu - - rmenu_specs = [ - ("Cut", "<>", "rmenu_check_cut"), - ("Copy", "<>", "rmenu_check_copy"), - ("Paste", "<>", "rmenu_check_paste"), - (None, None, None), - ("Go to file/line", "<>", None), - ] - - file_line_pats = [ - # order of patterns matters - r'file "([^"]*)", line (\d+)', - r'([^\s]+)$(\d+)$', - r'^(\s*\S.*?):\s*(\d+):', # Win filename, maybe starting with spaces - r'([^\s]+):\s*(\d+):', # filename or path, ltrim - r'^\s*(\S.*?):\s*(\d+):', # Win abs path with embedded spaces, ltrim - ] - - file_line_progs = None - - def goto_file_line(self, event=None): - if self.file_line_progs is None: - l = [] - for pat in self.file_line_pats: - l.append(re.compile(pat, re.IGNORECASE)) - self.file_line_progs = l - # x, y = self.event.x, self.event.y - # self.text.mark_set("insert", "@%d,%d" % (x, y)) - line = self.text.get("insert linestart", "insert lineend") - result = self._file_line_helper(line) - if not result: - # Try the previous line. This is handy e.g. in tracebacks, - # where you tend to right-click on the displayed source line - line = self.text.get("insert -1line linestart", - "insert -1line lineend") - result = self._file_line_helper(line) - if not result: - tkMessageBox.showerror( - "No special line", - "The line you point at doesn't look like " - "a valid file name followed by a line number.", - parent=self.text) - return - filename, lineno = result - edit = self.flist.open(filename) - edit.gotoline(lineno) - - def _file_line_helper(self, line): - for prog in self.file_line_progs: - match = prog.search(line) - if match: - filename, lineno = match.group(1, 2) - try: - f = open(filename, "r") - f.close() - break - except OSError: - continue - else: - return None - try: - return filename, int(lineno) - except TypeError: - return None - -# These classes are currently not used but might come in handy - -class OnDemandOutputWindow: - - tagdefs = { - # XXX Should use IdlePrefs.ColorPrefs - "stdout": {"foreground": "blue"}, - "stderr": {"foreground": "#007700"}, - } - - def __init__(self, flist): - self.flist = flist - self.owin = None - - def write(self, s, tags, mark): - if not self.owin: - self.setup() - self.owin.write(s, tags, mark) - - def setup(self): - self.owin = owin = OutputWindow(self.flist) - text = owin.text - for tag, cnf in self.tagdefs.items(): - if cnf: - text.tag_configure(tag, **cnf) - text.tag_raise('sel') - self.write = self.owin.write diff --git a/Lib/idlelib/ParenMatch.py b/Lib/idlelib/ParenMatch.py deleted file mode 100644 index 19bad8c..0000000 --- a/Lib/idlelib/ParenMatch.py +++ /dev/null @@ -1,178 +0,0 @@ -"""ParenMatch -- An IDLE extension for parenthesis matching. - -When you hit a right paren, the cursor should move briefly to the left -paren. Paren here is used generically; the matching applies to -parentheses, square brackets, and curly braces. -""" - -from idlelib.HyperParser import HyperParser -from idlelib.configHandler import idleConf - -_openers = {')':'(',']':'[','}':'{'} -CHECK_DELAY = 100 # miliseconds - -class ParenMatch: - """Highlight matching parentheses - - There are three supported style of paren matching, based loosely - on the Emacs options. The style is select based on the - HILITE_STYLE attribute; it can be changed used the set_style - method. - - The supported styles are: - - default -- When a right paren is typed, highlight the matching - left paren for 1/2 sec. - - expression -- When a right paren is typed, highlight the entire - expression from the left paren to the right paren. - - TODO: - - extend IDLE with configuration dialog to change options - - implement rest of Emacs highlight styles (see below) - - print mismatch warning in IDLE status window - - Note: In Emacs, there are several styles of highlight where the - matching paren is highlighted whenever the cursor is immediately - to the right of a right paren. I don't know how to do that in Tk, - so I haven't bothered. - """ - menudefs = [ - ('edit', [ - ("Show surrounding parens", "<>"), - ]) - ] - STYLE = idleConf.GetOption('extensions','ParenMatch','style', - default='expression') - FLASH_DELAY = idleConf.GetOption('extensions','ParenMatch','flash-delay', - type='int',default=500) - HILITE_CONFIG = idleConf.GetHighlight(idleConf.CurrentTheme(),'hilite') - BELL = idleConf.GetOption('extensions','ParenMatch','bell', - type='bool',default=1) - - RESTORE_VIRTUAL_EVENT_NAME = "<>" - # We want the restore event be called before the usual return and - # backspace events. - RESTORE_SEQUENCES = ("", "", - "", "") - - def __init__(self, editwin): - self.editwin = editwin - self.text = editwin.text - # Bind the check-restore event to the function restore_event, - # so that we can then use activate_restore (which calls event_add) - # and deactivate_restore (which calls event_delete). - editwin.text.bind(self.RESTORE_VIRTUAL_EVENT_NAME, - self.restore_event) - self.counter = 0 - self.is_restore_active = 0 - self.set_style(self.STYLE) - - def activate_restore(self): - if not self.is_restore_active: - for seq in self.RESTORE_SEQUENCES: - self.text.event_add(self.RESTORE_VIRTUAL_EVENT_NAME, seq) - self.is_restore_active = True - - def deactivate_restore(self): - if self.is_restore_active: - for seq in self.RESTORE_SEQUENCES: - self.text.event_delete(self.RESTORE_VIRTUAL_EVENT_NAME, seq) - self.is_restore_active = False - - def set_style(self, style): - self.STYLE = style - if style == "default": - self.create_tag = self.create_tag_default - self.set_timeout = self.set_timeout_last - elif style == "expression": - self.create_tag = self.create_tag_expression - self.set_timeout = self.set_timeout_none - - def flash_paren_event(self, event): - indices = (HyperParser(self.editwin, "insert") - .get_surrounding_brackets()) - if indices is None: - self.warn_mismatched() - return - self.activate_restore() - self.create_tag(indices) - self.set_timeout_last() - - def paren_closed_event(self, event): - # If it was a shortcut and not really a closing paren, quit. - closer = self.text.get("insert-1c") - if closer not in _openers: - return - hp = HyperParser(self.editwin, "insert-1c") - if not hp.is_in_code(): - return - indices = hp.get_surrounding_brackets(_openers[closer], True) - if indices is None: - self.warn_mismatched() - return - self.activate_restore() - self.create_tag(indices) - self.set_timeout() - - def restore_event(self, event=None): - self.text.tag_delete("paren") - self.deactivate_restore() - self.counter += 1 # disable the last timer, if there is one. - - def handle_restore_timer(self, timer_count): - if timer_count == self.counter: - self.restore_event() - - def warn_mismatched(self): - if self.BELL: - self.text.bell() - - # any one of the create_tag_XXX methods can be used depending on - # the style - - def create_tag_default(self, indices): - """Highlight the single paren that matches""" - self.text.tag_add("paren", indices[0]) - self.text.tag_config("paren", self.HILITE_CONFIG) - - def create_tag_expression(self, indices): - """Highlight the entire expression""" - if self.text.get(indices[1]) in (')', ']', '}'): - rightindex = indices[1]+"+1c" - else: - rightindex = indices[1] - self.text.tag_add("paren", indices[0], rightindex) - self.text.tag_config("paren", self.HILITE_CONFIG) - - # any one of the set_timeout_XXX methods can be used depending on - # the style - - def set_timeout_none(self): - """Highlight will remain until user input turns it off - or the insert has moved""" - # After CHECK_DELAY, call a function which disables the "paren" tag - # if the event is for the most recent timer and the insert has changed, - # or schedules another call for itself. - self.counter += 1 - def callme(callme, self=self, c=self.counter, - index=self.text.index("insert")): - if index != self.text.index("insert"): - self.handle_restore_timer(c) - else: - self.editwin.text_frame.after(CHECK_DELAY, callme, callme) - self.editwin.text_frame.after(CHECK_DELAY, callme, callme) - - def set_timeout_last(self): - """The last highlight created will be removed after .5 sec""" - # associate a counter with an event; only disable the "paren" - # tag if the event is for the most recent timer. - self.counter += 1 - self.editwin.text_frame.after( - self.FLASH_DELAY, - lambda self=self, c=self.counter: self.handle_restore_timer(c)) - - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_parenmatch', verbosity=2) diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py deleted file mode 100644 index 9ab7632..0000000 --- a/Lib/idlelib/PathBrowser.py +++ /dev/null @@ -1,108 +0,0 @@ -import os -import sys -import importlib.machinery - -from idlelib.TreeWidget import TreeItem -from idlelib.ClassBrowser import ClassBrowser, ModuleBrowserTreeItem -from idlelib.PyShell import PyShellFileList - - -class PathBrowser(ClassBrowser): - - def __init__(self, flist, _htest=False): - """ - _htest - bool, change box location when running htest - """ - self._htest = _htest - self.init(flist) - - def settitle(self): - "Set window titles." - self.top.wm_title("Path Browser") - self.top.wm_iconname("Path Browser") - - def rootnode(self): - return PathBrowserTreeItem() - -class PathBrowserTreeItem(TreeItem): - - def GetText(self): - return "sys.path" - - def GetSubList(self): - sublist = [] - for dir in sys.path: - item = DirBrowserTreeItem(dir) - sublist.append(item) - return sublist - -class DirBrowserTreeItem(TreeItem): - - def __init__(self, dir, packages=[]): - self.dir = dir - self.packages = packages - - def GetText(self): - if not self.packages: - return self.dir - else: - return self.packages[-1] + ": package" - - def GetSubList(self): - try: - names = os.listdir(self.dir or os.curdir) - except OSError: - return [] - packages = [] - for name in names: - file = os.path.join(self.dir, name) - if self.ispackagedir(file): - nn = os.path.normcase(name) - packages.append((nn, name, file)) - packages.sort() - sublist = [] - for nn, name, file in packages: - item = DirBrowserTreeItem(file, self.packages + [name]) - sublist.append(item) - for nn, name in self.listmodules(names): - item = ModuleBrowserTreeItem(os.path.join(self.dir, name)) - sublist.append(item) - return sublist - - def ispackagedir(self, file): - " Return true for directories that are packages." - if not os.path.isdir(file): - return False - init = os.path.join(file, "__init__.py") - return os.path.exists(init) - - def listmodules(self, allnames): - modules = {} - suffixes = importlib.machinery.EXTENSION_SUFFIXES[:] - suffixes += importlib.machinery.SOURCE_SUFFIXES - suffixes += importlib.machinery.BYTECODE_SUFFIXES - sorted = [] - for suff in suffixes: - i = -len(suff) - for name in allnames[:]: - normed_name = os.path.normcase(name) - if normed_name[i:] == suff: - mod_name = name[:i] - if mod_name not in modules: - modules[mod_name] = None - sorted.append((normed_name, name)) - allnames.remove(name) - sorted.sort() - return sorted - -def _path_browser(parent): # htest # - flist = PyShellFileList(parent) - PathBrowser(flist, _htest=True) - parent.mainloop() - -if __name__ == "__main__": - from unittest import main - main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(_path_browser) diff --git a/Lib/idlelib/Percolator.py b/Lib/idlelib/Percolator.py deleted file mode 100644 index b8be2aa..0000000 --- a/Lib/idlelib/Percolator.py +++ /dev/null @@ -1,105 +0,0 @@ -from idlelib.WidgetRedirector import WidgetRedirector -from idlelib.Delegator import Delegator - - -class Percolator: - - def __init__(self, text): - # XXX would be nice to inherit from Delegator - self.text = text - self.redir = WidgetRedirector(text) - self.top = self.bottom = Delegator(text) - self.bottom.insert = self.redir.register("insert", self.insert) - self.bottom.delete = self.redir.register("delete", self.delete) - self.filters = [] - - def close(self): - while self.top is not self.bottom: - self.removefilter(self.top) - self.top = None - self.bottom.setdelegate(None) - self.bottom = None - self.redir.close() - self.redir = None - self.text = None - - def insert(self, index, chars, tags=None): - # Could go away if inheriting from Delegator - self.top.insert(index, chars, tags) - - def delete(self, index1, index2=None): - # Could go away if inheriting from Delegator - self.top.delete(index1, index2) - - def insertfilter(self, filter): - # Perhaps rename to pushfilter()? - assert isinstance(filter, Delegator) - assert filter.delegate is None - filter.setdelegate(self.top) - self.top = filter - - def removefilter(self, filter): - # XXX Perhaps should only support popfilter()? - assert isinstance(filter, Delegator) - assert filter.delegate is not None - f = self.top - if f is filter: - self.top = filter.delegate - filter.setdelegate(None) - else: - while f.delegate is not filter: - assert f is not self.bottom - f.resetcache() - f = f.delegate - f.setdelegate(filter.delegate) - filter.setdelegate(None) - - -def _percolator(parent): # htest # - import tkinter as tk - import re - - class Tracer(Delegator): - def __init__(self, name): - self.name = name - Delegator.__init__(self, None) - - def insert(self, *args): - print(self.name, ": insert", args) - self.delegate.insert(*args) - - def delete(self, *args): - print(self.name, ": delete", args) - self.delegate.delete(*args) - - box = tk.Toplevel(parent) - box.title("Test Percolator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - box.geometry("+%d+%d" % (x, y + 150)) - text = tk.Text(box) - p = Percolator(text) - pin = p.insertfilter - pout = p.removefilter - t1 = Tracer("t1") - t2 = Tracer("t2") - - def toggle1(): - (pin if var1.get() else pout)(t1) - def toggle2(): - (pin if var2.get() else pout)(t2) - - text.pack() - var1 = tk.IntVar() - cb1 = tk.Checkbutton(box, text="Tracer1", command=toggle1, variable=var1) - cb1.pack() - var2 = tk.IntVar() - cb2 = tk.Checkbutton(box, text="Tracer2", command=toggle2, variable=var2) - cb2.pack() - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_percolator', verbosity=2, - exit=False) - - from idlelib.idle_test.htest import run - run(_percolator) diff --git a/Lib/idlelib/PyParse.py b/Lib/idlelib/PyParse.py deleted file mode 100644 index 9ccbb25..0000000 --- a/Lib/idlelib/PyParse.py +++ /dev/null @@ -1,617 +0,0 @@ -import re -import sys -from collections import Mapping - -# Reason last stmt is continued (or C_NONE if it's not). -(C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE, - C_STRING_NEXT_LINES, C_BRACKET) = range(5) - -if 0: # for throwaway debugging output - def dump(*stuff): - sys.__stdout__.write(" ".join(map(str, stuff)) + "\n") - -# Find what looks like the start of a popular stmt. - -_synchre = re.compile(r""" - ^ - [ \t]* - (?: while - | else - | def - | return - | assert - | break - | class - | continue - | elif - | try - | except - | raise - | import - | yield - ) - \b -""", re.VERBOSE | re.MULTILINE).search - -# Match blank line or non-indenting comment line. - -_junkre = re.compile(r""" - [ \t]* - (?: \# \S .* )? - \n -""", re.VERBOSE).match - -# Match any flavor of string; the terminating quote is optional -# so that we're robust in the face of incomplete program text. - -_match_stringre = re.compile(r""" - \""" [^"\\]* (?: - (?: \\. | "(?!"") ) - [^"\\]* - )* - (?: \""" )? - -| " [^"\\\n]* (?: \\. [^"\\\n]* )* "? - -| ''' [^'\\]* (?: - (?: \\. | '(?!'') ) - [^'\\]* - )* - (?: ''' )? - -| ' [^'\\\n]* (?: \\. [^'\\\n]* )* '? -""", re.VERBOSE | re.DOTALL).match - -# Match a line that starts with something interesting; -# used to find the first item of a bracket structure. - -_itemre = re.compile(r""" - [ \t]* - [^\s#\\] # if we match, m.end()-1 is the interesting char -""", re.VERBOSE).match - -# Match start of stmts that should be followed by a dedent. - -_closere = re.compile(r""" - \s* - (?: return - | break - | continue - | raise - | pass - ) - \b -""", re.VERBOSE).match - -# Chew up non-special chars as quickly as possible. If match is -# successful, m.end() less 1 is the index of the last boring char -# matched. If match is unsuccessful, the string starts with an -# interesting char. - -_chew_ordinaryre = re.compile(r""" - [^[\](){}#'"\\]+ -""", re.VERBOSE).match - - -class StringTranslatePseudoMapping(Mapping): - r"""Utility class to be used with str.translate() - - This Mapping class wraps a given dict. When a value for a key is - requested via __getitem__() or get(), the key is looked up in the - given dict. If found there, the value from the dict is returned. - Otherwise, the default value given upon initialization is returned. - - This allows using str.translate() to make some replacements, and to - replace all characters for which no replacement was specified with - a given character instead of leaving them as-is. - - For example, to replace everything except whitespace with 'x': - - >>> whitespace_chars = ' \t\n\r' - >>> preserve_dict = {ord(c): ord(c) for c in whitespace_chars} - >>> mapping = StringTranslatePseudoMapping(preserve_dict, ord('x')) - >>> text = "a + b\tc\nd" - >>> text.translate(mapping) - 'x x x\tx\nx' - """ - def __init__(self, non_defaults, default_value): - self._non_defaults = non_defaults - self._default_value = default_value - - def _get(key, _get=non_defaults.get, _default=default_value): - return _get(key, _default) - self._get = _get - - def __getitem__(self, item): - return self._get(item) - - def __len__(self): - return len(self._non_defaults) - - def __iter__(self): - return iter(self._non_defaults) - - def get(self, key, default=None): - return self._get(key) - - -class Parser: - - def __init__(self, indentwidth, tabwidth): - self.indentwidth = indentwidth - self.tabwidth = tabwidth - - def set_str(self, s): - assert len(s) == 0 or s[-1] == '\n' - self.str = s - self.study_level = 0 - - # Return index of a good place to begin parsing, as close to the - # end of the string as possible. This will be the start of some - # popular stmt like "if" or "def". Return None if none found: - # the caller should pass more prior context then, if possible, or - # if not (the entire program text up until the point of interest - # has already been tried) pass 0 to set_lo. - # - # This will be reliable iff given a reliable is_char_in_string - # function, meaning that when it says "no", it's absolutely - # guaranteed that the char is not in a string. - - def find_good_parse_start(self, is_char_in_string=None, - _synchre=_synchre): - str, pos = self.str, None - - if not is_char_in_string: - # no clue -- make the caller pass everything - return None - - # Peek back from the end for a good place to start, - # but don't try too often; pos will be left None, or - # bumped to a legitimate synch point. - limit = len(str) - for tries in range(5): - i = str.rfind(":\n", 0, limit) - if i < 0: - break - i = str.rfind('\n', 0, i) + 1 # start of colon line - m = _synchre(str, i, limit) - if m and not is_char_in_string(m.start()): - pos = m.start() - break - limit = i - if pos is None: - # Nothing looks like a block-opener, or stuff does - # but is_char_in_string keeps returning true; most likely - # we're in or near a giant string, the colorizer hasn't - # caught up enough to be helpful, or there simply *aren't* - # any interesting stmts. In any of these cases we're - # going to have to parse the whole thing to be sure, so - # give it one last try from the start, but stop wasting - # time here regardless of the outcome. - m = _synchre(str) - if m and not is_char_in_string(m.start()): - pos = m.start() - return pos - - # Peeking back worked; look forward until _synchre no longer - # matches. - i = pos + 1 - while 1: - m = _synchre(str, i) - if m: - s, i = m.span() - if not is_char_in_string(s): - pos = s - else: - break - return pos - - # Throw away the start of the string. Intended to be called with - # find_good_parse_start's result. - - def set_lo(self, lo): - assert lo == 0 or self.str[lo-1] == '\n' - if lo > 0: - self.str = self.str[lo:] - - # Build a translation table to map uninteresting chars to 'x', open - # brackets to '(', close brackets to ')' while preserving quotes, - # backslashes, newlines and hashes. This is to be passed to - # str.translate() in _study1(). - _tran = {} - _tran.update((ord(c), ord('(')) for c in "({[") - _tran.update((ord(c), ord(')')) for c in ")}]") - _tran.update((ord(c), ord(c)) for c in "\"'\\\n#") - _tran = StringTranslatePseudoMapping(_tran, default_value=ord('x')) - - # As quickly as humanly possible , find the line numbers (0- - # based) of the non-continuation lines. - # Creates self.{goodlines, continuation}. - - def _study1(self): - if self.study_level >= 1: - return - self.study_level = 1 - - # Map all uninteresting characters to "x", all open brackets - # to "(", all close brackets to ")", then collapse runs of - # uninteresting characters. This can cut the number of chars - # by a factor of 10-40, and so greatly speed the following loop. - str = self.str - str = str.translate(self._tran) - str = str.replace('xxxxxxxx', 'x') - str = str.replace('xxxx', 'x') - str = str.replace('xx', 'x') - str = str.replace('xx', 'x') - str = str.replace('\nx', '\n') - # note that replacing x\n with \n would be incorrect, because - # x may be preceded by a backslash - - # March over the squashed version of the program, accumulating - # the line numbers of non-continued stmts, and determining - # whether & why the last stmt is a continuation. - continuation = C_NONE - level = lno = 0 # level is nesting level; lno is line number - self.goodlines = goodlines = [0] - push_good = goodlines.append - i, n = 0, len(str) - while i < n: - ch = str[i] - i = i+1 - - # cases are checked in decreasing order of frequency - if ch == 'x': - continue - - if ch == '\n': - lno = lno + 1 - if level == 0: - push_good(lno) - # else we're in an unclosed bracket structure - continue - - if ch == '(': - level = level + 1 - continue - - if ch == ')': - if level: - level = level - 1 - # else the program is invalid, but we can't complain - continue - - if ch == '"' or ch == "'": - # consume the string - quote = ch - if str[i-1:i+2] == quote * 3: - quote = quote * 3 - firstlno = lno - w = len(quote) - 1 - i = i+w - while i < n: - ch = str[i] - i = i+1 - - if ch == 'x': - continue - - if str[i-1:i+w] == quote: - i = i+w - break - - if ch == '\n': - lno = lno + 1 - if w == 0: - # unterminated single-quoted string - if level == 0: - push_good(lno) - break - continue - - if ch == '\\': - assert i < n - if str[i] == '\n': - lno = lno + 1 - i = i+1 - continue - - # else comment char or paren inside string - - else: - # didn't break out of the loop, so we're still - # inside a string - if (lno - 1) == firstlno: - # before the previous \n in str, we were in the first - # line of the string - continuation = C_STRING_FIRST_LINE - else: - continuation = C_STRING_NEXT_LINES - continue # with outer loop - - if ch == '#': - # consume the comment - i = str.find('\n', i) - assert i >= 0 - continue - - assert ch == '\\' - assert i < n - if str[i] == '\n': - lno = lno + 1 - if i+1 == n: - continuation = C_BACKSLASH - i = i+1 - - # The last stmt may be continued for all 3 reasons. - # String continuation takes precedence over bracket - # continuation, which beats backslash continuation. - if (continuation != C_STRING_FIRST_LINE - and continuation != C_STRING_NEXT_LINES and level > 0): - continuation = C_BRACKET - self.continuation = continuation - - # Push the final line number as a sentinel value, regardless of - # whether it's continued. - assert (continuation == C_NONE) == (goodlines[-1] == lno) - if goodlines[-1] != lno: - push_good(lno) - - def get_continuation_type(self): - self._study1() - return self.continuation - - # study1 was sufficient to determine the continuation status, - # but doing more requires looking at every character. study2 - # does this for the last interesting statement in the block. - # Creates: - # self.stmt_start, stmt_end - # slice indices of last interesting stmt - # self.stmt_bracketing - # the bracketing structure of the last interesting stmt; - # for example, for the statement "say(boo) or die", stmt_bracketing - # will be [(0, 0), (3, 1), (8, 0)]. Strings and comments are - # treated as brackets, for the matter. - # self.lastch - # last non-whitespace character before optional trailing - # comment - # self.lastopenbracketpos - # if continuation is C_BRACKET, index of last open bracket - - def _study2(self): - if self.study_level >= 2: - return - self._study1() - self.study_level = 2 - - # Set p and q to slice indices of last interesting stmt. - str, goodlines = self.str, self.goodlines - i = len(goodlines) - 1 - p = len(str) # index of newest line - while i: - assert p - # p is the index of the stmt at line number goodlines[i]. - # Move p back to the stmt at line number goodlines[i-1]. - q = p - for nothing in range(goodlines[i-1], goodlines[i]): - # tricky: sets p to 0 if no preceding newline - p = str.rfind('\n', 0, p-1) + 1 - # The stmt str[p:q] isn't a continuation, but may be blank - # or a non-indenting comment line. - if _junkre(str, p): - i = i-1 - else: - break - if i == 0: - # nothing but junk! - assert p == 0 - q = p - self.stmt_start, self.stmt_end = p, q - - # Analyze this stmt, to find the last open bracket (if any) - # and last interesting character (if any). - lastch = "" - stack = [] # stack of open bracket indices - push_stack = stack.append - bracketing = [(p, 0)] - while p < q: - # suck up all except ()[]{}'"#\\ - m = _chew_ordinaryre(str, p, q) - if m: - # we skipped at least one boring char - newp = m.end() - # back up over totally boring whitespace - i = newp - 1 # index of last boring char - while i >= p and str[i] in " \t\n": - i = i-1 - if i >= p: - lastch = str[i] - p = newp - if p >= q: - break - - ch = str[p] - - if ch in "([{": - push_stack(p) - bracketing.append((p, len(stack))) - lastch = ch - p = p+1 - continue - - if ch in ")]}": - if stack: - del stack[-1] - lastch = ch - p = p+1 - bracketing.append((p, len(stack))) - continue - - if ch == '"' or ch == "'": - # consume string - # Note that study1 did this with a Python loop, but - # we use a regexp here; the reason is speed in both - # cases; the string may be huge, but study1 pre-squashed - # strings to a couple of characters per line. study1 - # also needed to keep track of newlines, and we don't - # have to. - bracketing.append((p, len(stack)+1)) - lastch = ch - p = _match_stringre(str, p, q).end() - bracketing.append((p, len(stack))) - continue - - if ch == '#': - # consume comment and trailing newline - bracketing.append((p, len(stack)+1)) - p = str.find('\n', p, q) + 1 - assert p > 0 - bracketing.append((p, len(stack))) - continue - - assert ch == '\\' - p = p+1 # beyond backslash - assert p < q - if str[p] != '\n': - # the program is invalid, but can't complain - lastch = ch + str[p] - p = p+1 # beyond escaped char - - # end while p < q: - - self.lastch = lastch - if stack: - self.lastopenbracketpos = stack[-1] - self.stmt_bracketing = tuple(bracketing) - - # Assuming continuation is C_BRACKET, return the number - # of spaces the next line should be indented. - - def compute_bracket_indent(self): - self._study2() - assert self.continuation == C_BRACKET - j = self.lastopenbracketpos - str = self.str - n = len(str) - origi = i = str.rfind('\n', 0, j) + 1 - j = j+1 # one beyond open bracket - # find first list item; set i to start of its line - while j < n: - m = _itemre(str, j) - if m: - j = m.end() - 1 # index of first interesting char - extra = 0 - break - else: - # this line is junk; advance to next line - i = j = str.find('\n', j) + 1 - else: - # nothing interesting follows the bracket; - # reproduce the bracket line's indentation + a level - j = i = origi - while str[j] in " \t": - j = j+1 - extra = self.indentwidth - return len(str[i:j].expandtabs(self.tabwidth)) + extra - - # Return number of physical lines in last stmt (whether or not - # it's an interesting stmt! this is intended to be called when - # continuation is C_BACKSLASH). - - def get_num_lines_in_stmt(self): - self._study1() - goodlines = self.goodlines - return goodlines[-1] - goodlines[-2] - - # Assuming continuation is C_BACKSLASH, return the number of spaces - # the next line should be indented. Also assuming the new line is - # the first one following the initial line of the stmt. - - def compute_backslash_indent(self): - self._study2() - assert self.continuation == C_BACKSLASH - str = self.str - i = self.stmt_start - while str[i] in " \t": - i = i+1 - startpos = i - - # See whether the initial line starts an assignment stmt; i.e., - # look for an = operator - endpos = str.find('\n', startpos) + 1 - found = level = 0 - while i < endpos: - ch = str[i] - if ch in "([{": - level = level + 1 - i = i+1 - elif ch in ")]}": - if level: - level = level - 1 - i = i+1 - elif ch == '"' or ch == "'": - i = _match_stringre(str, i, endpos).end() - elif ch == '#': - break - elif level == 0 and ch == '=' and \ - (i == 0 or str[i-1] not in "=<>!") and \ - str[i+1] != '=': - found = 1 - break - else: - i = i+1 - - if found: - # found a legit =, but it may be the last interesting - # thing on the line - i = i+1 # move beyond the = - found = re.match(r"\s*\\", str[i:endpos]) is None - - if not found: - # oh well ... settle for moving beyond the first chunk - # of non-whitespace chars - i = startpos - while str[i] not in " \t\n": - i = i+1 - - return len(str[self.stmt_start:i].expandtabs(\ - self.tabwidth)) + 1 - - # Return the leading whitespace on the initial line of the last - # interesting stmt. - - def get_base_indent_string(self): - self._study2() - i, n = self.stmt_start, self.stmt_end - j = i - str = self.str - while j < n and str[j] in " \t": - j = j + 1 - return str[i:j] - - # Did the last interesting stmt open a block? - - def is_block_opener(self): - self._study2() - return self.lastch == ':' - - # Did the last interesting stmt close a block? - - def is_block_closer(self): - self._study2() - return _closere(self.str, self.stmt_start) is not None - - # index of last open bracket ({[, or None if none - lastopenbracketpos = None - - def get_last_open_bracket_pos(self): - self._study2() - return self.lastopenbracketpos - - # the structure of the bracketing of the last interesting statement, - # in the format defined in _study2, or None if the text didn't contain - # anything - stmt_bracketing = None - - def get_last_stmt_bracketing(self): - self._study2() - return self.stmt_bracketing diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py deleted file mode 100755 index 1bcc9b6..0000000 --- a/Lib/idlelib/PyShell.py +++ /dev/null @@ -1,1619 +0,0 @@ -#! /usr/bin/env python3 - -import getopt -import os -import os.path -import re -import socket -import subprocess -import sys -import threading -import time -import tokenize -import io - -import linecache -from code import InteractiveInterpreter -from platform import python_version, system - -try: - from tkinter import * -except ImportError: - print("** IDLE can't import Tkinter.\n" - "Your Python may not be configured for Tk. **", file=sys.__stderr__) - sys.exit(1) -import tkinter.messagebox as tkMessageBox - -from idlelib.EditorWindow import EditorWindow, fixwordbreaks -from idlelib.FileList import FileList -from idlelib.ColorDelegator import ColorDelegator -from idlelib.UndoDelegator import UndoDelegator -from idlelib.OutputWindow import OutputWindow -from idlelib.configHandler import idleConf -from idlelib import rpc -from idlelib import Debugger -from idlelib import RemoteDebugger -from idlelib import macosxSupport - -HOST = '127.0.0.1' # python execution server on localhost loopback -PORT = 0 # someday pass in host, port for remote debug capability - -# Override warnings module to write to warning_stream. Initialize to send IDLE -# internal warnings to the console. ScriptBinding.check_syntax() will -# temporarily redirect the stream to the shell window to display warnings when -# checking user's code. -warning_stream = sys.__stderr__ # None, at least on Windows, if no console. -import warnings - -def idle_formatwarning(message, category, filename, lineno, line=None): - """Format warnings the IDLE way.""" - - s = "\nWarning (from warnings module):\n" - s += ' File \"%s\", line %s\n' % (filename, lineno) - if line is None: - line = linecache.getline(filename, lineno) - line = line.strip() - if line: - s += " %s\n" % line - s += "%s: %s\n" % (category.__name__, message) - return s - -def idle_showwarning( - message, category, filename, lineno, file=None, line=None): - """Show Idle-format warning (after replacing warnings.showwarning). - - The differences are the formatter called, the file=None replacement, - which can be None, the capture of the consequence AttributeError, - and the output of a hard-coded prompt. - """ - if file is None: - file = warning_stream - try: - file.write(idle_formatwarning( - message, category, filename, lineno, line=line)) - file.write(">>> ") - except (AttributeError, OSError): - pass # if file (probably __stderr__) is invalid, skip warning. - -_warnings_showwarning = None - -def capture_warnings(capture): - "Replace warning.showwarning with idle_showwarning, or reverse." - - global _warnings_showwarning - if capture: - if _warnings_showwarning is None: - _warnings_showwarning = warnings.showwarning - warnings.showwarning = idle_showwarning - else: - if _warnings_showwarning is not None: - warnings.showwarning = _warnings_showwarning - _warnings_showwarning = None - -capture_warnings(True) - -def extended_linecache_checkcache(filename=None, - orig_checkcache=linecache.checkcache): - """Extend linecache.checkcache to preserve the entries - - Rather than repeating the linecache code, patch it to save the - entries, call the original linecache.checkcache() - (skipping them), and then restore the saved entries. - - orig_checkcache is bound at definition time to the original - method, allowing it to be patched. - """ - cache = linecache.cache - save = {} - for key in list(cache): - if key[:1] + key[-1:] == '<>': - save[key] = cache.pop(key) - orig_checkcache(filename) - cache.update(save) - -# Patch linecache.checkcache(): -linecache.checkcache = extended_linecache_checkcache - - -class PyShellEditorWindow(EditorWindow): - "Regular text edit window in IDLE, supports breakpoints" - - def __init__(self, *args): - self.breakpoints = [] - EditorWindow.__init__(self, *args) - self.text.bind("<>", self.set_breakpoint_here) - self.text.bind("<>", self.clear_breakpoint_here) - self.text.bind("<>", self.flist.open_shell) - - self.breakpointPath = os.path.join(idleConf.GetUserCfgDir(), - 'breakpoints.lst') - # whenever a file is changed, restore breakpoints - def filename_changed_hook(old_hook=self.io.filename_change_hook, - self=self): - self.restore_file_breaks() - old_hook() - self.io.set_filename_change_hook(filename_changed_hook) - if self.io.filename: - self.restore_file_breaks() - self.color_breakpoint_text() - - rmenu_specs = [ - ("Cut", "<>", "rmenu_check_cut"), - ("Copy", "<>", "rmenu_check_copy"), - ("Paste", "<>", "rmenu_check_paste"), - (None, None, None), - ("Set Breakpoint", "<>", None), - ("Clear Breakpoint", "<>", None) - ] - - def color_breakpoint_text(self, color=True): - "Turn colorizing of breakpoint text on or off" - if self.io is None: - # possible due to update in restore_file_breaks - return - if color: - theme = idleConf.CurrentTheme() - cfg = idleConf.GetHighlight(theme, "break") - else: - cfg = {'foreground': '', 'background': ''} - self.text.tag_config('BREAK', cfg) - - def set_breakpoint(self, lineno): - text = self.text - filename = self.io.filename - text.tag_add("BREAK", "%d.0" % lineno, "%d.0" % (lineno+1)) - try: - self.breakpoints.index(lineno) - except ValueError: # only add if missing, i.e. do once - self.breakpoints.append(lineno) - try: # update the subprocess debugger - debug = self.flist.pyshell.interp.debugger - debug.set_breakpoint_here(filename, lineno) - except: # but debugger may not be active right now.... - pass - - def set_breakpoint_here(self, event=None): - text = self.text - filename = self.io.filename - if not filename: - text.bell() - return - lineno = int(float(text.index("insert"))) - self.set_breakpoint(lineno) - - def clear_breakpoint_here(self, event=None): - text = self.text - filename = self.io.filename - if not filename: - text.bell() - return - lineno = int(float(text.index("insert"))) - try: - self.breakpoints.remove(lineno) - except: - pass - text.tag_remove("BREAK", "insert linestart",\ - "insert lineend +1char") - try: - debug = self.flist.pyshell.interp.debugger - debug.clear_breakpoint_here(filename, lineno) - except: - pass - - def clear_file_breaks(self): - if self.breakpoints: - text = self.text - filename = self.io.filename - if not filename: - text.bell() - return - self.breakpoints = [] - text.tag_remove("BREAK", "1.0", END) - try: - debug = self.flist.pyshell.interp.debugger - debug.clear_file_breaks(filename) - except: - pass - - def store_file_breaks(self): - "Save breakpoints when file is saved" - # XXX 13 Dec 2002 KBK Currently the file must be saved before it can - # be run. The breaks are saved at that time. If we introduce - # a temporary file save feature the save breaks functionality - # needs to be re-verified, since the breaks at the time the - # temp file is created may differ from the breaks at the last - # permanent save of the file. Currently, a break introduced - # after a save will be effective, but not persistent. - # This is necessary to keep the saved breaks synched with the - # saved file. - # - # Breakpoints are set as tagged ranges in the text. - # Since a modified file has to be saved before it is - # run, and since self.breakpoints (from which the subprocess - # debugger is loaded) is updated during the save, the visible - # breaks stay synched with the subprocess even if one of these - # unexpected breakpoint deletions occurs. - breaks = self.breakpoints - filename = self.io.filename - try: - with open(self.breakpointPath, "r") as fp: - lines = fp.readlines() - except OSError: - lines = [] - try: - with open(self.breakpointPath, "w") as new_file: - for line in lines: - if not line.startswith(filename + '='): - new_file.write(line) - self.update_breakpoints() - breaks = self.breakpoints - if breaks: - new_file.write(filename + '=' + str(breaks) + '\n') - except OSError as err: - if not getattr(self.root, "breakpoint_error_displayed", False): - self.root.breakpoint_error_displayed = True - tkMessageBox.showerror(title='IDLE Error', - message='Unable to update breakpoint list:\n%s' - % str(err), - parent=self.text) - - def restore_file_breaks(self): - self.text.update() # this enables setting "BREAK" tags to be visible - if self.io is None: - # can happen if IDLE closes due to the .update() call - return - filename = self.io.filename - if filename is None: - return - if os.path.isfile(self.breakpointPath): - with open(self.breakpointPath, "r") as fp: - lines = fp.readlines() - for line in lines: - if line.startswith(filename + '='): - breakpoint_linenumbers = eval(line[len(filename)+1:]) - for breakpoint_linenumber in breakpoint_linenumbers: - self.set_breakpoint(breakpoint_linenumber) - - def update_breakpoints(self): - "Retrieves all the breakpoints in the current window" - text = self.text - ranges = text.tag_ranges("BREAK") - linenumber_list = self.ranges_to_linenumbers(ranges) - self.breakpoints = linenumber_list - - def ranges_to_linenumbers(self, ranges): - lines = [] - for index in range(0, len(ranges), 2): - lineno = int(float(ranges[index].string)) - end = int(float(ranges[index+1].string)) - while lineno < end: - lines.append(lineno) - lineno += 1 - return lines - -# XXX 13 Dec 2002 KBK Not used currently -# def saved_change_hook(self): -# "Extend base method - clear breaks if module is modified" -# if not self.get_saved(): -# self.clear_file_breaks() -# EditorWindow.saved_change_hook(self) - - def _close(self): - "Extend base method - clear breaks when module is closed" - self.clear_file_breaks() - EditorWindow._close(self) - - -class PyShellFileList(FileList): - "Extend base class: IDLE supports a shell and breakpoints" - - # override FileList's class variable, instances return PyShellEditorWindow - # instead of EditorWindow when new edit windows are created. - EditorWindow = PyShellEditorWindow - - pyshell = None - - def open_shell(self, event=None): - if self.pyshell: - self.pyshell.top.wakeup() - else: - self.pyshell = PyShell(self) - if self.pyshell: - if not self.pyshell.begin(): - return None - return self.pyshell - - -class ModifiedColorDelegator(ColorDelegator): - "Extend base class: colorizer for the shell window itself" - - def __init__(self): - ColorDelegator.__init__(self) - self.LoadTagDefs() - - def recolorize_main(self): - self.tag_remove("TODO", "1.0", "iomark") - self.tag_add("SYNC", "1.0", "iomark") - ColorDelegator.recolorize_main(self) - - def LoadTagDefs(self): - ColorDelegator.LoadTagDefs(self) - theme = idleConf.CurrentTheme() - self.tagdefs.update({ - "stdin": {'background':None,'foreground':None}, - "stdout": idleConf.GetHighlight(theme, "stdout"), - "stderr": idleConf.GetHighlight(theme, "stderr"), - "console": idleConf.GetHighlight(theme, "console"), - }) - - def removecolors(self): - # Don't remove shell color tags before "iomark" - for tag in self.tagdefs: - self.tag_remove(tag, "iomark", "end") - -class ModifiedUndoDelegator(UndoDelegator): - "Extend base class: forbid insert/delete before the I/O mark" - - def insert(self, index, chars, tags=None): - try: - if self.delegate.compare(index, "<", "iomark"): - self.delegate.bell() - return - except TclError: - pass - UndoDelegator.insert(self, index, chars, tags) - - def delete(self, index1, index2=None): - try: - if self.delegate.compare(index1, "<", "iomark"): - self.delegate.bell() - return - except TclError: - pass - UndoDelegator.delete(self, index1, index2) - - -class MyRPCClient(rpc.RPCClient): - - def handle_EOF(self): - "Override the base class - just re-raise EOFError" - raise EOFError - - -class ModifiedInterpreter(InteractiveInterpreter): - - def __init__(self, tkconsole): - self.tkconsole = tkconsole - locals = sys.modules['__main__'].__dict__ - InteractiveInterpreter.__init__(self, locals=locals) - self.save_warnings_filters = None - self.restarting = False - self.subprocess_arglist = None - self.port = PORT - self.original_compiler_flags = self.compile.compiler.flags - - _afterid = None - rpcclt = None - rpcsubproc = None - - def spawn_subprocess(self): - if self.subprocess_arglist is None: - self.subprocess_arglist = self.build_subprocess_arglist() - self.rpcsubproc = subprocess.Popen(self.subprocess_arglist) - - def build_subprocess_arglist(self): - assert (self.port!=0), ( - "Socket should have been assigned a port number.") - w = ['-W' + s for s in sys.warnoptions] - # Maybe IDLE is installed and is being accessed via sys.path, - # or maybe it's not installed and the idle.py script is being - # run from the IDLE source directory. - del_exitf = idleConf.GetOption('main', 'General', 'delete-exitfunc', - default=False, type='bool') - if __name__ == 'idlelib.PyShell': - command = "__import__('idlelib.run').run.main(%r)" % (del_exitf,) - else: - command = "__import__('run').main(%r)" % (del_exitf,) - return [sys.executable] + w + ["-c", command, str(self.port)] - - def start_subprocess(self): - addr = (HOST, self.port) - # GUI makes several attempts to acquire socket, listens for connection - for i in range(3): - time.sleep(i) - try: - self.rpcclt = MyRPCClient(addr) - break - except OSError: - pass - else: - self.display_port_binding_error() - return None - # if PORT was 0, system will assign an 'ephemeral' port. Find it out: - self.port = self.rpcclt.listening_sock.getsockname()[1] - # if PORT was not 0, probably working with a remote execution server - if PORT != 0: - # To allow reconnection within the 2MSL wait (cf. Stevens TCP - # V1, 18.6), set SO_REUSEADDR. Note that this can be problematic - # on Windows since the implementation allows two active sockets on - # the same address! - self.rpcclt.listening_sock.setsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR, 1) - self.spawn_subprocess() - #time.sleep(20) # test to simulate GUI not accepting connection - # Accept the connection from the Python execution server - self.rpcclt.listening_sock.settimeout(10) - try: - self.rpcclt.accept() - except socket.timeout: - self.display_no_subprocess_error() - return None - self.rpcclt.register("console", self.tkconsole) - self.rpcclt.register("stdin", self.tkconsole.stdin) - self.rpcclt.register("stdout", self.tkconsole.stdout) - self.rpcclt.register("stderr", self.tkconsole.stderr) - self.rpcclt.register("flist", self.tkconsole.flist) - self.rpcclt.register("linecache", linecache) - self.rpcclt.register("interp", self) - self.transfer_path(with_cwd=True) - self.poll_subprocess() - return self.rpcclt - - def restart_subprocess(self, with_cwd=False, filename=''): - if self.restarting: - return self.rpcclt - self.restarting = True - # close only the subprocess debugger - debug = self.getdebugger() - if debug: - try: - # Only close subprocess debugger, don't unregister gui_adap! - RemoteDebugger.close_subprocess_debugger(self.rpcclt) - except: - pass - # Kill subprocess, spawn a new one, accept connection. - self.rpcclt.close() - self.terminate_subprocess() - console = self.tkconsole - was_executing = console.executing - console.executing = False - self.spawn_subprocess() - try: - self.rpcclt.accept() - except socket.timeout: - self.display_no_subprocess_error() - return None - self.transfer_path(with_cwd=with_cwd) - console.stop_readline() - # annotate restart in shell window and mark it - console.text.delete("iomark", "end-1c") - tag = 'RESTART: ' + (filename if filename else 'Shell') - halfbar = ((int(console.width) -len(tag) - 4) // 2) * '=' - console.write("\n{0} {1} {0}".format(halfbar, tag)) - console.text.mark_set("restart", "end-1c") - console.text.mark_gravity("restart", "left") - if not filename: - console.showprompt() - # restart subprocess debugger - if debug: - # Restarted debugger connects to current instance of debug GUI - RemoteDebugger.restart_subprocess_debugger(self.rpcclt) - # reload remote debugger breakpoints for all PyShellEditWindows - debug.load_breakpoints() - self.compile.compiler.flags = self.original_compiler_flags - self.restarting = False - return self.rpcclt - - def __request_interrupt(self): - self.rpcclt.remotecall("exec", "interrupt_the_server", (), {}) - - def interrupt_subprocess(self): - threading.Thread(target=self.__request_interrupt).start() - - def kill_subprocess(self): - if self._afterid is not None: - self.tkconsole.text.after_cancel(self._afterid) - try: - self.rpcclt.listening_sock.close() - except AttributeError: # no socket - pass - try: - self.rpcclt.close() - except AttributeError: # no socket - pass - self.terminate_subprocess() - self.tkconsole.executing = False - self.rpcclt = None - - def terminate_subprocess(self): - "Make sure subprocess is terminated" - try: - self.rpcsubproc.kill() - except OSError: - # process already terminated - return - else: - try: - self.rpcsubproc.wait() - except OSError: - return - - def transfer_path(self, with_cwd=False): - if with_cwd: # Issue 13506 - path = [''] # include Current Working Directory - path.extend(sys.path) - else: - path = sys.path - - self.runcommand("""if 1: - import sys as _sys - _sys.path = %r - del _sys - \n""" % (path,)) - - active_seq = None - - def poll_subprocess(self): - clt = self.rpcclt - if clt is None: - return - try: - response = clt.pollresponse(self.active_seq, wait=0.05) - except (EOFError, OSError, KeyboardInterrupt): - # lost connection or subprocess terminated itself, restart - # [the KBI is from rpc.SocketIO.handle_EOF()] - if self.tkconsole.closing: - return - response = None - self.restart_subprocess() - if response: - self.tkconsole.resetoutput() - self.active_seq = None - how, what = response - console = self.tkconsole.console - if how == "OK": - if what is not None: - print(repr(what), file=console) - elif how == "EXCEPTION": - if self.tkconsole.getvar("<>"): - self.remote_stack_viewer() - elif how == "ERROR": - errmsg = "PyShell.ModifiedInterpreter: Subprocess ERROR:\n" - print(errmsg, what, file=sys.__stderr__) - print(errmsg, what, file=console) - # we received a response to the currently active seq number: - try: - self.tkconsole.endexecuting() - except AttributeError: # shell may have closed - pass - # Reschedule myself - if not self.tkconsole.closing: - self._afterid = self.tkconsole.text.after( - self.tkconsole.pollinterval, self.poll_subprocess) - - debugger = None - - def setdebugger(self, debugger): - self.debugger = debugger - - def getdebugger(self): - return self.debugger - - def open_remote_stack_viewer(self): - """Initiate the remote stack viewer from a separate thread. - - This method is called from the subprocess, and by returning from this - method we allow the subprocess to unblock. After a bit the shell - requests the subprocess to open the remote stack viewer which returns a - static object looking at the last exception. It is queried through - the RPC mechanism. - - """ - self.tkconsole.text.after(300, self.remote_stack_viewer) - return - - def remote_stack_viewer(self): - from idlelib import RemoteObjectBrowser - oid = self.rpcclt.remotequeue("exec", "stackviewer", ("flist",), {}) - if oid is None: - self.tkconsole.root.bell() - return - item = RemoteObjectBrowser.StubObjectTreeItem(self.rpcclt, oid) - from idlelib.TreeWidget import ScrolledCanvas, TreeNode - top = Toplevel(self.tkconsole.root) - theme = idleConf.CurrentTheme() - background = idleConf.GetHighlight(theme, 'normal')['background'] - sc = ScrolledCanvas(top, bg=background, highlightthickness=0) - sc.frame.pack(expand=1, fill="both") - node = TreeNode(sc.canvas, None, item) - node.expand() - # XXX Should GC the remote tree when closing the window - - gid = 0 - - def execsource(self, source): - "Like runsource() but assumes complete exec source" - filename = self.stuffsource(source) - self.execfile(filename, source) - - def execfile(self, filename, source=None): - "Execute an existing file" - if source is None: - with tokenize.open(filename) as fp: - source = fp.read() - try: - code = compile(source, filename, "exec") - except (OverflowError, SyntaxError): - self.tkconsole.resetoutput() - print('*** Error in script or command!\n' - 'Traceback (most recent call last):', - file=self.tkconsole.stderr) - InteractiveInterpreter.showsyntaxerror(self, filename) - self.tkconsole.showprompt() - else: - self.runcode(code) - - def runsource(self, source): - "Extend base class method: Stuff the source in the line cache first" - filename = self.stuffsource(source) - self.more = 0 - self.save_warnings_filters = warnings.filters[:] - warnings.filterwarnings(action="error", category=SyntaxWarning) - # at the moment, InteractiveInterpreter expects str - assert isinstance(source, str) - #if isinstance(source, str): - # from idlelib import IOBinding - # try: - # source = source.encode(IOBinding.encoding) - # except UnicodeError: - # self.tkconsole.resetoutput() - # self.write("Unsupported characters in input\n") - # return - try: - # InteractiveInterpreter.runsource() calls its runcode() method, - # which is overridden (see below) - return InteractiveInterpreter.runsource(self, source, filename) - finally: - if self.save_warnings_filters is not None: - warnings.filters[:] = self.save_warnings_filters - self.save_warnings_filters = None - - def stuffsource(self, source): - "Stuff source in the filename cache" - filename = "" % self.gid - self.gid = self.gid + 1 - lines = source.split("\n") - linecache.cache[filename] = len(source)+1, 0, lines, filename - return filename - - def prepend_syspath(self, filename): - "Prepend sys.path with file's directory if not already included" - self.runcommand("""if 1: - _filename = %r - import sys as _sys - from os.path import dirname as _dirname - _dir = _dirname(_filename) - if not _dir in _sys.path: - _sys.path.insert(0, _dir) - del _filename, _sys, _dirname, _dir - \n""" % (filename,)) - - def showsyntaxerror(self, filename=None): - """Override Interactive Interpreter method: Use Colorizing - - Color the offending position instead of printing it and pointing at it - with a caret. - - """ - tkconsole = self.tkconsole - text = tkconsole.text - text.tag_remove("ERROR", "1.0", "end") - type, value, tb = sys.exc_info() - msg = getattr(value, 'msg', '') or value or "" - lineno = getattr(value, 'lineno', '') or 1 - offset = getattr(value, 'offset', '') or 0 - if offset == 0: - lineno += 1 #mark end of offending line - if lineno == 1: - pos = "iomark + %d chars" % (offset-1) - else: - pos = "iomark linestart + %d lines + %d chars" % \ - (lineno-1, offset-1) - tkconsole.colorize_syntax_error(text, pos) - tkconsole.resetoutput() - self.write("SyntaxError: %s\n" % msg) - tkconsole.showprompt() - - def showtraceback(self): - "Extend base class method to reset output properly" - self.tkconsole.resetoutput() - self.checklinecache() - InteractiveInterpreter.showtraceback(self) - if self.tkconsole.getvar("<>"): - self.tkconsole.open_stack_viewer() - - def checklinecache(self): - c = linecache.cache - for key in list(c.keys()): - if key[:1] + key[-1:] != "<>": - del c[key] - - def runcommand(self, code): - "Run the code without invoking the debugger" - # The code better not raise an exception! - if self.tkconsole.executing: - self.display_executing_dialog() - return 0 - if self.rpcclt: - self.rpcclt.remotequeue("exec", "runcode", (code,), {}) - else: - exec(code, self.locals) - return 1 - - def runcode(self, code): - "Override base class method" - if self.tkconsole.executing: - self.interp.restart_subprocess() - self.checklinecache() - if self.save_warnings_filters is not None: - warnings.filters[:] = self.save_warnings_filters - self.save_warnings_filters = None - debugger = self.debugger - try: - self.tkconsole.beginexecuting() - if not debugger and self.rpcclt is not None: - self.active_seq = self.rpcclt.asyncqueue("exec", "runcode", - (code,), {}) - elif debugger: - debugger.run(code, self.locals) - else: - exec(code, self.locals) - except SystemExit: - if not self.tkconsole.closing: - if tkMessageBox.askyesno( - "Exit?", - "Do you want to exit altogether?", - default="yes", - parent=self.tkconsole.text): - raise - else: - self.showtraceback() - else: - raise - except: - if use_subprocess: - print("IDLE internal error in runcode()", - file=self.tkconsole.stderr) - self.showtraceback() - self.tkconsole.endexecuting() - else: - if self.tkconsole.canceled: - self.tkconsole.canceled = False - print("KeyboardInterrupt", file=self.tkconsole.stderr) - else: - self.showtraceback() - finally: - if not use_subprocess: - try: - self.tkconsole.endexecuting() - except AttributeError: # shell may have closed - pass - - def write(self, s): - "Override base class method" - return self.tkconsole.stderr.write(s) - - def display_port_binding_error(self): - tkMessageBox.showerror( - "Port Binding Error", - "IDLE can't bind to a TCP/IP port, which is necessary to " - "communicate with its Python execution server. This might be " - "because no networking is installed on this computer. " - "Run IDLE with the -n command line switch to start without a " - "subprocess and refer to Help/IDLE Help 'Running without a " - "subprocess' for further details.", - parent=self.tkconsole.text) - - def display_no_subprocess_error(self): - tkMessageBox.showerror( - "Subprocess Startup Error", - "IDLE's subprocess didn't make connection. Either IDLE can't " - "start a subprocess or personal firewall software is blocking " - "the connection.", - parent=self.tkconsole.text) - - def display_executing_dialog(self): - tkMessageBox.showerror( - "Already executing", - "The Python Shell window is already executing a command; " - "please wait until it is finished.", - parent=self.tkconsole.text) - - -class PyShell(OutputWindow): - - shell_title = "Python " + python_version() + " Shell" - - # Override classes - ColorDelegator = ModifiedColorDelegator - UndoDelegator = ModifiedUndoDelegator - - # Override menus - menu_specs = [ - ("file", "_File"), - ("edit", "_Edit"), - ("debug", "_Debug"), - ("options", "_Options"), - ("windows", "_Window"), - ("help", "_Help"), - ] - - - # New classes - from idlelib.IdleHistory import History - - def __init__(self, flist=None): - if use_subprocess: - ms = self.menu_specs - if ms[2][0] != "shell": - ms.insert(2, ("shell", "She_ll")) - self.interp = ModifiedInterpreter(self) - if flist is None: - root = Tk() - fixwordbreaks(root) - root.withdraw() - flist = PyShellFileList(root) - # - OutputWindow.__init__(self, flist, None, None) - # -## self.config(usetabs=1, indentwidth=8, context_use_ps1=1) - self.usetabs = True - # indentwidth must be 8 when using tabs. See note in EditorWindow: - self.indentwidth = 8 - self.context_use_ps1 = True - # - text = self.text - text.configure(wrap="char") - text.bind("<>", self.enter_callback) - text.bind("<>", self.linefeed_callback) - text.bind("<>", self.cancel_callback) - text.bind("<>", self.eof_callback) - text.bind("<>", self.open_stack_viewer) - text.bind("<>", self.toggle_debugger) - text.bind("<>", self.toggle_jit_stack_viewer) - if use_subprocess: - text.bind("<>", self.view_restart_mark) - text.bind("<>", self.restart_shell) - # - self.save_stdout = sys.stdout - self.save_stderr = sys.stderr - self.save_stdin = sys.stdin - from idlelib import IOBinding - self.stdin = PseudoInputFile(self, "stdin", IOBinding.encoding) - self.stdout = PseudoOutputFile(self, "stdout", IOBinding.encoding) - self.stderr = PseudoOutputFile(self, "stderr", IOBinding.encoding) - self.console = PseudoOutputFile(self, "console", IOBinding.encoding) - if not use_subprocess: - sys.stdout = self.stdout - sys.stderr = self.stderr - sys.stdin = self.stdin - try: - # page help() text to shell. - import pydoc # import must be done here to capture i/o rebinding. - # XXX KBK 27Dec07 use a textView someday, but must work w/o subproc - pydoc.pager = pydoc.plainpager - except: - sys.stderr = sys.__stderr__ - raise - # - self.history = self.History(self.text) - # - self.pollinterval = 50 # millisec - - def get_standard_extension_names(self): - return idleConf.GetExtensions(shell_only=True) - - reading = False - executing = False - canceled = False - endoffile = False - closing = False - _stop_readline_flag = False - - def set_warning_stream(self, stream): - global warning_stream - warning_stream = stream - - def get_warning_stream(self): - return warning_stream - - def toggle_debugger(self, event=None): - if self.executing: - tkMessageBox.showerror("Don't debug now", - "You can only toggle the debugger when idle", - parent=self.text) - self.set_debugger_indicator() - return "break" - else: - db = self.interp.getdebugger() - if db: - self.close_debugger() - else: - self.open_debugger() - - def set_debugger_indicator(self): - db = self.interp.getdebugger() - self.setvar("<>", not not db) - - def toggle_jit_stack_viewer(self, event=None): - pass # All we need is the variable - - def close_debugger(self): - db = self.interp.getdebugger() - if db: - self.interp.setdebugger(None) - db.close() - if self.interp.rpcclt: - RemoteDebugger.close_remote_debugger(self.interp.rpcclt) - self.resetoutput() - self.console.write("[DEBUG OFF]\n") - sys.ps1 = ">>> " - self.showprompt() - self.set_debugger_indicator() - - def open_debugger(self): - if self.interp.rpcclt: - dbg_gui = RemoteDebugger.start_remote_debugger(self.interp.rpcclt, - self) - else: - dbg_gui = Debugger.Debugger(self) - self.interp.setdebugger(dbg_gui) - dbg_gui.load_breakpoints() - sys.ps1 = "[DEBUG ON]\n>>> " - self.showprompt() - self.set_debugger_indicator() - - def beginexecuting(self): - "Helper for ModifiedInterpreter" - self.resetoutput() - self.executing = 1 - - def endexecuting(self): - "Helper for ModifiedInterpreter" - self.executing = 0 - self.canceled = 0 - self.showprompt() - - def close(self): - "Extend EditorWindow.close()" - if self.executing: - response = tkMessageBox.askokcancel( - "Kill?", - "Your program is still running!\n Do you want to kill it?", - default="ok", - parent=self.text) - if response is False: - return "cancel" - self.stop_readline() - self.canceled = True - self.closing = True - return EditorWindow.close(self) - - def _close(self): - "Extend EditorWindow._close(), shut down debugger and execution server" - self.close_debugger() - if use_subprocess: - self.interp.kill_subprocess() - # Restore std streams - sys.stdout = self.save_stdout - sys.stderr = self.save_stderr - sys.stdin = self.save_stdin - # Break cycles - self.interp = None - self.console = None - self.flist.pyshell = None - self.history = None - EditorWindow._close(self) - - def ispythonsource(self, filename): - "Override EditorWindow method: never remove the colorizer" - return True - - def short_title(self): - return self.shell_title - - COPYRIGHT = \ - 'Type "copyright", "credits" or "license()" for more information.' - - def begin(self): - self.text.mark_set("iomark", "insert") - self.resetoutput() - if use_subprocess: - nosub = '' - client = self.interp.start_subprocess() - if not client: - self.close() - return False - else: - nosub = ("==== No Subprocess ====\n\n" + - "WARNING: Running IDLE without a Subprocess is deprecated\n" + - "and will be removed in a later version. See Help/IDLE Help\n" + - "for details.\n\n") - sys.displayhook = rpc.displayhook - - self.write("Python %s on %s\n%s\n%s" % - (sys.version, sys.platform, self.COPYRIGHT, nosub)) - self.text.focus_force() - self.showprompt() - import tkinter - tkinter._default_root = None # 03Jan04 KBK What's this? - return True - - def stop_readline(self): - if not self.reading: # no nested mainloop to exit. - return - self._stop_readline_flag = True - self.top.quit() - - def readline(self): - save = self.reading - try: - self.reading = 1 - self.top.mainloop() # nested mainloop() - finally: - self.reading = save - if self._stop_readline_flag: - self._stop_readline_flag = False - return "" - line = self.text.get("iomark", "end-1c") - if len(line) == 0: # may be EOF if we quit our mainloop with Ctrl-C - line = "\n" - self.resetoutput() - if self.canceled: - self.canceled = 0 - if not use_subprocess: - raise KeyboardInterrupt - if self.endoffile: - self.endoffile = 0 - line = "" - return line - - def isatty(self): - return True - - def cancel_callback(self, event=None): - try: - if self.text.compare("sel.first", "!=", "sel.last"): - return # Active selection -- always use default binding - except: - pass - if not (self.executing or self.reading): - self.resetoutput() - self.interp.write("KeyboardInterrupt\n") - self.showprompt() - return "break" - self.endoffile = 0 - self.canceled = 1 - if (self.executing and self.interp.rpcclt): - if self.interp.getdebugger(): - self.interp.restart_subprocess() - else: - self.interp.interrupt_subprocess() - if self.reading: - self.top.quit() # exit the nested mainloop() in readline() - return "break" - - def eof_callback(self, event): - if self.executing and not self.reading: - return # Let the default binding (delete next char) take over - if not (self.text.compare("iomark", "==", "insert") and - self.text.compare("insert", "==", "end-1c")): - return # Let the default binding (delete next char) take over - if not self.executing: - self.resetoutput() - self.close() - else: - self.canceled = 0 - self.endoffile = 1 - self.top.quit() - return "break" - - def linefeed_callback(self, event): - # Insert a linefeed without entering anything (still autoindented) - if self.reading: - self.text.insert("insert", "\n") - self.text.see("insert") - else: - self.newline_and_indent_event(event) - return "break" - - def enter_callback(self, event): - if self.executing and not self.reading: - return # Let the default binding (insert '\n') take over - # If some text is selected, recall the selection - # (but only if this before the I/O mark) - try: - sel = self.text.get("sel.first", "sel.last") - if sel: - if self.text.compare("sel.last", "<=", "iomark"): - self.recall(sel, event) - return "break" - except: - pass - # If we're strictly before the line containing iomark, recall - # the current line, less a leading prompt, less leading or - # trailing whitespace - if self.text.compare("insert", "<", "iomark linestart"): - # Check if there's a relevant stdin range -- if so, use it - prev = self.text.tag_prevrange("stdin", "insert") - if prev and self.text.compare("insert", "<", prev[1]): - self.recall(self.text.get(prev[0], prev[1]), event) - return "break" - next = self.text.tag_nextrange("stdin", "insert") - if next and self.text.compare("insert lineend", ">=", next[0]): - self.recall(self.text.get(next[0], next[1]), event) - return "break" - # No stdin mark -- just get the current line, less any prompt - indices = self.text.tag_nextrange("console", "insert linestart") - if indices and \ - self.text.compare(indices[0], "<=", "insert linestart"): - self.recall(self.text.get(indices[1], "insert lineend"), event) - else: - self.recall(self.text.get("insert linestart", "insert lineend"), event) - return "break" - # If we're between the beginning of the line and the iomark, i.e. - # in the prompt area, move to the end of the prompt - if self.text.compare("insert", "<", "iomark"): - self.text.mark_set("insert", "iomark") - # If we're in the current input and there's only whitespace - # beyond the cursor, erase that whitespace first - s = self.text.get("insert", "end-1c") - if s and not s.strip(): - self.text.delete("insert", "end-1c") - # If we're in the current input before its last line, - # insert a newline right at the insert point - if self.text.compare("insert", "<", "end-1c linestart"): - self.newline_and_indent_event(event) - return "break" - # We're in the last line; append a newline and submit it - self.text.mark_set("insert", "end-1c") - if self.reading: - self.text.insert("insert", "\n") - self.text.see("insert") - else: - self.newline_and_indent_event(event) - self.text.tag_add("stdin", "iomark", "end-1c") - self.text.update_idletasks() - if self.reading: - self.top.quit() # Break out of recursive mainloop() - else: - self.runit() - return "break" - - def recall(self, s, event): - # remove leading and trailing empty or whitespace lines - s = re.sub(r'^\s*\n', '' , s) - s = re.sub(r'\n\s*$', '', s) - lines = s.split('\n') - self.text.undo_block_start() - try: - self.text.tag_remove("sel", "1.0", "end") - self.text.mark_set("insert", "end-1c") - prefix = self.text.get("insert linestart", "insert") - if prefix.rstrip().endswith(':'): - self.newline_and_indent_event(event) - prefix = self.text.get("insert linestart", "insert") - self.text.insert("insert", lines[0].strip()) - if len(lines) > 1: - orig_base_indent = re.search(r'^([ \t]*)', lines[0]).group(0) - new_base_indent = re.search(r'^([ \t]*)', prefix).group(0) - for line in lines[1:]: - if line.startswith(orig_base_indent): - # replace orig base indentation with new indentation - line = new_base_indent + line[len(orig_base_indent):] - self.text.insert('insert', '\n'+line.rstrip()) - finally: - self.text.see("insert") - self.text.undo_block_stop() - - def runit(self): - line = self.text.get("iomark", "end-1c") - # Strip off last newline and surrounding whitespace. - # (To allow you to hit return twice to end a statement.) - i = len(line) - while i > 0 and line[i-1] in " \t": - i = i-1 - if i > 0 and line[i-1] == "\n": - i = i-1 - while i > 0 and line[i-1] in " \t": - i = i-1 - line = line[:i] - self.interp.runsource(line) - - def open_stack_viewer(self, event=None): - if self.interp.rpcclt: - return self.interp.remote_stack_viewer() - try: - sys.last_traceback - except: - tkMessageBox.showerror("No stack trace", - "There is no stack trace yet.\n" - "(sys.last_traceback is not defined)", - parent=self.text) - return - from idlelib.StackViewer import StackBrowser - StackBrowser(self.root, self.flist) - - def view_restart_mark(self, event=None): - self.text.see("iomark") - self.text.see("restart") - - def restart_shell(self, event=None): - "Callback for Run/Restart Shell Cntl-F6" - self.interp.restart_subprocess(with_cwd=True) - - def showprompt(self): - self.resetoutput() - try: - s = str(sys.ps1) - except: - s = "" - self.console.write(s) - self.text.mark_set("insert", "end-1c") - self.set_line_and_column() - self.io.reset_undo() - - def resetoutput(self): - source = self.text.get("iomark", "end-1c") - if self.history: - self.history.store(source) - if self.text.get("end-2c") != "\n": - self.text.insert("end-1c", "\n") - self.text.mark_set("iomark", "end-1c") - self.set_line_and_column() - - def write(self, s, tags=()): - if isinstance(s, str) and len(s) and max(s) > '\uffff': - # Tk doesn't support outputting non-BMP characters - # Let's assume what printed string is not very long, - # find first non-BMP character and construct informative - # UnicodeEncodeError exception. - for start, char in enumerate(s): - if char > '\uffff': - break - raise UnicodeEncodeError("UCS-2", char, start, start+1, - 'Non-BMP character not supported in Tk') - try: - self.text.mark_gravity("iomark", "right") - count = OutputWindow.write(self, s, tags, "iomark") - self.text.mark_gravity("iomark", "left") - except: - raise ###pass # ### 11Aug07 KBK if we are expecting exceptions - # let's find out what they are and be specific. - if self.canceled: - self.canceled = 0 - if not use_subprocess: - raise KeyboardInterrupt - return count - - def rmenu_check_cut(self): - try: - if self.text.compare('sel.first', '<', 'iomark'): - return 'disabled' - except TclError: # no selection, so the index 'sel.first' doesn't exist - return 'disabled' - return super().rmenu_check_cut() - - def rmenu_check_paste(self): - if self.text.compare('insert','<','iomark'): - return 'disabled' - return super().rmenu_check_paste() - -class PseudoFile(io.TextIOBase): - - def __init__(self, shell, tags, encoding=None): - self.shell = shell - self.tags = tags - self._encoding = encoding - - @property - def encoding(self): - return self._encoding - - @property - def name(self): - return '<%s>' % self.tags - - def isatty(self): - return True - - -class PseudoOutputFile(PseudoFile): - - def writable(self): - return True - - def write(self, s): - if self.closed: - raise ValueError("write to closed file") - if type(s) is not str: - if not isinstance(s, str): - raise TypeError('must be str, not ' + type(s).__name__) - # See issue #19481 - s = str.__str__(s) - return self.shell.write(s, self.tags) - - -class PseudoInputFile(PseudoFile): - - def __init__(self, shell, tags, encoding=None): - PseudoFile.__init__(self, shell, tags, encoding) - self._line_buffer = '' - - def readable(self): - return True - - def read(self, size=-1): - if self.closed: - raise ValueError("read from closed file") - if size is None: - size = -1 - elif not isinstance(size, int): - raise TypeError('must be int, not ' + type(size).__name__) - result = self._line_buffer - self._line_buffer = '' - if size < 0: - while True: - line = self.shell.readline() - if not line: break - result += line - else: - while len(result) < size: - line = self.shell.readline() - if not line: break - result += line - self._line_buffer = result[size:] - result = result[:size] - return result - - def readline(self, size=-1): - if self.closed: - raise ValueError("read from closed file") - if size is None: - size = -1 - elif not isinstance(size, int): - raise TypeError('must be int, not ' + type(size).__name__) - line = self._line_buffer or self.shell.readline() - if size < 0: - size = len(line) - eol = line.find('\n', 0, size) - if eol >= 0: - size = eol + 1 - self._line_buffer = line[size:] - return line[:size] - - def close(self): - self.shell.close() - - -usage_msg = """\ - -USAGE: idle [-deins] [-t title] [file]* - idle [-dns] [-t title] (-c cmd | -r file) [arg]* - idle [-dns] [-t title] - [arg]* - - -h print this help message and exit - -n run IDLE without a subprocess (DEPRECATED, - see Help/IDLE Help for details) - -The following options will override the IDLE 'settings' configuration: - - -e open an edit window - -i open a shell window - -The following options imply -i and will open a shell: - - -c cmd run the command in a shell, or - -r file run script from file - - -d enable the debugger - -s run $IDLESTARTUP or $PYTHONSTARTUP before anything else - -t title set title of shell window - -A default edit window will be bypassed when -c, -r, or - are used. - -[arg]* are passed to the command (-c) or script (-r) in sys.argv[1:]. - -Examples: - -idle - Open an edit window or shell depending on IDLE's configuration. - -idle foo.py foobar.py - Edit the files, also open a shell if configured to start with shell. - -idle -est "Baz" foo.py - Run $IDLESTARTUP or $PYTHONSTARTUP, edit foo.py, and open a shell - window with the title "Baz". - -idle -c "import sys; print(sys.argv)" "foo" - Open a shell window and run the command, passing "-c" in sys.argv[0] - and "foo" in sys.argv[1]. - -idle -d -s -r foo.py "Hello World" - Open a shell window, run a startup script, enable the debugger, and - run foo.py, passing "foo.py" in sys.argv[0] and "Hello World" in - sys.argv[1]. - -echo "import sys; print(sys.argv)" | idle - "foobar" - Open a shell window, run the script piped in, passing '' in sys.argv[0] - and "foobar" in sys.argv[1]. -""" - -def main(): - global flist, root, use_subprocess - - capture_warnings(True) - use_subprocess = True - enable_shell = False - enable_edit = False - debug = False - cmd = None - script = None - startup = False - try: - opts, args = getopt.getopt(sys.argv[1:], "c:deihnr:st:") - except getopt.error as msg: - print("Error: %s\n%s" % (msg, usage_msg), file=sys.stderr) - sys.exit(2) - for o, a in opts: - if o == '-c': - cmd = a - enable_shell = True - if o == '-d': - debug = True - enable_shell = True - if o == '-e': - enable_edit = True - if o == '-h': - sys.stdout.write(usage_msg) - sys.exit() - if o == '-i': - enable_shell = True - if o == '-n': - print(" Warning: running IDLE without a subprocess is deprecated.", - file=sys.stderr) - use_subprocess = False - if o == '-r': - script = a - if os.path.isfile(script): - pass - else: - print("No script file: ", script) - sys.exit() - enable_shell = True - if o == '-s': - startup = True - enable_shell = True - if o == '-t': - PyShell.shell_title = a - enable_shell = True - if args and args[0] == '-': - cmd = sys.stdin.read() - enable_shell = True - # process sys.argv and sys.path: - for i in range(len(sys.path)): - sys.path[i] = os.path.abspath(sys.path[i]) - if args and args[0] == '-': - sys.argv = [''] + args[1:] - elif cmd: - sys.argv = ['-c'] + args - elif script: - sys.argv = [script] + args - elif args: - enable_edit = True - pathx = [] - for filename in args: - pathx.append(os.path.dirname(filename)) - for dir in pathx: - dir = os.path.abspath(dir) - if not dir in sys.path: - sys.path.insert(0, dir) - else: - dir = os.getcwd() - if dir not in sys.path: - sys.path.insert(0, dir) - # check the IDLE settings configuration (but command line overrides) - edit_start = idleConf.GetOption('main', 'General', - 'editor-on-startup', type='bool') - enable_edit = enable_edit or edit_start - enable_shell = enable_shell or not enable_edit - # start editor and/or shell windows: - root = Tk(className="Idle") - - # set application icon - icondir = os.path.join(os.path.dirname(__file__), 'Icons') - if system() == 'Windows': - iconfile = os.path.join(icondir, 'idle.ico') - root.wm_iconbitmap(default=iconfile) - elif TkVersion >= 8.5: - ext = '.png' if TkVersion >= 8.6 else '.gif' - iconfiles = [os.path.join(icondir, 'idle_%d%s' % (size, ext)) - for size in (16, 32, 48)] - icons = [PhotoImage(file=iconfile) for iconfile in iconfiles] - root.wm_iconphoto(True, *icons) - - fixwordbreaks(root) - root.withdraw() - flist = PyShellFileList(root) - macosxSupport.setupApp(root, flist) - - if macosxSupport.isAquaTk(): - # There are some screwed up <2> class bindings for text - # widgets defined in Tk which we need to do away with. - # See issue #24801. - root.unbind_class('Text', '') - root.unbind_class('Text', '') - root.unbind_class('Text', '<>') - - if enable_edit: - if not (cmd or script): - for filename in args[:]: - if flist.open(filename) is None: - # filename is a directory actually, disconsider it - args.remove(filename) - if not args: - flist.new() - - if enable_shell: - shell = flist.open_shell() - if not shell: - return # couldn't open shell - if macosxSupport.isAquaTk() and flist.dict: - # On OSX: when the user has double-clicked on a file that causes - # IDLE to be launched the shell window will open just in front of - # the file she wants to see. Lower the interpreter window when - # there are open files. - shell.top.lower() - else: - shell = flist.pyshell - - # Handle remaining options. If any of these are set, enable_shell - # was set also, so shell must be true to reach here. - if debug: - shell.open_debugger() - if startup: - filename = os.environ.get("IDLESTARTUP") or \ - os.environ.get("PYTHONSTARTUP") - if filename and os.path.isfile(filename): - shell.interp.execfile(filename) - if cmd or script: - shell.interp.runcommand("""if 1: - import sys as _sys - _sys.argv = %r - del _sys - \n""" % (sys.argv,)) - if cmd: - shell.interp.execsource(cmd) - elif script: - shell.interp.prepend_syspath(script) - shell.interp.execfile(script) - elif shell: - # If there is a shell window and no cmd or script in progress, - # check for problematic OS X Tk versions and print a warning - # message in the IDLE shell window; this is less intrusive - # than always opening a separate window. - tkversionwarning = macosxSupport.tkVersionWarning(root) - if tkversionwarning: - shell.interp.runcommand("print('%s')" % tkversionwarning) - - while flist.inversedict: # keep IDLE running while files are open. - root.mainloop() - root.destroy() - capture_warnings(False) - -if __name__ == "__main__": - sys.modules['PyShell'] = sys.modules['__main__'] - main() - -capture_warnings(False) # Make sure turned off; see issue 18081 diff --git a/Lib/idlelib/RemoteDebugger.py b/Lib/idlelib/RemoteDebugger.py deleted file mode 100644 index be2262f..0000000 --- a/Lib/idlelib/RemoteDebugger.py +++ /dev/null @@ -1,388 +0,0 @@ -"""Support for remote Python debugging. - -Some ASCII art to describe the structure: - - IN PYTHON SUBPROCESS # IN IDLE PROCESS - # - # oid='gui_adapter' - +----------+ # +------------+ +-----+ - | GUIProxy |--remote#call-->| GUIAdapter |--calls-->| GUI | -+-----+--calls-->+----------+ # +------------+ +-----+ -| Idb | # / -+-----+<-calls--+------------+ # +----------+<--calls-/ - | IdbAdapter |<--remote#call--| IdbProxy | - +------------+ # +----------+ - oid='idb_adapter' # - -The purpose of the Proxy and Adapter classes is to translate certain -arguments and return values that cannot be transported through the RPC -barrier, in particular frame and traceback objects. - -""" - -import types -from idlelib import Debugger - -debugging = 0 - -idb_adap_oid = "idb_adapter" -gui_adap_oid = "gui_adapter" - -#======================================= -# -# In the PYTHON subprocess: - -frametable = {} -dicttable = {} -codetable = {} -tracebacktable = {} - -def wrap_frame(frame): - fid = id(frame) - frametable[fid] = frame - return fid - -def wrap_info(info): - "replace info[2], a traceback instance, by its ID" - if info is None: - return None - else: - traceback = info[2] - assert isinstance(traceback, types.TracebackType) - traceback_id = id(traceback) - tracebacktable[traceback_id] = traceback - modified_info = (info[0], info[1], traceback_id) - return modified_info - -class GUIProxy: - - def __init__(self, conn, gui_adap_oid): - self.conn = conn - self.oid = gui_adap_oid - - def interaction(self, message, frame, info=None): - # calls rpc.SocketIO.remotecall() via run.MyHandler instance - # pass frame and traceback object IDs instead of the objects themselves - self.conn.remotecall(self.oid, "interaction", - (message, wrap_frame(frame), wrap_info(info)), - {}) - -class IdbAdapter: - - def __init__(self, idb): - self.idb = idb - - #----------called by an IdbProxy---------- - - def set_step(self): - self.idb.set_step() - - def set_quit(self): - self.idb.set_quit() - - def set_continue(self): - self.idb.set_continue() - - def set_next(self, fid): - frame = frametable[fid] - self.idb.set_next(frame) - - def set_return(self, fid): - frame = frametable[fid] - self.idb.set_return(frame) - - def get_stack(self, fid, tbid): - frame = frametable[fid] - if tbid is None: - tb = None - else: - tb = tracebacktable[tbid] - stack, i = self.idb.get_stack(frame, tb) - stack = [(wrap_frame(frame2), k) for frame2, k in stack] - return stack, i - - def run(self, cmd): - import __main__ - self.idb.run(cmd, __main__.__dict__) - - def set_break(self, filename, lineno): - msg = self.idb.set_break(filename, lineno) - return msg - - def clear_break(self, filename, lineno): - msg = self.idb.clear_break(filename, lineno) - return msg - - def clear_all_file_breaks(self, filename): - msg = self.idb.clear_all_file_breaks(filename) - return msg - - #----------called by a FrameProxy---------- - - def frame_attr(self, fid, name): - frame = frametable[fid] - return getattr(frame, name) - - def frame_globals(self, fid): - frame = frametable[fid] - dict = frame.f_globals - did = id(dict) - dicttable[did] = dict - return did - - def frame_locals(self, fid): - frame = frametable[fid] - dict = frame.f_locals - did = id(dict) - dicttable[did] = dict - return did - - def frame_code(self, fid): - frame = frametable[fid] - code = frame.f_code - cid = id(code) - codetable[cid] = code - return cid - - #----------called by a CodeProxy---------- - - def code_name(self, cid): - code = codetable[cid] - return code.co_name - - def code_filename(self, cid): - code = codetable[cid] - return code.co_filename - - #----------called by a DictProxy---------- - - def dict_keys(self, did): - raise NotImplemented("dict_keys not public or pickleable") -## dict = dicttable[did] -## return dict.keys() - - ### Needed until dict_keys is type is finished and pickealable. - ### Will probably need to extend rpc.py:SocketIO._proxify at that time. - def dict_keys_list(self, did): - dict = dicttable[did] - return list(dict.keys()) - - def dict_item(self, did, key): - dict = dicttable[did] - value = dict[key] - value = repr(value) ### can't pickle module 'builtins' - return value - -#----------end class IdbAdapter---------- - - -def start_debugger(rpchandler, gui_adap_oid): - """Start the debugger and its RPC link in the Python subprocess - - Start the subprocess side of the split debugger and set up that side of the - RPC link by instantiating the GUIProxy, Idb debugger, and IdbAdapter - objects and linking them together. Register the IdbAdapter with the - RPCServer to handle RPC requests from the split debugger GUI via the - IdbProxy. - - """ - gui_proxy = GUIProxy(rpchandler, gui_adap_oid) - idb = Debugger.Idb(gui_proxy) - idb_adap = IdbAdapter(idb) - rpchandler.register(idb_adap_oid, idb_adap) - return idb_adap_oid - - -#======================================= -# -# In the IDLE process: - - -class FrameProxy: - - def __init__(self, conn, fid): - self._conn = conn - self._fid = fid - self._oid = "idb_adapter" - self._dictcache = {} - - def __getattr__(self, name): - if name[:1] == "_": - raise AttributeError(name) - if name == "f_code": - return self._get_f_code() - if name == "f_globals": - return self._get_f_globals() - if name == "f_locals": - return self._get_f_locals() - return self._conn.remotecall(self._oid, "frame_attr", - (self._fid, name), {}) - - def _get_f_code(self): - cid = self._conn.remotecall(self._oid, "frame_code", (self._fid,), {}) - return CodeProxy(self._conn, self._oid, cid) - - def _get_f_globals(self): - did = self._conn.remotecall(self._oid, "frame_globals", - (self._fid,), {}) - return self._get_dict_proxy(did) - - def _get_f_locals(self): - did = self._conn.remotecall(self._oid, "frame_locals", - (self._fid,), {}) - return self._get_dict_proxy(did) - - def _get_dict_proxy(self, did): - if did in self._dictcache: - return self._dictcache[did] - dp = DictProxy(self._conn, self._oid, did) - self._dictcache[did] = dp - return dp - - -class CodeProxy: - - def __init__(self, conn, oid, cid): - self._conn = conn - self._oid = oid - self._cid = cid - - def __getattr__(self, name): - if name == "co_name": - return self._conn.remotecall(self._oid, "code_name", - (self._cid,), {}) - if name == "co_filename": - return self._conn.remotecall(self._oid, "code_filename", - (self._cid,), {}) - - -class DictProxy: - - def __init__(self, conn, oid, did): - self._conn = conn - self._oid = oid - self._did = did - -## def keys(self): -## return self._conn.remotecall(self._oid, "dict_keys", (self._did,), {}) - - # 'temporary' until dict_keys is a pickleable built-in type - def keys(self): - return self._conn.remotecall(self._oid, - "dict_keys_list", (self._did,), {}) - - def __getitem__(self, key): - return self._conn.remotecall(self._oid, "dict_item", - (self._did, key), {}) - - def __getattr__(self, name): - ##print("*** Failed DictProxy.__getattr__:", name) - raise AttributeError(name) - - -class GUIAdapter: - - def __init__(self, conn, gui): - self.conn = conn - self.gui = gui - - def interaction(self, message, fid, modified_info): - ##print("*** Interaction: (%s, %s, %s)" % (message, fid, modified_info)) - frame = FrameProxy(self.conn, fid) - self.gui.interaction(message, frame, modified_info) - - -class IdbProxy: - - def __init__(self, conn, shell, oid): - self.oid = oid - self.conn = conn - self.shell = shell - - def call(self, methodname, *args, **kwargs): - ##print("*** IdbProxy.call %s %s %s" % (methodname, args, kwargs)) - value = self.conn.remotecall(self.oid, methodname, args, kwargs) - ##print("*** IdbProxy.call %s returns %r" % (methodname, value)) - return value - - def run(self, cmd, locals): - # Ignores locals on purpose! - seq = self.conn.asyncqueue(self.oid, "run", (cmd,), {}) - self.shell.interp.active_seq = seq - - def get_stack(self, frame, tbid): - # passing frame and traceback IDs, not the objects themselves - stack, i = self.call("get_stack", frame._fid, tbid) - stack = [(FrameProxy(self.conn, fid), k) for fid, k in stack] - return stack, i - - def set_continue(self): - self.call("set_continue") - - def set_step(self): - self.call("set_step") - - def set_next(self, frame): - self.call("set_next", frame._fid) - - def set_return(self, frame): - self.call("set_return", frame._fid) - - def set_quit(self): - self.call("set_quit") - - def set_break(self, filename, lineno): - msg = self.call("set_break", filename, lineno) - return msg - - def clear_break(self, filename, lineno): - msg = self.call("clear_break", filename, lineno) - return msg - - def clear_all_file_breaks(self, filename): - msg = self.call("clear_all_file_breaks", filename) - return msg - -def start_remote_debugger(rpcclt, pyshell): - """Start the subprocess debugger, initialize the debugger GUI and RPC link - - Request the RPCServer start the Python subprocess debugger and link. Set - up the Idle side of the split debugger by instantiating the IdbProxy, - debugger GUI, and debugger GUIAdapter objects and linking them together. - - Register the GUIAdapter with the RPCClient to handle debugger GUI - interaction requests coming from the subprocess debugger via the GUIProxy. - - The IdbAdapter will pass execution and environment requests coming from the - Idle debugger GUI to the subprocess debugger via the IdbProxy. - - """ - global idb_adap_oid - - idb_adap_oid = rpcclt.remotecall("exec", "start_the_debugger",\ - (gui_adap_oid,), {}) - idb_proxy = IdbProxy(rpcclt, pyshell, idb_adap_oid) - gui = Debugger.Debugger(pyshell, idb_proxy) - gui_adap = GUIAdapter(rpcclt, gui) - rpcclt.register(gui_adap_oid, gui_adap) - return gui - -def close_remote_debugger(rpcclt): - """Shut down subprocess debugger and Idle side of debugger RPC link - - Request that the RPCServer shut down the subprocess debugger and link. - Unregister the GUIAdapter, which will cause a GC on the Idle process - debugger and RPC link objects. (The second reference to the debugger GUI - is deleted in PyShell.close_remote_debugger().) - - """ - close_subprocess_debugger(rpcclt) - rpcclt.unregister(gui_adap_oid) - -def close_subprocess_debugger(rpcclt): - rpcclt.remotecall("exec", "stop_the_debugger", (idb_adap_oid,), {}) - -def restart_subprocess_debugger(rpcclt): - idb_adap_oid_ret = rpcclt.remotecall("exec", "start_the_debugger",\ - (gui_adap_oid,), {}) - assert idb_adap_oid_ret == idb_adap_oid, 'Idb restarted with different oid' diff --git a/Lib/idlelib/RemoteObjectBrowser.py b/Lib/idlelib/RemoteObjectBrowser.py deleted file mode 100644 index 8031aae..0000000 --- a/Lib/idlelib/RemoteObjectBrowser.py +++ /dev/null @@ -1,36 +0,0 @@ -from idlelib import rpc - -def remote_object_tree_item(item): - wrapper = WrappedObjectTreeItem(item) - oid = id(wrapper) - rpc.objecttable[oid] = wrapper - return oid - -class WrappedObjectTreeItem: - # Lives in PYTHON subprocess - - def __init__(self, item): - self.__item = item - - def __getattr__(self, name): - value = getattr(self.__item, name) - return value - - def _GetSubList(self): - sub_list = self.__item._GetSubList() - return list(map(remote_object_tree_item, sub_list)) - -class StubObjectTreeItem: - # Lives in IDLE process - - def __init__(self, sockio, oid): - self.sockio = sockio - self.oid = oid - - def __getattr__(self, name): - value = rpc.MethodProxy(self.sockio, self.oid, name) - return value - - def _GetSubList(self): - sub_list = self.sockio.remotecall(self.oid, "_GetSubList", (), {}) - return [StubObjectTreeItem(self.sockio, oid) for oid in sub_list] diff --git a/Lib/idlelib/ReplaceDialog.py b/Lib/idlelib/ReplaceDialog.py deleted file mode 100644 index f2ea22e..0000000 --- a/Lib/idlelib/ReplaceDialog.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Replace dialog for IDLE. Inherits SearchDialogBase for GUI. -Uses idlelib.SearchEngine for search capability. -Defines various replace related functions like replace, replace all, -replace+find. -""" -from tkinter import * - -from idlelib import SearchEngine -from idlelib.SearchDialogBase import SearchDialogBase -import re - - -def replace(text): - """Returns a singleton ReplaceDialog instance.The single dialog - saves user entries and preferences across instances.""" - root = text._root() - engine = SearchEngine.get(root) - if not hasattr(engine, "_replacedialog"): - engine._replacedialog = ReplaceDialog(root, engine) - dialog = engine._replacedialog - dialog.open(text) - - -class ReplaceDialog(SearchDialogBase): - - title = "Replace Dialog" - icon = "Replace" - - def __init__(self, root, engine): - SearchDialogBase.__init__(self, root, engine) - self.replvar = StringVar(root) - - def open(self, text): - """Display the replace dialog""" - SearchDialogBase.open(self, text) - try: - first = text.index("sel.first") - except TclError: - first = None - try: - last = text.index("sel.last") - except TclError: - last = None - first = first or text.index("insert") - last = last or first - self.show_hit(first, last) - self.ok = 1 - - def create_entries(self): - """Create label and text entry widgets""" - SearchDialogBase.create_entries(self) - self.replent = self.make_entry("Replace with:", self.replvar)[0] - - def create_command_buttons(self): - SearchDialogBase.create_command_buttons(self) - self.make_button("Find", self.find_it) - self.make_button("Replace", self.replace_it) - self.make_button("Replace+Find", self.default_command, 1) - self.make_button("Replace All", self.replace_all) - - def find_it(self, event=None): - self.do_find(0) - - def replace_it(self, event=None): - if self.do_find(self.ok): - self.do_replace() - - def default_command(self, event=None): - "Replace and find next." - if self.do_find(self.ok): - if self.do_replace(): # Only find next match if replace succeeded. - # A bad re can cause it to fail. - self.do_find(0) - - def _replace_expand(self, m, repl): - """ Helper function for expanding a regular expression - in the replace field, if needed. """ - if self.engine.isre(): - try: - new = m.expand(repl) - except re.error: - self.engine.report_error(repl, 'Invalid Replace Expression') - new = None - else: - new = repl - - return new - - def replace_all(self, event=None): - """Replace all instances of patvar with replvar in text""" - prog = self.engine.getprog() - if not prog: - return - repl = self.replvar.get() - text = self.text - res = self.engine.search_text(text, prog) - if not res: - text.bell() - return - text.tag_remove("sel", "1.0", "end") - text.tag_remove("hit", "1.0", "end") - line = res[0] - col = res[1].start() - if self.engine.iswrap(): - line = 1 - col = 0 - ok = 1 - first = last = None - # XXX ought to replace circular instead of top-to-bottom when wrapping - text.undo_block_start() - while 1: - res = self.engine.search_forward(text, prog, line, col, 0, ok) - if not res: - break - line, m = res - chars = text.get("%d.0" % line, "%d.0" % (line+1)) - orig = m.group() - new = self._replace_expand(m, repl) - if new is None: - break - i, j = m.span() - first = "%d.%d" % (line, i) - last = "%d.%d" % (line, j) - if new == orig: - text.mark_set("insert", last) - else: - text.mark_set("insert", first) - if first != last: - text.delete(first, last) - if new: - text.insert(first, new) - col = i + len(new) - ok = 0 - text.undo_block_stop() - if first and last: - self.show_hit(first, last) - self.close() - - def do_find(self, ok=0): - if not self.engine.getprog(): - return False - text = self.text - res = self.engine.search_text(text, None, ok) - if not res: - text.bell() - return False - line, m = res - i, j = m.span() - first = "%d.%d" % (line, i) - last = "%d.%d" % (line, j) - self.show_hit(first, last) - self.ok = 1 - return True - - def do_replace(self): - prog = self.engine.getprog() - if not prog: - return False - text = self.text - try: - first = pos = text.index("sel.first") - last = text.index("sel.last") - except TclError: - pos = None - if not pos: - first = last = pos = text.index("insert") - line, col = SearchEngine.get_line_col(pos) - chars = text.get("%d.0" % line, "%d.0" % (line+1)) - m = prog.match(chars, col) - if not prog: - return False - new = self._replace_expand(m, self.replvar.get()) - if new is None: - return False - text.mark_set("insert", first) - text.undo_block_start() - if m.group(): - text.delete(first, last) - if new: - text.insert(first, new) - text.undo_block_stop() - self.show_hit(first, text.index("insert")) - self.ok = 0 - return True - - def show_hit(self, first, last): - """Highlight text from 'first' to 'last'. - 'first', 'last' - Text indices""" - text = self.text - text.mark_set("insert", first) - text.tag_remove("sel", "1.0", "end") - text.tag_add("sel", first, last) - text.tag_remove("hit", "1.0", "end") - if first == last: - text.tag_add("hit", first) - else: - text.tag_add("hit", first, last) - text.see("insert") - text.update_idletasks() - - def close(self, event=None): - SearchDialogBase.close(self, event) - self.text.tag_remove("hit", "1.0", "end") - - -def _replace_dialog(parent): # htest # - """htest wrapper function""" - box = Toplevel(parent) - box.title("Test ReplaceDialog") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - box.geometry("+%d+%d"%(x, y + 150)) - - # mock undo delegator methods - def undo_block_start(): - pass - - def undo_block_stop(): - pass - - text = Text(box, inactiveselectbackground='gray') - text.undo_block_start = undo_block_start - text.undo_block_stop = undo_block_stop - text.pack() - text.insert("insert","This is a sample sTring\nPlus MORE.") - text.focus_set() - - def show_replace(): - text.tag_add(SEL, "1.0", END) - replace(text) - text.tag_remove(SEL, "1.0", END) - - button = Button(box, text="Replace", command=show_replace) - button.pack() - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_replacedialog', - verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(_replace_dialog) diff --git a/Lib/idlelib/RstripExtension.py b/Lib/idlelib/RstripExtension.py deleted file mode 100644 index 2ce3c7e..0000000 --- a/Lib/idlelib/RstripExtension.py +++ /dev/null @@ -1,33 +0,0 @@ -'Provides "Strip trailing whitespace" under the "Format" menu.' - -class RstripExtension: - - menudefs = [ - ('format', [None, ('Strip trailing whitespace', '<>'), ] ), ] - - def __init__(self, editwin): - self.editwin = editwin - self.editwin.text.bind("<>", self.do_rstrip) - - def do_rstrip(self, event=None): - - text = self.editwin.text - undo = self.editwin.undo - - undo.undo_block_start() - - end_line = int(float(text.index('end'))) - for cur in range(1, end_line): - txt = text.get('%i.0' % cur, '%i.end' % cur) - raw = len(txt) - cut = len(txt.rstrip()) - # Since text.delete() marks file as changed, even if not, - # only call it when needed to actually delete something. - if cut < raw: - text.delete('%i.%i' % (cur, cut), '%i.end' % cur) - - undo.undo_block_stop() - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_rstrip', verbosity=2, exit=False) diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py deleted file mode 100644 index 5cb818d..0000000 --- a/Lib/idlelib/ScriptBinding.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Extension to execute code outside the Python shell window. - -This adds the following commands: - -- Check module does a full syntax check of the current module. - It also runs the tabnanny to catch any inconsistent tabs. - -- Run module executes the module's code in the __main__ namespace. The window - must have been saved previously. The module is added to sys.modules, and is - also added to the __main__ namespace. - -XXX GvR Redesign this interface (yet again) as follows: - -- Present a dialog box for ``Run Module'' - -- Allow specify command line arguments in the dialog box - -""" - -import os -import tabnanny -import tokenize -import tkinter.messagebox as tkMessageBox -from idlelib import PyShell - -from idlelib.configHandler import idleConf -from idlelib import macosxSupport - -indent_message = """Error: Inconsistent indentation detected! - -1) Your indentation is outright incorrect (easy to fix), OR - -2) Your indentation mixes tabs and spaces. - -To fix case 2, change all tabs to spaces by using Edit->Select All followed \ -by Format->Untabify Region and specify the number of columns used by each tab. -""" - - -class ScriptBinding: - - menudefs = [ - ('run', [None, - ('Check Module', '<>'), - ('Run Module', '<>'), ]), ] - - def __init__(self, editwin): - self.editwin = editwin - # Provide instance variables referenced by Debugger - # XXX This should be done differently - self.flist = self.editwin.flist - self.root = self.editwin.root - - if macosxSupport.isCocoaTk(): - self.editwin.text_frame.bind('<>', self._run_module_event) - - def check_module_event(self, event): - filename = self.getfilename() - if not filename: - return 'break' - if not self.checksyntax(filename): - return 'break' - if not self.tabnanny(filename): - return 'break' - - def tabnanny(self, filename): - # XXX: tabnanny should work on binary files as well - with tokenize.open(filename) as f: - try: - tabnanny.process_tokens(tokenize.generate_tokens(f.readline)) - except tokenize.TokenError as msg: - msgtxt, (lineno, start) = msg.args - self.editwin.gotoline(lineno) - self.errorbox("Tabnanny Tokenizing Error", - "Token Error: %s" % msgtxt) - return False - except tabnanny.NannyNag as nag: - # The error messages from tabnanny are too confusing... - self.editwin.gotoline(nag.get_lineno()) - self.errorbox("Tab/space error", indent_message) - return False - return True - - def checksyntax(self, filename): - self.shell = shell = self.flist.open_shell() - saved_stream = shell.get_warning_stream() - shell.set_warning_stream(shell.stderr) - with open(filename, 'rb') as f: - source = f.read() - if b'\r' in source: - source = source.replace(b'\r\n', b'\n') - source = source.replace(b'\r', b'\n') - if source and source[-1] != ord(b'\n'): - source = source + b'\n' - editwin = self.editwin - text = editwin.text - text.tag_remove("ERROR", "1.0", "end") - try: - # If successful, return the compiled code - return compile(source, filename, "exec") - except (SyntaxError, OverflowError, ValueError) as value: - msg = getattr(value, 'msg', '') or value or "" - lineno = getattr(value, 'lineno', '') or 1 - offset = getattr(value, 'offset', '') or 0 - if offset == 0: - lineno += 1 #mark end of offending line - pos = "0.0 + %d lines + %d chars" % (lineno-1, offset-1) - editwin.colorize_syntax_error(text, pos) - self.errorbox("SyntaxError", "%-20s" % msg) - return False - finally: - shell.set_warning_stream(saved_stream) - - def run_module_event(self, event): - if macosxSupport.isCocoaTk(): - # Tk-Cocoa in MacOSX is broken until at least - # Tk 8.5.9, and without this rather - # crude workaround IDLE would hang when a user - # tries to run a module using the keyboard shortcut - # (the menu item works fine). - self.editwin.text_frame.after(200, - lambda: self.editwin.text_frame.event_generate('<>')) - return 'break' - else: - return self._run_module_event(event) - - def _run_module_event(self, event): - """Run the module after setting up the environment. - - First check the syntax. If OK, make sure the shell is active and - then transfer the arguments, set the run environment's working - directory to the directory of the module being executed and also - add that directory to its sys.path if not already included. - """ - - filename = self.getfilename() - if not filename: - return 'break' - code = self.checksyntax(filename) - if not code: - return 'break' - if not self.tabnanny(filename): - return 'break' - interp = self.shell.interp - if PyShell.use_subprocess: - interp.restart_subprocess(with_cwd=False, filename= - self.editwin._filename_to_unicode(filename)) - dirname = os.path.dirname(filename) - # XXX Too often this discards arguments the user just set... - interp.runcommand("""if 1: - __file__ = {filename!r} - import sys as _sys - from os.path import basename as _basename - if (not _sys.argv or - _basename(_sys.argv[0]) != _basename(__file__)): - _sys.argv = [__file__] - import os as _os - _os.chdir({dirname!r}) - del _sys, _basename, _os - \n""".format(filename=filename, dirname=dirname)) - interp.prepend_syspath(filename) - # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still - # go to __stderr__. With subprocess, they go to the shell. - # Need to change streams in PyShell.ModifiedInterpreter. - interp.runcode(code) - return 'break' - - def getfilename(self): - """Get source filename. If not saved, offer to save (or create) file - - The debugger requires a source file. Make sure there is one, and that - the current version of the source buffer has been saved. If the user - declines to save or cancels the Save As dialog, return None. - - If the user has configured IDLE for Autosave, the file will be - silently saved if it already exists and is dirty. - - """ - filename = self.editwin.io.filename - if not self.editwin.get_saved(): - autosave = idleConf.GetOption('main', 'General', - 'autosave', type='bool') - if autosave and filename: - self.editwin.io.save(None) - else: - confirm = self.ask_save_dialog() - self.editwin.text.focus_set() - if confirm: - self.editwin.io.save(None) - filename = self.editwin.io.filename - else: - filename = None - return filename - - def ask_save_dialog(self): - msg = "Source Must Be Saved\n" + 5*' ' + "OK to Save?" - confirm = tkMessageBox.askokcancel(title="Save Before Run or Check", - message=msg, - default=tkMessageBox.OK, - parent=self.editwin.text) - return confirm - - def errorbox(self, title, message): - # XXX This should really be a function of EditorWindow... - tkMessageBox.showerror(title, message, parent=self.editwin.text) - self.editwin.text.focus_set() diff --git a/Lib/idlelib/ScrolledList.py b/Lib/idlelib/ScrolledList.py deleted file mode 100644 index 53576b5..0000000 --- a/Lib/idlelib/ScrolledList.py +++ /dev/null @@ -1,145 +0,0 @@ -from tkinter import * -from idlelib import macosxSupport - -class ScrolledList: - - default = "(None)" - - def __init__(self, master, **options): - # Create top frame, with scrollbar and listbox - self.master = master - self.frame = frame = Frame(master) - self.frame.pack(fill="both", expand=1) - self.vbar = vbar = Scrollbar(frame, name="vbar") - self.vbar.pack(side="right", fill="y") - self.listbox = listbox = Listbox(frame, exportselection=0, - background="white") - if options: - listbox.configure(options) - listbox.pack(expand=1, fill="both") - # Tie listbox and scrollbar together - vbar["command"] = listbox.yview - listbox["yscrollcommand"] = vbar.set - # Bind events to the list box - listbox.bind("", self.click_event) - listbox.bind("", self.double_click_event) - if macosxSupport.isAquaTk(): - listbox.bind("", self.popup_event) - listbox.bind("", self.popup_event) - else: - listbox.bind("", self.popup_event) - listbox.bind("", self.up_event) - listbox.bind("", self.down_event) - # Mark as empty - self.clear() - - def close(self): - self.frame.destroy() - - def clear(self): - self.listbox.delete(0, "end") - self.empty = 1 - self.listbox.insert("end", self.default) - - def append(self, item): - if self.empty: - self.listbox.delete(0, "end") - self.empty = 0 - self.listbox.insert("end", str(item)) - - def get(self, index): - return self.listbox.get(index) - - def click_event(self, event): - self.listbox.activate("@%d,%d" % (event.x, event.y)) - index = self.listbox.index("active") - self.select(index) - self.on_select(index) - return "break" - - def double_click_event(self, event): - index = self.listbox.index("active") - self.select(index) - self.on_double(index) - return "break" - - menu = None - - def popup_event(self, event): - if not self.menu: - self.make_menu() - menu = self.menu - self.listbox.activate("@%d,%d" % (event.x, event.y)) - index = self.listbox.index("active") - self.select(index) - menu.tk_popup(event.x_root, event.y_root) - - def make_menu(self): - menu = Menu(self.listbox, tearoff=0) - self.menu = menu - self.fill_menu() - - def up_event(self, event): - index = self.listbox.index("active") - if self.listbox.selection_includes(index): - index = index - 1 - else: - index = self.listbox.size() - 1 - if index < 0: - self.listbox.bell() - else: - self.select(index) - self.on_select(index) - return "break" - - def down_event(self, event): - index = self.listbox.index("active") - if self.listbox.selection_includes(index): - index = index + 1 - else: - index = 0 - if index >= self.listbox.size(): - self.listbox.bell() - else: - self.select(index) - self.on_select(index) - return "break" - - def select(self, index): - self.listbox.focus_set() - self.listbox.activate(index) - self.listbox.selection_clear(0, "end") - self.listbox.selection_set(index) - self.listbox.see(index) - - # Methods to override for specific actions - - def fill_menu(self): - pass - - def on_select(self, index): - pass - - def on_double(self, index): - pass - - -def _scrolled_list(parent): - root = Tk() - root.title("Test ScrolledList") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - class MyScrolledList(ScrolledList): - def fill_menu(self): self.menu.add_command(label="right click") - def on_select(self, index): print("select", self.get(index)) - def on_double(self, index): print("double", self.get(index)) - - scrolled_list = MyScrolledList(root) - for i in range(30): - scrolled_list.append("Item %02d" % i) - - root.mainloop() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_scrolled_list) diff --git a/Lib/idlelib/SearchDialog.py b/Lib/idlelib/SearchDialog.py deleted file mode 100644 index 765d53f..0000000 --- a/Lib/idlelib/SearchDialog.py +++ /dev/null @@ -1,97 +0,0 @@ -from tkinter import * - -from idlelib import SearchEngine -from idlelib.SearchDialogBase import SearchDialogBase - -def _setup(text): - "Create or find the singleton SearchDialog instance." - root = text._root() - engine = SearchEngine.get(root) - if not hasattr(engine, "_searchdialog"): - engine._searchdialog = SearchDialog(root, engine) - return engine._searchdialog - -def find(text): - "Handle the editor edit menu item and corresponding event." - pat = text.get("sel.first", "sel.last") - return _setup(text).open(text, pat) # Open is inherited from SDBase. - -def find_again(text): - "Handle the editor edit menu item and corresponding event." - return _setup(text).find_again(text) - -def find_selection(text): - "Handle the editor edit menu item and corresponding event." - return _setup(text).find_selection(text) - -class SearchDialog(SearchDialogBase): - - def create_widgets(self): - SearchDialogBase.create_widgets(self) - self.make_button("Find Next", self.default_command, 1) - - def default_command(self, event=None): - if not self.engine.getprog(): - return - self.find_again(self.text) - - def find_again(self, text): - if not self.engine.getpat(): - self.open(text) - return False - if not self.engine.getprog(): - return False - res = self.engine.search_text(text) - if res: - line, m = res - i, j = m.span() - first = "%d.%d" % (line, i) - last = "%d.%d" % (line, j) - try: - selfirst = text.index("sel.first") - sellast = text.index("sel.last") - if selfirst == first and sellast == last: - text.bell() - return False - except TclError: - pass - text.tag_remove("sel", "1.0", "end") - text.tag_add("sel", first, last) - text.mark_set("insert", self.engine.isback() and first or last) - text.see("insert") - return True - else: - text.bell() - return False - - def find_selection(self, text): - pat = text.get("sel.first", "sel.last") - if pat: - self.engine.setcookedpat(pat) - return self.find_again(text) - - -def _search_dialog(parent): # htest # - '''Display search test box.''' - box = Toplevel(parent) - box.title("Test SearchDialog") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - box.geometry("+%d+%d"%(x, y + 150)) - text = Text(box, inactiveselectbackground='gray') - text.pack() - text.insert("insert","This is a sample string.\n"*5) - - def show_find(): - text.tag_add(SEL, "1.0", END) - _setup(text).open(text) - text.tag_remove(SEL, "1.0", END) - - button = Button(box, text="Search (selection ignored)", command=show_find) - button.pack() - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_searchdialog', - verbosity=2, exit=False) - from idlelib.idle_test.htest import run - run(_search_dialog) diff --git a/Lib/idlelib/SearchDialogBase.py b/Lib/idlelib/SearchDialogBase.py deleted file mode 100644 index 5fa84e2..0000000 --- a/Lib/idlelib/SearchDialogBase.py +++ /dev/null @@ -1,184 +0,0 @@ -'''Define SearchDialogBase used by Search, Replace, and Grep dialogs.''' - -from tkinter import (Toplevel, Frame, Entry, Label, Button, - Checkbutton, Radiobutton) - -class SearchDialogBase: - '''Create most of a 3 or 4 row, 3 column search dialog. - - The left and wide middle column contain: - 1 or 2 labeled text entry lines (make_entry, create_entries); - a row of standard Checkbuttons (make_frame, create_option_buttons), - each of which corresponds to a search engine Variable; - a row of dialog-specific Check/Radiobuttons (create_other_buttons). - - The narrow right column contains command buttons - (make_button, create_command_buttons). - These are bound to functions that execute the command. - - Except for command buttons, this base class is not limited to items - common to all three subclasses. Rather, it is the Find dialog minus - the "Find Next" command, its execution function, and the - default_command attribute needed in create_widgets. The other - dialogs override attributes and methods, the latter to replace and - add widgets. - ''' - - title = "Search Dialog" # replace in subclasses - icon = "Search" - needwrapbutton = 1 # not in Find in Files - - def __init__(self, root, engine): - '''Initialize root, engine, and top attributes. - - top (level widget): set in create_widgets() called from open(). - text (Text searched): set in open(), only used in subclasses(). - ent (ry): created in make_entry() called from create_entry(). - row (of grid): 0 in create_widgets(), +1 in make_entry/frame(). - default_command: set in subclasses, used in create_widgers(). - - title (of dialog): class attribute, override in subclasses. - icon (of dialog): ditto, use unclear if cannot minimize dialog. - ''' - self.root = root - self.engine = engine - self.top = None - - def open(self, text, searchphrase=None): - "Make dialog visible on top of others and ready to use." - self.text = text - if not self.top: - self.create_widgets() - else: - self.top.deiconify() - self.top.tkraise() - if searchphrase: - self.ent.delete(0,"end") - self.ent.insert("end",searchphrase) - self.ent.focus_set() - self.ent.selection_range(0, "end") - self.ent.icursor(0) - self.top.grab_set() - - def close(self, event=None): - "Put dialog away for later use." - if self.top: - self.top.grab_release() - self.top.withdraw() - - def create_widgets(self): - '''Create basic 3 row x 3 col search (find) dialog. - - Other dialogs override subsidiary create_x methods as needed. - Replace and Find-in-Files add another entry row. - ''' - top = Toplevel(self.root) - top.bind("", self.default_command) - top.bind("", self.close) - top.protocol("WM_DELETE_WINDOW", self.close) - top.wm_title(self.title) - top.wm_iconname(self.icon) - self.top = top - - self.row = 0 - self.top.grid_columnconfigure(0, pad=2, weight=0) - self.top.grid_columnconfigure(1, pad=2, minsize=100, weight=100) - - self.create_entries() # row 0 (and maybe 1), cols 0, 1 - self.create_option_buttons() # next row, cols 0, 1 - self.create_other_buttons() # next row, cols 0, 1 - self.create_command_buttons() # col 2, all rows - - def make_entry(self, label_text, var): - '''Return (entry, label), . - - entry - gridded labeled Entry for text entry. - label - Label widget, returned for testing. - ''' - label = Label(self.top, text=label_text) - label.grid(row=self.row, column=0, sticky="nw") - entry = Entry(self.top, textvariable=var, exportselection=0) - entry.grid(row=self.row, column=1, sticky="nwe") - self.row = self.row + 1 - return entry, label - - def create_entries(self): - "Create one or more entry lines with make_entry." - self.ent = self.make_entry("Find:", self.engine.patvar)[0] - - def make_frame(self,labeltext=None): - '''Return (frame, label). - - frame - gridded labeled Frame for option or other buttons. - label - Label widget, returned for testing. - ''' - if labeltext: - label = Label(self.top, text=labeltext) - label.grid(row=self.row, column=0, sticky="nw") - else: - label = '' - frame = Frame(self.top) - frame.grid(row=self.row, column=1, columnspan=1, sticky="nwe") - self.row = self.row + 1 - return frame, label - - def create_option_buttons(self): - '''Return (filled frame, options) for testing. - - Options is a list of SearchEngine booleanvar, label pairs. - A gridded frame from make_frame is filled with a Checkbutton - for each pair, bound to the var, with the corresponding label. - ''' - frame = self.make_frame("Options")[0] - engine = self.engine - options = [(engine.revar, "Regular expression"), - (engine.casevar, "Match case"), - (engine.wordvar, "Whole word")] - if self.needwrapbutton: - options.append((engine.wrapvar, "Wrap around")) - for var, label in options: - btn = Checkbutton(frame, anchor="w", variable=var, text=label) - btn.pack(side="left", fill="both") - if var.get(): - btn.select() - return frame, options - - def create_other_buttons(self): - '''Return (frame, others) for testing. - - Others is a list of value, label pairs. - A gridded frame from make_frame is filled with radio buttons. - ''' - frame = self.make_frame("Direction")[0] - var = self.engine.backvar - others = [(1, 'Up'), (0, 'Down')] - for val, label in others: - btn = Radiobutton(frame, anchor="w", - variable=var, value=val, text=label) - btn.pack(side="left", fill="both") - if var.get() == val: - btn.select() - return frame, others - - def make_button(self, label, command, isdef=0): - "Return command button gridded in command frame." - b = Button(self.buttonframe, - text=label, command=command, - default=isdef and "active" or "normal") - cols,rows=self.buttonframe.grid_size() - b.grid(pady=1,row=rows,column=0,sticky="ew") - self.buttonframe.grid(rowspan=rows+1) - return b - - def create_command_buttons(self): - "Place buttons in vertical command frame gridded on right." - f = self.buttonframe = Frame(self.top) - f.grid(row=0,column=2,padx=2,pady=2,ipadx=2,ipady=2) - - b = self.make_button("close", self.close) - b.lower() - -if __name__ == '__main__': - import unittest - unittest.main( - 'idlelib.idle_test.test_searchdialogbase', verbosity=2) diff --git a/Lib/idlelib/SearchEngine.py b/Lib/idlelib/SearchEngine.py deleted file mode 100644 index 37883bf..0000000 --- a/Lib/idlelib/SearchEngine.py +++ /dev/null @@ -1,233 +0,0 @@ -'''Define SearchEngine for search dialogs.''' -import re -from tkinter import StringVar, BooleanVar, TclError -import tkinter.messagebox as tkMessageBox - -def get(root): - '''Return the singleton SearchEngine instance for the process. - - The single SearchEngine saves settings between dialog instances. - If there is not a SearchEngine already, make one. - ''' - if not hasattr(root, "_searchengine"): - root._searchengine = SearchEngine(root) - # This creates a cycle that persists until root is deleted. - return root._searchengine - -class SearchEngine: - """Handles searching a text widget for Find, Replace, and Grep.""" - - def __init__(self, root): - '''Initialize Variables that save search state. - - The dialogs bind these to the UI elements present in the dialogs. - ''' - self.root = root # need for report_error() - self.patvar = StringVar(root, '') # search pattern - self.revar = BooleanVar(root, False) # regular expression? - self.casevar = BooleanVar(root, False) # match case? - self.wordvar = BooleanVar(root, False) # match whole word? - self.wrapvar = BooleanVar(root, True) # wrap around buffer? - self.backvar = BooleanVar(root, False) # search backwards? - - # Access methods - - def getpat(self): - return self.patvar.get() - - def setpat(self, pat): - self.patvar.set(pat) - - def isre(self): - return self.revar.get() - - def iscase(self): - return self.casevar.get() - - def isword(self): - return self.wordvar.get() - - def iswrap(self): - return self.wrapvar.get() - - def isback(self): - return self.backvar.get() - - # Higher level access methods - - def setcookedpat(self, pat): - "Set pattern after escaping if re." - # called only in SearchDialog.py: 66 - if self.isre(): - pat = re.escape(pat) - self.setpat(pat) - - def getcookedpat(self): - pat = self.getpat() - if not self.isre(): # if True, see setcookedpat - pat = re.escape(pat) - if self.isword(): - pat = r"\b%s\b" % pat - return pat - - def getprog(self): - "Return compiled cooked search pattern." - pat = self.getpat() - if not pat: - self.report_error(pat, "Empty regular expression") - return None - pat = self.getcookedpat() - flags = 0 - if not self.iscase(): - flags = flags | re.IGNORECASE - try: - prog = re.compile(pat, flags) - except re.error as what: - args = what.args - msg = args[0] - col = args[1] if len(args) >= 2 else -1 - self.report_error(pat, msg, col) - return None - return prog - - def report_error(self, pat, msg, col=-1): - # Derived class could override this with something fancier - msg = "Error: " + str(msg) - if pat: - msg = msg + "\nPattern: " + str(pat) - if col >= 0: - msg = msg + "\nOffset: " + str(col) - tkMessageBox.showerror("Regular expression error", - msg, master=self.root) - - def search_text(self, text, prog=None, ok=0): - '''Return (lineno, matchobj) or None for forward/backward search. - - This function calls the right function with the right arguments. - It directly return the result of that call. - - Text is a text widget. Prog is a precompiled pattern. - The ok parameter is a bit complicated as it has two effects. - - If there is a selection, the search begin at either end, - depending on the direction setting and ok, with ok meaning that - the search starts with the selection. Otherwise, search begins - at the insert mark. - - To aid progress, the search functions do not return an empty - match at the starting position unless ok is True. - ''' - - if not prog: - prog = self.getprog() - if not prog: - return None # Compilation failed -- stop - wrap = self.wrapvar.get() - first, last = get_selection(text) - if self.isback(): - if ok: - start = last - else: - start = first - line, col = get_line_col(start) - res = self.search_backward(text, prog, line, col, wrap, ok) - else: - if ok: - start = first - else: - start = last - line, col = get_line_col(start) - res = self.search_forward(text, prog, line, col, wrap, ok) - return res - - def search_forward(self, text, prog, line, col, wrap, ok=0): - wrapped = 0 - startline = line - chars = text.get("%d.0" % line, "%d.0" % (line+1)) - while chars: - m = prog.search(chars[:-1], col) - if m: - if ok or m.end() > col: - return line, m - line = line + 1 - if wrapped and line > startline: - break - col = 0 - ok = 1 - chars = text.get("%d.0" % line, "%d.0" % (line+1)) - if not chars and wrap: - wrapped = 1 - wrap = 0 - line = 1 - chars = text.get("1.0", "2.0") - return None - - def search_backward(self, text, prog, line, col, wrap, ok=0): - wrapped = 0 - startline = line - chars = text.get("%d.0" % line, "%d.0" % (line+1)) - while 1: - m = search_reverse(prog, chars[:-1], col) - if m: - if ok or m.start() < col: - return line, m - line = line - 1 - if wrapped and line < startline: - break - ok = 1 - if line <= 0: - if not wrap: - break - wrapped = 1 - wrap = 0 - pos = text.index("end-1c") - line, col = map(int, pos.split(".")) - chars = text.get("%d.0" % line, "%d.0" % (line+1)) - col = len(chars) - 1 - return None - -def search_reverse(prog, chars, col): - '''Search backwards and return an re match object or None. - - This is done by searching forwards until there is no match. - Prog: compiled re object with a search method returning a match. - Chars: line of text, without \\n. - Col: stop index for the search; the limit for match.end(). - ''' - m = prog.search(chars) - if not m: - return None - found = None - i, j = m.span() # m.start(), m.end() == match slice indexes - while i < col and j <= col: - found = m - if i == j: - j = j+1 - m = prog.search(chars, j) - if not m: - break - i, j = m.span() - return found - -def get_selection(text): - '''Return tuple of 'line.col' indexes from selection or insert mark. - ''' - try: - first = text.index("sel.first") - last = text.index("sel.last") - except TclError: - first = last = None - if not first: - first = text.index("insert") - if not last: - last = first - return first, last - -def get_line_col(index): - '''Return (line, col) tuple of ints from 'line.col' string.''' - line, col = map(int, index.split(".")) # Fails on invalid index - return line, col - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_searchengine', verbosity=2, exit=False) diff --git a/Lib/idlelib/StackViewer.py b/Lib/idlelib/StackViewer.py deleted file mode 100644 index ccc755c..0000000 --- a/Lib/idlelib/StackViewer.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import sys -import linecache -import re -import tkinter as tk - -from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas -from idlelib.ObjectBrowser import ObjectTreeItem, make_objecttreeitem -from idlelib.PyShell import PyShellFileList - -def StackBrowser(root, flist=None, tb=None, top=None): - if top is None: - top = tk.Toplevel(root) - sc = ScrolledCanvas(top, bg="white", highlightthickness=0) - sc.frame.pack(expand=1, fill="both") - item = StackTreeItem(flist, tb) - node = TreeNode(sc.canvas, None, item) - node.expand() - -class StackTreeItem(TreeItem): - - def __init__(self, flist=None, tb=None): - self.flist = flist - self.stack = self.get_stack(tb) - self.text = self.get_exception() - - def get_stack(self, tb): - if tb is None: - tb = sys.last_traceback - stack = [] - if tb and tb.tb_frame is None: - tb = tb.tb_next - while tb is not None: - stack.append((tb.tb_frame, tb.tb_lineno)) - tb = tb.tb_next - return stack - - def get_exception(self): - type = sys.last_type - value = sys.last_value - if hasattr(type, "__name__"): - type = type.__name__ - s = str(type) - if value is not None: - s = s + ": " + str(value) - return s - - def GetText(self): - return self.text - - def GetSubList(self): - sublist = [] - for info in self.stack: - item = FrameTreeItem(info, self.flist) - sublist.append(item) - return sublist - -class FrameTreeItem(TreeItem): - - def __init__(self, info, flist): - self.info = info - self.flist = flist - - def GetText(self): - frame, lineno = self.info - try: - modname = frame.f_globals["__name__"] - except: - modname = "?" - code = frame.f_code - filename = code.co_filename - funcname = code.co_name - sourceline = linecache.getline(filename, lineno) - sourceline = sourceline.strip() - if funcname in ("?", "", None): - item = "%s, line %d: %s" % (modname, lineno, sourceline) - else: - item = "%s.%s(...), line %d: %s" % (modname, funcname, - lineno, sourceline) - return item - - def GetSubList(self): - frame, lineno = self.info - sublist = [] - if frame.f_globals is not frame.f_locals: - item = VariablesTreeItem("", frame.f_locals, self.flist) - sublist.append(item) - item = VariablesTreeItem("", frame.f_globals, self.flist) - sublist.append(item) - return sublist - - def OnDoubleClick(self): - if self.flist: - frame, lineno = self.info - filename = frame.f_code.co_filename - if os.path.isfile(filename): - self.flist.gotofileline(filename, lineno) - -class VariablesTreeItem(ObjectTreeItem): - - def GetText(self): - return self.labeltext - - def GetLabelText(self): - return None - - def IsExpandable(self): - return len(self.object) > 0 - - def GetSubList(self): - sublist = [] - for key in self.object.keys(): - try: - value = self.object[key] - except KeyError: - continue - def setfunction(value, key=key, object=self.object): - object[key] = value - item = make_objecttreeitem(key + " =", value, setfunction) - sublist.append(item) - return sublist - - def keys(self): # unused, left for possible 3rd party use - return list(self.object.keys()) - -def _stack_viewer(parent): - root = tk.Tk() - root.title("Test StackViewer") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - flist = PyShellFileList(root) - try: # to obtain a traceback object - intentional_name_error - except NameError: - exc_type, exc_value, exc_tb = sys.exc_info() - - # inject stack trace to sys - sys.last_type = exc_type - sys.last_value = exc_value - sys.last_traceback = exc_tb - - StackBrowser(root, flist=flist, top=root, tb=exc_tb) - - # restore sys to original state - del sys.last_type - del sys.last_value - del sys.last_traceback - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_stack_viewer) diff --git a/Lib/idlelib/ToolTip.py b/Lib/idlelib/ToolTip.py deleted file mode 100644 index 964107e..0000000 --- a/Lib/idlelib/ToolTip.py +++ /dev/null @@ -1,97 +0,0 @@ -# general purpose 'tooltip' routines - currently unused in idlefork -# (although the 'calltips' extension is partly based on this code) -# may be useful for some purposes in (or almost in ;) the current project scope -# Ideas gleaned from PySol - -from tkinter import * - -class ToolTipBase: - - def __init__(self, button): - self.button = button - self.tipwindow = None - self.id = None - self.x = self.y = 0 - self._id1 = self.button.bind("", self.enter) - self._id2 = self.button.bind("", self.leave) - self._id3 = self.button.bind("", self.leave) - - def enter(self, event=None): - self.schedule() - - def leave(self, event=None): - self.unschedule() - self.hidetip() - - def schedule(self): - self.unschedule() - self.id = self.button.after(1500, self.showtip) - - def unschedule(self): - id = self.id - self.id = None - if id: - self.button.after_cancel(id) - - def showtip(self): - if self.tipwindow: - return - # The tip window must be completely outside the button; - # otherwise when the mouse enters the tip window we get - # a leave event and it disappears, and then we get an enter - # event and it reappears, and so on forever :-( - x = self.button.winfo_rootx() + 20 - y = self.button.winfo_rooty() + self.button.winfo_height() + 1 - self.tipwindow = tw = Toplevel(self.button) - tw.wm_overrideredirect(1) - tw.wm_geometry("+%d+%d" % (x, y)) - self.showcontents() - - def showcontents(self, text="Your text here"): - # Override this in derived class - label = Label(self.tipwindow, text=text, justify=LEFT, - background="#ffffe0", relief=SOLID, borderwidth=1) - label.pack() - - def hidetip(self): - tw = self.tipwindow - self.tipwindow = None - if tw: - tw.destroy() - -class ToolTip(ToolTipBase): - def __init__(self, button, text): - ToolTipBase.__init__(self, button) - self.text = text - def showcontents(self): - ToolTipBase.showcontents(self, self.text) - -class ListboxToolTip(ToolTipBase): - def __init__(self, button, items): - ToolTipBase.__init__(self, button) - self.items = items - def showcontents(self): - listbox = Listbox(self.tipwindow, background="#ffffe0") - listbox.pack() - for item in self.items: - listbox.insert(END, item) - -def _tooltip(parent): - root = Tk() - root.title("Test tooltip") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - label = Label(root, text="Place your mouse over buttons") - label.pack() - button1 = Button(root, text="Button 1") - button2 = Button(root, text="Button 2") - button1.pack() - button2.pack() - ToolTip(button1, "This is tooltip text for button1.") - ListboxToolTip(button2, ["This is","multiple line", - "tooltip text","for button2"]) - root.mainloop() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tooltip) diff --git a/Lib/idlelib/TreeWidget.py b/Lib/idlelib/TreeWidget.py deleted file mode 100644 index a19578f..0000000 --- a/Lib/idlelib/TreeWidget.py +++ /dev/null @@ -1,466 +0,0 @@ -# XXX TO DO: -# - popup menu -# - support partial or total redisplay -# - key bindings (instead of quick-n-dirty bindings on Canvas): -# - up/down arrow keys to move focus around -# - ditto for page up/down, home/end -# - left/right arrows to expand/collapse & move out/in -# - more doc strings -# - add icons for "file", "module", "class", "method"; better "python" icon -# - callback for selection??? -# - multiple-item selection -# - tooltips -# - redo geometry without magic numbers -# - keep track of object ids to allow more careful cleaning -# - optimize tree redraw after expand of subnode - -import os -from tkinter import * - -from idlelib import ZoomHeight -from idlelib.configHandler import idleConf - -ICONDIR = "Icons" - -# Look for Icons subdirectory in the same directory as this module -try: - _icondir = os.path.join(os.path.dirname(__file__), ICONDIR) -except NameError: - _icondir = ICONDIR -if os.path.isdir(_icondir): - ICONDIR = _icondir -elif not os.path.isdir(ICONDIR): - raise RuntimeError("can't find icon directory (%r)" % (ICONDIR,)) - -def listicons(icondir=ICONDIR): - """Utility to display the available icons.""" - root = Tk() - import glob - list = glob.glob(os.path.join(icondir, "*.gif")) - list.sort() - images = [] - row = column = 0 - for file in list: - name = os.path.splitext(os.path.basename(file))[0] - image = PhotoImage(file=file, master=root) - images.append(image) - label = Label(root, image=image, bd=1, relief="raised") - label.grid(row=row, column=column) - label = Label(root, text=name) - label.grid(row=row+1, column=column) - column = column + 1 - if column >= 10: - row = row+2 - column = 0 - root.images = images - - -class TreeNode: - - def __init__(self, canvas, parent, item): - self.canvas = canvas - self.parent = parent - self.item = item - self.state = 'collapsed' - self.selected = False - self.children = [] - self.x = self.y = None - self.iconimages = {} # cache of PhotoImage instances for icons - - def destroy(self): - for c in self.children[:]: - self.children.remove(c) - c.destroy() - self.parent = None - - def geticonimage(self, name): - try: - return self.iconimages[name] - except KeyError: - pass - file, ext = os.path.splitext(name) - ext = ext or ".gif" - fullname = os.path.join(ICONDIR, file + ext) - image = PhotoImage(master=self.canvas, file=fullname) - self.iconimages[name] = image - return image - - def select(self, event=None): - if self.selected: - return - self.deselectall() - self.selected = True - self.canvas.delete(self.image_id) - self.drawicon() - self.drawtext() - - def deselect(self, event=None): - if not self.selected: - return - self.selected = False - self.canvas.delete(self.image_id) - self.drawicon() - self.drawtext() - - def deselectall(self): - if self.parent: - self.parent.deselectall() - else: - self.deselecttree() - - def deselecttree(self): - if self.selected: - self.deselect() - for child in self.children: - child.deselecttree() - - def flip(self, event=None): - if self.state == 'expanded': - self.collapse() - else: - self.expand() - self.item.OnDoubleClick() - return "break" - - def expand(self, event=None): - if not self.item._IsExpandable(): - return - if self.state != 'expanded': - self.state = 'expanded' - self.update() - self.view() - - def collapse(self, event=None): - if self.state != 'collapsed': - self.state = 'collapsed' - self.update() - - def view(self): - top = self.y - 2 - bottom = self.lastvisiblechild().y + 17 - height = bottom - top - visible_top = self.canvas.canvasy(0) - visible_height = self.canvas.winfo_height() - visible_bottom = self.canvas.canvasy(visible_height) - if visible_top <= top and bottom <= visible_bottom: - return - x0, y0, x1, y1 = self.canvas._getints(self.canvas['scrollregion']) - if top >= visible_top and height <= visible_height: - fraction = top + height - visible_height - else: - fraction = top - fraction = float(fraction) / y1 - self.canvas.yview_moveto(fraction) - - def lastvisiblechild(self): - if self.children and self.state == 'expanded': - return self.children[-1].lastvisiblechild() - else: - return self - - def update(self): - if self.parent: - self.parent.update() - else: - oldcursor = self.canvas['cursor'] - self.canvas['cursor'] = "watch" - self.canvas.update() - self.canvas.delete(ALL) # XXX could be more subtle - self.draw(7, 2) - x0, y0, x1, y1 = self.canvas.bbox(ALL) - self.canvas.configure(scrollregion=(0, 0, x1, y1)) - self.canvas['cursor'] = oldcursor - - def draw(self, x, y): - # XXX This hard-codes too many geometry constants! - dy = 20 - self.x, self.y = x, y - self.drawicon() - self.drawtext() - if self.state != 'expanded': - return y + dy - # draw children - if not self.children: - sublist = self.item._GetSubList() - if not sublist: - # _IsExpandable() was mistaken; that's allowed - return y+17 - for item in sublist: - child = self.__class__(self.canvas, self, item) - self.children.append(child) - cx = x+20 - cy = y + dy - cylast = 0 - for child in self.children: - cylast = cy - self.canvas.create_line(x+9, cy+7, cx, cy+7, fill="gray50") - cy = child.draw(cx, cy) - if child.item._IsExpandable(): - if child.state == 'expanded': - iconname = "minusnode" - callback = child.collapse - else: - iconname = "plusnode" - callback = child.expand - image = self.geticonimage(iconname) - id = self.canvas.create_image(x+9, cylast+7, image=image) - # XXX This leaks bindings until canvas is deleted: - self.canvas.tag_bind(id, "<1>", callback) - self.canvas.tag_bind(id, "", lambda x: None) - id = self.canvas.create_line(x+9, y+10, x+9, cylast+7, - ##stipple="gray50", # XXX Seems broken in Tk 8.0.x - fill="gray50") - self.canvas.tag_lower(id) # XXX .lower(id) before Python 1.5.2 - return cy - - def drawicon(self): - if self.selected: - imagename = (self.item.GetSelectedIconName() or - self.item.GetIconName() or - "openfolder") - else: - imagename = self.item.GetIconName() or "folder" - image = self.geticonimage(imagename) - id = self.canvas.create_image(self.x, self.y, anchor="nw", image=image) - self.image_id = id - self.canvas.tag_bind(id, "<1>", self.select) - self.canvas.tag_bind(id, "", self.flip) - - def drawtext(self): - textx = self.x+20-1 - texty = self.y-4 - labeltext = self.item.GetLabelText() - if labeltext: - id = self.canvas.create_text(textx, texty, anchor="nw", - text=labeltext) - self.canvas.tag_bind(id, "<1>", self.select) - self.canvas.tag_bind(id, "", self.flip) - x0, y0, x1, y1 = self.canvas.bbox(id) - textx = max(x1, 200) + 10 - text = self.item.GetText() or "" - try: - self.entry - except AttributeError: - pass - else: - self.edit_finish() - try: - self.label - except AttributeError: - # padding carefully selected (on Windows) to match Entry widget: - self.label = Label(self.canvas, text=text, bd=0, padx=2, pady=2) - theme = idleConf.CurrentTheme() - if self.selected: - self.label.configure(idleConf.GetHighlight(theme, 'hilite')) - else: - self.label.configure(idleConf.GetHighlight(theme, 'normal')) - id = self.canvas.create_window(textx, texty, - anchor="nw", window=self.label) - self.label.bind("<1>", self.select_or_edit) - self.label.bind("", self.flip) - self.text_id = id - - def select_or_edit(self, event=None): - if self.selected and self.item.IsEditable(): - self.edit(event) - else: - self.select(event) - - def edit(self, event=None): - self.entry = Entry(self.label, bd=0, highlightthickness=1, width=0) - self.entry.insert(0, self.label['text']) - self.entry.selection_range(0, END) - self.entry.pack(ipadx=5) - self.entry.focus_set() - self.entry.bind("", self.edit_finish) - self.entry.bind("", self.edit_cancel) - - def edit_finish(self, event=None): - try: - entry = self.entry - del self.entry - except AttributeError: - return - text = entry.get() - entry.destroy() - if text and text != self.item.GetText(): - self.item.SetText(text) - text = self.item.GetText() - self.label['text'] = text - self.drawtext() - self.canvas.focus_set() - - def edit_cancel(self, event=None): - try: - entry = self.entry - del self.entry - except AttributeError: - return - entry.destroy() - self.drawtext() - self.canvas.focus_set() - - -class TreeItem: - - """Abstract class representing tree items. - - Methods should typically be overridden, otherwise a default action - is used. - - """ - - def __init__(self): - """Constructor. Do whatever you need to do.""" - - def GetText(self): - """Return text string to display.""" - - def GetLabelText(self): - """Return label text string to display in front of text (if any).""" - - expandable = None - - def _IsExpandable(self): - """Do not override! Called by TreeNode.""" - if self.expandable is None: - self.expandable = self.IsExpandable() - return self.expandable - - def IsExpandable(self): - """Return whether there are subitems.""" - return 1 - - def _GetSubList(self): - """Do not override! Called by TreeNode.""" - if not self.IsExpandable(): - return [] - sublist = self.GetSubList() - if not sublist: - self.expandable = 0 - return sublist - - def IsEditable(self): - """Return whether the item's text may be edited.""" - - def SetText(self, text): - """Change the item's text (if it is editable).""" - - def GetIconName(self): - """Return name of icon to be displayed normally.""" - - def GetSelectedIconName(self): - """Return name of icon to be displayed when selected.""" - - def GetSubList(self): - """Return list of items forming sublist.""" - - def OnDoubleClick(self): - """Called on a double-click on the item.""" - - -# Example application - -class FileTreeItem(TreeItem): - - """Example TreeItem subclass -- browse the file system.""" - - def __init__(self, path): - self.path = path - - def GetText(self): - return os.path.basename(self.path) or self.path - - def IsEditable(self): - return os.path.basename(self.path) != "" - - def SetText(self, text): - newpath = os.path.dirname(self.path) - newpath = os.path.join(newpath, text) - if os.path.dirname(newpath) != os.path.dirname(self.path): - return - try: - os.rename(self.path, newpath) - self.path = newpath - except OSError: - pass - - def GetIconName(self): - if not self.IsExpandable(): - return "python" # XXX wish there was a "file" icon - - def IsExpandable(self): - return os.path.isdir(self.path) - - def GetSubList(self): - try: - names = os.listdir(self.path) - except OSError: - return [] - names.sort(key = os.path.normcase) - sublist = [] - for name in names: - item = FileTreeItem(os.path.join(self.path, name)) - sublist.append(item) - return sublist - - -# A canvas widget with scroll bars and some useful bindings - -class ScrolledCanvas: - def __init__(self, master, **opts): - if 'yscrollincrement' not in opts: - opts['yscrollincrement'] = 17 - self.master = master - self.frame = Frame(master) - self.frame.rowconfigure(0, weight=1) - self.frame.columnconfigure(0, weight=1) - self.canvas = Canvas(self.frame, **opts) - self.canvas.grid(row=0, column=0, sticky="nsew") - self.vbar = Scrollbar(self.frame, name="vbar") - self.vbar.grid(row=0, column=1, sticky="nse") - self.hbar = Scrollbar(self.frame, name="hbar", orient="horizontal") - self.hbar.grid(row=1, column=0, sticky="ews") - self.canvas['yscrollcommand'] = self.vbar.set - self.vbar['command'] = self.canvas.yview - self.canvas['xscrollcommand'] = self.hbar.set - self.hbar['command'] = self.canvas.xview - self.canvas.bind("", self.page_up) - self.canvas.bind("", self.page_down) - self.canvas.bind("", self.unit_up) - self.canvas.bind("", self.unit_down) - #if isinstance(master, Toplevel) or isinstance(master, Tk): - self.canvas.bind("", self.zoom_height) - self.canvas.focus_set() - def page_up(self, event): - self.canvas.yview_scroll(-1, "page") - return "break" - def page_down(self, event): - self.canvas.yview_scroll(1, "page") - return "break" - def unit_up(self, event): - self.canvas.yview_scroll(-1, "unit") - return "break" - def unit_down(self, event): - self.canvas.yview_scroll(1, "unit") - return "break" - def zoom_height(self, event): - ZoomHeight.zoom_height(self.master) - return "break" - - -def _tree_widget(parent): - root = Tk() - root.title("Test TreeWidget") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - sc = ScrolledCanvas(root, bg="white", highlightthickness=0, takefocus=1) - sc.frame.pack(expand=1, fill="both", side=LEFT) - item = FileTreeItem(os.getcwd()) - node = TreeNode(sc.canvas, None, item) - node.expand() - root.mainloop() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tree_widget) diff --git a/Lib/idlelib/UndoDelegator.py b/Lib/idlelib/UndoDelegator.py deleted file mode 100644 index 1c2502d..0000000 --- a/Lib/idlelib/UndoDelegator.py +++ /dev/null @@ -1,368 +0,0 @@ -import string -from tkinter import * - -from idlelib.Delegator import Delegator - -#$ event <> -#$ win -#$ unix - -#$ event <> -#$ win -#$ unix - -#$ event <> -#$ win -#$ unix - - -class UndoDelegator(Delegator): - - max_undo = 1000 - - def __init__(self): - Delegator.__init__(self) - self.reset_undo() - - def setdelegate(self, delegate): - if self.delegate is not None: - self.unbind("<>") - self.unbind("<>") - self.unbind("<>") - Delegator.setdelegate(self, delegate) - if delegate is not None: - self.bind("<>", self.undo_event) - self.bind("<>", self.redo_event) - self.bind("<>", self.dump_event) - - def dump_event(self, event): - from pprint import pprint - pprint(self.undolist[:self.pointer]) - print("pointer:", self.pointer, end=' ') - print("saved:", self.saved, end=' ') - print("can_merge:", self.can_merge, end=' ') - print("get_saved():", self.get_saved()) - pprint(self.undolist[self.pointer:]) - return "break" - - def reset_undo(self): - self.was_saved = -1 - self.pointer = 0 - self.undolist = [] - self.undoblock = 0 # or a CommandSequence instance - self.set_saved(1) - - def set_saved(self, flag): - if flag: - self.saved = self.pointer - else: - self.saved = -1 - self.can_merge = False - self.check_saved() - - def get_saved(self): - return self.saved == self.pointer - - saved_change_hook = None - - def set_saved_change_hook(self, hook): - self.saved_change_hook = hook - - was_saved = -1 - - def check_saved(self): - is_saved = self.get_saved() - if is_saved != self.was_saved: - self.was_saved = is_saved - if self.saved_change_hook: - self.saved_change_hook() - - def insert(self, index, chars, tags=None): - self.addcmd(InsertCommand(index, chars, tags)) - - def delete(self, index1, index2=None): - self.addcmd(DeleteCommand(index1, index2)) - - # Clients should call undo_block_start() and undo_block_stop() - # around a sequence of editing cmds to be treated as a unit by - # undo & redo. Nested matching calls are OK, and the inner calls - # then act like nops. OK too if no editing cmds, or only one - # editing cmd, is issued in between: if no cmds, the whole - # sequence has no effect; and if only one cmd, that cmd is entered - # directly into the undo list, as if undo_block_xxx hadn't been - # called. The intent of all that is to make this scheme easy - # to use: all the client has to worry about is making sure each - # _start() call is matched by a _stop() call. - - def undo_block_start(self): - if self.undoblock == 0: - self.undoblock = CommandSequence() - self.undoblock.bump_depth() - - def undo_block_stop(self): - if self.undoblock.bump_depth(-1) == 0: - cmd = self.undoblock - self.undoblock = 0 - if len(cmd) > 0: - if len(cmd) == 1: - # no need to wrap a single cmd - cmd = cmd.getcmd(0) - # this blk of cmds, or single cmd, has already - # been done, so don't execute it again - self.addcmd(cmd, 0) - - def addcmd(self, cmd, execute=True): - if execute: - cmd.do(self.delegate) - if self.undoblock != 0: - self.undoblock.append(cmd) - return - if self.can_merge and self.pointer > 0: - lastcmd = self.undolist[self.pointer-1] - if lastcmd.merge(cmd): - return - self.undolist[self.pointer:] = [cmd] - if self.saved > self.pointer: - self.saved = -1 - self.pointer = self.pointer + 1 - if len(self.undolist) > self.max_undo: - ##print "truncating undo list" - del self.undolist[0] - self.pointer = self.pointer - 1 - if self.saved >= 0: - self.saved = self.saved - 1 - self.can_merge = True - self.check_saved() - - def undo_event(self, event): - if self.pointer == 0: - self.bell() - return "break" - cmd = self.undolist[self.pointer - 1] - cmd.undo(self.delegate) - self.pointer = self.pointer - 1 - self.can_merge = False - self.check_saved() - return "break" - - def redo_event(self, event): - if self.pointer >= len(self.undolist): - self.bell() - return "break" - cmd = self.undolist[self.pointer] - cmd.redo(self.delegate) - self.pointer = self.pointer + 1 - self.can_merge = False - self.check_saved() - return "break" - - -class Command: - - # Base class for Undoable commands - - tags = None - - def __init__(self, index1, index2, chars, tags=None): - self.marks_before = {} - self.marks_after = {} - self.index1 = index1 - self.index2 = index2 - self.chars = chars - if tags: - self.tags = tags - - def __repr__(self): - s = self.__class__.__name__ - t = (self.index1, self.index2, self.chars, self.tags) - if self.tags is None: - t = t[:-1] - return s + repr(t) - - def do(self, text): - pass - - def redo(self, text): - pass - - def undo(self, text): - pass - - def merge(self, cmd): - return 0 - - def save_marks(self, text): - marks = {} - for name in text.mark_names(): - if name != "insert" and name != "current": - marks[name] = text.index(name) - return marks - - def set_marks(self, text, marks): - for name, index in marks.items(): - text.mark_set(name, index) - - -class InsertCommand(Command): - - # Undoable insert command - - def __init__(self, index1, chars, tags=None): - Command.__init__(self, index1, None, chars, tags) - - def do(self, text): - self.marks_before = self.save_marks(text) - self.index1 = text.index(self.index1) - if text.compare(self.index1, ">", "end-1c"): - # Insert before the final newline - self.index1 = text.index("end-1c") - text.insert(self.index1, self.chars, self.tags) - self.index2 = text.index("%s+%dc" % (self.index1, len(self.chars))) - self.marks_after = self.save_marks(text) - ##sys.__stderr__.write("do: %s\n" % self) - - def redo(self, text): - text.mark_set('insert', self.index1) - text.insert(self.index1, self.chars, self.tags) - self.set_marks(text, self.marks_after) - text.see('insert') - ##sys.__stderr__.write("redo: %s\n" % self) - - def undo(self, text): - text.mark_set('insert', self.index1) - text.delete(self.index1, self.index2) - self.set_marks(text, self.marks_before) - text.see('insert') - ##sys.__stderr__.write("undo: %s\n" % self) - - def merge(self, cmd): - if self.__class__ is not cmd.__class__: - return False - if self.index2 != cmd.index1: - return False - if self.tags != cmd.tags: - return False - if len(cmd.chars) != 1: - return False - if self.chars and \ - self.classify(self.chars[-1]) != self.classify(cmd.chars): - return False - self.index2 = cmd.index2 - self.chars = self.chars + cmd.chars - return True - - alphanumeric = string.ascii_letters + string.digits + "_" - - def classify(self, c): - if c in self.alphanumeric: - return "alphanumeric" - if c == "\n": - return "newline" - return "punctuation" - - -class DeleteCommand(Command): - - # Undoable delete command - - def __init__(self, index1, index2=None): - Command.__init__(self, index1, index2, None, None) - - def do(self, text): - self.marks_before = self.save_marks(text) - self.index1 = text.index(self.index1) - if self.index2: - self.index2 = text.index(self.index2) - else: - self.index2 = text.index(self.index1 + " +1c") - if text.compare(self.index2, ">", "end-1c"): - # Don't delete the final newline - self.index2 = text.index("end-1c") - self.chars = text.get(self.index1, self.index2) - text.delete(self.index1, self.index2) - self.marks_after = self.save_marks(text) - ##sys.__stderr__.write("do: %s\n" % self) - - def redo(self, text): - text.mark_set('insert', self.index1) - text.delete(self.index1, self.index2) - self.set_marks(text, self.marks_after) - text.see('insert') - ##sys.__stderr__.write("redo: %s\n" % self) - - def undo(self, text): - text.mark_set('insert', self.index1) - text.insert(self.index1, self.chars) - self.set_marks(text, self.marks_before) - text.see('insert') - ##sys.__stderr__.write("undo: %s\n" % self) - -class CommandSequence(Command): - - # Wrapper for a sequence of undoable cmds to be undone/redone - # as a unit - - def __init__(self): - self.cmds = [] - self.depth = 0 - - def __repr__(self): - s = self.__class__.__name__ - strs = [] - for cmd in self.cmds: - strs.append(" %r" % (cmd,)) - return s + "(\n" + ",\n".join(strs) + "\n)" - - def __len__(self): - return len(self.cmds) - - def append(self, cmd): - self.cmds.append(cmd) - - def getcmd(self, i): - return self.cmds[i] - - def redo(self, text): - for cmd in self.cmds: - cmd.redo(text) - - def undo(self, text): - cmds = self.cmds[:] - cmds.reverse() - for cmd in cmds: - cmd.undo(text) - - def bump_depth(self, incr=1): - self.depth = self.depth + incr - return self.depth - - -def _undo_delegator(parent): # htest # - import re - import tkinter as tk - from idlelib.Percolator import Percolator - undowin = tk.Toplevel() - undowin.title("Test UndoDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - undowin.geometry("+%d+%d"%(x, y + 150)) - - text = Text(undowin, height=10) - text.pack() - text.focus_set() - p = Percolator(text) - d = UndoDelegator() - p.insertfilter(d) - - undo = Button(undowin, text="Undo", command=lambda:d.undo_event(None)) - undo.pack(side='left') - redo = Button(undowin, text="Redo", command=lambda:d.redo_event(None)) - redo.pack(side='left') - dump = Button(undowin, text="Dump", command=lambda:d.dump_event(None)) - dump.pack(side='left') - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_undodelegator', verbosity=2, - exit=False) - from idlelib.idle_test.htest import run - run(_undo_delegator) diff --git a/Lib/idlelib/WidgetRedirector.py b/Lib/idlelib/WidgetRedirector.py deleted file mode 100644 index b66be9e..0000000 --- a/Lib/idlelib/WidgetRedirector.py +++ /dev/null @@ -1,176 +0,0 @@ -from tkinter import TclError - -class WidgetRedirector: - """Support for redirecting arbitrary widget subcommands. - - Some Tk operations don't normally pass through tkinter. For example, if a - character is inserted into a Text widget by pressing a key, a default Tk - binding to the widget's 'insert' operation is activated, and the Tk library - processes the insert without calling back into tkinter. - - Although a binding to could be made via tkinter, what we really want - to do is to hook the Tk 'insert' operation itself. For one thing, we want - a text.insert call in idle code to have the same effect as a key press. - - When a widget is instantiated, a Tcl command is created whose name is the - same as the pathname widget._w. This command is used to invoke the various - widget operations, e.g. insert (for a Text widget). We are going to hook - this command and provide a facility ('register') to intercept the widget - operation. We will also intercept method calls on the tkinter class - instance that represents the tk widget. - - In IDLE, WidgetRedirector is used in Percolator to intercept Text - commands. The function being registered provides access to the top - of a Percolator chain. At the bottom of the chain is a call to the - original Tk widget operation. - """ - def __init__(self, widget): - '''Initialize attributes and setup redirection. - - _operations: dict mapping operation name to new function. - widget: the widget whose tcl command is to be intercepted. - tk: widget.tk, a convenience attribute, probably not needed. - orig: new name of the original tcl command. - - Since renaming to orig fails with TclError when orig already - exists, only one WidgetDirector can exist for a given widget. - ''' - self._operations = {} - self.widget = widget # widget instance - self.tk = tk = widget.tk # widget's root - w = widget._w # widget's (full) Tk pathname - self.orig = w + "_orig" - # Rename the Tcl command within Tcl: - tk.call("rename", w, self.orig) - # Create a new Tcl command whose name is the widget's pathname, and - # whose action is to dispatch on the operation passed to the widget: - tk.createcommand(w, self.dispatch) - - def __repr__(self): - return "%s(%s<%s>)" % (self.__class__.__name__, - self.widget.__class__.__name__, - self.widget._w) - - def close(self): - "Unregister operations and revert redirection created by .__init__." - for operation in list(self._operations): - self.unregister(operation) - widget = self.widget - tk = widget.tk - w = widget._w - # Restore the original widget Tcl command. - tk.deletecommand(w) - tk.call("rename", self.orig, w) - del self.widget, self.tk # Should not be needed - # if instance is deleted after close, as in Percolator. - - def register(self, operation, function): - '''Return OriginalCommand(operation) after registering function. - - Registration adds an operation: function pair to ._operations. - It also adds a widget function attribute that masks the tkinter - class instance method. Method masking operates independently - from command dispatch. - - If a second function is registered for the same operation, the - first function is replaced in both places. - ''' - self._operations[operation] = function - setattr(self.widget, operation, function) - return OriginalCommand(self, operation) - - def unregister(self, operation): - '''Return the function for the operation, or None. - - Deleting the instance attribute unmasks the class attribute. - ''' - if operation in self._operations: - function = self._operations[operation] - del self._operations[operation] - try: - delattr(self.widget, operation) - except AttributeError: - pass - return function - else: - return None - - def dispatch(self, operation, *args): - '''Callback from Tcl which runs when the widget is referenced. - - If an operation has been registered in self._operations, apply the - associated function to the args passed into Tcl. Otherwise, pass the - operation through to Tk via the original Tcl function. - - Note that if a registered function is called, the operation is not - passed through to Tk. Apply the function returned by self.register() - to *args to accomplish that. For an example, see ColorDelegator.py. - - ''' - m = self._operations.get(operation) - try: - if m: - return m(*args) - else: - return self.tk.call((self.orig, operation) + args) - except TclError: - return "" - - -class OriginalCommand: - '''Callable for original tk command that has been redirected. - - Returned by .register; can be used in the function registered. - redir = WidgetRedirector(text) - def my_insert(*args): - print("insert", args) - original_insert(*args) - original_insert = redir.register("insert", my_insert) - ''' - - def __init__(self, redir, operation): - '''Create .tk_call and .orig_and_operation for .__call__ method. - - .redir and .operation store the input args for __repr__. - .tk and .orig copy attributes of .redir (probably not needed). - ''' - self.redir = redir - self.operation = operation - self.tk = redir.tk # redundant with self.redir - self.orig = redir.orig # redundant with self.redir - # These two could be deleted after checking recipient code. - self.tk_call = redir.tk.call - self.orig_and_operation = (redir.orig, operation) - - def __repr__(self): - return "%s(%r, %r)" % (self.__class__.__name__, - self.redir, self.operation) - - def __call__(self, *args): - return self.tk_call(self.orig_and_operation + args) - - -def _widget_redirector(parent): # htest # - from tkinter import Tk, Text - import re - - root = Tk() - root.title("Test WidgetRedirector") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - text = Text(root) - text.pack() - text.focus_set() - redir = WidgetRedirector(text) - def my_insert(*args): - print("insert", args) - original_insert(*args) - original_insert = redir.register("insert", my_insert) - root.mainloop() - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_widgetredir', - verbosity=2, exit=False) - from idlelib.idle_test.htest import run - run(_widget_redirector) diff --git a/Lib/idlelib/WindowList.py b/Lib/idlelib/WindowList.py deleted file mode 100644 index bc74348..0000000 --- a/Lib/idlelib/WindowList.py +++ /dev/null @@ -1,90 +0,0 @@ -from tkinter import * - -class WindowList: - - def __init__(self): - self.dict = {} - self.callbacks = [] - - def add(self, window): - window.after_idle(self.call_callbacks) - self.dict[str(window)] = window - - def delete(self, window): - try: - del self.dict[str(window)] - except KeyError: - # Sometimes, destroy() is called twice - pass - self.call_callbacks() - - def add_windows_to_menu(self, menu): - list = [] - for key in self.dict: - window = self.dict[key] - try: - title = window.get_title() - except TclError: - continue - list.append((title, key, window)) - list.sort() - for title, key, window in list: - menu.add_command(label=title, command=window.wakeup) - - def register_callback(self, callback): - self.callbacks.append(callback) - - def unregister_callback(self, callback): - try: - self.callbacks.remove(callback) - except ValueError: - pass - - def call_callbacks(self): - for callback in self.callbacks: - try: - callback() - except: - t, v, tb = sys.exc_info() - print("warning: callback failed in WindowList", t, ":", v) - -registry = WindowList() - -add_windows_to_menu = registry.add_windows_to_menu -register_callback = registry.register_callback -unregister_callback = registry.unregister_callback - - -class ListedToplevel(Toplevel): - - def __init__(self, master, **kw): - Toplevel.__init__(self, master, kw) - registry.add(self) - self.focused_widget = self - - def destroy(self): - registry.delete(self) - Toplevel.destroy(self) - # If this is Idle's last window then quit the mainloop - # (Needed for clean exit on Windows 98) - if not registry.dict: - self.quit() - - def update_windowlist_registry(self, window): - registry.call_callbacks() - - def get_title(self): - # Subclass can override - return self.wm_title() - - def wakeup(self): - try: - if self.wm_state() == "iconic": - self.wm_withdraw() - self.wm_deiconify() - self.tkraise() - self.focused_widget.focus_set() - except TclError: - # This can happen when the window menu was torn off. - # Simply ignore it. - pass diff --git a/Lib/idlelib/ZoomHeight.py b/Lib/idlelib/ZoomHeight.py deleted file mode 100644 index a5d679e..0000000 --- a/Lib/idlelib/ZoomHeight.py +++ /dev/null @@ -1,51 +0,0 @@ -# Sample extension: zoom a window to maximum height - -import re -import sys - -from idlelib import macosxSupport - -class ZoomHeight: - - menudefs = [ - ('windows', [ - ('_Zoom Height', '<>'), - ]) - ] - - def __init__(self, editwin): - self.editwin = editwin - - def zoom_height_event(self, event): - top = self.editwin.top - zoom_height(top) - -def zoom_height(top): - geom = top.wm_geometry() - m = re.match(r"(\d+)x(\d+)\+(-?\d+)\+(-?\d+)", geom) - if not m: - top.bell() - return - width, height, x, y = map(int, m.groups()) - newheight = top.winfo_screenheight() - if sys.platform == 'win32': - newy = 0 - newheight = newheight - 72 - - elif macosxSupport.isAquaTk(): - # The '88' below is a magic number that avoids placing the bottom - # of the window below the panel on my machine. I don't know how - # to calculate the correct value for this with tkinter. - newy = 22 - newheight = newheight - newy - 88 - - else: - #newy = 24 - newy = 0 - #newheight = newheight - 96 - newheight = newheight - 88 - if height >= newheight: - newgeom = "" - else: - newgeom = "%dx%d+%d+%d" % (width, newheight, x, newy) - top.wm_geometry(newgeom) diff --git a/Lib/idlelib/aboutDialog.py b/Lib/idlelib/aboutDialog.py deleted file mode 100644 index 3112e6a..0000000 --- a/Lib/idlelib/aboutDialog.py +++ /dev/null @@ -1,149 +0,0 @@ -"""About Dialog for IDLE - -""" - -import os -from sys import version -from tkinter import * -from idlelib import textView - -class AboutDialog(Toplevel): - """Modal about dialog for idle - - """ - def __init__(self, parent, title, _htest=False): - """ - _htest - bool, change box location when running htest - """ - Toplevel.__init__(self, parent) - self.configure(borderwidth=5) - # place dialog below parent if running htest - self.geometry("+%d+%d" % ( - parent.winfo_rootx()+30, - parent.winfo_rooty()+(30 if not _htest else 100))) - self.bg = "#707070" - self.fg = "#ffffff" - self.CreateWidgets() - self.resizable(height=FALSE, width=FALSE) - self.title(title) - self.transient(parent) - self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Ok) - self.parent = parent - self.buttonOk.focus_set() - self.bind('',self.Ok) #dismiss dialog - self.bind('',self.Ok) #dismiss dialog - self.wait_window() - - def CreateWidgets(self): - release = version[:version.index(' ')] - frameMain = Frame(self, borderwidth=2, relief=SUNKEN) - frameButtons = Frame(self) - frameButtons.pack(side=BOTTOM, fill=X) - frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) - self.buttonOk = Button(frameButtons, text='Close', - command=self.Ok) - self.buttonOk.pack(padx=5, pady=5) - #self.picture = Image('photo', data=self.pictureData) - frameBg = Frame(frameMain, bg=self.bg) - frameBg.pack(expand=TRUE, fill=BOTH) - labelTitle = Label(frameBg, text='IDLE', fg=self.fg, bg=self.bg, - font=('courier', 24, 'bold')) - labelTitle.grid(row=0, column=0, sticky=W, padx=10, pady=10) - #labelPicture = Label(frameBg, text='[picture]') - #image=self.picture, bg=self.bg) - #labelPicture.grid(row=1, column=1, sticky=W, rowspan=2, - # padx=0, pady=3) - byline = "Python's Integrated DeveLopment Environment" + 5*'\n' - labelDesc = Label(frameBg, text=byline, justify=LEFT, - fg=self.fg, bg=self.bg) - labelDesc.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) - labelEmail = Label(frameBg, text='email: idle-dev@python.org', - justify=LEFT, fg=self.fg, bg=self.bg) - labelEmail.grid(row=6, column=0, columnspan=2, - sticky=W, padx=10, pady=0) - labelWWW = Label(frameBg, text='https://docs.python.org/' + - version[:3] + '/library/idle.html', - justify=LEFT, fg=self.fg, bg=self.bg) - labelWWW.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) - Frame(frameBg, borderwidth=1, relief=SUNKEN, - height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, - columnspan=3, padx=5, pady=5) - labelPythonVer = Label(frameBg, text='Python version: ' + - release, fg=self.fg, bg=self.bg) - labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) - tkVer = self.tk.call('info', 'patchlevel') - labelTkVer = Label(frameBg, text='Tk version: '+ - tkVer, fg=self.fg, bg=self.bg) - labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) - py_button_f = Frame(frameBg, bg=self.bg) - py_button_f.grid(row=10, column=0, columnspan=2, sticky=NSEW) - buttonLicense = Button(py_button_f, text='License', width=8, - highlightbackground=self.bg, - command=self.ShowLicense) - buttonLicense.pack(side=LEFT, padx=10, pady=10) - buttonCopyright = Button(py_button_f, text='Copyright', width=8, - highlightbackground=self.bg, - command=self.ShowCopyright) - buttonCopyright.pack(side=LEFT, padx=10, pady=10) - buttonCredits = Button(py_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowPythonCredits) - buttonCredits.pack(side=LEFT, padx=10, pady=10) - Frame(frameBg, borderwidth=1, relief=SUNKEN, - height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, - columnspan=3, padx=5, pady=5) - idle_v = Label(frameBg, text='IDLE version: ' + release, - fg=self.fg, bg=self.bg) - idle_v.grid(row=12, column=0, sticky=W, padx=10, pady=0) - idle_button_f = Frame(frameBg, bg=self.bg) - idle_button_f.grid(row=13, column=0, columnspan=3, sticky=NSEW) - idle_about_b = Button(idle_button_f, text='README', width=8, - highlightbackground=self.bg, - command=self.ShowIDLEAbout) - idle_about_b.pack(side=LEFT, padx=10, pady=10) - idle_news_b = Button(idle_button_f, text='NEWS', width=8, - highlightbackground=self.bg, - command=self.ShowIDLENEWS) - idle_news_b.pack(side=LEFT, padx=10, pady=10) - idle_credits_b = Button(idle_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowIDLECredits) - idle_credits_b.pack(side=LEFT, padx=10, pady=10) - - # License, et all, are of type _sitebuiltins._Printer - def ShowLicense(self): - self.display_printer_text('About - License', license) - - def ShowCopyright(self): - self.display_printer_text('About - Copyright', copyright) - - def ShowPythonCredits(self): - self.display_printer_text('About - Python Credits', credits) - - # Encode CREDITS.txt to utf-8 for proper version of Loewis. - # Specify others as ascii until need utf-8, so catch errors. - def ShowIDLECredits(self): - self.display_file_text('About - Credits', 'CREDITS.txt', 'utf-8') - - def ShowIDLEAbout(self): - self.display_file_text('About - Readme', 'README.txt', 'ascii') - - def ShowIDLENEWS(self): - self.display_file_text('About - NEWS', 'NEWS.txt', 'ascii') - - def display_printer_text(self, title, printer): - printer._Printer__setup() - text = '\n'.join(printer._Printer__lines) - textView.view_text(self, title, text) - - def display_file_text(self, title, filename, encoding=None): - fn = os.path.join(os.path.abspath(os.path.dirname(__file__)), filename) - textView.view_file(self, title, fn, encoding) - - def Ok(self, event=None): - self.destroy() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(AboutDialog) diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py new file mode 100644 index 0000000..b9ec539 --- /dev/null +++ b/Lib/idlelib/autocomplete.py @@ -0,0 +1,233 @@ +"""AutoComplete.py - An IDLE extension for automatically completing names. + +This extension can complete either attribute names of file names. It can pop +a window with all available names, for the user to select from. +""" +import os +import sys +import string + +from idlelib.configHandler import idleConf + +# This string includes all chars that may be in an identifier +ID_CHARS = string.ascii_letters + string.digits + "_" + +# These constants represent the two different types of completions +COMPLETE_ATTRIBUTES, COMPLETE_FILES = range(1, 2+1) + +from idlelib import AutoCompleteWindow +from idlelib.HyperParser import HyperParser + +import __main__ + +SEPS = os.sep +if os.altsep: # e.g. '/' on Windows... + SEPS += os.altsep + +class AutoComplete: + + menudefs = [ + ('edit', [ + ("Show Completions", "<>"), + ]) + ] + + popupwait = idleConf.GetOption("extensions", "AutoComplete", + "popupwait", type="int", default=0) + + def __init__(self, editwin=None): + self.editwin = editwin + if editwin is None: # subprocess and test + return + self.text = editwin.text + self.autocompletewindow = None + + # id of delayed call, and the index of the text insert when the delayed + # call was issued. If _delayed_completion_id is None, there is no + # delayed call. + self._delayed_completion_id = None + self._delayed_completion_index = None + + def _make_autocomplete_window(self): + return AutoCompleteWindow.AutoCompleteWindow(self.text) + + def _remove_autocomplete_window(self, event=None): + if self.autocompletewindow: + self.autocompletewindow.hide_window() + self.autocompletewindow = None + + def force_open_completions_event(self, event): + """Happens when the user really wants to open a completion list, even + if a function call is needed. + """ + self.open_completions(True, False, True) + + def try_open_completions_event(self, event): + """Happens when it would be nice to open a completion list, but not + really necessary, for example after a dot, so function + calls won't be made. + """ + lastchar = self.text.get("insert-1c") + if lastchar == ".": + self._open_completions_later(False, False, False, + COMPLETE_ATTRIBUTES) + elif lastchar in SEPS: + self._open_completions_later(False, False, False, + COMPLETE_FILES) + + def autocomplete_event(self, event): + """Happens when the user wants to complete his word, and if necessary, + open a completion list after that (if there is more than one + completion) + """ + if hasattr(event, "mc_state") and event.mc_state: + # A modifier was pressed along with the tab, continue as usual. + return + if self.autocompletewindow and self.autocompletewindow.is_active(): + self.autocompletewindow.complete() + return "break" + else: + opened = self.open_completions(False, True, True) + if opened: + return "break" + + def _open_completions_later(self, *args): + self._delayed_completion_index = self.text.index("insert") + if self._delayed_completion_id is not None: + self.text.after_cancel(self._delayed_completion_id) + self._delayed_completion_id = \ + self.text.after(self.popupwait, self._delayed_open_completions, + *args) + + def _delayed_open_completions(self, *args): + self._delayed_completion_id = None + if self.text.index("insert") != self._delayed_completion_index: + return + self.open_completions(*args) + + def open_completions(self, evalfuncs, complete, userWantsWin, mode=None): + """Find the completions and create the AutoCompleteWindow. + Return True if successful (no syntax error or so found). + if complete is True, then if there's nothing to complete and no + start of completion, won't open completions and return False. + If mode is given, will open a completion list only in this mode. + """ + # Cancel another delayed call, if it exists. + if self._delayed_completion_id is not None: + self.text.after_cancel(self._delayed_completion_id) + self._delayed_completion_id = None + + hp = HyperParser(self.editwin, "insert") + curline = self.text.get("insert linestart", "insert") + i = j = len(curline) + if hp.is_in_string() and (not mode or mode==COMPLETE_FILES): + # Find the beginning of the string + # fetch_completions will look at the file system to determine whether the + # string value constitutes an actual file name + # XXX could consider raw strings here and unescape the string value if it's + # not raw. + self._remove_autocomplete_window() + mode = COMPLETE_FILES + # Find last separator or string start + while i and curline[i-1] not in "'\"" + SEPS: + i -= 1 + comp_start = curline[i:j] + j = i + # Find string start + while i and curline[i-1] not in "'\"": + i -= 1 + comp_what = curline[i:j] + elif hp.is_in_code() and (not mode or mode==COMPLETE_ATTRIBUTES): + self._remove_autocomplete_window() + mode = COMPLETE_ATTRIBUTES + while i and (curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127): + i -= 1 + comp_start = curline[i:j] + if i and curline[i-1] == '.': + hp.set_index("insert-%dc" % (len(curline)-(i-1))) + comp_what = hp.get_expression() + if not comp_what or \ + (not evalfuncs and comp_what.find('(') != -1): + return + else: + comp_what = "" + else: + return + + if complete and not comp_what and not comp_start: + return + comp_lists = self.fetch_completions(comp_what, mode) + if not comp_lists[0]: + return + self.autocompletewindow = self._make_autocomplete_window() + return not self.autocompletewindow.show_window( + comp_lists, "insert-%dc" % len(comp_start), + complete, mode, userWantsWin) + + def fetch_completions(self, what, mode): + """Return a pair of lists of completions for something. The first list + is a sublist of the second. Both are sorted. + + If there is a Python subprocess, get the comp. list there. Otherwise, + either fetch_completions() is running in the subprocess itself or it + was called in an IDLE EditorWindow before any script had been run. + + The subprocess environment is that of the most recently run script. If + two unrelated modules are being edited some calltips in the current + module may be inoperative if the module was not the last to run. + """ + try: + rpcclt = self.editwin.flist.pyshell.interp.rpcclt + except: + rpcclt = None + if rpcclt: + return rpcclt.remotecall("exec", "get_the_completion_list", + (what, mode), {}) + else: + if mode == COMPLETE_ATTRIBUTES: + if what == "": + namespace = __main__.__dict__.copy() + namespace.update(__main__.__builtins__.__dict__) + bigl = eval("dir()", namespace) + bigl.sort() + if "__all__" in bigl: + smalll = sorted(eval("__all__", namespace)) + else: + smalll = [s for s in bigl if s[:1] != '_'] + else: + try: + entity = self.get_entity(what) + bigl = dir(entity) + bigl.sort() + if "__all__" in bigl: + smalll = sorted(entity.__all__) + else: + smalll = [s for s in bigl if s[:1] != '_'] + except: + return [], [] + + elif mode == COMPLETE_FILES: + if what == "": + what = "." + try: + expandedpath = os.path.expanduser(what) + bigl = os.listdir(expandedpath) + bigl.sort() + smalll = [s for s in bigl if s[:1] != '.'] + except OSError: + return [], [] + + if not smalll: + smalll = bigl + return smalll, bigl + + def get_entity(self, name): + """Lookup name in a namespace spanning sys.modules and __main.dict__""" + namespace = sys.modules.copy() + namespace.update(__main__.__dict__) + return eval(name, namespace) + + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_autocomplete', verbosity=2) diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py new file mode 100644 index 0000000..2ee6878 --- /dev/null +++ b/Lib/idlelib/autocomplete_w.py @@ -0,0 +1,416 @@ +""" +An auto-completion window for IDLE, used by the AutoComplete extension +""" +from tkinter import * +from idlelib.MultiCall import MC_SHIFT +from idlelib.AutoComplete import COMPLETE_FILES, COMPLETE_ATTRIBUTES + +HIDE_VIRTUAL_EVENT_NAME = "<>" +HIDE_SEQUENCES = ("", "") +KEYPRESS_VIRTUAL_EVENT_NAME = "<>" +# We need to bind event beyond so that the function will be called +# before the default specific IDLE function +KEYPRESS_SEQUENCES = ("", "", "", "", + "", "", "", "", + "", "") +KEYRELEASE_VIRTUAL_EVENT_NAME = "<>" +KEYRELEASE_SEQUENCE = "" +LISTUPDATE_SEQUENCE = "" +WINCONFIG_SEQUENCE = "" +DOUBLECLICK_SEQUENCE = "" + +class AutoCompleteWindow: + + def __init__(self, widget): + # The widget (Text) on which we place the AutoCompleteWindow + self.widget = widget + # The widgets we create + self.autocompletewindow = self.listbox = self.scrollbar = None + # The default foreground and background of a selection. Saved because + # they are changed to the regular colors of list items when the + # completion start is not a prefix of the selected completion + self.origselforeground = self.origselbackground = None + # The list of completions + self.completions = None + # A list with more completions, or None + self.morecompletions = None + # The completion mode. Either AutoComplete.COMPLETE_ATTRIBUTES or + # AutoComplete.COMPLETE_FILES + self.mode = None + # The current completion start, on the text box (a string) + self.start = None + # The index of the start of the completion + self.startindex = None + # The last typed start, used so that when the selection changes, + # the new start will be as close as possible to the last typed one. + self.lasttypedstart = None + # Do we have an indication that the user wants the completion window + # (for example, he clicked the list) + self.userwantswindow = None + # event ids + self.hideid = self.keypressid = self.listupdateid = self.winconfigid \ + = self.keyreleaseid = self.doubleclickid = None + # Flag set if last keypress was a tab + self.lastkey_was_tab = False + + def _change_start(self, newstart): + min_len = min(len(self.start), len(newstart)) + i = 0 + while i < min_len and self.start[i] == newstart[i]: + i += 1 + if i < len(self.start): + self.widget.delete("%s+%dc" % (self.startindex, i), + "%s+%dc" % (self.startindex, len(self.start))) + if i < len(newstart): + self.widget.insert("%s+%dc" % (self.startindex, i), + newstart[i:]) + self.start = newstart + + def _binary_search(self, s): + """Find the first index in self.completions where completions[i] is + greater or equal to s, or the last index if there is no such + one.""" + i = 0; j = len(self.completions) + while j > i: + m = (i + j) // 2 + if self.completions[m] >= s: + j = m + else: + i = m + 1 + return min(i, len(self.completions)-1) + + def _complete_string(self, s): + """Assuming that s is the prefix of a string in self.completions, + return the longest string which is a prefix of all the strings which + s is a prefix of them. If s is not a prefix of a string, return s.""" + first = self._binary_search(s) + if self.completions[first][:len(s)] != s: + # There is not even one completion which s is a prefix of. + return s + # Find the end of the range of completions where s is a prefix of. + i = first + 1 + j = len(self.completions) + while j > i: + m = (i + j) // 2 + if self.completions[m][:len(s)] != s: + j = m + else: + i = m + 1 + last = i-1 + + if first == last: # only one possible completion + return self.completions[first] + + # We should return the maximum prefix of first and last + first_comp = self.completions[first] + last_comp = self.completions[last] + min_len = min(len(first_comp), len(last_comp)) + i = len(s) + while i < min_len and first_comp[i] == last_comp[i]: + i += 1 + return first_comp[:i] + + def _selection_changed(self): + """Should be called when the selection of the Listbox has changed. + Updates the Listbox display and calls _change_start.""" + cursel = int(self.listbox.curselection()[0]) + + self.listbox.see(cursel) + + lts = self.lasttypedstart + selstart = self.completions[cursel] + if self._binary_search(lts) == cursel: + newstart = lts + else: + min_len = min(len(lts), len(selstart)) + i = 0 + while i < min_len and lts[i] == selstart[i]: + i += 1 + newstart = selstart[:i] + self._change_start(newstart) + + if self.completions[cursel][:len(self.start)] == self.start: + # start is a prefix of the selected completion + self.listbox.configure(selectbackground=self.origselbackground, + selectforeground=self.origselforeground) + else: + self.listbox.configure(selectbackground=self.listbox.cget("bg"), + selectforeground=self.listbox.cget("fg")) + # If there are more completions, show them, and call me again. + if self.morecompletions: + self.completions = self.morecompletions + self.morecompletions = None + self.listbox.delete(0, END) + for item in self.completions: + self.listbox.insert(END, item) + self.listbox.select_set(self._binary_search(self.start)) + self._selection_changed() + + def show_window(self, comp_lists, index, complete, mode, userWantsWin): + """Show the autocomplete list, bind events. + If complete is True, complete the text, and if there is exactly one + matching completion, don't open a list.""" + # Handle the start we already have + self.completions, self.morecompletions = comp_lists + self.mode = mode + self.startindex = self.widget.index(index) + self.start = self.widget.get(self.startindex, "insert") + if complete: + completed = self._complete_string(self.start) + start = self.start + self._change_start(completed) + i = self._binary_search(completed) + if self.completions[i] == completed and \ + (i == len(self.completions)-1 or + self.completions[i+1][:len(completed)] != completed): + # There is exactly one matching completion + return completed == start + self.userwantswindow = userWantsWin + self.lasttypedstart = self.start + + # Put widgets in place + self.autocompletewindow = acw = Toplevel(self.widget) + # Put it in a position so that it is not seen. + acw.wm_geometry("+10000+10000") + # Make it float + acw.wm_overrideredirect(1) + try: + # This command is only needed and available on Tk >= 8.4.0 for OSX + # Without it, call tips intrude on the typing process by grabbing + # the focus. + acw.tk.call("::tk::unsupported::MacWindowStyle", "style", acw._w, + "help", "noActivates") + except TclError: + pass + self.scrollbar = scrollbar = Scrollbar(acw, orient=VERTICAL) + self.listbox = listbox = Listbox(acw, yscrollcommand=scrollbar.set, + exportselection=False, bg="white") + for item in self.completions: + listbox.insert(END, item) + self.origselforeground = listbox.cget("selectforeground") + self.origselbackground = listbox.cget("selectbackground") + scrollbar.config(command=listbox.yview) + scrollbar.pack(side=RIGHT, fill=Y) + listbox.pack(side=LEFT, fill=BOTH, expand=True) + acw.lift() # work around bug in Tk 8.5.18+ (issue #24570) + + # Initialize the listbox selection + self.listbox.select_set(self._binary_search(self.start)) + self._selection_changed() + + # bind events + self.hideid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, + self.hide_event) + for seq in HIDE_SEQUENCES: + self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq) + self.keypressid = self.widget.bind(KEYPRESS_VIRTUAL_EVENT_NAME, + self.keypress_event) + for seq in KEYPRESS_SEQUENCES: + self.widget.event_add(KEYPRESS_VIRTUAL_EVENT_NAME, seq) + self.keyreleaseid = self.widget.bind(KEYRELEASE_VIRTUAL_EVENT_NAME, + self.keyrelease_event) + self.widget.event_add(KEYRELEASE_VIRTUAL_EVENT_NAME,KEYRELEASE_SEQUENCE) + self.listupdateid = listbox.bind(LISTUPDATE_SEQUENCE, + self.listselect_event) + self.winconfigid = acw.bind(WINCONFIG_SEQUENCE, self.winconfig_event) + self.doubleclickid = listbox.bind(DOUBLECLICK_SEQUENCE, + self.doubleclick_event) + + def winconfig_event(self, event): + if not self.is_active(): + return + # Position the completion list window + text = self.widget + text.see(self.startindex) + x, y, cx, cy = text.bbox(self.startindex) + acw = self.autocompletewindow + acw_width, acw_height = acw.winfo_width(), acw.winfo_height() + text_width, text_height = text.winfo_width(), text.winfo_height() + new_x = text.winfo_rootx() + min(x, max(0, text_width - acw_width)) + new_y = text.winfo_rooty() + y + if (text_height - (y + cy) >= acw_height # enough height below + or y < acw_height): # not enough height above + # place acw below current line + new_y += cy + else: + # place acw above current line + new_y -= acw_height + acw.wm_geometry("+%d+%d" % (new_x, new_y)) + + def hide_event(self, event): + if not self.is_active(): + return + self.hide_window() + + def listselect_event(self, event): + if not self.is_active(): + return + self.userwantswindow = True + cursel = int(self.listbox.curselection()[0]) + self._change_start(self.completions[cursel]) + + def doubleclick_event(self, event): + # Put the selected completion in the text, and close the list + cursel = int(self.listbox.curselection()[0]) + self._change_start(self.completions[cursel]) + self.hide_window() + + def keypress_event(self, event): + if not self.is_active(): + return + keysym = event.keysym + if hasattr(event, "mc_state"): + state = event.mc_state + else: + state = 0 + if keysym != "Tab": + self.lastkey_was_tab = False + if (len(keysym) == 1 or keysym in ("underscore", "BackSpace") + or (self.mode == COMPLETE_FILES and keysym in + ("period", "minus"))) \ + and not (state & ~MC_SHIFT): + # Normal editing of text + if len(keysym) == 1: + self._change_start(self.start + keysym) + elif keysym == "underscore": + self._change_start(self.start + '_') + elif keysym == "period": + self._change_start(self.start + '.') + elif keysym == "minus": + self._change_start(self.start + '-') + else: + # keysym == "BackSpace" + if len(self.start) == 0: + self.hide_window() + return + self._change_start(self.start[:-1]) + self.lasttypedstart = self.start + self.listbox.select_clear(0, int(self.listbox.curselection()[0])) + self.listbox.select_set(self._binary_search(self.start)) + self._selection_changed() + return "break" + + elif keysym == "Return": + self.hide_window() + return + + elif (self.mode == COMPLETE_ATTRIBUTES and keysym in + ("period", "space", "parenleft", "parenright", "bracketleft", + "bracketright")) or \ + (self.mode == COMPLETE_FILES and keysym in + ("slash", "backslash", "quotedbl", "apostrophe")) \ + and not (state & ~MC_SHIFT): + # If start is a prefix of the selection, but is not '' when + # completing file names, put the whole + # selected completion. Anyway, close the list. + cursel = int(self.listbox.curselection()[0]) + if self.completions[cursel][:len(self.start)] == self.start \ + and (self.mode == COMPLETE_ATTRIBUTES or self.start): + self._change_start(self.completions[cursel]) + self.hide_window() + return + + elif keysym in ("Home", "End", "Prior", "Next", "Up", "Down") and \ + not state: + # Move the selection in the listbox + self.userwantswindow = True + cursel = int(self.listbox.curselection()[0]) + if keysym == "Home": + newsel = 0 + elif keysym == "End": + newsel = len(self.completions)-1 + elif keysym in ("Prior", "Next"): + jump = self.listbox.nearest(self.listbox.winfo_height()) - \ + self.listbox.nearest(0) + if keysym == "Prior": + newsel = max(0, cursel-jump) + else: + assert keysym == "Next" + newsel = min(len(self.completions)-1, cursel+jump) + elif keysym == "Up": + newsel = max(0, cursel-1) + else: + assert keysym == "Down" + newsel = min(len(self.completions)-1, cursel+1) + self.listbox.select_clear(cursel) + self.listbox.select_set(newsel) + self._selection_changed() + self._change_start(self.completions[newsel]) + return "break" + + elif (keysym == "Tab" and not state): + if self.lastkey_was_tab: + # two tabs in a row; insert current selection and close acw + cursel = int(self.listbox.curselection()[0]) + self._change_start(self.completions[cursel]) + self.hide_window() + return "break" + else: + # first tab; let AutoComplete handle the completion + self.userwantswindow = True + self.lastkey_was_tab = True + return + + elif any(s in keysym for s in ("Shift", "Control", "Alt", + "Meta", "Command", "Option")): + # A modifier key, so ignore + return + + elif event.char and event.char >= ' ': + # Regular character with a non-length-1 keycode + self._change_start(self.start + event.char) + self.lasttypedstart = self.start + self.listbox.select_clear(0, int(self.listbox.curselection()[0])) + self.listbox.select_set(self._binary_search(self.start)) + self._selection_changed() + return "break" + + else: + # Unknown event, close the window and let it through. + self.hide_window() + return + + def keyrelease_event(self, event): + if not self.is_active(): + return + if self.widget.index("insert") != \ + self.widget.index("%s+%dc" % (self.startindex, len(self.start))): + # If we didn't catch an event which moved the insert, close window + self.hide_window() + + def is_active(self): + return self.autocompletewindow is not None + + def complete(self): + self._change_start(self._complete_string(self.start)) + # The selection doesn't change. + + def hide_window(self): + if not self.is_active(): + return + + # unbind events + for seq in HIDE_SEQUENCES: + self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq) + self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideid) + self.hideid = None + for seq in KEYPRESS_SEQUENCES: + self.widget.event_delete(KEYPRESS_VIRTUAL_EVENT_NAME, seq) + self.widget.unbind(KEYPRESS_VIRTUAL_EVENT_NAME, self.keypressid) + self.keypressid = None + self.widget.event_delete(KEYRELEASE_VIRTUAL_EVENT_NAME, + KEYRELEASE_SEQUENCE) + self.widget.unbind(KEYRELEASE_VIRTUAL_EVENT_NAME, self.keyreleaseid) + self.keyreleaseid = None + self.listbox.unbind(LISTUPDATE_SEQUENCE, self.listupdateid) + self.listupdateid = None + self.autocompletewindow.unbind(WINCONFIG_SEQUENCE, self.winconfigid) + self.winconfigid = None + + # destroy widgets + self.scrollbar.destroy() + self.scrollbar = None + self.listbox.destroy() + self.listbox = None + self.autocompletewindow.destroy() + self.autocompletewindow = None diff --git a/Lib/idlelib/autoexpand.py b/Lib/idlelib/autoexpand.py new file mode 100644 index 0000000..7059054 --- /dev/null +++ b/Lib/idlelib/autoexpand.py @@ -0,0 +1,104 @@ +'''Complete the current word before the cursor with words in the editor. + +Each menu selection or shortcut key selection replaces the word with a +different word with the same prefix. The search for matches begins +before the target and moves toward the top of the editor. It then starts +after the cursor and moves down. It then returns to the original word and +the cycle starts again. + +Changing the current text line or leaving the cursor in a different +place before requesting the next selection causes AutoExpand to reset +its state. + +This is an extension file and there is only one instance of AutoExpand. +''' +import string +import re + +###$ event <> +###$ win +###$ unix + +class AutoExpand: + + menudefs = [ + ('edit', [ + ('E_xpand Word', '<>'), + ]), + ] + + wordchars = string.ascii_letters + string.digits + "_" + + def __init__(self, editwin): + self.text = editwin.text + self.state = None + + def expand_word_event(self, event): + "Replace the current word with the next expansion." + curinsert = self.text.index("insert") + curline = self.text.get("insert linestart", "insert lineend") + if not self.state: + words = self.getwords() + index = 0 + else: + words, index, insert, line = self.state + if insert != curinsert or line != curline: + words = self.getwords() + index = 0 + if not words: + self.text.bell() + return "break" + word = self.getprevword() + self.text.delete("insert - %d chars" % len(word), "insert") + newword = words[index] + index = (index + 1) % len(words) + if index == 0: + self.text.bell() # Warn we cycled around + self.text.insert("insert", newword) + curinsert = self.text.index("insert") + curline = self.text.get("insert linestart", "insert lineend") + self.state = words, index, curinsert, curline + return "break" + + def getwords(self): + "Return a list of words that match the prefix before the cursor." + word = self.getprevword() + if not word: + return [] + before = self.text.get("1.0", "insert wordstart") + wbefore = re.findall(r"\b" + word + r"\w+\b", before) + del before + after = self.text.get("insert wordend", "end") + wafter = re.findall(r"\b" + word + r"\w+\b", after) + del after + if not wbefore and not wafter: + return [] + words = [] + dict = {} + # search backwards through words before + wbefore.reverse() + for w in wbefore: + if dict.get(w): + continue + words.append(w) + dict[w] = w + # search onwards through words after + for w in wafter: + if dict.get(w): + continue + words.append(w) + dict[w] = w + words.append(word) + return words + + def getprevword(self): + "Return the word prefix before the cursor." + line = self.text.get("insert linestart", "insert") + i = len(line) + while i > 0 and line[i-1] in self.wordchars: + i = i-1 + return line[i:] + +if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_autoexpand', verbosity=2) diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py new file mode 100644 index 0000000..d09c52f --- /dev/null +++ b/Lib/idlelib/browser.py @@ -0,0 +1,236 @@ +"""Class browser. + +XXX TO DO: + +- reparse when source changed (maybe just a button would be OK?) + (or recheck on window popup) +- add popup menu with more options (e.g. doc strings, base classes, imports) +- show function argument list? (have to do pattern matching on source) +- should the classes and methods lists also be in the module's menu bar? +- add base classes to class browser tree +""" + +import os +import sys +import pyclbr + +from idlelib import PyShell +from idlelib.WindowList import ListedToplevel +from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas +from idlelib.configHandler import idleConf + +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + +class ClassBrowser: + + def __init__(self, flist, name, path, _htest=False): + # XXX This API should change, if the file doesn't end in ".py" + # XXX the code here is bogus! + """ + _htest - bool, change box when location running htest. + """ + global file_open + if not _htest: + file_open = PyShell.flist.open + self.name = name + self.file = os.path.join(path[0], self.name + ".py") + self._htest = _htest + self.init(flist) + + def close(self, event=None): + self.top.destroy() + self.node.destroy() + + def init(self, flist): + self.flist = flist + # reset pyclbr + pyclbr._modules.clear() + # create top + self.top = top = ListedToplevel(flist.root) + top.protocol("WM_DELETE_WINDOW", self.close) + top.bind("", self.close) + if self._htest: # place dialog below parent if running htest + top.geometry("+%d+%d" % + (flist.root.winfo_rootx(), flist.root.winfo_rooty() + 200)) + self.settitle() + top.focus_set() + # create scrolled canvas + theme = idleConf.CurrentTheme() + background = idleConf.GetHighlight(theme, 'normal')['background'] + sc = ScrolledCanvas(top, bg=background, highlightthickness=0, takefocus=1) + sc.frame.pack(expand=1, fill="both") + item = self.rootnode() + self.node = node = TreeNode(sc.canvas, None, item) + node.update() + node.expand() + + def settitle(self): + self.top.wm_title("Class Browser - " + self.name) + self.top.wm_iconname("Class Browser") + + def rootnode(self): + return ModuleBrowserTreeItem(self.file) + +class ModuleBrowserTreeItem(TreeItem): + + def __init__(self, file): + self.file = file + + def GetText(self): + return os.path.basename(self.file) + + def GetIconName(self): + return "python" + + def GetSubList(self): + sublist = [] + for name in self.listclasses(): + item = ClassBrowserTreeItem(name, self.classes, self.file) + sublist.append(item) + return sublist + + def OnDoubleClick(self): + if os.path.normcase(self.file[-3:]) != ".py": + return + if not os.path.exists(self.file): + return + PyShell.flist.open(self.file) + + def IsExpandable(self): + return os.path.normcase(self.file[-3:]) == ".py" + + def listclasses(self): + dir, file = os.path.split(self.file) + name, ext = os.path.splitext(file) + if os.path.normcase(ext) != ".py": + return [] + try: + dict = pyclbr.readmodule_ex(name, [dir] + sys.path) + except ImportError: + return [] + items = [] + self.classes = {} + for key, cl in dict.items(): + if cl.module == name: + s = key + if hasattr(cl, 'super') and cl.super: + supers = [] + for sup in cl.super: + if type(sup) is type(''): + sname = sup + else: + sname = sup.name + if sup.module != cl.module: + sname = "%s.%s" % (sup.module, sname) + supers.append(sname) + s = s + "(%s)" % ", ".join(supers) + items.append((cl.lineno, s)) + self.classes[s] = cl + items.sort() + list = [] + for item, s in items: + list.append(s) + return list + +class ClassBrowserTreeItem(TreeItem): + + def __init__(self, name, classes, file): + self.name = name + self.classes = classes + self.file = file + try: + self.cl = self.classes[self.name] + except (IndexError, KeyError): + self.cl = None + self.isfunction = isinstance(self.cl, pyclbr.Function) + + def GetText(self): + if self.isfunction: + return "def " + self.name + "(...)" + else: + return "class " + self.name + + def GetIconName(self): + if self.isfunction: + return "python" + else: + return "folder" + + def IsExpandable(self): + if self.cl: + try: + return not not self.cl.methods + except AttributeError: + return False + + def GetSubList(self): + if not self.cl: + return [] + sublist = [] + for name in self.listmethods(): + item = MethodBrowserTreeItem(name, self.cl, self.file) + sublist.append(item) + return sublist + + def OnDoubleClick(self): + if not os.path.exists(self.file): + return + edit = file_open(self.file) + if hasattr(self.cl, 'lineno'): + lineno = self.cl.lineno + edit.gotoline(lineno) + + def listmethods(self): + if not self.cl: + return [] + items = [] + for name, lineno in self.cl.methods.items(): + items.append((lineno, name)) + items.sort() + list = [] + for item, name in items: + list.append(name) + return list + +class MethodBrowserTreeItem(TreeItem): + + def __init__(self, name, cl, file): + self.name = name + self.cl = cl + self.file = file + + def GetText(self): + return "def " + self.name + "(...)" + + def GetIconName(self): + return "python" # XXX + + def IsExpandable(self): + return 0 + + def OnDoubleClick(self): + if not os.path.exists(self.file): + return + edit = file_open(self.file) + edit.gotoline(self.cl.methods[self.name]) + +def _class_browser(parent): #Wrapper for htest + try: + file = __file__ + except NameError: + file = sys.argv[0] + if sys.argv[1:]: + file = sys.argv[1] + else: + file = sys.argv[0] + dir, file = os.path.split(file) + name = os.path.splitext(file)[0] + flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open + ClassBrowser(flist, name, [dir], _htest=True) + +if __name__ == "__main__": + from idlelib.idle_test.htest import run + run(_class_browser) diff --git a/Lib/idlelib/calltip_w.py b/Lib/idlelib/calltip_w.py new file mode 100644 index 0000000..8e68a76 --- /dev/null +++ b/Lib/idlelib/calltip_w.py @@ -0,0 +1,161 @@ +"""A CallTip window class for Tkinter/IDLE. + +After ToolTip.py, which uses ideas gleaned from PySol +Used by the CallTips IDLE extension. +""" +from tkinter import Toplevel, Label, LEFT, SOLID, TclError + +HIDE_VIRTUAL_EVENT_NAME = "<>" +HIDE_SEQUENCES = ("", "") +CHECKHIDE_VIRTUAL_EVENT_NAME = "<>" +CHECKHIDE_SEQUENCES = ("", "") +CHECKHIDE_TIME = 100 # miliseconds + +MARK_RIGHT = "calltipwindowregion_right" + +class CallTip: + + def __init__(self, widget): + self.widget = widget + self.tipwindow = self.label = None + self.parenline = self.parencol = None + self.lastline = None + self.hideid = self.checkhideid = None + self.checkhide_after_id = None + + def position_window(self): + """Check if needs to reposition the window, and if so - do it.""" + curline = int(self.widget.index("insert").split('.')[0]) + if curline == self.lastline: + return + self.lastline = curline + self.widget.see("insert") + if curline == self.parenline: + box = self.widget.bbox("%d.%d" % (self.parenline, + self.parencol)) + else: + box = self.widget.bbox("%d.0" % curline) + if not box: + box = list(self.widget.bbox("insert")) + # align to left of window + box[0] = 0 + box[2] = 0 + x = box[0] + self.widget.winfo_rootx() + 2 + y = box[1] + box[3] + self.widget.winfo_rooty() + self.tipwindow.wm_geometry("+%d+%d" % (x, y)) + + def showtip(self, text, parenleft, parenright): + """Show the calltip, bind events which will close it and reposition it. + """ + # Only called in CallTips, where lines are truncated + self.text = text + if self.tipwindow or not self.text: + return + + self.widget.mark_set(MARK_RIGHT, parenright) + self.parenline, self.parencol = map( + int, self.widget.index(parenleft).split(".")) + + self.tipwindow = tw = Toplevel(self.widget) + self.position_window() + # remove border on calltip window + tw.wm_overrideredirect(1) + try: + # This command is only needed and available on Tk >= 8.4.0 for OSX + # Without it, call tips intrude on the typing process by grabbing + # the focus. + tw.tk.call("::tk::unsupported::MacWindowStyle", "style", tw._w, + "help", "noActivates") + except TclError: + pass + self.label = Label(tw, text=self.text, justify=LEFT, + background="#ffffe0", relief=SOLID, borderwidth=1, + font = self.widget['font']) + self.label.pack() + tw.lift() # work around bug in Tk 8.5.18+ (issue #24570) + + self.checkhideid = self.widget.bind(CHECKHIDE_VIRTUAL_EVENT_NAME, + self.checkhide_event) + for seq in CHECKHIDE_SEQUENCES: + self.widget.event_add(CHECKHIDE_VIRTUAL_EVENT_NAME, seq) + self.widget.after(CHECKHIDE_TIME, self.checkhide_event) + self.hideid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, + self.hide_event) + for seq in HIDE_SEQUENCES: + self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq) + + def checkhide_event(self, event=None): + if not self.tipwindow: + # If the event was triggered by the same event that unbinded + # this function, the function will be called nevertheless, + # so do nothing in this case. + return + curline, curcol = map(int, self.widget.index("insert").split('.')) + if curline < self.parenline or \ + (curline == self.parenline and curcol <= self.parencol) or \ + self.widget.compare("insert", ">", MARK_RIGHT): + self.hidetip() + else: + self.position_window() + if self.checkhide_after_id is not None: + self.widget.after_cancel(self.checkhide_after_id) + self.checkhide_after_id = \ + self.widget.after(CHECKHIDE_TIME, self.checkhide_event) + + def hide_event(self, event): + if not self.tipwindow: + # See the explanation in checkhide_event. + return + self.hidetip() + + def hidetip(self): + if not self.tipwindow: + return + + for seq in CHECKHIDE_SEQUENCES: + self.widget.event_delete(CHECKHIDE_VIRTUAL_EVENT_NAME, seq) + self.widget.unbind(CHECKHIDE_VIRTUAL_EVENT_NAME, self.checkhideid) + self.checkhideid = None + for seq in HIDE_SEQUENCES: + self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq) + self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideid) + self.hideid = None + + self.label.destroy() + self.label = None + self.tipwindow.destroy() + self.tipwindow = None + + self.widget.mark_unset(MARK_RIGHT) + self.parenline = self.parencol = self.lastline = None + + def is_active(self): + return bool(self.tipwindow) + + +def _calltip_window(parent): # htest # + from tkinter import Toplevel, Text, LEFT, BOTH + + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) + + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() + +if __name__=='__main__': + from idlelib.idle_test.htest import run + run(_calltip_window) diff --git a/Lib/idlelib/calltips.py b/Lib/idlelib/calltips.py new file mode 100644 index 0000000..81bd5f1 --- /dev/null +++ b/Lib/idlelib/calltips.py @@ -0,0 +1,175 @@ +"""CallTips.py - An IDLE Extension to Jog Your Memory + +Call Tips are floating windows which display function, class, and method +parameter and docstring information when you type an opening parenthesis, and +which disappear when you type a closing parenthesis. + +""" +import __main__ +import inspect +import re +import sys +import textwrap +import types + +from idlelib import CallTipWindow +from idlelib.HyperParser import HyperParser + +class CallTips: + + menudefs = [ + ('edit', [ + ("Show call tip", "<>"), + ]) + ] + + def __init__(self, editwin=None): + if editwin is None: # subprocess and test + self.editwin = None + else: + self.editwin = editwin + self.text = editwin.text + self.active_calltip = None + self._calltip_window = self._make_tk_calltip_window + + def close(self): + self._calltip_window = None + + def _make_tk_calltip_window(self): + # See __init__ for usage + return CallTipWindow.CallTip(self.text) + + def _remove_calltip_window(self, event=None): + if self.active_calltip: + self.active_calltip.hidetip() + self.active_calltip = None + + def force_open_calltip_event(self, event): + "The user selected the menu entry or hotkey, open the tip." + self.open_calltip(True) + + def try_open_calltip_event(self, event): + """Happens when it would be nice to open a CallTip, but not really + necessary, for example after an opening bracket, so function calls + won't be made. + """ + self.open_calltip(False) + + def refresh_calltip_event(self, event): + if self.active_calltip and self.active_calltip.is_active(): + self.open_calltip(False) + + def open_calltip(self, evalfuncs): + self._remove_calltip_window() + + hp = HyperParser(self.editwin, "insert") + sur_paren = hp.get_surrounding_brackets('(') + if not sur_paren: + return + hp.set_index(sur_paren[0]) + expression = hp.get_expression() + if not expression: + return + if not evalfuncs and (expression.find('(') != -1): + return + argspec = self.fetch_tip(expression) + if not argspec: + return + self.active_calltip = self._calltip_window() + self.active_calltip.showtip(argspec, sur_paren[0], sur_paren[1]) + + def fetch_tip(self, expression): + """Return the argument list and docstring of a function or class. + + If there is a Python subprocess, get the calltip there. Otherwise, + either this fetch_tip() is running in the subprocess or it was + called in an IDLE running without the subprocess. + + The subprocess environment is that of the most recently run script. If + two unrelated modules are being edited some calltips in the current + module may be inoperative if the module was not the last to run. + + To find methods, fetch_tip must be fed a fully qualified name. + + """ + try: + rpcclt = self.editwin.flist.pyshell.interp.rpcclt + except AttributeError: + rpcclt = None + if rpcclt: + return rpcclt.remotecall("exec", "get_the_calltip", + (expression,), {}) + else: + return get_argspec(get_entity(expression)) + +def get_entity(expression): + """Return the object corresponding to expression evaluated + in a namespace spanning sys.modules and __main.dict__. + """ + if expression: + namespace = sys.modules.copy() + namespace.update(__main__.__dict__) + try: + return eval(expression, namespace) + except BaseException: + # An uncaught exception closes idle, and eval can raise any + # exception, especially if user classes are involved. + return None + +# The following are used in get_argspec and some in tests +_MAX_COLS = 85 +_MAX_LINES = 5 # enough for bytes +_INDENT = ' '*4 # for wrapped signatures +_first_param = re.compile('(?<=\()\w*\,?\s*') +_default_callable_argspec = "See source or doc" + + +def get_argspec(ob): + '''Return a string describing the signature of a callable object, or ''. + + For Python-coded functions and methods, the first line is introspected. + Delete 'self' parameter for classes (.__init__) and bound methods. + The next lines are the first lines of the doc string up to the first + empty line or _MAX_LINES. For builtins, this typically includes + the arguments in addition to the return value. + ''' + argspec = "" + try: + ob_call = ob.__call__ + except BaseException: + return argspec + if isinstance(ob, type): + fob = ob.__init__ + elif isinstance(ob_call, types.MethodType): + fob = ob_call + else: + fob = ob + if isinstance(fob, (types.FunctionType, types.MethodType)): + argspec = inspect.formatargspec(*inspect.getfullargspec(fob)) + if (isinstance(ob, (type, types.MethodType)) or + isinstance(ob_call, types.MethodType)): + argspec = _first_param.sub("", argspec) + + lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) + if len(argspec) > _MAX_COLS else [argspec] if argspec else []) + + if isinstance(ob_call, types.MethodType): + doc = ob_call.__doc__ + else: + doc = getattr(ob, "__doc__", "") + if doc: + for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: + line = line.strip() + if not line: + break + if len(line) > _MAX_COLS: + line = line[: _MAX_COLS - 3] + '...' + lines.append(line) + argspec = '\n'.join(lines) + if not argspec: + argspec = _default_callable_argspec + return argspec + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_calltips', verbosity=2) diff --git a/Lib/idlelib/codecontext.py b/Lib/idlelib/codecontext.py new file mode 100644 index 0000000..7d25ada --- /dev/null +++ b/Lib/idlelib/codecontext.py @@ -0,0 +1,176 @@ +"""CodeContext - Extension to display the block context above the edit window + +Once code has scrolled off the top of a window, it can be difficult to +determine which block you are in. This extension implements a pane at the top +of each IDLE edit window which provides block structure hints. These hints are +the lines which contain the block opening keywords, e.g. 'if', for the +enclosing block. The number of hint lines is determined by the numlines +variable in the CodeContext section of config-extensions.def. Lines which do +not open blocks are not shown in the context hints pane. + +""" +import tkinter +from tkinter.constants import TOP, LEFT, X, W, SUNKEN +import re +from sys import maxsize as INFINITY +from idlelib.configHandler import idleConf + +BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", + "if", "try", "while", "with"} +UPDATEINTERVAL = 100 # millisec +FONTUPDATEINTERVAL = 1000 # millisec + +getspacesfirstword =\ + lambda s, c=re.compile(r"^(\s*)(\w*)"): c.match(s).groups() + +class CodeContext: + menudefs = [('options', [('!Code Conte_xt', '<>')])] + context_depth = idleConf.GetOption("extensions", "CodeContext", + "numlines", type="int", default=3) + bgcolor = idleConf.GetOption("extensions", "CodeContext", + "bgcolor", type="str", default="LightGray") + fgcolor = idleConf.GetOption("extensions", "CodeContext", + "fgcolor", type="str", default="Black") + def __init__(self, editwin): + self.editwin = editwin + self.text = editwin.text + self.textfont = self.text["font"] + self.label = None + # self.info is a list of (line number, indent level, line text, block + # keyword) tuples providing the block structure associated with + # self.topvisible (the linenumber of the line displayed at the top of + # the edit window). self.info[0] is initialized as a 'dummy' line which + # starts the toplevel 'block' of the module. + self.info = [(0, -1, "", False)] + self.topvisible = 1 + visible = idleConf.GetOption("extensions", "CodeContext", + "visible", type="bool", default=False) + if visible: + self.toggle_code_context_event() + self.editwin.setvar('<>', True) + # Start two update cycles, one for context lines, one for font changes. + self.text.after(UPDATEINTERVAL, self.timer_event) + self.text.after(FONTUPDATEINTERVAL, self.font_timer_event) + + def toggle_code_context_event(self, event=None): + if not self.label: + # Calculate the border width and horizontal padding required to + # align the context with the text in the main Text widget. + # + # All values are passed through getint(), since some + # values may be pixel objects, which can't simply be added to ints. + widgets = self.editwin.text, self.editwin.text_frame + # Calculate the required vertical padding + padx = 0 + for widget in widgets: + padx += widget.tk.getint(widget.pack_info()['padx']) + padx += widget.tk.getint(widget.cget('padx')) + # Calculate the required border width + border = 0 + for widget in widgets: + border += widget.tk.getint(widget.cget('border')) + self.label = tkinter.Label(self.editwin.top, + text="\n" * (self.context_depth - 1), + anchor=W, justify=LEFT, + font=self.textfont, + bg=self.bgcolor, fg=self.fgcolor, + width=1, #don't request more than we get + padx=padx, border=border, + relief=SUNKEN) + # Pack the label widget before and above the text_frame widget, + # thus ensuring that it will appear directly above text_frame + self.label.pack(side=TOP, fill=X, expand=False, + before=self.editwin.text_frame) + else: + self.label.destroy() + self.label = None + idleConf.SetOption("extensions", "CodeContext", "visible", + str(self.label is not None)) + idleConf.SaveUserCfgFiles() + + def get_line_info(self, linenum): + """Get the line indent value, text, and any block start keyword + + If the line does not start a block, the keyword value is False. + The indentation of empty lines (or comment lines) is INFINITY. + + """ + text = self.text.get("%d.0" % linenum, "%d.end" % linenum) + spaces, firstword = getspacesfirstword(text) + opener = firstword in BLOCKOPENERS and firstword + if len(text) == len(spaces) or text[len(spaces)] == '#': + indent = INFINITY + else: + indent = len(spaces) + return indent, text, opener + + def get_context(self, new_topvisible, stopline=1, stopindent=0): + """Get context lines, starting at new_topvisible and working backwards. + + Stop when stopline or stopindent is reached. Return a tuple of context + data and the indent level at the top of the region inspected. + + """ + assert stopline > 0 + lines = [] + # The indentation level we are currently in: + lastindent = INFINITY + # For a line to be interesting, it must begin with a block opening + # keyword, and have less indentation than lastindent. + for linenum in range(new_topvisible, stopline-1, -1): + indent, text, opener = self.get_line_info(linenum) + if indent < lastindent: + lastindent = indent + if opener in ("else", "elif"): + # We also show the if statement + lastindent += 1 + if opener and linenum < new_topvisible and indent >= stopindent: + lines.append((linenum, indent, text, opener)) + if lastindent <= stopindent: + break + lines.reverse() + return lines, lastindent + + def update_code_context(self): + """Update context information and lines visible in the context pane. + + """ + new_topvisible = int(self.text.index("@0,0").split('.')[0]) + if self.topvisible == new_topvisible: # haven't scrolled + return + if self.topvisible < new_topvisible: # scroll down + lines, lastindent = self.get_context(new_topvisible, + self.topvisible) + # retain only context info applicable to the region + # between topvisible and new_topvisible: + while self.info[-1][1] >= lastindent: + del self.info[-1] + elif self.topvisible > new_topvisible: # scroll up + stopindent = self.info[-1][1] + 1 + # retain only context info associated + # with lines above new_topvisible: + while self.info[-1][0] >= new_topvisible: + stopindent = self.info[-1][1] + del self.info[-1] + lines, lastindent = self.get_context(new_topvisible, + self.info[-1][0]+1, + stopindent) + self.info.extend(lines) + self.topvisible = new_topvisible + # empty lines in context pane: + context_strings = [""] * max(0, self.context_depth - len(self.info)) + # followed by the context hint lines: + context_strings += [x[2] for x in self.info[-self.context_depth:]] + self.label["text"] = '\n'.join(context_strings) + + def timer_event(self): + if self.label: + self.update_code_context() + self.text.after(UPDATEINTERVAL, self.timer_event) + + def font_timer_event(self): + newtextfont = self.text["font"] + if self.label and newtextfont != self.textfont: + self.textfont = newtextfont + self.label["font"] = self.textfont + self.text.after(FONTUPDATEINTERVAL, self.font_timer_event) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py new file mode 100644 index 0000000..9f31349 --- /dev/null +++ b/Lib/idlelib/colorizer.py @@ -0,0 +1,256 @@ +import time +import re +import keyword +import builtins +from idlelib.Delegator import Delegator +from idlelib.configHandler import idleConf + +DEBUG = False + +def any(name, alternates): + "Return a named group pattern matching list of alternates." + return "(?P<%s>" % name + "|".join(alternates) + ")" + +def make_pat(): + kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" + builtinlist = [str(name) for name in dir(builtins) + if not name.startswith('_') and \ + name not in keyword.kwlist] + # self.file = open("file") : + # 1st 'file' colorized normal, 2nd as builtin, 3rd as string + builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b" + comment = any("COMMENT", [r"#[^\n]*"]) + stringprefix = r"(\br|u|ur|R|U|UR|Ur|uR|b|B|br|Br|bR|BR|rb|rB|Rb|RB)?" + sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?" + dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?' + sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" + dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' + string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) + return kw + "|" + builtin + "|" + comment + "|" + string +\ + "|" + any("SYNC", [r"\n"]) + +prog = re.compile(make_pat(), re.S) +idprog = re.compile(r"\s+(\w+)", re.S) + +class ColorDelegator(Delegator): + + def __init__(self): + Delegator.__init__(self) + self.prog = prog + self.idprog = idprog + self.LoadTagDefs() + + def setdelegate(self, delegate): + if self.delegate is not None: + self.unbind("<>") + Delegator.setdelegate(self, delegate) + if delegate is not None: + self.config_colors() + self.bind("<>", self.toggle_colorize_event) + self.notify_range("1.0", "end") + else: + # No delegate - stop any colorizing + self.stop_colorizing = True + self.allow_colorizing = False + + def config_colors(self): + for tag, cnf in self.tagdefs.items(): + if cnf: + self.tag_configure(tag, **cnf) + self.tag_raise('sel') + + def LoadTagDefs(self): + theme = idleConf.CurrentTheme() + self.tagdefs = { + "COMMENT": idleConf.GetHighlight(theme, "comment"), + "KEYWORD": idleConf.GetHighlight(theme, "keyword"), + "BUILTIN": idleConf.GetHighlight(theme, "builtin"), + "STRING": idleConf.GetHighlight(theme, "string"), + "DEFINITION": idleConf.GetHighlight(theme, "definition"), + "SYNC": {'background':None,'foreground':None}, + "TODO": {'background':None,'foreground':None}, + "ERROR": idleConf.GetHighlight(theme, "error"), + # The following is used by ReplaceDialog: + "hit": idleConf.GetHighlight(theme, "hit"), + } + + if DEBUG: print('tagdefs',self.tagdefs) + + def insert(self, index, chars, tags=None): + index = self.index(index) + self.delegate.insert(index, chars, tags) + self.notify_range(index, index + "+%dc" % len(chars)) + + def delete(self, index1, index2=None): + index1 = self.index(index1) + self.delegate.delete(index1, index2) + self.notify_range(index1) + + after_id = None + allow_colorizing = True + colorizing = False + + def notify_range(self, index1, index2=None): + self.tag_add("TODO", index1, index2) + if self.after_id: + if DEBUG: print("colorizing already scheduled") + return + if self.colorizing: + self.stop_colorizing = True + if DEBUG: print("stop colorizing") + if self.allow_colorizing: + if DEBUG: print("schedule colorizing") + self.after_id = self.after(1, self.recolorize) + + close_when_done = None # Window to be closed when done colorizing + + def close(self, close_when_done=None): + if self.after_id: + after_id = self.after_id + self.after_id = None + if DEBUG: print("cancel scheduled recolorizer") + self.after_cancel(after_id) + self.allow_colorizing = False + self.stop_colorizing = True + if close_when_done: + if not self.colorizing: + close_when_done.destroy() + else: + self.close_when_done = close_when_done + + def toggle_colorize_event(self, event): + if self.after_id: + after_id = self.after_id + self.after_id = None + if DEBUG: print("cancel scheduled recolorizer") + self.after_cancel(after_id) + if self.allow_colorizing and self.colorizing: + if DEBUG: print("stop colorizing") + self.stop_colorizing = True + self.allow_colorizing = not self.allow_colorizing + if self.allow_colorizing and not self.colorizing: + self.after_id = self.after(1, self.recolorize) + if DEBUG: + print("auto colorizing turned",\ + self.allow_colorizing and "on" or "off") + return "break" + + def recolorize(self): + self.after_id = None + if not self.delegate: + if DEBUG: print("no delegate") + return + if not self.allow_colorizing: + if DEBUG: print("auto colorizing is off") + return + if self.colorizing: + if DEBUG: print("already colorizing") + return + try: + self.stop_colorizing = False + self.colorizing = True + if DEBUG: print("colorizing...") + t0 = time.perf_counter() + self.recolorize_main() + t1 = time.perf_counter() + if DEBUG: print("%.3f seconds" % (t1-t0)) + finally: + self.colorizing = False + if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"): + if DEBUG: print("reschedule colorizing") + self.after_id = self.after(1, self.recolorize) + if self.close_when_done: + top = self.close_when_done + self.close_when_done = None + top.destroy() + + def recolorize_main(self): + next = "1.0" + while True: + item = self.tag_nextrange("TODO", next) + if not item: + break + head, tail = item + self.tag_remove("SYNC", head, tail) + item = self.tag_prevrange("SYNC", head) + if item: + head = item[1] + else: + head = "1.0" + + chars = "" + next = head + lines_to_get = 1 + ok = False + while not ok: + mark = next + next = self.index(mark + "+%d lines linestart" % + lines_to_get) + lines_to_get = min(lines_to_get * 2, 100) + ok = "SYNC" in self.tag_names(next + "-1c") + line = self.get(mark, next) + ##print head, "get", mark, next, "->", repr(line) + if not line: + return + for tag in self.tagdefs: + self.tag_remove(tag, mark, next) + chars = chars + line + m = self.prog.search(chars) + while m: + for key, value in m.groupdict().items(): + if value: + a, b = m.span(key) + self.tag_add(key, + head + "+%dc" % a, + head + "+%dc" % b) + if value in ("def", "class"): + m1 = self.idprog.match(chars, b) + if m1: + a, b = m1.span(1) + self.tag_add("DEFINITION", + head + "+%dc" % a, + head + "+%dc" % b) + m = self.prog.search(chars, m.end()) + if "SYNC" in self.tag_names(next + "-1c"): + head = next + chars = "" + else: + ok = False + if not ok: + # We're in an inconsistent state, and the call to + # update may tell us to stop. It may also change + # the correct value for "next" (since this is a + # line.col string, not a true mark). So leave a + # crumb telling the next invocation to resume here + # in case update tells us to leave. + self.tag_add("TODO", next) + self.update() + if self.stop_colorizing: + if DEBUG: print("colorizing stopped") + return + + def removecolors(self): + for tag in self.tagdefs: + self.tag_remove(tag, "1.0", "end") + +def _color_delegator(parent): # htest # + from tkinter import Toplevel, Text + from idlelib.Percolator import Percolator + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") + text.insert("insert", source) + text.focus_set() + + p = Percolator(text) + d = ColorDelegator() + p.insertfilter(d) + +if __name__ == "__main__": + from idlelib.idle_test.htest import run + run(_color_delegator) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py new file mode 100644 index 0000000..8ac1f60 --- /dev/null +++ b/Lib/idlelib/config.py @@ -0,0 +1,760 @@ +"""Provides access to stored IDLE configuration information. + +Refer to the comments at the beginning of config-main.def for a description of +the available configuration files and the design implemented to update user +configuration information. In particular, user configuration choices which +duplicate the defaults will be removed from the user's configuration files, +and if a file becomes empty, it will be deleted. + +The contents of the user files may be altered using the Options/Configure IDLE +menu to access the configuration GUI (configDialog.py), or manually. + +Throughout this module there is an emphasis on returning useable defaults +when a problem occurs in returning a requested configuration value back to +idle. This is to allow IDLE to continue to function in spite of errors in +the retrieval of config information. When a default is returned instead of +a requested config value, a message is printed to stderr to aid in +configuration problem notification and resolution. +""" +# TODOs added Oct 2014, tjr + +import os +import sys + +from configparser import ConfigParser +from tkinter import TkVersion +from tkinter.font import Font, nametofont + +class InvalidConfigType(Exception): pass +class InvalidConfigSet(Exception): pass +class InvalidFgBg(Exception): pass +class InvalidTheme(Exception): pass + +class IdleConfParser(ConfigParser): + """ + A ConfigParser specialised for idle configuration file handling + """ + def __init__(self, cfgFile, cfgDefaults=None): + """ + cfgFile - string, fully specified configuration file name + """ + self.file = cfgFile + ConfigParser.__init__(self, defaults=cfgDefaults, strict=False) + + def Get(self, section, option, type=None, default=None, raw=False): + """ + Get an option value for given section/option or return default. + If type is specified, return as type. + """ + # TODO Use default as fallback, at least if not None + # Should also print Warning(file, section, option). + # Currently may raise ValueError + if not self.has_option(section, option): + return default + if type == 'bool': + return self.getboolean(section, option) + elif type == 'int': + return self.getint(section, option) + else: + return self.get(section, option, raw=raw) + + def GetOptionList(self, section): + "Return a list of options for given section, else []." + if self.has_section(section): + return self.options(section) + else: #return a default value + return [] + + def Load(self): + "Load the configuration file from disk." + self.read(self.file) + +class IdleUserConfParser(IdleConfParser): + """ + IdleConfigParser specialised for user configuration handling. + """ + + def AddSection(self, section): + "If section doesn't exist, add it." + if not self.has_section(section): + self.add_section(section) + + def RemoveEmptySections(self): + "Remove any sections that have no options." + for section in self.sections(): + if not self.GetOptionList(section): + self.remove_section(section) + + def IsEmpty(self): + "Return True if no sections after removing empty sections." + self.RemoveEmptySections() + return not self.sections() + + def RemoveOption(self, section, option): + """Return True if option is removed from section, else False. + + False if either section does not exist or did not have option. + """ + if self.has_section(section): + return self.remove_option(section, option) + return False + + def SetOption(self, section, option, value): + """Return True if option is added or changed to value, else False. + + Add section if required. False means option already had value. + """ + if self.has_option(section, option): + if self.get(section, option) == value: + return False + else: + self.set(section, option, value) + return True + else: + if not self.has_section(section): + self.add_section(section) + self.set(section, option, value) + return True + + def RemoveFile(self): + "Remove user config file self.file from disk if it exists." + if os.path.exists(self.file): + os.remove(self.file) + + def Save(self): + """Update user configuration file. + + Remove empty sections. If resulting config isn't empty, write the file + to disk. If config is empty, remove the file from disk if it exists. + + """ + if not self.IsEmpty(): + fname = self.file + try: + cfgFile = open(fname, 'w') + except OSError: + os.unlink(fname) + cfgFile = open(fname, 'w') + with cfgFile: + self.write(cfgFile) + else: + self.RemoveFile() + +class IdleConf: + """Hold config parsers for all idle config files in singleton instance. + + Default config files, self.defaultCfg -- + for config_type in self.config_types: + (idle install dir)/config-{config-type}.def + + User config files, self.userCfg -- + for config_type in self.config_types: + (user home dir)/.idlerc/config-{config-type}.cfg + """ + def __init__(self): + self.config_types = ('main', 'extensions', 'highlight', 'keys') + self.defaultCfg = {} + self.userCfg = {} + self.cfg = {} # TODO use to select userCfg vs defaultCfg + self.CreateConfigHandlers() + self.LoadCfgFiles() + + + def CreateConfigHandlers(self): + "Populate default and user config parser dictionaries." + #build idle install path + if __name__ != '__main__': # we were imported + idleDir=os.path.dirname(__file__) + else: # we were exec'ed (for testing only) + idleDir=os.path.abspath(sys.path[0]) + userDir=self.GetUserCfgDir() + + defCfgFiles = {} + usrCfgFiles = {} + # TODO eliminate these temporaries by combining loops + for cfgType in self.config_types: #build config file names + defCfgFiles[cfgType] = os.path.join( + idleDir, 'config-' + cfgType + '.def') + usrCfgFiles[cfgType] = os.path.join( + userDir, 'config-' + cfgType + '.cfg') + for cfgType in self.config_types: #create config parsers + self.defaultCfg[cfgType] = IdleConfParser(defCfgFiles[cfgType]) + self.userCfg[cfgType] = IdleUserConfParser(usrCfgFiles[cfgType]) + + def GetUserCfgDir(self): + """Return a filesystem directory for storing user config files. + + Creates it if required. + """ + cfgDir = '.idlerc' + userDir = os.path.expanduser('~') + if userDir != '~': # expanduser() found user home dir + if not os.path.exists(userDir): + warn = ('\n Warning: os.path.expanduser("~") points to\n ' + + userDir + ',\n but the path does not exist.') + try: + print(warn, file=sys.stderr) + except OSError: + pass + userDir = '~' + if userDir == "~": # still no path to home! + # traditionally IDLE has defaulted to os.getcwd(), is this adequate? + userDir = os.getcwd() + userDir = os.path.join(userDir, cfgDir) + if not os.path.exists(userDir): + try: + os.mkdir(userDir) + except OSError: + warn = ('\n Warning: unable to create user config directory\n' + + userDir + '\n Check path and permissions.\n Exiting!\n') + print(warn, file=sys.stderr) + raise SystemExit + # TODO continue without userDIr instead of exit + return userDir + + def GetOption(self, configType, section, option, default=None, type=None, + warn_on_default=True, raw=False): + """Return a value for configType section option, or default. + + If type is not None, return a value of that type. Also pass raw + to the config parser. First try to return a valid value + (including type) from a user configuration. If that fails, try + the default configuration. If that fails, return default, with a + default of None. + + Warn if either user or default configurations have an invalid value. + Warn if default is returned and warn_on_default is True. + """ + try: + if self.userCfg[configType].has_option(section, option): + return self.userCfg[configType].Get(section, option, + type=type, raw=raw) + except ValueError: + warning = ('\n Warning: configHandler.py - IdleConf.GetOption -\n' + ' invalid %r value for configuration option %r\n' + ' from section %r: %r' % + (type, option, section, + self.userCfg[configType].Get(section, option, raw=raw))) + try: + print(warning, file=sys.stderr) + except OSError: + pass + try: + if self.defaultCfg[configType].has_option(section,option): + return self.defaultCfg[configType].Get( + section, option, type=type, raw=raw) + except ValueError: + pass + #returning default, print warning + if warn_on_default: + warning = ('\n Warning: configHandler.py - IdleConf.GetOption -\n' + ' problem retrieving configuration option %r\n' + ' from section %r.\n' + ' returning default value: %r' % + (option, section, default)) + try: + print(warning, file=sys.stderr) + except OSError: + pass + return default + + def SetOption(self, configType, section, option, value): + """Set section option to value in user config file.""" + self.userCfg[configType].SetOption(section, option, value) + + def GetSectionList(self, configSet, configType): + """Return sections for configSet configType configuration. + + configSet must be either 'user' or 'default' + configType must be in self.config_types. + """ + if not (configType in self.config_types): + raise InvalidConfigType('Invalid configType specified') + if configSet == 'user': + cfgParser = self.userCfg[configType] + elif configSet == 'default': + cfgParser=self.defaultCfg[configType] + else: + raise InvalidConfigSet('Invalid configSet specified') + return cfgParser.sections() + + def GetHighlight(self, theme, element, fgBg=None): + """Return individual theme element highlight color(s). + + fgBg - string ('fg' or 'bg') or None. + If None, return a dictionary containing fg and bg colors with + keys 'foreground' and 'background'. Otherwise, only return + fg or bg color, as specified. Colors are intended to be + appropriate for passing to Tkinter in, e.g., a tag_config call). + """ + if self.defaultCfg['highlight'].has_section(theme): + themeDict = self.GetThemeDict('default', theme) + else: + themeDict = self.GetThemeDict('user', theme) + fore = themeDict[element + '-foreground'] + if element == 'cursor': # There is no config value for cursor bg + back = themeDict['normal-background'] + else: + back = themeDict[element + '-background'] + highlight = {"foreground": fore, "background": back} + if not fgBg: # Return dict of both colors + return highlight + else: # Return specified color only + if fgBg == 'fg': + return highlight["foreground"] + if fgBg == 'bg': + return highlight["background"] + else: + raise InvalidFgBg('Invalid fgBg specified') + + def GetThemeDict(self, type, themeName): + """Return {option:value} dict for elements in themeName. + + type - string, 'default' or 'user' theme type + themeName - string, theme name + Values are loaded over ultimate fallback defaults to guarantee + that all theme elements are present in a newly created theme. + """ + if type == 'user': + cfgParser = self.userCfg['highlight'] + elif type == 'default': + cfgParser = self.defaultCfg['highlight'] + else: + raise InvalidTheme('Invalid theme type specified') + # Provide foreground and background colors for each theme + # element (other than cursor) even though some values are not + # yet used by idle, to allow for their use in the future. + # Default values are generally black and white. + # TODO copy theme from a class attribute. + theme ={'normal-foreground':'#000000', + 'normal-background':'#ffffff', + 'keyword-foreground':'#000000', + 'keyword-background':'#ffffff', + 'builtin-foreground':'#000000', + 'builtin-background':'#ffffff', + 'comment-foreground':'#000000', + 'comment-background':'#ffffff', + 'string-foreground':'#000000', + 'string-background':'#ffffff', + 'definition-foreground':'#000000', + 'definition-background':'#ffffff', + 'hilite-foreground':'#000000', + 'hilite-background':'gray', + 'break-foreground':'#ffffff', + 'break-background':'#000000', + 'hit-foreground':'#ffffff', + 'hit-background':'#000000', + 'error-foreground':'#ffffff', + 'error-background':'#000000', + #cursor (only foreground can be set) + 'cursor-foreground':'#000000', + #shell window + 'stdout-foreground':'#000000', + 'stdout-background':'#ffffff', + 'stderr-foreground':'#000000', + 'stderr-background':'#ffffff', + 'console-foreground':'#000000', + 'console-background':'#ffffff' } + for element in theme: + if not cfgParser.has_option(themeName, element): + # Print warning that will return a default color + warning = ('\n Warning: configHandler.IdleConf.GetThemeDict' + ' -\n problem retrieving theme element %r' + '\n from theme %r.\n' + ' returning default color: %r' % + (element, themeName, theme[element])) + try: + print(warning, file=sys.stderr) + except OSError: + pass + theme[element] = cfgParser.Get( + themeName, element, default=theme[element]) + return theme + + def CurrentTheme(self): + """Return the name of the currently active text color theme. + + idlelib.config-main.def includes this section + [Theme] + default= 1 + name= IDLE Classic + name2= + # name2 set in user config-main.cfg for themes added after 2015 Oct 1 + + Item name2 is needed because setting name to a new builtin + causes older IDLEs to display multiple error messages or quit. + See https://bugs.python.org/issue25313. + When default = True, name2 takes precedence over name, + while older IDLEs will just use name. + """ + default = self.GetOption('main', 'Theme', 'default', + type='bool', default=True) + if default: + theme = self.GetOption('main', 'Theme', 'name2', default='') + if default and not theme or not default: + theme = self.GetOption('main', 'Theme', 'name', default='') + source = self.defaultCfg if default else self.userCfg + if source['highlight'].has_section(theme): + return theme + else: + return "IDLE Classic" + + def CurrentKeys(self): + "Return the name of the currently active key set." + return self.GetOption('main', 'Keys', 'name', default='') + + def GetExtensions(self, active_only=True, editor_only=False, shell_only=False): + """Return extensions in default and user config-extensions files. + + If active_only True, only return active (enabled) extensions + and optionally only editor or shell extensions. + If active_only False, return all extensions. + """ + extns = self.RemoveKeyBindNames( + self.GetSectionList('default', 'extensions')) + userExtns = self.RemoveKeyBindNames( + self.GetSectionList('user', 'extensions')) + for extn in userExtns: + if extn not in extns: #user has added own extension + extns.append(extn) + if active_only: + activeExtns = [] + for extn in extns: + if self.GetOption('extensions', extn, 'enable', default=True, + type='bool'): + #the extension is enabled + if editor_only or shell_only: # TODO if both, contradictory + if editor_only: + option = "enable_editor" + else: + option = "enable_shell" + if self.GetOption('extensions', extn,option, + default=True, type='bool', + warn_on_default=False): + activeExtns.append(extn) + else: + activeExtns.append(extn) + return activeExtns + else: + return extns + + def RemoveKeyBindNames(self, extnNameList): + "Return extnNameList with keybinding section names removed." + # TODO Easier to return filtered copy with list comp + names = extnNameList + kbNameIndicies = [] + for name in names: + if name.endswith(('_bindings', '_cfgBindings')): + kbNameIndicies.append(names.index(name)) + kbNameIndicies.sort(reverse=True) + for index in kbNameIndicies: #delete each keybinding section name + del(names[index]) + return names + + def GetExtnNameForEvent(self, virtualEvent): + """Return the name of the extension binding virtualEvent, or None. + + virtualEvent - string, name of the virtual event to test for, + without the enclosing '<< >>' + """ + extName = None + vEvent = '<<' + virtualEvent + '>>' + for extn in self.GetExtensions(active_only=0): + for event in self.GetExtensionKeys(extn): + if event == vEvent: + extName = extn # TODO return here? + return extName + + def GetExtensionKeys(self, extensionName): + """Return dict: {configurable extensionName event : active keybinding}. + + Events come from default config extension_cfgBindings section. + Keybindings come from GetCurrentKeySet() active key dict, + where previously used bindings are disabled. + """ + keysName = extensionName + '_cfgBindings' + activeKeys = self.GetCurrentKeySet() + extKeys = {} + if self.defaultCfg['extensions'].has_section(keysName): + eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) + for eventName in eventNames: + event = '<<' + eventName + '>>' + binding = activeKeys[event] + extKeys[event] = binding + return extKeys + + def __GetRawExtensionKeys(self,extensionName): + """Return dict {configurable extensionName event : keybinding list}. + + Events come from default config extension_cfgBindings section. + Keybindings list come from the splitting of GetOption, which + tries user config before default config. + """ + keysName = extensionName+'_cfgBindings' + extKeys = {} + if self.defaultCfg['extensions'].has_section(keysName): + eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) + for eventName in eventNames: + binding = self.GetOption( + 'extensions', keysName, eventName, default='').split() + event = '<<' + eventName + '>>' + extKeys[event] = binding + return extKeys + + def GetExtensionBindings(self, extensionName): + """Return dict {extensionName event : active or defined keybinding}. + + Augment self.GetExtensionKeys(extensionName) with mapping of non- + configurable events (from default config) to GetOption splits, + as in self.__GetRawExtensionKeys. + """ + bindsName = extensionName + '_bindings' + extBinds = self.GetExtensionKeys(extensionName) + #add the non-configurable bindings + if self.defaultCfg['extensions'].has_section(bindsName): + eventNames = self.defaultCfg['extensions'].GetOptionList(bindsName) + for eventName in eventNames: + binding = self.GetOption( + 'extensions', bindsName, eventName, default='').split() + event = '<<' + eventName + '>>' + extBinds[event] = binding + + return extBinds + + def GetKeyBinding(self, keySetName, eventStr): + """Return the keybinding list for keySetName eventStr. + + keySetName - name of key binding set (config-keys section). + eventStr - virtual event, including brackets, as in '<>'. + """ + eventName = eventStr[2:-2] #trim off the angle brackets + binding = self.GetOption('keys', keySetName, eventName, default='').split() + return binding + + def GetCurrentKeySet(self): + "Return CurrentKeys with 'darwin' modifications." + result = self.GetKeySet(self.CurrentKeys()) + + if sys.platform == "darwin": + # OS X Tk variants do not support the "Alt" keyboard modifier. + # So replace all keybingings that use "Alt" with ones that + # use the "Option" keyboard modifier. + # TODO (Ned?): the "Option" modifier does not work properly for + # Cocoa Tk and XQuartz Tk so we should not use it + # in default OS X KeySets. + for k, v in result.items(): + v2 = [ x.replace('>' + """ + return ('<<'+virtualEvent+'>>') in self.GetCoreKeys() + +# TODO make keyBindins a file or class attribute used for test above +# and copied in function below + + def GetCoreKeys(self, keySetName=None): + """Return dict of core virtual-key keybindings for keySetName. + + The default keySetName None corresponds to the keyBindings base + dict. If keySetName is not None, bindings from the config + file(s) are loaded _over_ these defaults, so if there is a + problem getting any core binding there will be an 'ultimate last + resort fallback' to the CUA-ish bindings defined here. + """ + keyBindings={ + '<>': ['', ''], + '<>': ['', ''], + '<>': ['', ''], + '<>': ['', ''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': ['', ''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': ['', ''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''], + '<>': [''] + } + if keySetName: + for event in keyBindings: + binding = self.GetKeyBinding(keySetName, event) + if binding: + keyBindings[event] = binding + else: #we are going to return a default, print warning + warning=('\n Warning: configHandler.py - IdleConf.GetCoreKeys' + ' -\n problem retrieving key binding for event %r' + '\n from key set %r.\n' + ' returning default value: %r' % + (event, keySetName, keyBindings[event])) + try: + print(warning, file=sys.stderr) + except OSError: + pass + return keyBindings + + def GetExtraHelpSourceList(self, configSet): + """Return list of extra help sources from a given configSet. + + Valid configSets are 'user' or 'default'. Return a list of tuples of + the form (menu_item , path_to_help_file , option), or return the empty + list. 'option' is the sequence number of the help resource. 'option' + values determine the position of the menu items on the Help menu, + therefore the returned list must be sorted by 'option'. + + """ + helpSources = [] + if configSet == 'user': + cfgParser = self.userCfg['main'] + elif configSet == 'default': + cfgParser = self.defaultCfg['main'] + else: + raise InvalidConfigSet('Invalid configSet specified') + options=cfgParser.GetOptionList('HelpFiles') + for option in options: + value=cfgParser.Get('HelpFiles', option, default=';') + if value.find(';') == -1: #malformed config entry with no ';' + menuItem = '' #make these empty + helpPath = '' #so value won't be added to list + else: #config entry contains ';' as expected + value=value.split(';') + menuItem=value[0].strip() + helpPath=value[1].strip() + if menuItem and helpPath: #neither are empty strings + helpSources.append( (menuItem,helpPath,option) ) + helpSources.sort(key=lambda x: x[2]) + return helpSources + + def GetAllExtraHelpSourcesList(self): + """Return a list of the details of all additional help sources. + + Tuples in the list are those of GetExtraHelpSourceList. + """ + allHelpSources = (self.GetExtraHelpSourceList('default') + + self.GetExtraHelpSourceList('user') ) + return allHelpSources + + def GetFont(self, root, configType, section): + """Retrieve a font from configuration (font, font-size, font-bold) + Intercept the special value 'TkFixedFont' and substitute + the actual font, factoring in some tweaks if needed for + appearance sakes. + + The 'root' parameter can normally be any valid Tkinter widget. + + Return a tuple (family, size, weight) suitable for passing + to tkinter.Font + """ + family = self.GetOption(configType, section, 'font', default='courier') + size = self.GetOption(configType, section, 'font-size', type='int', + default='10') + bold = self.GetOption(configType, section, 'font-bold', default=0, + type='bool') + if (family == 'TkFixedFont'): + if TkVersion < 8.5: + family = 'Courier' + else: + f = Font(name='TkFixedFont', exists=True, root=root) + actualFont = Font.actual(f) + family = actualFont['family'] + size = actualFont['size'] + if size <= 0: + size = 10 # if font in pixels, ignore actual size + bold = actualFont['weight']=='bold' + return (family, size, 'bold' if bold else 'normal') + + def LoadCfgFiles(self): + "Load all configuration files." + for key in self.defaultCfg: + self.defaultCfg[key].Load() + self.userCfg[key].Load() #same keys + + def SaveUserCfgFiles(self): + "Write all loaded user configuration files to disk." + for key in self.userCfg: + self.userCfg[key].Save() + + +idleConf = IdleConf() + +# TODO Revise test output, write expanded unittest +### module test +if __name__ == '__main__': + def dumpCfg(cfg): + print('\n', cfg, '\n') + for key in cfg: + sections = cfg[key].sections() + print(key) + print(sections) + for section in sections: + options = cfg[key].options(section) + print(section) + print(options) + for option in options: + print(option, '=', cfg[key].Get(section, option)) + dumpCfg(idleConf.defaultCfg) + dumpCfg(idleConf.userCfg) + print(idleConf.userCfg['main'].Get('Theme', 'name')) + #print idleConf.userCfg['highlight'].GetDefHighlight('Foo','normal') diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py deleted file mode 100644 index b702253..0000000 --- a/Lib/idlelib/configDialog.py +++ /dev/null @@ -1,1434 +0,0 @@ -"""IDLE Configuration Dialog: support user customization of IDLE by GUI - -Customize font faces, sizes, and colorization attributes. Set indentation -defaults. Customize keybindings. Colorization and keybindings can be -saved as user defined sets. Select startup options including shell/editor -and default window size. Define additional help sources. - -Note that tab width in IDLE is currently fixed at eight due to Tk issues. -Refer to comments in EditorWindow autoindent code for details. - -""" -from tkinter import * -import tkinter.messagebox as tkMessageBox -import tkinter.colorchooser as tkColorChooser -import tkinter.font as tkFont - -from idlelib.configHandler import idleConf -from idlelib.dynOptionMenuWidget import DynOptionMenu -from idlelib.keybindingDialog import GetKeysDialog -from idlelib.configSectionNameDialog import GetCfgSectionNameDialog -from idlelib.configHelpSourceEdit import GetHelpSourceDialog -from idlelib.tabbedpages import TabbedPageSet -from idlelib.textView import view_text -from idlelib import macosxSupport - -class ConfigDialog(Toplevel): - - def __init__(self, parent, title='', _htest=False, _utest=False): - """ - _htest - bool, change box location when running htest - _utest - bool, don't wait_window when running unittest - """ - Toplevel.__init__(self, parent) - self.parent = parent - if _htest: - parent.instance_dict = {} - self.wm_withdraw() - - self.configure(borderwidth=5) - self.title(title or 'IDLE Preferences') - self.geometry( - "+%d+%d" % (parent.winfo_rootx() + 20, - parent.winfo_rooty() + (30 if not _htest else 150))) - #Theme Elements. Each theme element key is its display name. - #The first value of the tuple is the sample area tag name. - #The second value is the display name list sort index. - self.themeElements={ - 'Normal Text': ('normal', '00'), - 'Python Keywords': ('keyword', '01'), - 'Python Definitions': ('definition', '02'), - 'Python Builtins': ('builtin', '03'), - 'Python Comments': ('comment', '04'), - 'Python Strings': ('string', '05'), - 'Selected Text': ('hilite', '06'), - 'Found Text': ('hit', '07'), - 'Cursor': ('cursor', '08'), - 'Editor Breakpoint': ('break', '09'), - 'Shell Normal Text': ('console', '10'), - 'Shell Error Text': ('error', '11'), - 'Shell Stdout Text': ('stdout', '12'), - 'Shell Stderr Text': ('stderr', '13'), - } - self.ResetChangedItems() #load initial values in changed items dict - self.CreateWidgets() - self.resizable(height=FALSE, width=FALSE) - self.transient(parent) - self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Cancel) - self.tabPages.focus_set() - #key bindings for this dialog - #self.bind('', self.Cancel) #dismiss dialog, no save - #self.bind('', self.Apply) #apply changes, save - #self.bind('', self.Help) #context help - self.LoadConfigs() - self.AttachVarCallbacks() #avoid callbacks during LoadConfigs - - if not _utest: - self.wm_deiconify() - self.wait_window() - - def CreateWidgets(self): - self.tabPages = TabbedPageSet(self, - page_names=['Fonts/Tabs', 'Highlighting', 'Keys', 'General', - 'Extensions']) - self.tabPages.pack(side=TOP, expand=TRUE, fill=BOTH) - self.CreatePageFontTab() - self.CreatePageHighlight() - self.CreatePageKeys() - self.CreatePageGeneral() - self.CreatePageExtensions() - self.create_action_buttons().pack(side=BOTTOM) - - def create_action_buttons(self): - if macosxSupport.isAquaTk(): - # Changing the default padding on OSX results in unreadable - # text in the buttons - paddingArgs = {} - else: - paddingArgs = {'padx':6, 'pady':3} - outer = Frame(self, pady=2) - buttons = Frame(outer, pady=2) - for txt, cmd in ( - ('Ok', self.Ok), - ('Apply', self.Apply), - ('Cancel', self.Cancel), - ('Help', self.Help)): - Button(buttons, text=txt, command=cmd, takefocus=FALSE, - **paddingArgs).pack(side=LEFT, padx=5) - # add space above buttons - Frame(outer, height=2, borderwidth=0).pack(side=TOP) - buttons.pack(side=BOTTOM) - return outer - - def CreatePageFontTab(self): - parent = self.parent - self.fontSize = StringVar(parent) - self.fontBold = BooleanVar(parent) - self.fontName = StringVar(parent) - self.spaceNum = IntVar(parent) - self.editFont = tkFont.Font(parent, ('courier', 10, 'normal')) - - ##widget creation - #body frame - frame = self.tabPages.pages['Fonts/Tabs'].frame - #body section frames - frameFont = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') - frameIndent = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') - #frameFont - frameFontName = Frame(frameFont) - frameFontParam = Frame(frameFont) - labelFontNameTitle = Label( - frameFontName, justify=LEFT, text='Font Face :') - self.listFontName = Listbox( - frameFontName, height=5, takefocus=FALSE, exportselection=FALSE) - self.listFontName.bind( - '', self.OnListFontButtonRelease) - scrollFont = Scrollbar(frameFontName) - scrollFont.config(command=self.listFontName.yview) - self.listFontName.config(yscrollcommand=scrollFont.set) - labelFontSizeTitle = Label(frameFontParam, text='Size :') - self.optMenuFontSize = DynOptionMenu( - frameFontParam, self.fontSize, None, command=self.SetFontSample) - checkFontBold = Checkbutton( - frameFontParam, variable=self.fontBold, onvalue=1, - offvalue=0, text='Bold', command=self.SetFontSample) - frameFontSample = Frame(frameFont, relief=SOLID, borderwidth=1) - self.labelFontSample = Label( - frameFontSample, justify=LEFT, font=self.editFont, - text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]') - #frameIndent - frameIndentSize = Frame(frameIndent) - labelSpaceNumTitle = Label( - frameIndentSize, justify=LEFT, - text='Python Standard: 4 Spaces!') - self.scaleSpaceNum = Scale( - frameIndentSize, variable=self.spaceNum, - orient='horizontal', tickinterval=2, from_=2, to=16) - - #widget packing - #body - frameFont.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameIndent.pack(side=LEFT, padx=5, pady=5, fill=Y) - #frameFont - frameFontName.pack(side=TOP, padx=5, pady=5, fill=X) - frameFontParam.pack(side=TOP, padx=5, pady=5, fill=X) - labelFontNameTitle.pack(side=TOP, anchor=W) - self.listFontName.pack(side=LEFT, expand=TRUE, fill=X) - scrollFont.pack(side=LEFT, fill=Y) - labelFontSizeTitle.pack(side=LEFT, anchor=W) - self.optMenuFontSize.pack(side=LEFT, anchor=W) - checkFontBold.pack(side=LEFT, anchor=W, padx=20) - frameFontSample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - self.labelFontSample.pack(expand=TRUE, fill=BOTH) - #frameIndent - frameIndentSize.pack(side=TOP, fill=X) - labelSpaceNumTitle.pack(side=TOP, anchor=W, padx=5) - self.scaleSpaceNum.pack(side=TOP, padx=5, fill=X) - return frame - - def CreatePageHighlight(self): - parent = self.parent - self.builtinTheme = StringVar(parent) - self.customTheme = StringVar(parent) - self.fgHilite = BooleanVar(parent) - self.colour = StringVar(parent) - self.fontName = StringVar(parent) - self.themeIsBuiltin = BooleanVar(parent) - self.highlightTarget = StringVar(parent) - - ##widget creation - #body frame - frame = self.tabPages.pages['Highlighting'].frame - #body section frames - frameCustom = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Custom Highlighting ') - frameTheme = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Highlighting Theme ') - #frameCustom - self.textHighlightSample=Text( - frameCustom, relief=SOLID, borderwidth=1, - font=('courier', 12, ''), cursor='hand2', width=21, height=11, - takefocus=FALSE, highlightthickness=0, wrap=NONE) - text=self.textHighlightSample - text.bind('', lambda e: 'break') - text.bind('', lambda e: 'break') - textAndTags=( - ('#you can click here', 'comment'), ('\n', 'normal'), - ('#to choose items', 'comment'), ('\n', 'normal'), - ('def', 'keyword'), (' ', 'normal'), - ('func', 'definition'), ('(param):\n ', 'normal'), - ('"""string"""', 'string'), ('\n var0 = ', 'normal'), - ("'string'", 'string'), ('\n var1 = ', 'normal'), - ("'selected'", 'hilite'), ('\n var2 = ', 'normal'), - ("'found'", 'hit'), ('\n var3 = ', 'normal'), - ('list', 'builtin'), ('(', 'normal'), - ('None', 'keyword'), (')\n', 'normal'), - (' breakpoint("line")', 'break'), ('\n\n', 'normal'), - (' error ', 'error'), (' ', 'normal'), - ('cursor |', 'cursor'), ('\n ', 'normal'), - ('shell', 'console'), (' ', 'normal'), - ('stdout', 'stdout'), (' ', 'normal'), - ('stderr', 'stderr'), ('\n', 'normal')) - for txTa in textAndTags: - text.insert(END, txTa[0], txTa[1]) - for element in self.themeElements: - def tem(event, elem=element): - event.widget.winfo_toplevel().highlightTarget.set(elem) - text.tag_bind( - self.themeElements[element][0], '', tem) - text.config(state=DISABLED) - self.frameColourSet = Frame(frameCustom, relief=SOLID, borderwidth=1) - frameFgBg = Frame(frameCustom) - buttonSetColour = Button( - self.frameColourSet, text='Choose Colour for :', - command=self.GetColour, highlightthickness=0) - self.optMenuHighlightTarget = DynOptionMenu( - self.frameColourSet, self.highlightTarget, None, - highlightthickness=0) #, command=self.SetHighlightTargetBinding - self.radioFg = Radiobutton( - frameFgBg, variable=self.fgHilite, value=1, - text='Foreground', command=self.SetColourSampleBinding) - self.radioBg=Radiobutton( - frameFgBg, variable=self.fgHilite, value=0, - text='Background', command=self.SetColourSampleBinding) - self.fgHilite.set(1) - buttonSaveCustomTheme = Button( - frameCustom, text='Save as New Custom Theme', - command=self.SaveAsNewTheme) - #frameTheme - labelTypeTitle = Label(frameTheme, text='Select : ') - self.radioThemeBuiltin = Radiobutton( - frameTheme, variable=self.themeIsBuiltin, value=1, - command=self.SetThemeType, text='a Built-in Theme') - self.radioThemeCustom = Radiobutton( - frameTheme, variable=self.themeIsBuiltin, value=0, - command=self.SetThemeType, text='a Custom Theme') - self.optMenuThemeBuiltin = DynOptionMenu( - frameTheme, self.builtinTheme, None, command=None) - self.optMenuThemeCustom=DynOptionMenu( - frameTheme, self.customTheme, None, command=None) - self.buttonDeleteCustomTheme=Button( - frameTheme, text='Delete Custom Theme', - command=self.DeleteCustomTheme) - self.new_custom_theme = Label(frameTheme, bd=2) - - ##widget packing - #body - frameCustom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameTheme.pack(side=LEFT, padx=5, pady=5, fill=Y) - #frameCustom - self.frameColourSet.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) - frameFgBg.pack(side=TOP, padx=5, pady=0) - self.textHighlightSample.pack( - side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - buttonSetColour.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) - self.optMenuHighlightTarget.pack( - side=TOP, expand=TRUE, fill=X, padx=8, pady=3) - self.radioFg.pack(side=LEFT, anchor=E) - self.radioBg.pack(side=RIGHT, anchor=W) - buttonSaveCustomTheme.pack(side=BOTTOM, fill=X, padx=5, pady=5) - #frameTheme - labelTypeTitle.pack(side=TOP, anchor=W, padx=5, pady=5) - self.radioThemeBuiltin.pack(side=TOP, anchor=W, padx=5) - self.radioThemeCustom.pack(side=TOP, anchor=W, padx=5, pady=2) - self.optMenuThemeBuiltin.pack(side=TOP, fill=X, padx=5, pady=5) - self.optMenuThemeCustom.pack(side=TOP, fill=X, anchor=W, padx=5, pady=5) - self.buttonDeleteCustomTheme.pack(side=TOP, fill=X, padx=5, pady=5) - self.new_custom_theme.pack(side=TOP, fill=X, pady=5) - return frame - - def CreatePageKeys(self): - parent = self.parent - self.bindingTarget = StringVar(parent) - self.builtinKeys = StringVar(parent) - self.customKeys = StringVar(parent) - self.keysAreBuiltin = BooleanVar(parent) - self.keyBinding = StringVar(parent) - - ##widget creation - #body frame - frame = self.tabPages.pages['Keys'].frame - #body section frames - frameCustom = LabelFrame( - frame, borderwidth=2, relief=GROOVE, - text=' Custom Key Bindings ') - frameKeySets = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Key Set ') - #frameCustom - frameTarget = Frame(frameCustom) - labelTargetTitle = Label(frameTarget, text='Action - Key(s)') - scrollTargetY = Scrollbar(frameTarget) - scrollTargetX = Scrollbar(frameTarget, orient=HORIZONTAL) - self.listBindings = Listbox( - frameTarget, takefocus=FALSE, exportselection=FALSE) - self.listBindings.bind('', self.KeyBindingSelected) - scrollTargetY.config(command=self.listBindings.yview) - scrollTargetX.config(command=self.listBindings.xview) - self.listBindings.config(yscrollcommand=scrollTargetY.set) - self.listBindings.config(xscrollcommand=scrollTargetX.set) - self.buttonNewKeys = Button( - frameCustom, text='Get New Keys for Selection', - command=self.GetNewKeys, state=DISABLED) - #frameKeySets - frames = [Frame(frameKeySets, padx=2, pady=2, borderwidth=0) - for i in range(2)] - self.radioKeysBuiltin = Radiobutton( - frames[0], variable=self.keysAreBuiltin, value=1, - command=self.SetKeysType, text='Use a Built-in Key Set') - self.radioKeysCustom = Radiobutton( - frames[0], variable=self.keysAreBuiltin, value=0, - command=self.SetKeysType, text='Use a Custom Key Set') - self.optMenuKeysBuiltin = DynOptionMenu( - frames[0], self.builtinKeys, None, command=None) - self.optMenuKeysCustom = DynOptionMenu( - frames[0], self.customKeys, None, command=None) - self.buttonDeleteCustomKeys = Button( - frames[1], text='Delete Custom Key Set', - command=self.DeleteCustomKeys) - buttonSaveCustomKeys = Button( - frames[1], text='Save as New Custom Key Set', - command=self.SaveAsNewKeySet) - - ##widget packing - #body - frameCustom.pack(side=BOTTOM, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameKeySets.pack(side=BOTTOM, padx=5, pady=5, fill=BOTH) - #frameCustom - self.buttonNewKeys.pack(side=BOTTOM, fill=X, padx=5, pady=5) - frameTarget.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - #frame target - frameTarget.columnconfigure(0, weight=1) - frameTarget.rowconfigure(1, weight=1) - labelTargetTitle.grid(row=0, column=0, columnspan=2, sticky=W) - self.listBindings.grid(row=1, column=0, sticky=NSEW) - scrollTargetY.grid(row=1, column=1, sticky=NS) - scrollTargetX.grid(row=2, column=0, sticky=EW) - #frameKeySets - self.radioKeysBuiltin.grid(row=0, column=0, sticky=W+NS) - self.radioKeysCustom.grid(row=1, column=0, sticky=W+NS) - self.optMenuKeysBuiltin.grid(row=0, column=1, sticky=NSEW) - self.optMenuKeysCustom.grid(row=1, column=1, sticky=NSEW) - self.buttonDeleteCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) - buttonSaveCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) - frames[0].pack(side=TOP, fill=BOTH, expand=True) - frames[1].pack(side=TOP, fill=X, expand=True, pady=2) - return frame - - def CreatePageGeneral(self): - parent = self.parent - self.winWidth = StringVar(parent) - self.winHeight = StringVar(parent) - self.startupEdit = IntVar(parent) - self.autoSave = IntVar(parent) - self.encoding = StringVar(parent) - self.userHelpBrowser = BooleanVar(parent) - self.helpBrowser = StringVar(parent) - - #widget creation - #body - frame = self.tabPages.pages['General'].frame - #body section frames - frameRun = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Startup Preferences ') - frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Autosave Preferences ') - frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE) - frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Additional Help Sources ') - #frameRun - labelRunChoiceTitle = Label(frameRun, text='At Startup') - radioStartupEdit = Radiobutton( - frameRun, variable=self.startupEdit, value=1, - command=self.SetKeysType, text="Open Edit Window") - radioStartupShell = Radiobutton( - frameRun, variable=self.startupEdit, value=0, - command=self.SetKeysType, text='Open Shell Window') - #frameSave - labelRunSaveTitle = Label(frameSave, text='At Start of Run (F5) ') - radioSaveAsk = Radiobutton( - frameSave, variable=self.autoSave, value=0, - command=self.SetKeysType, text="Prompt to Save") - radioSaveAuto = Radiobutton( - frameSave, variable=self.autoSave, value=1, - command=self.SetKeysType, text='No Prompt') - #frameWinSize - labelWinSizeTitle = Label( - frameWinSize, text='Initial Window Size (in characters)') - labelWinWidthTitle = Label(frameWinSize, text='Width') - entryWinWidth = Entry( - frameWinSize, textvariable=self.winWidth, width=3) - labelWinHeightTitle = Label(frameWinSize, text='Height') - entryWinHeight = Entry( - frameWinSize, textvariable=self.winHeight, width=3) - #frameHelp - frameHelpList = Frame(frameHelp) - frameHelpListButtons = Frame(frameHelpList) - scrollHelpList = Scrollbar(frameHelpList) - self.listHelp = Listbox( - frameHelpList, height=5, takefocus=FALSE, - exportselection=FALSE) - scrollHelpList.config(command=self.listHelp.yview) - self.listHelp.config(yscrollcommand=scrollHelpList.set) - self.listHelp.bind('', self.HelpSourceSelected) - self.buttonHelpListEdit = Button( - frameHelpListButtons, text='Edit', state=DISABLED, - width=8, command=self.HelpListItemEdit) - self.buttonHelpListAdd = Button( - frameHelpListButtons, text='Add', - width=8, command=self.HelpListItemAdd) - self.buttonHelpListRemove = Button( - frameHelpListButtons, text='Remove', state=DISABLED, - width=8, command=self.HelpListItemRemove) - - #widget packing - #body - frameRun.pack(side=TOP, padx=5, pady=5, fill=X) - frameSave.pack(side=TOP, padx=5, pady=5, fill=X) - frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X) - frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - #frameRun - labelRunChoiceTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - radioStartupShell.pack(side=RIGHT, anchor=W, padx=5, pady=5) - radioStartupEdit.pack(side=RIGHT, anchor=W, padx=5, pady=5) - #frameSave - labelRunSaveTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - radioSaveAuto.pack(side=RIGHT, anchor=W, padx=5, pady=5) - radioSaveAsk.pack(side=RIGHT, anchor=W, padx=5, pady=5) - #frameWinSize - labelWinSizeTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - entryWinHeight.pack(side=RIGHT, anchor=E, padx=10, pady=5) - labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5) - entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5) - labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5) - #frameHelp - frameHelpListButtons.pack(side=RIGHT, padx=5, pady=5, fill=Y) - frameHelpList.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - scrollHelpList.pack(side=RIGHT, anchor=W, fill=Y) - self.listHelp.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) - self.buttonHelpListEdit.pack(side=TOP, anchor=W, pady=5) - self.buttonHelpListAdd.pack(side=TOP, anchor=W) - self.buttonHelpListRemove.pack(side=TOP, anchor=W, pady=5) - return frame - - def AttachVarCallbacks(self): - self.fontSize.trace_variable('w', self.VarChanged_font) - self.fontName.trace_variable('w', self.VarChanged_font) - self.fontBold.trace_variable('w', self.VarChanged_font) - self.spaceNum.trace_variable('w', self.VarChanged_spaceNum) - self.colour.trace_variable('w', self.VarChanged_colour) - self.builtinTheme.trace_variable('w', self.VarChanged_builtinTheme) - self.customTheme.trace_variable('w', self.VarChanged_customTheme) - self.themeIsBuiltin.trace_variable('w', self.VarChanged_themeIsBuiltin) - self.highlightTarget.trace_variable('w', self.VarChanged_highlightTarget) - self.keyBinding.trace_variable('w', self.VarChanged_keyBinding) - self.builtinKeys.trace_variable('w', self.VarChanged_builtinKeys) - self.customKeys.trace_variable('w', self.VarChanged_customKeys) - self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin) - self.winWidth.trace_variable('w', self.VarChanged_winWidth) - self.winHeight.trace_variable('w', self.VarChanged_winHeight) - self.startupEdit.trace_variable('w', self.VarChanged_startupEdit) - self.autoSave.trace_variable('w', self.VarChanged_autoSave) - self.encoding.trace_variable('w', self.VarChanged_encoding) - - def remove_var_callbacks(self): - "Remove callbacks to prevent memory leaks." - for var in ( - self.fontSize, self.fontName, self.fontBold, - self.spaceNum, self.colour, self.builtinTheme, - self.customTheme, self.themeIsBuiltin, self.highlightTarget, - self.keyBinding, self.builtinKeys, self.customKeys, - self.keysAreBuiltin, self.winWidth, self.winHeight, - self.startupEdit, self.autoSave, self.encoding,): - var.trace_vdelete('w', var.trace_vinfo()[0][1]) - - def VarChanged_font(self, *params): - '''When one font attribute changes, save them all, as they are - not independent from each other. In particular, when we are - overriding the default font, we need to write out everything. - ''' - value = self.fontName.get() - self.AddChangedItem('main', 'EditorWindow', 'font', value) - value = self.fontSize.get() - self.AddChangedItem('main', 'EditorWindow', 'font-size', value) - value = self.fontBold.get() - self.AddChangedItem('main', 'EditorWindow', 'font-bold', value) - - def VarChanged_spaceNum(self, *params): - value = self.spaceNum.get() - self.AddChangedItem('main', 'Indent', 'num-spaces', value) - - def VarChanged_colour(self, *params): - self.OnNewColourSet() - - def VarChanged_builtinTheme(self, *params): - value = self.builtinTheme.get() - if value == 'IDLE Dark': - if idleConf.GetOption('main', 'Theme', 'name') != 'IDLE New': - self.AddChangedItem('main', 'Theme', 'name', 'IDLE Classic') - self.AddChangedItem('main', 'Theme', 'name2', value) - self.new_custom_theme.config(text='New theme, see Help', - fg='#500000') - else: - self.AddChangedItem('main', 'Theme', 'name', value) - self.AddChangedItem('main', 'Theme', 'name2', '') - self.new_custom_theme.config(text='', fg='black') - self.PaintThemeSample() - - def VarChanged_customTheme(self, *params): - value = self.customTheme.get() - if value != '- no custom themes -': - self.AddChangedItem('main', 'Theme', 'name', value) - self.PaintThemeSample() - - def VarChanged_themeIsBuiltin(self, *params): - value = self.themeIsBuiltin.get() - self.AddChangedItem('main', 'Theme', 'default', value) - if value: - self.VarChanged_builtinTheme() - else: - self.VarChanged_customTheme() - - def VarChanged_highlightTarget(self, *params): - self.SetHighlightTarget() - - def VarChanged_keyBinding(self, *params): - value = self.keyBinding.get() - keySet = self.customKeys.get() - event = self.listBindings.get(ANCHOR).split()[0] - if idleConf.IsCoreBinding(event): - #this is a core keybinding - self.AddChangedItem('keys', keySet, event, value) - else: #this is an extension key binding - extName = idleConf.GetExtnNameForEvent(event) - extKeybindSection = extName + '_cfgBindings' - self.AddChangedItem('extensions', extKeybindSection, event, value) - - def VarChanged_builtinKeys(self, *params): - value = self.builtinKeys.get() - self.AddChangedItem('main', 'Keys', 'name', value) - self.LoadKeysList(value) - - def VarChanged_customKeys(self, *params): - value = self.customKeys.get() - if value != '- no custom keys -': - self.AddChangedItem('main', 'Keys', 'name', value) - self.LoadKeysList(value) - - def VarChanged_keysAreBuiltin(self, *params): - value = self.keysAreBuiltin.get() - self.AddChangedItem('main', 'Keys', 'default', value) - if value: - self.VarChanged_builtinKeys() - else: - self.VarChanged_customKeys() - - def VarChanged_winWidth(self, *params): - value = self.winWidth.get() - self.AddChangedItem('main', 'EditorWindow', 'width', value) - - def VarChanged_winHeight(self, *params): - value = self.winHeight.get() - self.AddChangedItem('main', 'EditorWindow', 'height', value) - - def VarChanged_startupEdit(self, *params): - value = self.startupEdit.get() - self.AddChangedItem('main', 'General', 'editor-on-startup', value) - - def VarChanged_autoSave(self, *params): - value = self.autoSave.get() - self.AddChangedItem('main', 'General', 'autosave', value) - - def VarChanged_encoding(self, *params): - value = self.encoding.get() - self.AddChangedItem('main', 'EditorWindow', 'encoding', value) - - def ResetChangedItems(self): - #When any config item is changed in this dialog, an entry - #should be made in the relevant section (config type) of this - #dictionary. The key should be the config file section name and the - #value a dictionary, whose key:value pairs are item=value pairs for - #that config file section. - self.changedItems = {'main':{}, 'highlight':{}, 'keys':{}, - 'extensions':{}} - - def AddChangedItem(self, typ, section, item, value): - value = str(value) #make sure we use a string - if section not in self.changedItems[typ]: - self.changedItems[typ][section] = {} - self.changedItems[typ][section][item] = value - - def GetDefaultItems(self): - dItems={'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} - for configType in dItems: - sections = idleConf.GetSectionList('default', configType) - for section in sections: - dItems[configType][section] = {} - options = idleConf.defaultCfg[configType].GetOptionList(section) - for option in options: - dItems[configType][section][option] = ( - idleConf.defaultCfg[configType].Get(section, option)) - return dItems - - def SetThemeType(self): - if self.themeIsBuiltin.get(): - self.optMenuThemeBuiltin.config(state=NORMAL) - self.optMenuThemeCustom.config(state=DISABLED) - self.buttonDeleteCustomTheme.config(state=DISABLED) - else: - self.optMenuThemeBuiltin.config(state=DISABLED) - self.radioThemeCustom.config(state=NORMAL) - self.optMenuThemeCustom.config(state=NORMAL) - self.buttonDeleteCustomTheme.config(state=NORMAL) - - def SetKeysType(self): - if self.keysAreBuiltin.get(): - self.optMenuKeysBuiltin.config(state=NORMAL) - self.optMenuKeysCustom.config(state=DISABLED) - self.buttonDeleteCustomKeys.config(state=DISABLED) - else: - self.optMenuKeysBuiltin.config(state=DISABLED) - self.radioKeysCustom.config(state=NORMAL) - self.optMenuKeysCustom.config(state=NORMAL) - self.buttonDeleteCustomKeys.config(state=NORMAL) - - def GetNewKeys(self): - listIndex = self.listBindings.index(ANCHOR) - binding = self.listBindings.get(listIndex) - bindName = binding.split()[0] #first part, up to first space - if self.keysAreBuiltin.get(): - currentKeySetName = self.builtinKeys.get() - else: - currentKeySetName = self.customKeys.get() - currentBindings = idleConf.GetCurrentKeySet() - if currentKeySetName in self.changedItems['keys']: #unsaved changes - keySetChanges = self.changedItems['keys'][currentKeySetName] - for event in keySetChanges: - currentBindings[event] = keySetChanges[event].split() - currentKeySequences = list(currentBindings.values()) - newKeys = GetKeysDialog(self, 'Get New Keys', bindName, - currentKeySequences).result - if newKeys: #new keys were specified - if self.keysAreBuiltin.get(): #current key set is a built-in - message = ('Your changes will be saved as a new Custom Key Set.' - ' Enter a name for your new Custom Key Set below.') - newKeySet = self.GetNewKeysName(message) - if not newKeySet: #user cancelled custom key set creation - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) - return - else: #create new custom key set based on previously active key set - self.CreateNewKeySet(newKeySet) - self.listBindings.delete(listIndex) - self.listBindings.insert(listIndex, bindName+' - '+newKeys) - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) - self.keyBinding.set(newKeys) - else: - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) - - def GetNewKeysName(self, message): - usedNames = (idleConf.GetSectionList('user', 'keys') + - idleConf.GetSectionList('default', 'keys')) - newKeySet = GetCfgSectionNameDialog( - self, 'New Custom Key Set', message, usedNames).result - return newKeySet - - def SaveAsNewKeySet(self): - newKeysName = self.GetNewKeysName('New Key Set Name:') - if newKeysName: - self.CreateNewKeySet(newKeysName) - - def KeyBindingSelected(self, event): - self.buttonNewKeys.config(state=NORMAL) - - def CreateNewKeySet(self, newKeySetName): - #creates new custom key set based on the previously active key set, - #and makes the new key set active - if self.keysAreBuiltin.get(): - prevKeySetName = self.builtinKeys.get() - else: - prevKeySetName = self.customKeys.get() - prevKeys = idleConf.GetCoreKeys(prevKeySetName) - newKeys = {} - for event in prevKeys: #add key set to changed items - eventName = event[2:-2] #trim off the angle brackets - binding = ' '.join(prevKeys[event]) - newKeys[eventName] = binding - #handle any unsaved changes to prev key set - if prevKeySetName in self.changedItems['keys']: - keySetChanges = self.changedItems['keys'][prevKeySetName] - for event in keySetChanges: - newKeys[event] = keySetChanges[event] - #save the new theme - self.SaveNewKeySet(newKeySetName, newKeys) - #change gui over to the new key set - customKeyList = idleConf.GetSectionList('user', 'keys') - customKeyList.sort() - self.optMenuKeysCustom.SetMenu(customKeyList, newKeySetName) - self.keysAreBuiltin.set(0) - self.SetKeysType() - - def LoadKeysList(self, keySetName): - reselect = 0 - newKeySet = 0 - if self.listBindings.curselection(): - reselect = 1 - listIndex = self.listBindings.index(ANCHOR) - keySet = idleConf.GetKeySet(keySetName) - bindNames = list(keySet.keys()) - bindNames.sort() - self.listBindings.delete(0, END) - for bindName in bindNames: - key = ' '.join(keySet[bindName]) #make key(s) into a string - bindName = bindName[2:-2] #trim off the angle brackets - if keySetName in self.changedItems['keys']: - #handle any unsaved changes to this key set - if bindName in self.changedItems['keys'][keySetName]: - key = self.changedItems['keys'][keySetName][bindName] - self.listBindings.insert(END, bindName+' - '+key) - if reselect: - self.listBindings.see(listIndex) - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) - - def DeleteCustomKeys(self): - keySetName=self.customKeys.get() - delmsg = 'Are you sure you wish to delete the key set %r ?' - if not tkMessageBox.askyesno( - 'Delete Key Set', delmsg % keySetName, parent=self): - return - #remove key set from config - idleConf.userCfg['keys'].remove_section(keySetName) - if keySetName in self.changedItems['keys']: - del(self.changedItems['keys'][keySetName]) - #write changes - idleConf.userCfg['keys'].Save() - #reload user key set list - itemList = idleConf.GetSectionList('user', 'keys') - itemList.sort() - if not itemList: - self.radioKeysCustom.config(state=DISABLED) - self.optMenuKeysCustom.SetMenu(itemList, '- no custom keys -') - else: - self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) - #revert to default key set - self.keysAreBuiltin.set(idleConf.defaultCfg['main'].Get('Keys', 'default')) - self.builtinKeys.set(idleConf.defaultCfg['main'].Get('Keys', 'name')) - #user can't back out of these changes, they must be applied now - self.Apply() - self.SetKeysType() - - def DeleteCustomTheme(self): - themeName = self.customTheme.get() - delmsg = 'Are you sure you wish to delete the theme %r ?' - if not tkMessageBox.askyesno( - 'Delete Theme', delmsg % themeName, parent=self): - return - #remove theme from config - idleConf.userCfg['highlight'].remove_section(themeName) - if themeName in self.changedItems['highlight']: - del(self.changedItems['highlight'][themeName]) - #write changes - idleConf.userCfg['highlight'].Save() - #reload user theme list - itemList = idleConf.GetSectionList('user', 'highlight') - itemList.sort() - if not itemList: - self.radioThemeCustom.config(state=DISABLED) - self.optMenuThemeCustom.SetMenu(itemList, '- no custom themes -') - else: - self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) - #revert to default theme - self.themeIsBuiltin.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) - self.builtinTheme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) - #user can't back out of these changes, they must be applied now - self.Apply() - self.SetThemeType() - - def GetColour(self): - target = self.highlightTarget.get() - prevColour = self.frameColourSet.cget('bg') - rgbTuplet, colourString = tkColorChooser.askcolor( - parent=self, title='Pick new colour for : '+target, - initialcolor=prevColour) - if colourString and (colourString != prevColour): - #user didn't cancel, and they chose a new colour - if self.themeIsBuiltin.get(): #current theme is a built-in - message = ('Your changes will be saved as a new Custom Theme. ' - 'Enter a name for your new Custom Theme below.') - newTheme = self.GetNewThemeName(message) - if not newTheme: #user cancelled custom theme creation - return - else: #create new custom theme based on previously active theme - self.CreateNewTheme(newTheme) - self.colour.set(colourString) - else: #current theme is user defined - self.colour.set(colourString) - - def OnNewColourSet(self): - newColour=self.colour.get() - self.frameColourSet.config(bg=newColour) #set sample - plane ='foreground' if self.fgHilite.get() else 'background' - sampleElement = self.themeElements[self.highlightTarget.get()][0] - self.textHighlightSample.tag_config(sampleElement, **{plane:newColour}) - theme = self.customTheme.get() - themeElement = sampleElement + '-' + plane - self.AddChangedItem('highlight', theme, themeElement, newColour) - - def GetNewThemeName(self, message): - usedNames = (idleConf.GetSectionList('user', 'highlight') + - idleConf.GetSectionList('default', 'highlight')) - newTheme = GetCfgSectionNameDialog( - self, 'New Custom Theme', message, usedNames).result - return newTheme - - def SaveAsNewTheme(self): - newThemeName = self.GetNewThemeName('New Theme Name:') - if newThemeName: - self.CreateNewTheme(newThemeName) - - def CreateNewTheme(self, newThemeName): - #creates new custom theme based on the previously active theme, - #and makes the new theme active - if self.themeIsBuiltin.get(): - themeType = 'default' - themeName = self.builtinTheme.get() - else: - themeType = 'user' - themeName = self.customTheme.get() - newTheme = idleConf.GetThemeDict(themeType, themeName) - #apply any of the old theme's unsaved changes to the new theme - if themeName in self.changedItems['highlight']: - themeChanges = self.changedItems['highlight'][themeName] - for element in themeChanges: - newTheme[element] = themeChanges[element] - #save the new theme - self.SaveNewTheme(newThemeName, newTheme) - #change gui over to the new theme - customThemeList = idleConf.GetSectionList('user', 'highlight') - customThemeList.sort() - self.optMenuThemeCustom.SetMenu(customThemeList, newThemeName) - self.themeIsBuiltin.set(0) - self.SetThemeType() - - def OnListFontButtonRelease(self, event): - font = self.listFontName.get(ANCHOR) - self.fontName.set(font.lower()) - self.SetFontSample() - - def SetFontSample(self, event=None): - fontName = self.fontName.get() - fontWeight = tkFont.BOLD if self.fontBold.get() else tkFont.NORMAL - newFont = (fontName, self.fontSize.get(), fontWeight) - self.labelFontSample.config(font=newFont) - self.textHighlightSample.configure(font=newFont) - - def SetHighlightTarget(self): - if self.highlightTarget.get() == 'Cursor': #bg not possible - self.radioFg.config(state=DISABLED) - self.radioBg.config(state=DISABLED) - self.fgHilite.set(1) - else: #both fg and bg can be set - self.radioFg.config(state=NORMAL) - self.radioBg.config(state=NORMAL) - self.fgHilite.set(1) - self.SetColourSample() - - def SetColourSampleBinding(self, *args): - self.SetColourSample() - - def SetColourSample(self): - #set the colour smaple area - tag = self.themeElements[self.highlightTarget.get()][0] - plane = 'foreground' if self.fgHilite.get() else 'background' - colour = self.textHighlightSample.tag_cget(tag, plane) - self.frameColourSet.config(bg=colour) - - def PaintThemeSample(self): - if self.themeIsBuiltin.get(): #a default theme - theme = self.builtinTheme.get() - else: #a user theme - theme = self.customTheme.get() - for elementTitle in self.themeElements: - element = self.themeElements[elementTitle][0] - colours = idleConf.GetHighlight(theme, element) - if element == 'cursor': #cursor sample needs special painting - colours['background'] = idleConf.GetHighlight( - theme, 'normal', fgBg='bg') - #handle any unsaved changes to this theme - if theme in self.changedItems['highlight']: - themeDict = self.changedItems['highlight'][theme] - if element + '-foreground' in themeDict: - colours['foreground'] = themeDict[element + '-foreground'] - if element + '-background' in themeDict: - colours['background'] = themeDict[element + '-background'] - self.textHighlightSample.tag_config(element, **colours) - self.SetColourSample() - - def HelpSourceSelected(self, event): - self.SetHelpListButtonStates() - - def SetHelpListButtonStates(self): - if self.listHelp.size() < 1: #no entries in list - self.buttonHelpListEdit.config(state=DISABLED) - self.buttonHelpListRemove.config(state=DISABLED) - else: #there are some entries - if self.listHelp.curselection(): #there currently is a selection - self.buttonHelpListEdit.config(state=NORMAL) - self.buttonHelpListRemove.config(state=NORMAL) - else: #there currently is not a selection - self.buttonHelpListEdit.config(state=DISABLED) - self.buttonHelpListRemove.config(state=DISABLED) - - def HelpListItemAdd(self): - helpSource = GetHelpSourceDialog(self, 'New Help Source').result - if helpSource: - self.userHelpList.append((helpSource[0], helpSource[1])) - self.listHelp.insert(END, helpSource[0]) - self.UpdateUserHelpChangedItems() - self.SetHelpListButtonStates() - - def HelpListItemEdit(self): - itemIndex = self.listHelp.index(ANCHOR) - helpSource = self.userHelpList[itemIndex] - newHelpSource = GetHelpSourceDialog( - self, 'Edit Help Source', menuItem=helpSource[0], - filePath=helpSource[1]).result - if (not newHelpSource) or (newHelpSource == helpSource): - return #no changes - self.userHelpList[itemIndex] = newHelpSource - self.listHelp.delete(itemIndex) - self.listHelp.insert(itemIndex, newHelpSource[0]) - self.UpdateUserHelpChangedItems() - self.SetHelpListButtonStates() - - def HelpListItemRemove(self): - itemIndex = self.listHelp.index(ANCHOR) - del(self.userHelpList[itemIndex]) - self.listHelp.delete(itemIndex) - self.UpdateUserHelpChangedItems() - self.SetHelpListButtonStates() - - def UpdateUserHelpChangedItems(self): - "Clear and rebuild the HelpFiles section in self.changedItems" - self.changedItems['main']['HelpFiles'] = {} - for num in range(1, len(self.userHelpList) + 1): - self.AddChangedItem( - 'main', 'HelpFiles', str(num), - ';'.join(self.userHelpList[num-1][:2])) - - def LoadFontCfg(self): - ##base editor font selection list - fonts = list(tkFont.families(self)) - fonts.sort() - for font in fonts: - self.listFontName.insert(END, font) - configuredFont = idleConf.GetFont(self, 'main', 'EditorWindow') - fontName = configuredFont[0].lower() - fontSize = configuredFont[1] - fontBold = configuredFont[2]=='bold' - self.fontName.set(fontName) - lc_fonts = [s.lower() for s in fonts] - try: - currentFontIndex = lc_fonts.index(fontName) - self.listFontName.see(currentFontIndex) - self.listFontName.select_set(currentFontIndex) - self.listFontName.select_anchor(currentFontIndex) - except ValueError: - pass - ##font size dropdown - self.optMenuFontSize.SetMenu(('7', '8', '9', '10', '11', '12', '13', - '14', '16', '18', '20', '22'), fontSize ) - ##fontWeight - self.fontBold.set(fontBold) - ##font sample - self.SetFontSample() - - def LoadTabCfg(self): - ##indent sizes - spaceNum = idleConf.GetOption( - 'main', 'Indent', 'num-spaces', default=4, type='int') - self.spaceNum.set(spaceNum) - - def LoadThemeCfg(self): - ##current theme type radiobutton - self.themeIsBuiltin.set(idleConf.GetOption( - 'main', 'Theme', 'default', type='bool', default=1)) - ##currently set theme - currentOption = idleConf.CurrentTheme() - ##load available theme option menus - if self.themeIsBuiltin.get(): #default theme selected - itemList = idleConf.GetSectionList('default', 'highlight') - itemList.sort() - self.optMenuThemeBuiltin.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('user', 'highlight') - itemList.sort() - if not itemList: - self.radioThemeCustom.config(state=DISABLED) - self.customTheme.set('- no custom themes -') - else: - self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) - else: #user theme selected - itemList = idleConf.GetSectionList('user', 'highlight') - itemList.sort() - self.optMenuThemeCustom.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('default', 'highlight') - itemList.sort() - self.optMenuThemeBuiltin.SetMenu(itemList, itemList[0]) - self.SetThemeType() - ##load theme element option menu - themeNames = list(self.themeElements.keys()) - themeNames.sort(key=lambda x: self.themeElements[x][1]) - self.optMenuHighlightTarget.SetMenu(themeNames, themeNames[0]) - self.PaintThemeSample() - self.SetHighlightTarget() - - def LoadKeyCfg(self): - ##current keys type radiobutton - self.keysAreBuiltin.set(idleConf.GetOption( - 'main', 'Keys', 'default', type='bool', default=1)) - ##currently set keys - currentOption = idleConf.CurrentKeys() - ##load available keyset option menus - if self.keysAreBuiltin.get(): #default theme selected - itemList = idleConf.GetSectionList('default', 'keys') - itemList.sort() - self.optMenuKeysBuiltin.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('user', 'keys') - itemList.sort() - if not itemList: - self.radioKeysCustom.config(state=DISABLED) - self.customKeys.set('- no custom keys -') - else: - self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) - else: #user key set selected - itemList = idleConf.GetSectionList('user', 'keys') - itemList.sort() - self.optMenuKeysCustom.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('default', 'keys') - itemList.sort() - self.optMenuKeysBuiltin.SetMenu(itemList, itemList[0]) - self.SetKeysType() - ##load keyset element list - keySetName = idleConf.CurrentKeys() - self.LoadKeysList(keySetName) - - def LoadGeneralCfg(self): - #startup state - self.startupEdit.set(idleConf.GetOption( - 'main', 'General', 'editor-on-startup', default=1, type='bool')) - #autosave state - self.autoSave.set(idleConf.GetOption( - 'main', 'General', 'autosave', default=0, type='bool')) - #initial window size - self.winWidth.set(idleConf.GetOption( - 'main', 'EditorWindow', 'width', type='int')) - self.winHeight.set(idleConf.GetOption( - 'main', 'EditorWindow', 'height', type='int')) - # default source encoding - self.encoding.set(idleConf.GetOption( - 'main', 'EditorWindow', 'encoding', default='none')) - # additional help sources - self.userHelpList = idleConf.GetAllExtraHelpSourcesList() - for helpItem in self.userHelpList: - self.listHelp.insert(END, helpItem[0]) - self.SetHelpListButtonStates() - - def LoadConfigs(self): - """ - load configuration from default and user config files and populate - the widgets on the config dialog pages. - """ - ### fonts / tabs page - self.LoadFontCfg() - self.LoadTabCfg() - ### highlighting page - self.LoadThemeCfg() - ### keys page - self.LoadKeyCfg() - ### general page - self.LoadGeneralCfg() - # note: extension page handled separately - - def SaveNewKeySet(self, keySetName, keySet): - """ - save a newly created core key set. - keySetName - string, the name of the new key set - keySet - dictionary containing the new key set - """ - if not idleConf.userCfg['keys'].has_section(keySetName): - idleConf.userCfg['keys'].add_section(keySetName) - for event in keySet: - value = keySet[event] - idleConf.userCfg['keys'].SetOption(keySetName, event, value) - - def SaveNewTheme(self, themeName, theme): - """ - save a newly created theme. - themeName - string, the name of the new theme - theme - dictionary containing the new theme - """ - if not idleConf.userCfg['highlight'].has_section(themeName): - idleConf.userCfg['highlight'].add_section(themeName) - for element in theme: - value = theme[element] - idleConf.userCfg['highlight'].SetOption(themeName, element, value) - - def SetUserValue(self, configType, section, item, value): - if idleConf.defaultCfg[configType].has_option(section, item): - if idleConf.defaultCfg[configType].Get(section, item) == value: - #the setting equals a default setting, remove it from user cfg - return idleConf.userCfg[configType].RemoveOption(section, item) - #if we got here set the option - return idleConf.userCfg[configType].SetOption(section, item, value) - - def SaveAllChangedConfigs(self): - "Save configuration changes to the user config file." - idleConf.userCfg['main'].Save() - for configType in self.changedItems: - cfgTypeHasChanges = False - for section in self.changedItems[configType]: - if section == 'HelpFiles': - #this section gets completely replaced - idleConf.userCfg['main'].remove_section('HelpFiles') - cfgTypeHasChanges = True - for item in self.changedItems[configType][section]: - value = self.changedItems[configType][section][item] - if self.SetUserValue(configType, section, item, value): - cfgTypeHasChanges = True - if cfgTypeHasChanges: - idleConf.userCfg[configType].Save() - for configType in ['keys', 'highlight']: - # save these even if unchanged! - idleConf.userCfg[configType].Save() - self.ResetChangedItems() #clear the changed items dict - self.save_all_changed_extensions() # uses a different mechanism - - def DeactivateCurrentConfig(self): - #Before a config is saved, some cleanup of current - #config must be done - remove the previous keybindings - winInstances = self.parent.instance_dict.keys() - for instance in winInstances: - instance.RemoveKeybindings() - - def ActivateConfigChanges(self): - "Dynamically apply configuration changes" - winInstances = self.parent.instance_dict.keys() - for instance in winInstances: - instance.ResetColorizer() - instance.ResetFont() - instance.set_notabs_indentwidth() - instance.ApplyKeybindings() - instance.reset_help_menu_entries() - - def Cancel(self): - self.destroy() - - def Ok(self): - self.Apply() - self.destroy() - - def Apply(self): - self.DeactivateCurrentConfig() - self.SaveAllChangedConfigs() - self.ActivateConfigChanges() - - def Help(self): - page = self.tabPages._current_page - view_text(self, title='Help for IDLE preferences', - text=help_common+help_pages.get(page, '')) - - def CreatePageExtensions(self): - """Part of the config dialog used for configuring IDLE extensions. - - This code is generic - it works for any and all IDLE extensions. - - IDLE extensions save their configuration options using idleConf. - This code reads the current configuration using idleConf, supplies a - GUI interface to change the configuration values, and saves the - changes using idleConf. - - Not all changes take effect immediately - some may require restarting IDLE. - This depends on each extension's implementation. - - All values are treated as text, and it is up to the user to supply - reasonable values. The only exception to this are the 'enable*' options, - which are boolean, and can be toggled with a True/False button. - """ - parent = self.parent - frame = self.tabPages.pages['Extensions'].frame - self.ext_defaultCfg = idleConf.defaultCfg['extensions'] - self.ext_userCfg = idleConf.userCfg['extensions'] - self.is_int = self.register(is_int) - self.load_extensions() - # create widgets - a listbox shows all available extensions, with the - # controls for the extension selected in the listbox to the right - self.extension_names = StringVar(self) - frame.rowconfigure(0, weight=1) - frame.columnconfigure(2, weight=1) - self.extension_list = Listbox(frame, listvariable=self.extension_names, - selectmode='browse') - self.extension_list.bind('<>', self.extension_selected) - scroll = Scrollbar(frame, command=self.extension_list.yview) - self.extension_list.yscrollcommand=scroll.set - self.details_frame = LabelFrame(frame, width=250, height=250) - self.extension_list.grid(column=0, row=0, sticky='nws') - scroll.grid(column=1, row=0, sticky='ns') - self.details_frame.grid(column=2, row=0, sticky='nsew', padx=[10, 0]) - frame.configure(padx=10, pady=10) - self.config_frame = {} - self.current_extension = None - - self.outerframe = self # TEMPORARY - self.tabbed_page_set = self.extension_list # TEMPORARY - - # create the frame holding controls for each extension - ext_names = '' - for ext_name in sorted(self.extensions): - self.create_extension_frame(ext_name) - ext_names = ext_names + '{' + ext_name + '} ' - self.extension_names.set(ext_names) - self.extension_list.selection_set(0) - self.extension_selected(None) - - def load_extensions(self): - "Fill self.extensions with data from the default and user configs." - self.extensions = {} - for ext_name in idleConf.GetExtensions(active_only=False): - self.extensions[ext_name] = [] - - for ext_name in self.extensions: - opt_list = sorted(self.ext_defaultCfg.GetOptionList(ext_name)) - - # bring 'enable' options to the beginning of the list - enables = [opt_name for opt_name in opt_list - if opt_name.startswith('enable')] - for opt_name in enables: - opt_list.remove(opt_name) - opt_list = enables + opt_list - - for opt_name in opt_list: - def_str = self.ext_defaultCfg.Get( - ext_name, opt_name, raw=True) - try: - def_obj = {'True':True, 'False':False}[def_str] - opt_type = 'bool' - except KeyError: - try: - def_obj = int(def_str) - opt_type = 'int' - except ValueError: - def_obj = def_str - opt_type = None - try: - value = self.ext_userCfg.Get( - ext_name, opt_name, type=opt_type, raw=True, - default=def_obj) - except ValueError: # Need this until .Get fixed - value = def_obj # bad values overwritten by entry - var = StringVar(self) - var.set(str(value)) - - self.extensions[ext_name].append({'name': opt_name, - 'type': opt_type, - 'default': def_str, - 'value': value, - 'var': var, - }) - - def extension_selected(self, event): - newsel = self.extension_list.curselection() - if newsel: - newsel = self.extension_list.get(newsel) - if newsel is None or newsel != self.current_extension: - if self.current_extension: - self.details_frame.config(text='') - self.config_frame[self.current_extension].grid_forget() - self.current_extension = None - if newsel: - self.details_frame.config(text=newsel) - self.config_frame[newsel].grid(column=0, row=0, sticky='nsew') - self.current_extension = newsel - - def create_extension_frame(self, ext_name): - """Create a frame holding the widgets to configure one extension""" - f = VerticalScrolledFrame(self.details_frame, height=250, width=250) - self.config_frame[ext_name] = f - entry_area = f.interior - # create an entry for each configuration option - for row, opt in enumerate(self.extensions[ext_name]): - # create a row with a label and entry/checkbutton - label = Label(entry_area, text=opt['name']) - label.grid(row=row, column=0, sticky=NW) - var = opt['var'] - if opt['type'] == 'bool': - Checkbutton(entry_area, textvariable=var, variable=var, - onvalue='True', offvalue='False', - indicatoron=FALSE, selectcolor='', width=8 - ).grid(row=row, column=1, sticky=W, padx=7) - elif opt['type'] == 'int': - Entry(entry_area, textvariable=var, validate='key', - validatecommand=(self.is_int, '%P') - ).grid(row=row, column=1, sticky=NSEW, padx=7) - - else: - Entry(entry_area, textvariable=var - ).grid(row=row, column=1, sticky=NSEW, padx=7) - return - - def set_extension_value(self, section, opt): - name = opt['name'] - default = opt['default'] - value = opt['var'].get().strip() or default - opt['var'].set(value) - # if self.defaultCfg.has_section(section): - # Currently, always true; if not, indent to return - if (value == default): - return self.ext_userCfg.RemoveOption(section, name) - # set the option - return self.ext_userCfg.SetOption(section, name, value) - - def save_all_changed_extensions(self): - """Save configuration changes to the user config file.""" - has_changes = False - for ext_name in self.extensions: - options = self.extensions[ext_name] - for opt in options: - if self.set_extension_value(ext_name, opt): - has_changes = True - if has_changes: - self.ext_userCfg.Save() - - -help_common = '''\ -When you click either the Apply or Ok buttons, settings in this -dialog that are different from IDLE's default are saved in -a .idlerc directory in your home directory. Except as noted, -these changes apply to all versions of IDLE installed on this -machine. Some do not take affect until IDLE is restarted. -[Cancel] only cancels changes made since the last save. -''' -help_pages = { - 'Highlighting':''' -Highlighting: -The IDLE Dark color theme is new in October 2015. It can only -be used with older IDLE releases if it is saved as a custom -theme, with a different name. -''' -} - - -def is_int(s): - "Return 's is blank or represents an int'" - if not s: - return True - try: - int(s) - return True - except ValueError: - return False - - -class VerticalScrolledFrame(Frame): - """A pure Tkinter vertically scrollable frame. - - * Use the 'interior' attribute to place widgets inside the scrollable frame - * Construct and pack/place/grid normally - * This frame only allows vertical scrolling - """ - def __init__(self, parent, *args, **kw): - Frame.__init__(self, parent, *args, **kw) - - # create a canvas object and a vertical scrollbar for scrolling it - vscrollbar = Scrollbar(self, orient=VERTICAL) - vscrollbar.pack(fill=Y, side=RIGHT, expand=FALSE) - canvas = Canvas(self, bd=0, highlightthickness=0, - yscrollcommand=vscrollbar.set, width=240) - canvas.pack(side=LEFT, fill=BOTH, expand=TRUE) - vscrollbar.config(command=canvas.yview) - - # reset the view - canvas.xview_moveto(0) - canvas.yview_moveto(0) - - # create a frame inside the canvas which will be scrolled with it - self.interior = interior = Frame(canvas) - interior_id = canvas.create_window(0, 0, window=interior, anchor=NW) - - # track changes to the canvas and frame width and sync them, - # also updating the scrollbar - def _configure_interior(event): - # update the scrollbars to match the size of the inner frame - size = (interior.winfo_reqwidth(), interior.winfo_reqheight()) - canvas.config(scrollregion="0 0 %s %s" % size) - interior.bind('', _configure_interior) - - def _configure_canvas(event): - if interior.winfo_reqwidth() != canvas.winfo_width(): - # update the inner frame's width to fill the canvas - canvas.itemconfigure(interior_id, width=canvas.winfo_width()) - canvas.bind('', _configure_canvas) - - return - - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_configdialog', - verbosity=2, exit=False) - from idlelib.idle_test.htest import run - run(ConfigDialog) diff --git a/Lib/idlelib/configHandler.py b/Lib/idlelib/configHandler.py deleted file mode 100644 index 8ac1f60..0000000 --- a/Lib/idlelib/configHandler.py +++ /dev/null @@ -1,760 +0,0 @@ -"""Provides access to stored IDLE configuration information. - -Refer to the comments at the beginning of config-main.def for a description of -the available configuration files and the design implemented to update user -configuration information. In particular, user configuration choices which -duplicate the defaults will be removed from the user's configuration files, -and if a file becomes empty, it will be deleted. - -The contents of the user files may be altered using the Options/Configure IDLE -menu to access the configuration GUI (configDialog.py), or manually. - -Throughout this module there is an emphasis on returning useable defaults -when a problem occurs in returning a requested configuration value back to -idle. This is to allow IDLE to continue to function in spite of errors in -the retrieval of config information. When a default is returned instead of -a requested config value, a message is printed to stderr to aid in -configuration problem notification and resolution. -""" -# TODOs added Oct 2014, tjr - -import os -import sys - -from configparser import ConfigParser -from tkinter import TkVersion -from tkinter.font import Font, nametofont - -class InvalidConfigType(Exception): pass -class InvalidConfigSet(Exception): pass -class InvalidFgBg(Exception): pass -class InvalidTheme(Exception): pass - -class IdleConfParser(ConfigParser): - """ - A ConfigParser specialised for idle configuration file handling - """ - def __init__(self, cfgFile, cfgDefaults=None): - """ - cfgFile - string, fully specified configuration file name - """ - self.file = cfgFile - ConfigParser.__init__(self, defaults=cfgDefaults, strict=False) - - def Get(self, section, option, type=None, default=None, raw=False): - """ - Get an option value for given section/option or return default. - If type is specified, return as type. - """ - # TODO Use default as fallback, at least if not None - # Should also print Warning(file, section, option). - # Currently may raise ValueError - if not self.has_option(section, option): - return default - if type == 'bool': - return self.getboolean(section, option) - elif type == 'int': - return self.getint(section, option) - else: - return self.get(section, option, raw=raw) - - def GetOptionList(self, section): - "Return a list of options for given section, else []." - if self.has_section(section): - return self.options(section) - else: #return a default value - return [] - - def Load(self): - "Load the configuration file from disk." - self.read(self.file) - -class IdleUserConfParser(IdleConfParser): - """ - IdleConfigParser specialised for user configuration handling. - """ - - def AddSection(self, section): - "If section doesn't exist, add it." - if not self.has_section(section): - self.add_section(section) - - def RemoveEmptySections(self): - "Remove any sections that have no options." - for section in self.sections(): - if not self.GetOptionList(section): - self.remove_section(section) - - def IsEmpty(self): - "Return True if no sections after removing empty sections." - self.RemoveEmptySections() - return not self.sections() - - def RemoveOption(self, section, option): - """Return True if option is removed from section, else False. - - False if either section does not exist or did not have option. - """ - if self.has_section(section): - return self.remove_option(section, option) - return False - - def SetOption(self, section, option, value): - """Return True if option is added or changed to value, else False. - - Add section if required. False means option already had value. - """ - if self.has_option(section, option): - if self.get(section, option) == value: - return False - else: - self.set(section, option, value) - return True - else: - if not self.has_section(section): - self.add_section(section) - self.set(section, option, value) - return True - - def RemoveFile(self): - "Remove user config file self.file from disk if it exists." - if os.path.exists(self.file): - os.remove(self.file) - - def Save(self): - """Update user configuration file. - - Remove empty sections. If resulting config isn't empty, write the file - to disk. If config is empty, remove the file from disk if it exists. - - """ - if not self.IsEmpty(): - fname = self.file - try: - cfgFile = open(fname, 'w') - except OSError: - os.unlink(fname) - cfgFile = open(fname, 'w') - with cfgFile: - self.write(cfgFile) - else: - self.RemoveFile() - -class IdleConf: - """Hold config parsers for all idle config files in singleton instance. - - Default config files, self.defaultCfg -- - for config_type in self.config_types: - (idle install dir)/config-{config-type}.def - - User config files, self.userCfg -- - for config_type in self.config_types: - (user home dir)/.idlerc/config-{config-type}.cfg - """ - def __init__(self): - self.config_types = ('main', 'extensions', 'highlight', 'keys') - self.defaultCfg = {} - self.userCfg = {} - self.cfg = {} # TODO use to select userCfg vs defaultCfg - self.CreateConfigHandlers() - self.LoadCfgFiles() - - - def CreateConfigHandlers(self): - "Populate default and user config parser dictionaries." - #build idle install path - if __name__ != '__main__': # we were imported - idleDir=os.path.dirname(__file__) - else: # we were exec'ed (for testing only) - idleDir=os.path.abspath(sys.path[0]) - userDir=self.GetUserCfgDir() - - defCfgFiles = {} - usrCfgFiles = {} - # TODO eliminate these temporaries by combining loops - for cfgType in self.config_types: #build config file names - defCfgFiles[cfgType] = os.path.join( - idleDir, 'config-' + cfgType + '.def') - usrCfgFiles[cfgType] = os.path.join( - userDir, 'config-' + cfgType + '.cfg') - for cfgType in self.config_types: #create config parsers - self.defaultCfg[cfgType] = IdleConfParser(defCfgFiles[cfgType]) - self.userCfg[cfgType] = IdleUserConfParser(usrCfgFiles[cfgType]) - - def GetUserCfgDir(self): - """Return a filesystem directory for storing user config files. - - Creates it if required. - """ - cfgDir = '.idlerc' - userDir = os.path.expanduser('~') - if userDir != '~': # expanduser() found user home dir - if not os.path.exists(userDir): - warn = ('\n Warning: os.path.expanduser("~") points to\n ' + - userDir + ',\n but the path does not exist.') - try: - print(warn, file=sys.stderr) - except OSError: - pass - userDir = '~' - if userDir == "~": # still no path to home! - # traditionally IDLE has defaulted to os.getcwd(), is this adequate? - userDir = os.getcwd() - userDir = os.path.join(userDir, cfgDir) - if not os.path.exists(userDir): - try: - os.mkdir(userDir) - except OSError: - warn = ('\n Warning: unable to create user config directory\n' + - userDir + '\n Check path and permissions.\n Exiting!\n') - print(warn, file=sys.stderr) - raise SystemExit - # TODO continue without userDIr instead of exit - return userDir - - def GetOption(self, configType, section, option, default=None, type=None, - warn_on_default=True, raw=False): - """Return a value for configType section option, or default. - - If type is not None, return a value of that type. Also pass raw - to the config parser. First try to return a valid value - (including type) from a user configuration. If that fails, try - the default configuration. If that fails, return default, with a - default of None. - - Warn if either user or default configurations have an invalid value. - Warn if default is returned and warn_on_default is True. - """ - try: - if self.userCfg[configType].has_option(section, option): - return self.userCfg[configType].Get(section, option, - type=type, raw=raw) - except ValueError: - warning = ('\n Warning: configHandler.py - IdleConf.GetOption -\n' - ' invalid %r value for configuration option %r\n' - ' from section %r: %r' % - (type, option, section, - self.userCfg[configType].Get(section, option, raw=raw))) - try: - print(warning, file=sys.stderr) - except OSError: - pass - try: - if self.defaultCfg[configType].has_option(section,option): - return self.defaultCfg[configType].Get( - section, option, type=type, raw=raw) - except ValueError: - pass - #returning default, print warning - if warn_on_default: - warning = ('\n Warning: configHandler.py - IdleConf.GetOption -\n' - ' problem retrieving configuration option %r\n' - ' from section %r.\n' - ' returning default value: %r' % - (option, section, default)) - try: - print(warning, file=sys.stderr) - except OSError: - pass - return default - - def SetOption(self, configType, section, option, value): - """Set section option to value in user config file.""" - self.userCfg[configType].SetOption(section, option, value) - - def GetSectionList(self, configSet, configType): - """Return sections for configSet configType configuration. - - configSet must be either 'user' or 'default' - configType must be in self.config_types. - """ - if not (configType in self.config_types): - raise InvalidConfigType('Invalid configType specified') - if configSet == 'user': - cfgParser = self.userCfg[configType] - elif configSet == 'default': - cfgParser=self.defaultCfg[configType] - else: - raise InvalidConfigSet('Invalid configSet specified') - return cfgParser.sections() - - def GetHighlight(self, theme, element, fgBg=None): - """Return individual theme element highlight color(s). - - fgBg - string ('fg' or 'bg') or None. - If None, return a dictionary containing fg and bg colors with - keys 'foreground' and 'background'. Otherwise, only return - fg or bg color, as specified. Colors are intended to be - appropriate for passing to Tkinter in, e.g., a tag_config call). - """ - if self.defaultCfg['highlight'].has_section(theme): - themeDict = self.GetThemeDict('default', theme) - else: - themeDict = self.GetThemeDict('user', theme) - fore = themeDict[element + '-foreground'] - if element == 'cursor': # There is no config value for cursor bg - back = themeDict['normal-background'] - else: - back = themeDict[element + '-background'] - highlight = {"foreground": fore, "background": back} - if not fgBg: # Return dict of both colors - return highlight - else: # Return specified color only - if fgBg == 'fg': - return highlight["foreground"] - if fgBg == 'bg': - return highlight["background"] - else: - raise InvalidFgBg('Invalid fgBg specified') - - def GetThemeDict(self, type, themeName): - """Return {option:value} dict for elements in themeName. - - type - string, 'default' or 'user' theme type - themeName - string, theme name - Values are loaded over ultimate fallback defaults to guarantee - that all theme elements are present in a newly created theme. - """ - if type == 'user': - cfgParser = self.userCfg['highlight'] - elif type == 'default': - cfgParser = self.defaultCfg['highlight'] - else: - raise InvalidTheme('Invalid theme type specified') - # Provide foreground and background colors for each theme - # element (other than cursor) even though some values are not - # yet used by idle, to allow for their use in the future. - # Default values are generally black and white. - # TODO copy theme from a class attribute. - theme ={'normal-foreground':'#000000', - 'normal-background':'#ffffff', - 'keyword-foreground':'#000000', - 'keyword-background':'#ffffff', - 'builtin-foreground':'#000000', - 'builtin-background':'#ffffff', - 'comment-foreground':'#000000', - 'comment-background':'#ffffff', - 'string-foreground':'#000000', - 'string-background':'#ffffff', - 'definition-foreground':'#000000', - 'definition-background':'#ffffff', - 'hilite-foreground':'#000000', - 'hilite-background':'gray', - 'break-foreground':'#ffffff', - 'break-background':'#000000', - 'hit-foreground':'#ffffff', - 'hit-background':'#000000', - 'error-foreground':'#ffffff', - 'error-background':'#000000', - #cursor (only foreground can be set) - 'cursor-foreground':'#000000', - #shell window - 'stdout-foreground':'#000000', - 'stdout-background':'#ffffff', - 'stderr-foreground':'#000000', - 'stderr-background':'#ffffff', - 'console-foreground':'#000000', - 'console-background':'#ffffff' } - for element in theme: - if not cfgParser.has_option(themeName, element): - # Print warning that will return a default color - warning = ('\n Warning: configHandler.IdleConf.GetThemeDict' - ' -\n problem retrieving theme element %r' - '\n from theme %r.\n' - ' returning default color: %r' % - (element, themeName, theme[element])) - try: - print(warning, file=sys.stderr) - except OSError: - pass - theme[element] = cfgParser.Get( - themeName, element, default=theme[element]) - return theme - - def CurrentTheme(self): - """Return the name of the currently active text color theme. - - idlelib.config-main.def includes this section - [Theme] - default= 1 - name= IDLE Classic - name2= - # name2 set in user config-main.cfg for themes added after 2015 Oct 1 - - Item name2 is needed because setting name to a new builtin - causes older IDLEs to display multiple error messages or quit. - See https://bugs.python.org/issue25313. - When default = True, name2 takes precedence over name, - while older IDLEs will just use name. - """ - default = self.GetOption('main', 'Theme', 'default', - type='bool', default=True) - if default: - theme = self.GetOption('main', 'Theme', 'name2', default='') - if default and not theme or not default: - theme = self.GetOption('main', 'Theme', 'name', default='') - source = self.defaultCfg if default else self.userCfg - if source['highlight'].has_section(theme): - return theme - else: - return "IDLE Classic" - - def CurrentKeys(self): - "Return the name of the currently active key set." - return self.GetOption('main', 'Keys', 'name', default='') - - def GetExtensions(self, active_only=True, editor_only=False, shell_only=False): - """Return extensions in default and user config-extensions files. - - If active_only True, only return active (enabled) extensions - and optionally only editor or shell extensions. - If active_only False, return all extensions. - """ - extns = self.RemoveKeyBindNames( - self.GetSectionList('default', 'extensions')) - userExtns = self.RemoveKeyBindNames( - self.GetSectionList('user', 'extensions')) - for extn in userExtns: - if extn not in extns: #user has added own extension - extns.append(extn) - if active_only: - activeExtns = [] - for extn in extns: - if self.GetOption('extensions', extn, 'enable', default=True, - type='bool'): - #the extension is enabled - if editor_only or shell_only: # TODO if both, contradictory - if editor_only: - option = "enable_editor" - else: - option = "enable_shell" - if self.GetOption('extensions', extn,option, - default=True, type='bool', - warn_on_default=False): - activeExtns.append(extn) - else: - activeExtns.append(extn) - return activeExtns - else: - return extns - - def RemoveKeyBindNames(self, extnNameList): - "Return extnNameList with keybinding section names removed." - # TODO Easier to return filtered copy with list comp - names = extnNameList - kbNameIndicies = [] - for name in names: - if name.endswith(('_bindings', '_cfgBindings')): - kbNameIndicies.append(names.index(name)) - kbNameIndicies.sort(reverse=True) - for index in kbNameIndicies: #delete each keybinding section name - del(names[index]) - return names - - def GetExtnNameForEvent(self, virtualEvent): - """Return the name of the extension binding virtualEvent, or None. - - virtualEvent - string, name of the virtual event to test for, - without the enclosing '<< >>' - """ - extName = None - vEvent = '<<' + virtualEvent + '>>' - for extn in self.GetExtensions(active_only=0): - for event in self.GetExtensionKeys(extn): - if event == vEvent: - extName = extn # TODO return here? - return extName - - def GetExtensionKeys(self, extensionName): - """Return dict: {configurable extensionName event : active keybinding}. - - Events come from default config extension_cfgBindings section. - Keybindings come from GetCurrentKeySet() active key dict, - where previously used bindings are disabled. - """ - keysName = extensionName + '_cfgBindings' - activeKeys = self.GetCurrentKeySet() - extKeys = {} - if self.defaultCfg['extensions'].has_section(keysName): - eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) - for eventName in eventNames: - event = '<<' + eventName + '>>' - binding = activeKeys[event] - extKeys[event] = binding - return extKeys - - def __GetRawExtensionKeys(self,extensionName): - """Return dict {configurable extensionName event : keybinding list}. - - Events come from default config extension_cfgBindings section. - Keybindings list come from the splitting of GetOption, which - tries user config before default config. - """ - keysName = extensionName+'_cfgBindings' - extKeys = {} - if self.defaultCfg['extensions'].has_section(keysName): - eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) - for eventName in eventNames: - binding = self.GetOption( - 'extensions', keysName, eventName, default='').split() - event = '<<' + eventName + '>>' - extKeys[event] = binding - return extKeys - - def GetExtensionBindings(self, extensionName): - """Return dict {extensionName event : active or defined keybinding}. - - Augment self.GetExtensionKeys(extensionName) with mapping of non- - configurable events (from default config) to GetOption splits, - as in self.__GetRawExtensionKeys. - """ - bindsName = extensionName + '_bindings' - extBinds = self.GetExtensionKeys(extensionName) - #add the non-configurable bindings - if self.defaultCfg['extensions'].has_section(bindsName): - eventNames = self.defaultCfg['extensions'].GetOptionList(bindsName) - for eventName in eventNames: - binding = self.GetOption( - 'extensions', bindsName, eventName, default='').split() - event = '<<' + eventName + '>>' - extBinds[event] = binding - - return extBinds - - def GetKeyBinding(self, keySetName, eventStr): - """Return the keybinding list for keySetName eventStr. - - keySetName - name of key binding set (config-keys section). - eventStr - virtual event, including brackets, as in '<>'. - """ - eventName = eventStr[2:-2] #trim off the angle brackets - binding = self.GetOption('keys', keySetName, eventName, default='').split() - return binding - - def GetCurrentKeySet(self): - "Return CurrentKeys with 'darwin' modifications." - result = self.GetKeySet(self.CurrentKeys()) - - if sys.platform == "darwin": - # OS X Tk variants do not support the "Alt" keyboard modifier. - # So replace all keybingings that use "Alt" with ones that - # use the "Option" keyboard modifier. - # TODO (Ned?): the "Option" modifier does not work properly for - # Cocoa Tk and XQuartz Tk so we should not use it - # in default OS X KeySets. - for k, v in result.items(): - v2 = [ x.replace('>' - """ - return ('<<'+virtualEvent+'>>') in self.GetCoreKeys() - -# TODO make keyBindins a file or class attribute used for test above -# and copied in function below - - def GetCoreKeys(self, keySetName=None): - """Return dict of core virtual-key keybindings for keySetName. - - The default keySetName None corresponds to the keyBindings base - dict. If keySetName is not None, bindings from the config - file(s) are loaded _over_ these defaults, so if there is a - problem getting any core binding there will be an 'ultimate last - resort fallback' to the CUA-ish bindings defined here. - """ - keyBindings={ - '<>': ['', ''], - '<>': ['', ''], - '<>': ['', ''], - '<>': ['', ''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': ['', ''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': ['', ''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''], - '<>': [''] - } - if keySetName: - for event in keyBindings: - binding = self.GetKeyBinding(keySetName, event) - if binding: - keyBindings[event] = binding - else: #we are going to return a default, print warning - warning=('\n Warning: configHandler.py - IdleConf.GetCoreKeys' - ' -\n problem retrieving key binding for event %r' - '\n from key set %r.\n' - ' returning default value: %r' % - (event, keySetName, keyBindings[event])) - try: - print(warning, file=sys.stderr) - except OSError: - pass - return keyBindings - - def GetExtraHelpSourceList(self, configSet): - """Return list of extra help sources from a given configSet. - - Valid configSets are 'user' or 'default'. Return a list of tuples of - the form (menu_item , path_to_help_file , option), or return the empty - list. 'option' is the sequence number of the help resource. 'option' - values determine the position of the menu items on the Help menu, - therefore the returned list must be sorted by 'option'. - - """ - helpSources = [] - if configSet == 'user': - cfgParser = self.userCfg['main'] - elif configSet == 'default': - cfgParser = self.defaultCfg['main'] - else: - raise InvalidConfigSet('Invalid configSet specified') - options=cfgParser.GetOptionList('HelpFiles') - for option in options: - value=cfgParser.Get('HelpFiles', option, default=';') - if value.find(';') == -1: #malformed config entry with no ';' - menuItem = '' #make these empty - helpPath = '' #so value won't be added to list - else: #config entry contains ';' as expected - value=value.split(';') - menuItem=value[0].strip() - helpPath=value[1].strip() - if menuItem and helpPath: #neither are empty strings - helpSources.append( (menuItem,helpPath,option) ) - helpSources.sort(key=lambda x: x[2]) - return helpSources - - def GetAllExtraHelpSourcesList(self): - """Return a list of the details of all additional help sources. - - Tuples in the list are those of GetExtraHelpSourceList. - """ - allHelpSources = (self.GetExtraHelpSourceList('default') + - self.GetExtraHelpSourceList('user') ) - return allHelpSources - - def GetFont(self, root, configType, section): - """Retrieve a font from configuration (font, font-size, font-bold) - Intercept the special value 'TkFixedFont' and substitute - the actual font, factoring in some tweaks if needed for - appearance sakes. - - The 'root' parameter can normally be any valid Tkinter widget. - - Return a tuple (family, size, weight) suitable for passing - to tkinter.Font - """ - family = self.GetOption(configType, section, 'font', default='courier') - size = self.GetOption(configType, section, 'font-size', type='int', - default='10') - bold = self.GetOption(configType, section, 'font-bold', default=0, - type='bool') - if (family == 'TkFixedFont'): - if TkVersion < 8.5: - family = 'Courier' - else: - f = Font(name='TkFixedFont', exists=True, root=root) - actualFont = Font.actual(f) - family = actualFont['family'] - size = actualFont['size'] - if size <= 0: - size = 10 # if font in pixels, ignore actual size - bold = actualFont['weight']=='bold' - return (family, size, 'bold' if bold else 'normal') - - def LoadCfgFiles(self): - "Load all configuration files." - for key in self.defaultCfg: - self.defaultCfg[key].Load() - self.userCfg[key].Load() #same keys - - def SaveUserCfgFiles(self): - "Write all loaded user configuration files to disk." - for key in self.userCfg: - self.userCfg[key].Save() - - -idleConf = IdleConf() - -# TODO Revise test output, write expanded unittest -### module test -if __name__ == '__main__': - def dumpCfg(cfg): - print('\n', cfg, '\n') - for key in cfg: - sections = cfg[key].sections() - print(key) - print(sections) - for section in sections: - options = cfg[key].options(section) - print(section) - print(options) - for option in options: - print(option, '=', cfg[key].Get(section, option)) - dumpCfg(idleConf.defaultCfg) - dumpCfg(idleConf.userCfg) - print(idleConf.userCfg['main'].Get('Theme', 'name')) - #print idleConf.userCfg['highlight'].GetDefHighlight('Foo','normal') diff --git a/Lib/idlelib/configHelpSourceEdit.py b/Lib/idlelib/configHelpSourceEdit.py deleted file mode 100644 index cde8118..0000000 --- a/Lib/idlelib/configHelpSourceEdit.py +++ /dev/null @@ -1,170 +0,0 @@ -"Dialog to specify or edit the parameters for a user configured help source." - -import os -import sys - -from tkinter import * -import tkinter.messagebox as tkMessageBox -import tkinter.filedialog as tkFileDialog - -class GetHelpSourceDialog(Toplevel): - def __init__(self, parent, title, menuItem='', filePath='', _htest=False): - """Get menu entry and url/ local file location for Additional Help - - User selects a name for the Help resource and provides a web url - or a local file as its source. The user can enter a url or browse - for the file. - - _htest - bool, change box location when running htest - """ - Toplevel.__init__(self, parent) - self.configure(borderwidth=5) - self.resizable(height=FALSE, width=FALSE) - self.title(title) - self.transient(parent) - self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.cancel) - self.parent = parent - self.result = None - self.create_widgets() - self.menu.set(menuItem) - self.path.set(filePath) - self.withdraw() #hide while setting geometry - #needs to be done here so that the winfo_reqwidth is valid - self.update_idletasks() - #centre dialog over parent. below parent if running htest. - self.geometry( - "+%d+%d" % ( - parent.winfo_rootx() + - (parent.winfo_width()/2 - self.winfo_reqwidth()/2), - parent.winfo_rooty() + - ((parent.winfo_height()/2 - self.winfo_reqheight()/2) - if not _htest else 150))) - self.deiconify() #geometry set, unhide - self.bind('', self.ok) - self.wait_window() - - def create_widgets(self): - self.menu = StringVar(self) - self.path = StringVar(self) - self.fontSize = StringVar(self) - self.frameMain = Frame(self, borderwidth=2, relief=GROOVE) - self.frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) - labelMenu = Label(self.frameMain, anchor=W, justify=LEFT, - text='Menu Item:') - self.entryMenu = Entry(self.frameMain, textvariable=self.menu, - width=30) - self.entryMenu.focus_set() - labelPath = Label(self.frameMain, anchor=W, justify=LEFT, - text='Help File Path: Enter URL or browse for file') - self.entryPath = Entry(self.frameMain, textvariable=self.path, - width=40) - self.entryMenu.focus_set() - labelMenu.pack(anchor=W, padx=5, pady=3) - self.entryMenu.pack(anchor=W, padx=5, pady=3) - labelPath.pack(anchor=W, padx=5, pady=3) - self.entryPath.pack(anchor=W, padx=5, pady=3) - browseButton = Button(self.frameMain, text='Browse', width=8, - command=self.browse_file) - browseButton.pack(pady=3) - frameButtons = Frame(self) - frameButtons.pack(side=BOTTOM, fill=X) - self.buttonOk = Button(frameButtons, text='OK', - width=8, default=ACTIVE, command=self.ok) - self.buttonOk.grid(row=0, column=0, padx=5,pady=5) - self.buttonCancel = Button(frameButtons, text='Cancel', - width=8, command=self.cancel) - self.buttonCancel.grid(row=0, column=1, padx=5, pady=5) - - def browse_file(self): - filetypes = [ - ("HTML Files", "*.htm *.html", "TEXT"), - ("PDF Files", "*.pdf", "TEXT"), - ("Windows Help Files", "*.chm"), - ("Text Files", "*.txt", "TEXT"), - ("All Files", "*")] - path = self.path.get() - if path: - dir, base = os.path.split(path) - else: - base = None - if sys.platform[:3] == 'win': - dir = os.path.join(os.path.dirname(sys.executable), 'Doc') - if not os.path.isdir(dir): - dir = os.getcwd() - else: - dir = os.getcwd() - opendialog = tkFileDialog.Open(parent=self, filetypes=filetypes) - file = opendialog.show(initialdir=dir, initialfile=base) - if file: - self.path.set(file) - - def menu_ok(self): - "Simple validity check for a sensible menu item name" - menu_ok = True - menu = self.menu.get() - menu.strip() - if not menu: - tkMessageBox.showerror(title='Menu Item Error', - message='No menu item specified', - parent=self) - self.entryMenu.focus_set() - menu_ok = False - elif len(menu) > 30: - tkMessageBox.showerror(title='Menu Item Error', - message='Menu item too long:' - '\nLimit 30 characters.', - parent=self) - self.entryMenu.focus_set() - menu_ok = False - return menu_ok - - def path_ok(self): - "Simple validity check for menu file path" - path_ok = True - path = self.path.get() - path.strip() - if not path: #no path specified - tkMessageBox.showerror(title='File Path Error', - message='No help file path specified.', - parent=self) - self.entryPath.focus_set() - path_ok = False - elif path.startswith(('www.', 'http')): - pass - else: - if path[:5] == 'file:': - path = path[5:] - if not os.path.exists(path): - tkMessageBox.showerror(title='File Path Error', - message='Help file path does not exist.', - parent=self) - self.entryPath.focus_set() - path_ok = False - return path_ok - - def ok(self, event=None): - if self.menu_ok() and self.path_ok(): - self.result = (self.menu.get().strip(), - self.path.get().strip()) - if sys.platform == 'darwin': - path = self.result[1] - if path.startswith(('www', 'file:', 'http:', 'https:')): - pass - else: - # Mac Safari insists on using the URI form for local files - self.result = list(self.result) - self.result[1] = "file://" + path - self.destroy() - - def cancel(self, event=None): - self.result = None - self.destroy() - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_config_help', - verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(GetHelpSourceDialog) diff --git a/Lib/idlelib/configSectionNameDialog.py b/Lib/idlelib/configSectionNameDialog.py deleted file mode 100644 index 5137836..0000000 --- a/Lib/idlelib/configSectionNameDialog.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Dialog that allows user to specify a new config file section name. -Used to get new highlight theme and keybinding set names. -The 'return value' for the dialog, used two placed in configDialog.py, -is the .result attribute set in the Ok and Cancel methods. -""" -from tkinter import * -import tkinter.messagebox as tkMessageBox - -class GetCfgSectionNameDialog(Toplevel): - def __init__(self, parent, title, message, used_names, _htest=False): - """ - message - string, informational message to display - used_names - string collection, names already in use for validity check - _htest - bool, change box location when running htest - """ - Toplevel.__init__(self, parent) - self.configure(borderwidth=5) - self.resizable(height=FALSE, width=FALSE) - self.title(title) - self.transient(parent) - self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Cancel) - self.parent = parent - self.message = message - self.used_names = used_names - self.create_widgets() - self.withdraw() #hide while setting geometry - self.update_idletasks() - #needs to be done here so that the winfo_reqwidth is valid - self.messageInfo.config(width=self.frameMain.winfo_reqwidth()) - self.geometry( - "+%d+%d" % ( - parent.winfo_rootx() + - (parent.winfo_width()/2 - self.winfo_reqwidth()/2), - parent.winfo_rooty() + - ((parent.winfo_height()/2 - self.winfo_reqheight()/2) - if not _htest else 100) - ) ) #centre dialog over parent (or below htest box) - self.deiconify() #geometry set, unhide - self.wait_window() - - def create_widgets(self): - self.name = StringVar(self.parent) - self.fontSize = StringVar(self.parent) - self.frameMain = Frame(self, borderwidth=2, relief=SUNKEN) - self.frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) - self.messageInfo = Message(self.frameMain, anchor=W, justify=LEFT, - padx=5, pady=5, text=self.message) #,aspect=200) - entryName = Entry(self.frameMain, textvariable=self.name, width=30) - entryName.focus_set() - self.messageInfo.pack(padx=5, pady=5) #, expand=TRUE, fill=BOTH) - entryName.pack(padx=5, pady=5) - - frameButtons = Frame(self, pady=2) - frameButtons.pack(side=BOTTOM) - self.buttonOk = Button(frameButtons, text='Ok', - width=8, command=self.Ok) - self.buttonOk.pack(side=LEFT, padx=5) - self.buttonCancel = Button(frameButtons, text='Cancel', - width=8, command=self.Cancel) - self.buttonCancel.pack(side=RIGHT, padx=5) - - def name_ok(self): - ''' After stripping entered name, check that it is a sensible - ConfigParser file section name. Return it if it is, '' if not. - ''' - name = self.name.get().strip() - if not name: #no name specified - tkMessageBox.showerror(title='Name Error', - message='No name specified.', parent=self) - elif len(name)>30: #name too long - tkMessageBox.showerror(title='Name Error', - message='Name too long. It should be no more than '+ - '30 characters.', parent=self) - name = '' - elif name in self.used_names: - tkMessageBox.showerror(title='Name Error', - message='This name is already in use.', parent=self) - name = '' - return name - - def Ok(self, event=None): - name = self.name_ok() - if name: - self.result = name - self.destroy() - - def Cancel(self, event=None): - self.result = '' - self.destroy() - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_config_name', verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(GetCfgSectionNameDialog) diff --git a/Lib/idlelib/config_help.py b/Lib/idlelib/config_help.py new file mode 100644 index 0000000..cde8118 --- /dev/null +++ b/Lib/idlelib/config_help.py @@ -0,0 +1,170 @@ +"Dialog to specify or edit the parameters for a user configured help source." + +import os +import sys + +from tkinter import * +import tkinter.messagebox as tkMessageBox +import tkinter.filedialog as tkFileDialog + +class GetHelpSourceDialog(Toplevel): + def __init__(self, parent, title, menuItem='', filePath='', _htest=False): + """Get menu entry and url/ local file location for Additional Help + + User selects a name for the Help resource and provides a web url + or a local file as its source. The user can enter a url or browse + for the file. + + _htest - bool, change box location when running htest + """ + Toplevel.__init__(self, parent) + self.configure(borderwidth=5) + self.resizable(height=FALSE, width=FALSE) + self.title(title) + self.transient(parent) + self.grab_set() + self.protocol("WM_DELETE_WINDOW", self.cancel) + self.parent = parent + self.result = None + self.create_widgets() + self.menu.set(menuItem) + self.path.set(filePath) + self.withdraw() #hide while setting geometry + #needs to be done here so that the winfo_reqwidth is valid + self.update_idletasks() + #centre dialog over parent. below parent if running htest. + self.geometry( + "+%d+%d" % ( + parent.winfo_rootx() + + (parent.winfo_width()/2 - self.winfo_reqwidth()/2), + parent.winfo_rooty() + + ((parent.winfo_height()/2 - self.winfo_reqheight()/2) + if not _htest else 150))) + self.deiconify() #geometry set, unhide + self.bind('', self.ok) + self.wait_window() + + def create_widgets(self): + self.menu = StringVar(self) + self.path = StringVar(self) + self.fontSize = StringVar(self) + self.frameMain = Frame(self, borderwidth=2, relief=GROOVE) + self.frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) + labelMenu = Label(self.frameMain, anchor=W, justify=LEFT, + text='Menu Item:') + self.entryMenu = Entry(self.frameMain, textvariable=self.menu, + width=30) + self.entryMenu.focus_set() + labelPath = Label(self.frameMain, anchor=W, justify=LEFT, + text='Help File Path: Enter URL or browse for file') + self.entryPath = Entry(self.frameMain, textvariable=self.path, + width=40) + self.entryMenu.focus_set() + labelMenu.pack(anchor=W, padx=5, pady=3) + self.entryMenu.pack(anchor=W, padx=5, pady=3) + labelPath.pack(anchor=W, padx=5, pady=3) + self.entryPath.pack(anchor=W, padx=5, pady=3) + browseButton = Button(self.frameMain, text='Browse', width=8, + command=self.browse_file) + browseButton.pack(pady=3) + frameButtons = Frame(self) + frameButtons.pack(side=BOTTOM, fill=X) + self.buttonOk = Button(frameButtons, text='OK', + width=8, default=ACTIVE, command=self.ok) + self.buttonOk.grid(row=0, column=0, padx=5,pady=5) + self.buttonCancel = Button(frameButtons, text='Cancel', + width=8, command=self.cancel) + self.buttonCancel.grid(row=0, column=1, padx=5, pady=5) + + def browse_file(self): + filetypes = [ + ("HTML Files", "*.htm *.html", "TEXT"), + ("PDF Files", "*.pdf", "TEXT"), + ("Windows Help Files", "*.chm"), + ("Text Files", "*.txt", "TEXT"), + ("All Files", "*")] + path = self.path.get() + if path: + dir, base = os.path.split(path) + else: + base = None + if sys.platform[:3] == 'win': + dir = os.path.join(os.path.dirname(sys.executable), 'Doc') + if not os.path.isdir(dir): + dir = os.getcwd() + else: + dir = os.getcwd() + opendialog = tkFileDialog.Open(parent=self, filetypes=filetypes) + file = opendialog.show(initialdir=dir, initialfile=base) + if file: + self.path.set(file) + + def menu_ok(self): + "Simple validity check for a sensible menu item name" + menu_ok = True + menu = self.menu.get() + menu.strip() + if not menu: + tkMessageBox.showerror(title='Menu Item Error', + message='No menu item specified', + parent=self) + self.entryMenu.focus_set() + menu_ok = False + elif len(menu) > 30: + tkMessageBox.showerror(title='Menu Item Error', + message='Menu item too long:' + '\nLimit 30 characters.', + parent=self) + self.entryMenu.focus_set() + menu_ok = False + return menu_ok + + def path_ok(self): + "Simple validity check for menu file path" + path_ok = True + path = self.path.get() + path.strip() + if not path: #no path specified + tkMessageBox.showerror(title='File Path Error', + message='No help file path specified.', + parent=self) + self.entryPath.focus_set() + path_ok = False + elif path.startswith(('www.', 'http')): + pass + else: + if path[:5] == 'file:': + path = path[5:] + if not os.path.exists(path): + tkMessageBox.showerror(title='File Path Error', + message='Help file path does not exist.', + parent=self) + self.entryPath.focus_set() + path_ok = False + return path_ok + + def ok(self, event=None): + if self.menu_ok() and self.path_ok(): + self.result = (self.menu.get().strip(), + self.path.get().strip()) + if sys.platform == 'darwin': + path = self.result[1] + if path.startswith(('www', 'file:', 'http:', 'https:')): + pass + else: + # Mac Safari insists on using the URI form for local files + self.result = list(self.result) + self.result[1] = "file://" + path + self.destroy() + + def cancel(self, event=None): + self.result = None + self.destroy() + +if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_config_help', + verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(GetHelpSourceDialog) diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py new file mode 100644 index 0000000..e6438bf --- /dev/null +++ b/Lib/idlelib/config_key.py @@ -0,0 +1,266 @@ +""" +Dialog for building Tkinter accelerator key bindings +""" +from tkinter import * +import tkinter.messagebox as tkMessageBox +import string +import sys + +class GetKeysDialog(Toplevel): + def __init__(self,parent,title,action,currentKeySequences,_htest=False): + """ + action - string, the name of the virtual event these keys will be + mapped to + currentKeys - list, a list of all key sequence lists currently mapped + to virtual events, for overlap checking + _htest - bool, change box location when running htest + """ + Toplevel.__init__(self, parent) + self.configure(borderwidth=5) + self.resizable(height=FALSE,width=FALSE) + self.title(title) + self.transient(parent) + self.grab_set() + self.protocol("WM_DELETE_WINDOW", self.Cancel) + self.parent = parent + self.action=action + self.currentKeySequences=currentKeySequences + self.result='' + self.keyString=StringVar(self) + self.keyString.set('') + self.SetModifiersForPlatform() # set self.modifiers, self.modifier_label + self.modifier_vars = [] + for modifier in self.modifiers: + variable = StringVar(self) + variable.set('') + self.modifier_vars.append(variable) + self.advanced = False + self.CreateWidgets() + self.LoadFinalKeyList() + self.withdraw() #hide while setting geometry + self.update_idletasks() + self.geometry( + "+%d+%d" % ( + parent.winfo_rootx() + + (parent.winfo_width()/2 - self.winfo_reqwidth()/2), + parent.winfo_rooty() + + ((parent.winfo_height()/2 - self.winfo_reqheight()/2) + if not _htest else 150) + ) ) #centre dialog over parent (or below htest box) + self.deiconify() #geometry set, unhide + self.wait_window() + + def CreateWidgets(self): + frameMain = Frame(self,borderwidth=2,relief=SUNKEN) + frameMain.pack(side=TOP,expand=TRUE,fill=BOTH) + frameButtons=Frame(self) + frameButtons.pack(side=BOTTOM,fill=X) + self.buttonOK = Button(frameButtons,text='OK', + width=8,command=self.OK) + self.buttonOK.grid(row=0,column=0,padx=5,pady=5) + self.buttonCancel = Button(frameButtons,text='Cancel', + width=8,command=self.Cancel) + self.buttonCancel.grid(row=0,column=1,padx=5,pady=5) + self.frameKeySeqBasic = Frame(frameMain) + self.frameKeySeqAdvanced = Frame(frameMain) + self.frameControlsBasic = Frame(frameMain) + self.frameHelpAdvanced = Frame(frameMain) + self.frameKeySeqAdvanced.grid(row=0,column=0,sticky=NSEW,padx=5,pady=5) + self.frameKeySeqBasic.grid(row=0,column=0,sticky=NSEW,padx=5,pady=5) + self.frameKeySeqBasic.lift() + self.frameHelpAdvanced.grid(row=1,column=0,sticky=NSEW,padx=5) + self.frameControlsBasic.grid(row=1,column=0,sticky=NSEW,padx=5) + self.frameControlsBasic.lift() + self.buttonLevel = Button(frameMain,command=self.ToggleLevel, + text='Advanced Key Binding Entry >>') + self.buttonLevel.grid(row=2,column=0,stick=EW,padx=5,pady=5) + labelTitleBasic = Label(self.frameKeySeqBasic, + text="New keys for '"+self.action+"' :") + labelTitleBasic.pack(anchor=W) + labelKeysBasic = Label(self.frameKeySeqBasic,justify=LEFT, + textvariable=self.keyString,relief=GROOVE,borderwidth=2) + labelKeysBasic.pack(ipadx=5,ipady=5,fill=X) + self.modifier_checkbuttons = {} + column = 0 + for modifier, variable in zip(self.modifiers, self.modifier_vars): + label = self.modifier_label.get(modifier, modifier) + check=Checkbutton(self.frameControlsBasic, + command=self.BuildKeyString, + text=label,variable=variable,onvalue=modifier,offvalue='') + check.grid(row=0,column=column,padx=2,sticky=W) + self.modifier_checkbuttons[modifier] = check + column += 1 + labelFnAdvice=Label(self.frameControlsBasic,justify=LEFT, + text=\ + "Select the desired modifier keys\n"+ + "above, and the final key from the\n"+ + "list on the right.\n\n" + + "Use upper case Symbols when using\n" + + "the Shift modifier. (Letters will be\n" + + "converted automatically.)") + labelFnAdvice.grid(row=1,column=0,columnspan=4,padx=2,sticky=W) + self.listKeysFinal=Listbox(self.frameControlsBasic,width=15,height=10, + selectmode=SINGLE) + self.listKeysFinal.bind('',self.FinalKeySelected) + self.listKeysFinal.grid(row=0,column=4,rowspan=4,sticky=NS) + scrollKeysFinal=Scrollbar(self.frameControlsBasic,orient=VERTICAL, + command=self.listKeysFinal.yview) + self.listKeysFinal.config(yscrollcommand=scrollKeysFinal.set) + scrollKeysFinal.grid(row=0,column=5,rowspan=4,sticky=NS) + self.buttonClear=Button(self.frameControlsBasic, + text='Clear Keys',command=self.ClearKeySeq) + self.buttonClear.grid(row=2,column=0,columnspan=4) + labelTitleAdvanced = Label(self.frameKeySeqAdvanced,justify=LEFT, + text="Enter new binding(s) for '"+self.action+"' :\n"+ + "(These bindings will not be checked for validity!)") + labelTitleAdvanced.pack(anchor=W) + self.entryKeysAdvanced=Entry(self.frameKeySeqAdvanced, + textvariable=self.keyString) + self.entryKeysAdvanced.pack(fill=X) + labelHelpAdvanced=Label(self.frameHelpAdvanced,justify=LEFT, + text="Key bindings are specified using Tkinter keysyms as\n"+ + "in these samples: , , ,\n" + ", , .\n" + "Upper case is used when the Shift modifier is present!\n\n" + + "'Emacs style' multi-keystroke bindings are specified as\n" + + "follows: , where the first key\n" + + "is the 'do-nothing' keybinding.\n\n" + + "Multiple separate bindings for one action should be\n"+ + "separated by a space, eg., ." ) + labelHelpAdvanced.grid(row=0,column=0,sticky=NSEW) + + def SetModifiersForPlatform(self): + """Determine list of names of key modifiers for this platform. + + The names are used to build Tk bindings -- it doesn't matter if the + keyboard has these keys, it matters if Tk understands them. The + order is also important: key binding equality depends on it, so + config-keys.def must use the same ordering. + """ + if sys.platform == "darwin": + self.modifiers = ['Shift', 'Control', 'Option', 'Command'] + else: + self.modifiers = ['Control', 'Alt', 'Shift'] + self.modifier_label = {'Control': 'Ctrl'} # short name + + def ToggleLevel(self): + if self.buttonLevel.cget('text')[:8]=='Advanced': + self.ClearKeySeq() + self.buttonLevel.config(text='<< Basic Key Binding Entry') + self.frameKeySeqAdvanced.lift() + self.frameHelpAdvanced.lift() + self.entryKeysAdvanced.focus_set() + self.advanced = True + else: + self.ClearKeySeq() + self.buttonLevel.config(text='Advanced Key Binding Entry >>') + self.frameKeySeqBasic.lift() + self.frameControlsBasic.lift() + self.advanced = False + + def FinalKeySelected(self,event): + self.BuildKeyString() + + def BuildKeyString(self): + keyList = modifiers = self.GetModifiers() + finalKey = self.listKeysFinal.get(ANCHOR) + if finalKey: + finalKey = self.TranslateKey(finalKey, modifiers) + keyList.append(finalKey) + self.keyString.set('<' + '-'.join(keyList) + '>') + + def GetModifiers(self): + modList = [variable.get() for variable in self.modifier_vars] + return [mod for mod in modList if mod] + + def ClearKeySeq(self): + self.listKeysFinal.select_clear(0,END) + self.listKeysFinal.yview(MOVETO, '0.0') + for variable in self.modifier_vars: + variable.set('') + self.keyString.set('') + + def LoadFinalKeyList(self): + #these tuples are also available for use in validity checks + self.functionKeys=('F1','F2','F2','F4','F5','F6','F7','F8','F9', + 'F10','F11','F12') + self.alphanumKeys=tuple(string.ascii_lowercase+string.digits) + self.punctuationKeys=tuple('~!@#%^&*()_-+={}[]|;:,.<>/?') + self.whitespaceKeys=('Tab','Space','Return') + self.editKeys=('BackSpace','Delete','Insert') + self.moveKeys=('Home','End','Page Up','Page Down','Left Arrow', + 'Right Arrow','Up Arrow','Down Arrow') + #make a tuple of most of the useful common 'final' keys + keys=(self.alphanumKeys+self.punctuationKeys+self.functionKeys+ + self.whitespaceKeys+self.editKeys+self.moveKeys) + self.listKeysFinal.insert(END, *keys) + + def TranslateKey(self, key, modifiers): + "Translate from keycap symbol to the Tkinter keysym" + translateDict = {'Space':'space', + '~':'asciitilde','!':'exclam','@':'at','#':'numbersign', + '%':'percent','^':'asciicircum','&':'ampersand','*':'asterisk', + '(':'parenleft',')':'parenright','_':'underscore','-':'minus', + '+':'plus','=':'equal','{':'braceleft','}':'braceright', + '[':'bracketleft',']':'bracketright','|':'bar',';':'semicolon', + ':':'colon',',':'comma','.':'period','<':'less','>':'greater', + '/':'slash','?':'question','Page Up':'Prior','Page Down':'Next', + 'Left Arrow':'Left','Right Arrow':'Right','Up Arrow':'Up', + 'Down Arrow': 'Down', 'Tab':'Tab'} + if key in translateDict: + key = translateDict[key] + if 'Shift' in modifiers and key in string.ascii_lowercase: + key = key.upper() + key = 'Key-' + key + return key + + def OK(self, event=None): + if self.advanced or self.KeysOK(): # doesn't check advanced string yet + self.result=self.keyString.get() + self.destroy() + + def Cancel(self, event=None): + self.result='' + self.destroy() + + def KeysOK(self): + '''Validity check on user's 'basic' keybinding selection. + + Doesn't check the string produced by the advanced dialog because + 'modifiers' isn't set. + + ''' + keys = self.keyString.get() + keys.strip() + finalKey = self.listKeysFinal.get(ANCHOR) + modifiers = self.GetModifiers() + # create a key sequence list for overlap check: + keySequence = keys.split() + keysOK = False + title = 'Key Sequence Error' + if not keys: + tkMessageBox.showerror(title=title, parent=self, + message='No keys specified.') + elif not keys.endswith('>'): + tkMessageBox.showerror(title=title, parent=self, + message='Missing the final Key') + elif (not modifiers + and finalKey not in self.functionKeys + self.moveKeys): + tkMessageBox.showerror(title=title, parent=self, + message='No modifier key(s) specified.') + elif (modifiers == ['Shift']) \ + and (finalKey not in + self.functionKeys + self.moveKeys + ('Tab', 'Space')): + msg = 'The shift modifier by itself may not be used with'\ + ' this key symbol.' + tkMessageBox.showerror(title=title, parent=self, message=msg) + elif keySequence in self.currentKeySequences: + msg = 'This key combination is already in use.' + tkMessageBox.showerror(title=title, parent=self, message=msg) + else: + keysOK = True + return keysOK + +if __name__ == '__main__': + from idlelib.idle_test.htest import run + run(GetKeysDialog) diff --git a/Lib/idlelib/config_sec.py b/Lib/idlelib/config_sec.py new file mode 100644 index 0000000..5137836 --- /dev/null +++ b/Lib/idlelib/config_sec.py @@ -0,0 +1,98 @@ +""" +Dialog that allows user to specify a new config file section name. +Used to get new highlight theme and keybinding set names. +The 'return value' for the dialog, used two placed in configDialog.py, +is the .result attribute set in the Ok and Cancel methods. +""" +from tkinter import * +import tkinter.messagebox as tkMessageBox + +class GetCfgSectionNameDialog(Toplevel): + def __init__(self, parent, title, message, used_names, _htest=False): + """ + message - string, informational message to display + used_names - string collection, names already in use for validity check + _htest - bool, change box location when running htest + """ + Toplevel.__init__(self, parent) + self.configure(borderwidth=5) + self.resizable(height=FALSE, width=FALSE) + self.title(title) + self.transient(parent) + self.grab_set() + self.protocol("WM_DELETE_WINDOW", self.Cancel) + self.parent = parent + self.message = message + self.used_names = used_names + self.create_widgets() + self.withdraw() #hide while setting geometry + self.update_idletasks() + #needs to be done here so that the winfo_reqwidth is valid + self.messageInfo.config(width=self.frameMain.winfo_reqwidth()) + self.geometry( + "+%d+%d" % ( + parent.winfo_rootx() + + (parent.winfo_width()/2 - self.winfo_reqwidth()/2), + parent.winfo_rooty() + + ((parent.winfo_height()/2 - self.winfo_reqheight()/2) + if not _htest else 100) + ) ) #centre dialog over parent (or below htest box) + self.deiconify() #geometry set, unhide + self.wait_window() + + def create_widgets(self): + self.name = StringVar(self.parent) + self.fontSize = StringVar(self.parent) + self.frameMain = Frame(self, borderwidth=2, relief=SUNKEN) + self.frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) + self.messageInfo = Message(self.frameMain, anchor=W, justify=LEFT, + padx=5, pady=5, text=self.message) #,aspect=200) + entryName = Entry(self.frameMain, textvariable=self.name, width=30) + entryName.focus_set() + self.messageInfo.pack(padx=5, pady=5) #, expand=TRUE, fill=BOTH) + entryName.pack(padx=5, pady=5) + + frameButtons = Frame(self, pady=2) + frameButtons.pack(side=BOTTOM) + self.buttonOk = Button(frameButtons, text='Ok', + width=8, command=self.Ok) + self.buttonOk.pack(side=LEFT, padx=5) + self.buttonCancel = Button(frameButtons, text='Cancel', + width=8, command=self.Cancel) + self.buttonCancel.pack(side=RIGHT, padx=5) + + def name_ok(self): + ''' After stripping entered name, check that it is a sensible + ConfigParser file section name. Return it if it is, '' if not. + ''' + name = self.name.get().strip() + if not name: #no name specified + tkMessageBox.showerror(title='Name Error', + message='No name specified.', parent=self) + elif len(name)>30: #name too long + tkMessageBox.showerror(title='Name Error', + message='Name too long. It should be no more than '+ + '30 characters.', parent=self) + name = '' + elif name in self.used_names: + tkMessageBox.showerror(title='Name Error', + message='This name is already in use.', parent=self) + name = '' + return name + + def Ok(self, event=None): + name = self.name_ok() + if name: + self.result = name + self.destroy() + + def Cancel(self, event=None): + self.result = '' + self.destroy() + +if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_config_name', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(GetCfgSectionNameDialog) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py new file mode 100644 index 0000000..b702253 --- /dev/null +++ b/Lib/idlelib/configdialog.py @@ -0,0 +1,1434 @@ +"""IDLE Configuration Dialog: support user customization of IDLE by GUI + +Customize font faces, sizes, and colorization attributes. Set indentation +defaults. Customize keybindings. Colorization and keybindings can be +saved as user defined sets. Select startup options including shell/editor +and default window size. Define additional help sources. + +Note that tab width in IDLE is currently fixed at eight due to Tk issues. +Refer to comments in EditorWindow autoindent code for details. + +""" +from tkinter import * +import tkinter.messagebox as tkMessageBox +import tkinter.colorchooser as tkColorChooser +import tkinter.font as tkFont + +from idlelib.configHandler import idleConf +from idlelib.dynOptionMenuWidget import DynOptionMenu +from idlelib.keybindingDialog import GetKeysDialog +from idlelib.configSectionNameDialog import GetCfgSectionNameDialog +from idlelib.configHelpSourceEdit import GetHelpSourceDialog +from idlelib.tabbedpages import TabbedPageSet +from idlelib.textView import view_text +from idlelib import macosxSupport + +class ConfigDialog(Toplevel): + + def __init__(self, parent, title='', _htest=False, _utest=False): + """ + _htest - bool, change box location when running htest + _utest - bool, don't wait_window when running unittest + """ + Toplevel.__init__(self, parent) + self.parent = parent + if _htest: + parent.instance_dict = {} + self.wm_withdraw() + + self.configure(borderwidth=5) + self.title(title or 'IDLE Preferences') + self.geometry( + "+%d+%d" % (parent.winfo_rootx() + 20, + parent.winfo_rooty() + (30 if not _htest else 150))) + #Theme Elements. Each theme element key is its display name. + #The first value of the tuple is the sample area tag name. + #The second value is the display name list sort index. + self.themeElements={ + 'Normal Text': ('normal', '00'), + 'Python Keywords': ('keyword', '01'), + 'Python Definitions': ('definition', '02'), + 'Python Builtins': ('builtin', '03'), + 'Python Comments': ('comment', '04'), + 'Python Strings': ('string', '05'), + 'Selected Text': ('hilite', '06'), + 'Found Text': ('hit', '07'), + 'Cursor': ('cursor', '08'), + 'Editor Breakpoint': ('break', '09'), + 'Shell Normal Text': ('console', '10'), + 'Shell Error Text': ('error', '11'), + 'Shell Stdout Text': ('stdout', '12'), + 'Shell Stderr Text': ('stderr', '13'), + } + self.ResetChangedItems() #load initial values in changed items dict + self.CreateWidgets() + self.resizable(height=FALSE, width=FALSE) + self.transient(parent) + self.grab_set() + self.protocol("WM_DELETE_WINDOW", self.Cancel) + self.tabPages.focus_set() + #key bindings for this dialog + #self.bind('', self.Cancel) #dismiss dialog, no save + #self.bind('', self.Apply) #apply changes, save + #self.bind('', self.Help) #context help + self.LoadConfigs() + self.AttachVarCallbacks() #avoid callbacks during LoadConfigs + + if not _utest: + self.wm_deiconify() + self.wait_window() + + def CreateWidgets(self): + self.tabPages = TabbedPageSet(self, + page_names=['Fonts/Tabs', 'Highlighting', 'Keys', 'General', + 'Extensions']) + self.tabPages.pack(side=TOP, expand=TRUE, fill=BOTH) + self.CreatePageFontTab() + self.CreatePageHighlight() + self.CreatePageKeys() + self.CreatePageGeneral() + self.CreatePageExtensions() + self.create_action_buttons().pack(side=BOTTOM) + + def create_action_buttons(self): + if macosxSupport.isAquaTk(): + # Changing the default padding on OSX results in unreadable + # text in the buttons + paddingArgs = {} + else: + paddingArgs = {'padx':6, 'pady':3} + outer = Frame(self, pady=2) + buttons = Frame(outer, pady=2) + for txt, cmd in ( + ('Ok', self.Ok), + ('Apply', self.Apply), + ('Cancel', self.Cancel), + ('Help', self.Help)): + Button(buttons, text=txt, command=cmd, takefocus=FALSE, + **paddingArgs).pack(side=LEFT, padx=5) + # add space above buttons + Frame(outer, height=2, borderwidth=0).pack(side=TOP) + buttons.pack(side=BOTTOM) + return outer + + def CreatePageFontTab(self): + parent = self.parent + self.fontSize = StringVar(parent) + self.fontBold = BooleanVar(parent) + self.fontName = StringVar(parent) + self.spaceNum = IntVar(parent) + self.editFont = tkFont.Font(parent, ('courier', 10, 'normal')) + + ##widget creation + #body frame + frame = self.tabPages.pages['Fonts/Tabs'].frame + #body section frames + frameFont = LabelFrame( + frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') + frameIndent = LabelFrame( + frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') + #frameFont + frameFontName = Frame(frameFont) + frameFontParam = Frame(frameFont) + labelFontNameTitle = Label( + frameFontName, justify=LEFT, text='Font Face :') + self.listFontName = Listbox( + frameFontName, height=5, takefocus=FALSE, exportselection=FALSE) + self.listFontName.bind( + '', self.OnListFontButtonRelease) + scrollFont = Scrollbar(frameFontName) + scrollFont.config(command=self.listFontName.yview) + self.listFontName.config(yscrollcommand=scrollFont.set) + labelFontSizeTitle = Label(frameFontParam, text='Size :') + self.optMenuFontSize = DynOptionMenu( + frameFontParam, self.fontSize, None, command=self.SetFontSample) + checkFontBold = Checkbutton( + frameFontParam, variable=self.fontBold, onvalue=1, + offvalue=0, text='Bold', command=self.SetFontSample) + frameFontSample = Frame(frameFont, relief=SOLID, borderwidth=1) + self.labelFontSample = Label( + frameFontSample, justify=LEFT, font=self.editFont, + text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]') + #frameIndent + frameIndentSize = Frame(frameIndent) + labelSpaceNumTitle = Label( + frameIndentSize, justify=LEFT, + text='Python Standard: 4 Spaces!') + self.scaleSpaceNum = Scale( + frameIndentSize, variable=self.spaceNum, + orient='horizontal', tickinterval=2, from_=2, to=16) + + #widget packing + #body + frameFont.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + frameIndent.pack(side=LEFT, padx=5, pady=5, fill=Y) + #frameFont + frameFontName.pack(side=TOP, padx=5, pady=5, fill=X) + frameFontParam.pack(side=TOP, padx=5, pady=5, fill=X) + labelFontNameTitle.pack(side=TOP, anchor=W) + self.listFontName.pack(side=LEFT, expand=TRUE, fill=X) + scrollFont.pack(side=LEFT, fill=Y) + labelFontSizeTitle.pack(side=LEFT, anchor=W) + self.optMenuFontSize.pack(side=LEFT, anchor=W) + checkFontBold.pack(side=LEFT, anchor=W, padx=20) + frameFontSample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + self.labelFontSample.pack(expand=TRUE, fill=BOTH) + #frameIndent + frameIndentSize.pack(side=TOP, fill=X) + labelSpaceNumTitle.pack(side=TOP, anchor=W, padx=5) + self.scaleSpaceNum.pack(side=TOP, padx=5, fill=X) + return frame + + def CreatePageHighlight(self): + parent = self.parent + self.builtinTheme = StringVar(parent) + self.customTheme = StringVar(parent) + self.fgHilite = BooleanVar(parent) + self.colour = StringVar(parent) + self.fontName = StringVar(parent) + self.themeIsBuiltin = BooleanVar(parent) + self.highlightTarget = StringVar(parent) + + ##widget creation + #body frame + frame = self.tabPages.pages['Highlighting'].frame + #body section frames + frameCustom = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' Custom Highlighting ') + frameTheme = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' Highlighting Theme ') + #frameCustom + self.textHighlightSample=Text( + frameCustom, relief=SOLID, borderwidth=1, + font=('courier', 12, ''), cursor='hand2', width=21, height=11, + takefocus=FALSE, highlightthickness=0, wrap=NONE) + text=self.textHighlightSample + text.bind('', lambda e: 'break') + text.bind('', lambda e: 'break') + textAndTags=( + ('#you can click here', 'comment'), ('\n', 'normal'), + ('#to choose items', 'comment'), ('\n', 'normal'), + ('def', 'keyword'), (' ', 'normal'), + ('func', 'definition'), ('(param):\n ', 'normal'), + ('"""string"""', 'string'), ('\n var0 = ', 'normal'), + ("'string'", 'string'), ('\n var1 = ', 'normal'), + ("'selected'", 'hilite'), ('\n var2 = ', 'normal'), + ("'found'", 'hit'), ('\n var3 = ', 'normal'), + ('list', 'builtin'), ('(', 'normal'), + ('None', 'keyword'), (')\n', 'normal'), + (' breakpoint("line")', 'break'), ('\n\n', 'normal'), + (' error ', 'error'), (' ', 'normal'), + ('cursor |', 'cursor'), ('\n ', 'normal'), + ('shell', 'console'), (' ', 'normal'), + ('stdout', 'stdout'), (' ', 'normal'), + ('stderr', 'stderr'), ('\n', 'normal')) + for txTa in textAndTags: + text.insert(END, txTa[0], txTa[1]) + for element in self.themeElements: + def tem(event, elem=element): + event.widget.winfo_toplevel().highlightTarget.set(elem) + text.tag_bind( + self.themeElements[element][0], '', tem) + text.config(state=DISABLED) + self.frameColourSet = Frame(frameCustom, relief=SOLID, borderwidth=1) + frameFgBg = Frame(frameCustom) + buttonSetColour = Button( + self.frameColourSet, text='Choose Colour for :', + command=self.GetColour, highlightthickness=0) + self.optMenuHighlightTarget = DynOptionMenu( + self.frameColourSet, self.highlightTarget, None, + highlightthickness=0) #, command=self.SetHighlightTargetBinding + self.radioFg = Radiobutton( + frameFgBg, variable=self.fgHilite, value=1, + text='Foreground', command=self.SetColourSampleBinding) + self.radioBg=Radiobutton( + frameFgBg, variable=self.fgHilite, value=0, + text='Background', command=self.SetColourSampleBinding) + self.fgHilite.set(1) + buttonSaveCustomTheme = Button( + frameCustom, text='Save as New Custom Theme', + command=self.SaveAsNewTheme) + #frameTheme + labelTypeTitle = Label(frameTheme, text='Select : ') + self.radioThemeBuiltin = Radiobutton( + frameTheme, variable=self.themeIsBuiltin, value=1, + command=self.SetThemeType, text='a Built-in Theme') + self.radioThemeCustom = Radiobutton( + frameTheme, variable=self.themeIsBuiltin, value=0, + command=self.SetThemeType, text='a Custom Theme') + self.optMenuThemeBuiltin = DynOptionMenu( + frameTheme, self.builtinTheme, None, command=None) + self.optMenuThemeCustom=DynOptionMenu( + frameTheme, self.customTheme, None, command=None) + self.buttonDeleteCustomTheme=Button( + frameTheme, text='Delete Custom Theme', + command=self.DeleteCustomTheme) + self.new_custom_theme = Label(frameTheme, bd=2) + + ##widget packing + #body + frameCustom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + frameTheme.pack(side=LEFT, padx=5, pady=5, fill=Y) + #frameCustom + self.frameColourSet.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) + frameFgBg.pack(side=TOP, padx=5, pady=0) + self.textHighlightSample.pack( + side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + buttonSetColour.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) + self.optMenuHighlightTarget.pack( + side=TOP, expand=TRUE, fill=X, padx=8, pady=3) + self.radioFg.pack(side=LEFT, anchor=E) + self.radioBg.pack(side=RIGHT, anchor=W) + buttonSaveCustomTheme.pack(side=BOTTOM, fill=X, padx=5, pady=5) + #frameTheme + labelTypeTitle.pack(side=TOP, anchor=W, padx=5, pady=5) + self.radioThemeBuiltin.pack(side=TOP, anchor=W, padx=5) + self.radioThemeCustom.pack(side=TOP, anchor=W, padx=5, pady=2) + self.optMenuThemeBuiltin.pack(side=TOP, fill=X, padx=5, pady=5) + self.optMenuThemeCustom.pack(side=TOP, fill=X, anchor=W, padx=5, pady=5) + self.buttonDeleteCustomTheme.pack(side=TOP, fill=X, padx=5, pady=5) + self.new_custom_theme.pack(side=TOP, fill=X, pady=5) + return frame + + def CreatePageKeys(self): + parent = self.parent + self.bindingTarget = StringVar(parent) + self.builtinKeys = StringVar(parent) + self.customKeys = StringVar(parent) + self.keysAreBuiltin = BooleanVar(parent) + self.keyBinding = StringVar(parent) + + ##widget creation + #body frame + frame = self.tabPages.pages['Keys'].frame + #body section frames + frameCustom = LabelFrame( + frame, borderwidth=2, relief=GROOVE, + text=' Custom Key Bindings ') + frameKeySets = LabelFrame( + frame, borderwidth=2, relief=GROOVE, text=' Key Set ') + #frameCustom + frameTarget = Frame(frameCustom) + labelTargetTitle = Label(frameTarget, text='Action - Key(s)') + scrollTargetY = Scrollbar(frameTarget) + scrollTargetX = Scrollbar(frameTarget, orient=HORIZONTAL) + self.listBindings = Listbox( + frameTarget, takefocus=FALSE, exportselection=FALSE) + self.listBindings.bind('', self.KeyBindingSelected) + scrollTargetY.config(command=self.listBindings.yview) + scrollTargetX.config(command=self.listBindings.xview) + self.listBindings.config(yscrollcommand=scrollTargetY.set) + self.listBindings.config(xscrollcommand=scrollTargetX.set) + self.buttonNewKeys = Button( + frameCustom, text='Get New Keys for Selection', + command=self.GetNewKeys, state=DISABLED) + #frameKeySets + frames = [Frame(frameKeySets, padx=2, pady=2, borderwidth=0) + for i in range(2)] + self.radioKeysBuiltin = Radiobutton( + frames[0], variable=self.keysAreBuiltin, value=1, + command=self.SetKeysType, text='Use a Built-in Key Set') + self.radioKeysCustom = Radiobutton( + frames[0], variable=self.keysAreBuiltin, value=0, + command=self.SetKeysType, text='Use a Custom Key Set') + self.optMenuKeysBuiltin = DynOptionMenu( + frames[0], self.builtinKeys, None, command=None) + self.optMenuKeysCustom = DynOptionMenu( + frames[0], self.customKeys, None, command=None) + self.buttonDeleteCustomKeys = Button( + frames[1], text='Delete Custom Key Set', + command=self.DeleteCustomKeys) + buttonSaveCustomKeys = Button( + frames[1], text='Save as New Custom Key Set', + command=self.SaveAsNewKeySet) + + ##widget packing + #body + frameCustom.pack(side=BOTTOM, padx=5, pady=5, expand=TRUE, fill=BOTH) + frameKeySets.pack(side=BOTTOM, padx=5, pady=5, fill=BOTH) + #frameCustom + self.buttonNewKeys.pack(side=BOTTOM, fill=X, padx=5, pady=5) + frameTarget.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + #frame target + frameTarget.columnconfigure(0, weight=1) + frameTarget.rowconfigure(1, weight=1) + labelTargetTitle.grid(row=0, column=0, columnspan=2, sticky=W) + self.listBindings.grid(row=1, column=0, sticky=NSEW) + scrollTargetY.grid(row=1, column=1, sticky=NS) + scrollTargetX.grid(row=2, column=0, sticky=EW) + #frameKeySets + self.radioKeysBuiltin.grid(row=0, column=0, sticky=W+NS) + self.radioKeysCustom.grid(row=1, column=0, sticky=W+NS) + self.optMenuKeysBuiltin.grid(row=0, column=1, sticky=NSEW) + self.optMenuKeysCustom.grid(row=1, column=1, sticky=NSEW) + self.buttonDeleteCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) + buttonSaveCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) + frames[0].pack(side=TOP, fill=BOTH, expand=True) + frames[1].pack(side=TOP, fill=X, expand=True, pady=2) + return frame + + def CreatePageGeneral(self): + parent = self.parent + self.winWidth = StringVar(parent) + self.winHeight = StringVar(parent) + self.startupEdit = IntVar(parent) + self.autoSave = IntVar(parent) + self.encoding = StringVar(parent) + self.userHelpBrowser = BooleanVar(parent) + self.helpBrowser = StringVar(parent) + + #widget creation + #body + frame = self.tabPages.pages['General'].frame + #body section frames + frameRun = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' Startup Preferences ') + frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' Autosave Preferences ') + frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE) + frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' Additional Help Sources ') + #frameRun + labelRunChoiceTitle = Label(frameRun, text='At Startup') + radioStartupEdit = Radiobutton( + frameRun, variable=self.startupEdit, value=1, + command=self.SetKeysType, text="Open Edit Window") + radioStartupShell = Radiobutton( + frameRun, variable=self.startupEdit, value=0, + command=self.SetKeysType, text='Open Shell Window') + #frameSave + labelRunSaveTitle = Label(frameSave, text='At Start of Run (F5) ') + radioSaveAsk = Radiobutton( + frameSave, variable=self.autoSave, value=0, + command=self.SetKeysType, text="Prompt to Save") + radioSaveAuto = Radiobutton( + frameSave, variable=self.autoSave, value=1, + command=self.SetKeysType, text='No Prompt') + #frameWinSize + labelWinSizeTitle = Label( + frameWinSize, text='Initial Window Size (in characters)') + labelWinWidthTitle = Label(frameWinSize, text='Width') + entryWinWidth = Entry( + frameWinSize, textvariable=self.winWidth, width=3) + labelWinHeightTitle = Label(frameWinSize, text='Height') + entryWinHeight = Entry( + frameWinSize, textvariable=self.winHeight, width=3) + #frameHelp + frameHelpList = Frame(frameHelp) + frameHelpListButtons = Frame(frameHelpList) + scrollHelpList = Scrollbar(frameHelpList) + self.listHelp = Listbox( + frameHelpList, height=5, takefocus=FALSE, + exportselection=FALSE) + scrollHelpList.config(command=self.listHelp.yview) + self.listHelp.config(yscrollcommand=scrollHelpList.set) + self.listHelp.bind('', self.HelpSourceSelected) + self.buttonHelpListEdit = Button( + frameHelpListButtons, text='Edit', state=DISABLED, + width=8, command=self.HelpListItemEdit) + self.buttonHelpListAdd = Button( + frameHelpListButtons, text='Add', + width=8, command=self.HelpListItemAdd) + self.buttonHelpListRemove = Button( + frameHelpListButtons, text='Remove', state=DISABLED, + width=8, command=self.HelpListItemRemove) + + #widget packing + #body + frameRun.pack(side=TOP, padx=5, pady=5, fill=X) + frameSave.pack(side=TOP, padx=5, pady=5, fill=X) + frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X) + frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + #frameRun + labelRunChoiceTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) + radioStartupShell.pack(side=RIGHT, anchor=W, padx=5, pady=5) + radioStartupEdit.pack(side=RIGHT, anchor=W, padx=5, pady=5) + #frameSave + labelRunSaveTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) + radioSaveAuto.pack(side=RIGHT, anchor=W, padx=5, pady=5) + radioSaveAsk.pack(side=RIGHT, anchor=W, padx=5, pady=5) + #frameWinSize + labelWinSizeTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) + entryWinHeight.pack(side=RIGHT, anchor=E, padx=10, pady=5) + labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5) + entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5) + labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5) + #frameHelp + frameHelpListButtons.pack(side=RIGHT, padx=5, pady=5, fill=Y) + frameHelpList.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + scrollHelpList.pack(side=RIGHT, anchor=W, fill=Y) + self.listHelp.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) + self.buttonHelpListEdit.pack(side=TOP, anchor=W, pady=5) + self.buttonHelpListAdd.pack(side=TOP, anchor=W) + self.buttonHelpListRemove.pack(side=TOP, anchor=W, pady=5) + return frame + + def AttachVarCallbacks(self): + self.fontSize.trace_variable('w', self.VarChanged_font) + self.fontName.trace_variable('w', self.VarChanged_font) + self.fontBold.trace_variable('w', self.VarChanged_font) + self.spaceNum.trace_variable('w', self.VarChanged_spaceNum) + self.colour.trace_variable('w', self.VarChanged_colour) + self.builtinTheme.trace_variable('w', self.VarChanged_builtinTheme) + self.customTheme.trace_variable('w', self.VarChanged_customTheme) + self.themeIsBuiltin.trace_variable('w', self.VarChanged_themeIsBuiltin) + self.highlightTarget.trace_variable('w', self.VarChanged_highlightTarget) + self.keyBinding.trace_variable('w', self.VarChanged_keyBinding) + self.builtinKeys.trace_variable('w', self.VarChanged_builtinKeys) + self.customKeys.trace_variable('w', self.VarChanged_customKeys) + self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin) + self.winWidth.trace_variable('w', self.VarChanged_winWidth) + self.winHeight.trace_variable('w', self.VarChanged_winHeight) + self.startupEdit.trace_variable('w', self.VarChanged_startupEdit) + self.autoSave.trace_variable('w', self.VarChanged_autoSave) + self.encoding.trace_variable('w', self.VarChanged_encoding) + + def remove_var_callbacks(self): + "Remove callbacks to prevent memory leaks." + for var in ( + self.fontSize, self.fontName, self.fontBold, + self.spaceNum, self.colour, self.builtinTheme, + self.customTheme, self.themeIsBuiltin, self.highlightTarget, + self.keyBinding, self.builtinKeys, self.customKeys, + self.keysAreBuiltin, self.winWidth, self.winHeight, + self.startupEdit, self.autoSave, self.encoding,): + var.trace_vdelete('w', var.trace_vinfo()[0][1]) + + def VarChanged_font(self, *params): + '''When one font attribute changes, save them all, as they are + not independent from each other. In particular, when we are + overriding the default font, we need to write out everything. + ''' + value = self.fontName.get() + self.AddChangedItem('main', 'EditorWindow', 'font', value) + value = self.fontSize.get() + self.AddChangedItem('main', 'EditorWindow', 'font-size', value) + value = self.fontBold.get() + self.AddChangedItem('main', 'EditorWindow', 'font-bold', value) + + def VarChanged_spaceNum(self, *params): + value = self.spaceNum.get() + self.AddChangedItem('main', 'Indent', 'num-spaces', value) + + def VarChanged_colour(self, *params): + self.OnNewColourSet() + + def VarChanged_builtinTheme(self, *params): + value = self.builtinTheme.get() + if value == 'IDLE Dark': + if idleConf.GetOption('main', 'Theme', 'name') != 'IDLE New': + self.AddChangedItem('main', 'Theme', 'name', 'IDLE Classic') + self.AddChangedItem('main', 'Theme', 'name2', value) + self.new_custom_theme.config(text='New theme, see Help', + fg='#500000') + else: + self.AddChangedItem('main', 'Theme', 'name', value) + self.AddChangedItem('main', 'Theme', 'name2', '') + self.new_custom_theme.config(text='', fg='black') + self.PaintThemeSample() + + def VarChanged_customTheme(self, *params): + value = self.customTheme.get() + if value != '- no custom themes -': + self.AddChangedItem('main', 'Theme', 'name', value) + self.PaintThemeSample() + + def VarChanged_themeIsBuiltin(self, *params): + value = self.themeIsBuiltin.get() + self.AddChangedItem('main', 'Theme', 'default', value) + if value: + self.VarChanged_builtinTheme() + else: + self.VarChanged_customTheme() + + def VarChanged_highlightTarget(self, *params): + self.SetHighlightTarget() + + def VarChanged_keyBinding(self, *params): + value = self.keyBinding.get() + keySet = self.customKeys.get() + event = self.listBindings.get(ANCHOR).split()[0] + if idleConf.IsCoreBinding(event): + #this is a core keybinding + self.AddChangedItem('keys', keySet, event, value) + else: #this is an extension key binding + extName = idleConf.GetExtnNameForEvent(event) + extKeybindSection = extName + '_cfgBindings' + self.AddChangedItem('extensions', extKeybindSection, event, value) + + def VarChanged_builtinKeys(self, *params): + value = self.builtinKeys.get() + self.AddChangedItem('main', 'Keys', 'name', value) + self.LoadKeysList(value) + + def VarChanged_customKeys(self, *params): + value = self.customKeys.get() + if value != '- no custom keys -': + self.AddChangedItem('main', 'Keys', 'name', value) + self.LoadKeysList(value) + + def VarChanged_keysAreBuiltin(self, *params): + value = self.keysAreBuiltin.get() + self.AddChangedItem('main', 'Keys', 'default', value) + if value: + self.VarChanged_builtinKeys() + else: + self.VarChanged_customKeys() + + def VarChanged_winWidth(self, *params): + value = self.winWidth.get() + self.AddChangedItem('main', 'EditorWindow', 'width', value) + + def VarChanged_winHeight(self, *params): + value = self.winHeight.get() + self.AddChangedItem('main', 'EditorWindow', 'height', value) + + def VarChanged_startupEdit(self, *params): + value = self.startupEdit.get() + self.AddChangedItem('main', 'General', 'editor-on-startup', value) + + def VarChanged_autoSave(self, *params): + value = self.autoSave.get() + self.AddChangedItem('main', 'General', 'autosave', value) + + def VarChanged_encoding(self, *params): + value = self.encoding.get() + self.AddChangedItem('main', 'EditorWindow', 'encoding', value) + + def ResetChangedItems(self): + #When any config item is changed in this dialog, an entry + #should be made in the relevant section (config type) of this + #dictionary. The key should be the config file section name and the + #value a dictionary, whose key:value pairs are item=value pairs for + #that config file section. + self.changedItems = {'main':{}, 'highlight':{}, 'keys':{}, + 'extensions':{}} + + def AddChangedItem(self, typ, section, item, value): + value = str(value) #make sure we use a string + if section not in self.changedItems[typ]: + self.changedItems[typ][section] = {} + self.changedItems[typ][section][item] = value + + def GetDefaultItems(self): + dItems={'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} + for configType in dItems: + sections = idleConf.GetSectionList('default', configType) + for section in sections: + dItems[configType][section] = {} + options = idleConf.defaultCfg[configType].GetOptionList(section) + for option in options: + dItems[configType][section][option] = ( + idleConf.defaultCfg[configType].Get(section, option)) + return dItems + + def SetThemeType(self): + if self.themeIsBuiltin.get(): + self.optMenuThemeBuiltin.config(state=NORMAL) + self.optMenuThemeCustom.config(state=DISABLED) + self.buttonDeleteCustomTheme.config(state=DISABLED) + else: + self.optMenuThemeBuiltin.config(state=DISABLED) + self.radioThemeCustom.config(state=NORMAL) + self.optMenuThemeCustom.config(state=NORMAL) + self.buttonDeleteCustomTheme.config(state=NORMAL) + + def SetKeysType(self): + if self.keysAreBuiltin.get(): + self.optMenuKeysBuiltin.config(state=NORMAL) + self.optMenuKeysCustom.config(state=DISABLED) + self.buttonDeleteCustomKeys.config(state=DISABLED) + else: + self.optMenuKeysBuiltin.config(state=DISABLED) + self.radioKeysCustom.config(state=NORMAL) + self.optMenuKeysCustom.config(state=NORMAL) + self.buttonDeleteCustomKeys.config(state=NORMAL) + + def GetNewKeys(self): + listIndex = self.listBindings.index(ANCHOR) + binding = self.listBindings.get(listIndex) + bindName = binding.split()[0] #first part, up to first space + if self.keysAreBuiltin.get(): + currentKeySetName = self.builtinKeys.get() + else: + currentKeySetName = self.customKeys.get() + currentBindings = idleConf.GetCurrentKeySet() + if currentKeySetName in self.changedItems['keys']: #unsaved changes + keySetChanges = self.changedItems['keys'][currentKeySetName] + for event in keySetChanges: + currentBindings[event] = keySetChanges[event].split() + currentKeySequences = list(currentBindings.values()) + newKeys = GetKeysDialog(self, 'Get New Keys', bindName, + currentKeySequences).result + if newKeys: #new keys were specified + if self.keysAreBuiltin.get(): #current key set is a built-in + message = ('Your changes will be saved as a new Custom Key Set.' + ' Enter a name for your new Custom Key Set below.') + newKeySet = self.GetNewKeysName(message) + if not newKeySet: #user cancelled custom key set creation + self.listBindings.select_set(listIndex) + self.listBindings.select_anchor(listIndex) + return + else: #create new custom key set based on previously active key set + self.CreateNewKeySet(newKeySet) + self.listBindings.delete(listIndex) + self.listBindings.insert(listIndex, bindName+' - '+newKeys) + self.listBindings.select_set(listIndex) + self.listBindings.select_anchor(listIndex) + self.keyBinding.set(newKeys) + else: + self.listBindings.select_set(listIndex) + self.listBindings.select_anchor(listIndex) + + def GetNewKeysName(self, message): + usedNames = (idleConf.GetSectionList('user', 'keys') + + idleConf.GetSectionList('default', 'keys')) + newKeySet = GetCfgSectionNameDialog( + self, 'New Custom Key Set', message, usedNames).result + return newKeySet + + def SaveAsNewKeySet(self): + newKeysName = self.GetNewKeysName('New Key Set Name:') + if newKeysName: + self.CreateNewKeySet(newKeysName) + + def KeyBindingSelected(self, event): + self.buttonNewKeys.config(state=NORMAL) + + def CreateNewKeySet(self, newKeySetName): + #creates new custom key set based on the previously active key set, + #and makes the new key set active + if self.keysAreBuiltin.get(): + prevKeySetName = self.builtinKeys.get() + else: + prevKeySetName = self.customKeys.get() + prevKeys = idleConf.GetCoreKeys(prevKeySetName) + newKeys = {} + for event in prevKeys: #add key set to changed items + eventName = event[2:-2] #trim off the angle brackets + binding = ' '.join(prevKeys[event]) + newKeys[eventName] = binding + #handle any unsaved changes to prev key set + if prevKeySetName in self.changedItems['keys']: + keySetChanges = self.changedItems['keys'][prevKeySetName] + for event in keySetChanges: + newKeys[event] = keySetChanges[event] + #save the new theme + self.SaveNewKeySet(newKeySetName, newKeys) + #change gui over to the new key set + customKeyList = idleConf.GetSectionList('user', 'keys') + customKeyList.sort() + self.optMenuKeysCustom.SetMenu(customKeyList, newKeySetName) + self.keysAreBuiltin.set(0) + self.SetKeysType() + + def LoadKeysList(self, keySetName): + reselect = 0 + newKeySet = 0 + if self.listBindings.curselection(): + reselect = 1 + listIndex = self.listBindings.index(ANCHOR) + keySet = idleConf.GetKeySet(keySetName) + bindNames = list(keySet.keys()) + bindNames.sort() + self.listBindings.delete(0, END) + for bindName in bindNames: + key = ' '.join(keySet[bindName]) #make key(s) into a string + bindName = bindName[2:-2] #trim off the angle brackets + if keySetName in self.changedItems['keys']: + #handle any unsaved changes to this key set + if bindName in self.changedItems['keys'][keySetName]: + key = self.changedItems['keys'][keySetName][bindName] + self.listBindings.insert(END, bindName+' - '+key) + if reselect: + self.listBindings.see(listIndex) + self.listBindings.select_set(listIndex) + self.listBindings.select_anchor(listIndex) + + def DeleteCustomKeys(self): + keySetName=self.customKeys.get() + delmsg = 'Are you sure you wish to delete the key set %r ?' + if not tkMessageBox.askyesno( + 'Delete Key Set', delmsg % keySetName, parent=self): + return + #remove key set from config + idleConf.userCfg['keys'].remove_section(keySetName) + if keySetName in self.changedItems['keys']: + del(self.changedItems['keys'][keySetName]) + #write changes + idleConf.userCfg['keys'].Save() + #reload user key set list + itemList = idleConf.GetSectionList('user', 'keys') + itemList.sort() + if not itemList: + self.radioKeysCustom.config(state=DISABLED) + self.optMenuKeysCustom.SetMenu(itemList, '- no custom keys -') + else: + self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) + #revert to default key set + self.keysAreBuiltin.set(idleConf.defaultCfg['main'].Get('Keys', 'default')) + self.builtinKeys.set(idleConf.defaultCfg['main'].Get('Keys', 'name')) + #user can't back out of these changes, they must be applied now + self.Apply() + self.SetKeysType() + + def DeleteCustomTheme(self): + themeName = self.customTheme.get() + delmsg = 'Are you sure you wish to delete the theme %r ?' + if not tkMessageBox.askyesno( + 'Delete Theme', delmsg % themeName, parent=self): + return + #remove theme from config + idleConf.userCfg['highlight'].remove_section(themeName) + if themeName in self.changedItems['highlight']: + del(self.changedItems['highlight'][themeName]) + #write changes + idleConf.userCfg['highlight'].Save() + #reload user theme list + itemList = idleConf.GetSectionList('user', 'highlight') + itemList.sort() + if not itemList: + self.radioThemeCustom.config(state=DISABLED) + self.optMenuThemeCustom.SetMenu(itemList, '- no custom themes -') + else: + self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) + #revert to default theme + self.themeIsBuiltin.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) + self.builtinTheme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) + #user can't back out of these changes, they must be applied now + self.Apply() + self.SetThemeType() + + def GetColour(self): + target = self.highlightTarget.get() + prevColour = self.frameColourSet.cget('bg') + rgbTuplet, colourString = tkColorChooser.askcolor( + parent=self, title='Pick new colour for : '+target, + initialcolor=prevColour) + if colourString and (colourString != prevColour): + #user didn't cancel, and they chose a new colour + if self.themeIsBuiltin.get(): #current theme is a built-in + message = ('Your changes will be saved as a new Custom Theme. ' + 'Enter a name for your new Custom Theme below.') + newTheme = self.GetNewThemeName(message) + if not newTheme: #user cancelled custom theme creation + return + else: #create new custom theme based on previously active theme + self.CreateNewTheme(newTheme) + self.colour.set(colourString) + else: #current theme is user defined + self.colour.set(colourString) + + def OnNewColourSet(self): + newColour=self.colour.get() + self.frameColourSet.config(bg=newColour) #set sample + plane ='foreground' if self.fgHilite.get() else 'background' + sampleElement = self.themeElements[self.highlightTarget.get()][0] + self.textHighlightSample.tag_config(sampleElement, **{plane:newColour}) + theme = self.customTheme.get() + themeElement = sampleElement + '-' + plane + self.AddChangedItem('highlight', theme, themeElement, newColour) + + def GetNewThemeName(self, message): + usedNames = (idleConf.GetSectionList('user', 'highlight') + + idleConf.GetSectionList('default', 'highlight')) + newTheme = GetCfgSectionNameDialog( + self, 'New Custom Theme', message, usedNames).result + return newTheme + + def SaveAsNewTheme(self): + newThemeName = self.GetNewThemeName('New Theme Name:') + if newThemeName: + self.CreateNewTheme(newThemeName) + + def CreateNewTheme(self, newThemeName): + #creates new custom theme based on the previously active theme, + #and makes the new theme active + if self.themeIsBuiltin.get(): + themeType = 'default' + themeName = self.builtinTheme.get() + else: + themeType = 'user' + themeName = self.customTheme.get() + newTheme = idleConf.GetThemeDict(themeType, themeName) + #apply any of the old theme's unsaved changes to the new theme + if themeName in self.changedItems['highlight']: + themeChanges = self.changedItems['highlight'][themeName] + for element in themeChanges: + newTheme[element] = themeChanges[element] + #save the new theme + self.SaveNewTheme(newThemeName, newTheme) + #change gui over to the new theme + customThemeList = idleConf.GetSectionList('user', 'highlight') + customThemeList.sort() + self.optMenuThemeCustom.SetMenu(customThemeList, newThemeName) + self.themeIsBuiltin.set(0) + self.SetThemeType() + + def OnListFontButtonRelease(self, event): + font = self.listFontName.get(ANCHOR) + self.fontName.set(font.lower()) + self.SetFontSample() + + def SetFontSample(self, event=None): + fontName = self.fontName.get() + fontWeight = tkFont.BOLD if self.fontBold.get() else tkFont.NORMAL + newFont = (fontName, self.fontSize.get(), fontWeight) + self.labelFontSample.config(font=newFont) + self.textHighlightSample.configure(font=newFont) + + def SetHighlightTarget(self): + if self.highlightTarget.get() == 'Cursor': #bg not possible + self.radioFg.config(state=DISABLED) + self.radioBg.config(state=DISABLED) + self.fgHilite.set(1) + else: #both fg and bg can be set + self.radioFg.config(state=NORMAL) + self.radioBg.config(state=NORMAL) + self.fgHilite.set(1) + self.SetColourSample() + + def SetColourSampleBinding(self, *args): + self.SetColourSample() + + def SetColourSample(self): + #set the colour smaple area + tag = self.themeElements[self.highlightTarget.get()][0] + plane = 'foreground' if self.fgHilite.get() else 'background' + colour = self.textHighlightSample.tag_cget(tag, plane) + self.frameColourSet.config(bg=colour) + + def PaintThemeSample(self): + if self.themeIsBuiltin.get(): #a default theme + theme = self.builtinTheme.get() + else: #a user theme + theme = self.customTheme.get() + for elementTitle in self.themeElements: + element = self.themeElements[elementTitle][0] + colours = idleConf.GetHighlight(theme, element) + if element == 'cursor': #cursor sample needs special painting + colours['background'] = idleConf.GetHighlight( + theme, 'normal', fgBg='bg') + #handle any unsaved changes to this theme + if theme in self.changedItems['highlight']: + themeDict = self.changedItems['highlight'][theme] + if element + '-foreground' in themeDict: + colours['foreground'] = themeDict[element + '-foreground'] + if element + '-background' in themeDict: + colours['background'] = themeDict[element + '-background'] + self.textHighlightSample.tag_config(element, **colours) + self.SetColourSample() + + def HelpSourceSelected(self, event): + self.SetHelpListButtonStates() + + def SetHelpListButtonStates(self): + if self.listHelp.size() < 1: #no entries in list + self.buttonHelpListEdit.config(state=DISABLED) + self.buttonHelpListRemove.config(state=DISABLED) + else: #there are some entries + if self.listHelp.curselection(): #there currently is a selection + self.buttonHelpListEdit.config(state=NORMAL) + self.buttonHelpListRemove.config(state=NORMAL) + else: #there currently is not a selection + self.buttonHelpListEdit.config(state=DISABLED) + self.buttonHelpListRemove.config(state=DISABLED) + + def HelpListItemAdd(self): + helpSource = GetHelpSourceDialog(self, 'New Help Source').result + if helpSource: + self.userHelpList.append((helpSource[0], helpSource[1])) + self.listHelp.insert(END, helpSource[0]) + self.UpdateUserHelpChangedItems() + self.SetHelpListButtonStates() + + def HelpListItemEdit(self): + itemIndex = self.listHelp.index(ANCHOR) + helpSource = self.userHelpList[itemIndex] + newHelpSource = GetHelpSourceDialog( + self, 'Edit Help Source', menuItem=helpSource[0], + filePath=helpSource[1]).result + if (not newHelpSource) or (newHelpSource == helpSource): + return #no changes + self.userHelpList[itemIndex] = newHelpSource + self.listHelp.delete(itemIndex) + self.listHelp.insert(itemIndex, newHelpSource[0]) + self.UpdateUserHelpChangedItems() + self.SetHelpListButtonStates() + + def HelpListItemRemove(self): + itemIndex = self.listHelp.index(ANCHOR) + del(self.userHelpList[itemIndex]) + self.listHelp.delete(itemIndex) + self.UpdateUserHelpChangedItems() + self.SetHelpListButtonStates() + + def UpdateUserHelpChangedItems(self): + "Clear and rebuild the HelpFiles section in self.changedItems" + self.changedItems['main']['HelpFiles'] = {} + for num in range(1, len(self.userHelpList) + 1): + self.AddChangedItem( + 'main', 'HelpFiles', str(num), + ';'.join(self.userHelpList[num-1][:2])) + + def LoadFontCfg(self): + ##base editor font selection list + fonts = list(tkFont.families(self)) + fonts.sort() + for font in fonts: + self.listFontName.insert(END, font) + configuredFont = idleConf.GetFont(self, 'main', 'EditorWindow') + fontName = configuredFont[0].lower() + fontSize = configuredFont[1] + fontBold = configuredFont[2]=='bold' + self.fontName.set(fontName) + lc_fonts = [s.lower() for s in fonts] + try: + currentFontIndex = lc_fonts.index(fontName) + self.listFontName.see(currentFontIndex) + self.listFontName.select_set(currentFontIndex) + self.listFontName.select_anchor(currentFontIndex) + except ValueError: + pass + ##font size dropdown + self.optMenuFontSize.SetMenu(('7', '8', '9', '10', '11', '12', '13', + '14', '16', '18', '20', '22'), fontSize ) + ##fontWeight + self.fontBold.set(fontBold) + ##font sample + self.SetFontSample() + + def LoadTabCfg(self): + ##indent sizes + spaceNum = idleConf.GetOption( + 'main', 'Indent', 'num-spaces', default=4, type='int') + self.spaceNum.set(spaceNum) + + def LoadThemeCfg(self): + ##current theme type radiobutton + self.themeIsBuiltin.set(idleConf.GetOption( + 'main', 'Theme', 'default', type='bool', default=1)) + ##currently set theme + currentOption = idleConf.CurrentTheme() + ##load available theme option menus + if self.themeIsBuiltin.get(): #default theme selected + itemList = idleConf.GetSectionList('default', 'highlight') + itemList.sort() + self.optMenuThemeBuiltin.SetMenu(itemList, currentOption) + itemList = idleConf.GetSectionList('user', 'highlight') + itemList.sort() + if not itemList: + self.radioThemeCustom.config(state=DISABLED) + self.customTheme.set('- no custom themes -') + else: + self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) + else: #user theme selected + itemList = idleConf.GetSectionList('user', 'highlight') + itemList.sort() + self.optMenuThemeCustom.SetMenu(itemList, currentOption) + itemList = idleConf.GetSectionList('default', 'highlight') + itemList.sort() + self.optMenuThemeBuiltin.SetMenu(itemList, itemList[0]) + self.SetThemeType() + ##load theme element option menu + themeNames = list(self.themeElements.keys()) + themeNames.sort(key=lambda x: self.themeElements[x][1]) + self.optMenuHighlightTarget.SetMenu(themeNames, themeNames[0]) + self.PaintThemeSample() + self.SetHighlightTarget() + + def LoadKeyCfg(self): + ##current keys type radiobutton + self.keysAreBuiltin.set(idleConf.GetOption( + 'main', 'Keys', 'default', type='bool', default=1)) + ##currently set keys + currentOption = idleConf.CurrentKeys() + ##load available keyset option menus + if self.keysAreBuiltin.get(): #default theme selected + itemList = idleConf.GetSectionList('default', 'keys') + itemList.sort() + self.optMenuKeysBuiltin.SetMenu(itemList, currentOption) + itemList = idleConf.GetSectionList('user', 'keys') + itemList.sort() + if not itemList: + self.radioKeysCustom.config(state=DISABLED) + self.customKeys.set('- no custom keys -') + else: + self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) + else: #user key set selected + itemList = idleConf.GetSectionList('user', 'keys') + itemList.sort() + self.optMenuKeysCustom.SetMenu(itemList, currentOption) + itemList = idleConf.GetSectionList('default', 'keys') + itemList.sort() + self.optMenuKeysBuiltin.SetMenu(itemList, itemList[0]) + self.SetKeysType() + ##load keyset element list + keySetName = idleConf.CurrentKeys() + self.LoadKeysList(keySetName) + + def LoadGeneralCfg(self): + #startup state + self.startupEdit.set(idleConf.GetOption( + 'main', 'General', 'editor-on-startup', default=1, type='bool')) + #autosave state + self.autoSave.set(idleConf.GetOption( + 'main', 'General', 'autosave', default=0, type='bool')) + #initial window size + self.winWidth.set(idleConf.GetOption( + 'main', 'EditorWindow', 'width', type='int')) + self.winHeight.set(idleConf.GetOption( + 'main', 'EditorWindow', 'height', type='int')) + # default source encoding + self.encoding.set(idleConf.GetOption( + 'main', 'EditorWindow', 'encoding', default='none')) + # additional help sources + self.userHelpList = idleConf.GetAllExtraHelpSourcesList() + for helpItem in self.userHelpList: + self.listHelp.insert(END, helpItem[0]) + self.SetHelpListButtonStates() + + def LoadConfigs(self): + """ + load configuration from default and user config files and populate + the widgets on the config dialog pages. + """ + ### fonts / tabs page + self.LoadFontCfg() + self.LoadTabCfg() + ### highlighting page + self.LoadThemeCfg() + ### keys page + self.LoadKeyCfg() + ### general page + self.LoadGeneralCfg() + # note: extension page handled separately + + def SaveNewKeySet(self, keySetName, keySet): + """ + save a newly created core key set. + keySetName - string, the name of the new key set + keySet - dictionary containing the new key set + """ + if not idleConf.userCfg['keys'].has_section(keySetName): + idleConf.userCfg['keys'].add_section(keySetName) + for event in keySet: + value = keySet[event] + idleConf.userCfg['keys'].SetOption(keySetName, event, value) + + def SaveNewTheme(self, themeName, theme): + """ + save a newly created theme. + themeName - string, the name of the new theme + theme - dictionary containing the new theme + """ + if not idleConf.userCfg['highlight'].has_section(themeName): + idleConf.userCfg['highlight'].add_section(themeName) + for element in theme: + value = theme[element] + idleConf.userCfg['highlight'].SetOption(themeName, element, value) + + def SetUserValue(self, configType, section, item, value): + if idleConf.defaultCfg[configType].has_option(section, item): + if idleConf.defaultCfg[configType].Get(section, item) == value: + #the setting equals a default setting, remove it from user cfg + return idleConf.userCfg[configType].RemoveOption(section, item) + #if we got here set the option + return idleConf.userCfg[configType].SetOption(section, item, value) + + def SaveAllChangedConfigs(self): + "Save configuration changes to the user config file." + idleConf.userCfg['main'].Save() + for configType in self.changedItems: + cfgTypeHasChanges = False + for section in self.changedItems[configType]: + if section == 'HelpFiles': + #this section gets completely replaced + idleConf.userCfg['main'].remove_section('HelpFiles') + cfgTypeHasChanges = True + for item in self.changedItems[configType][section]: + value = self.changedItems[configType][section][item] + if self.SetUserValue(configType, section, item, value): + cfgTypeHasChanges = True + if cfgTypeHasChanges: + idleConf.userCfg[configType].Save() + for configType in ['keys', 'highlight']: + # save these even if unchanged! + idleConf.userCfg[configType].Save() + self.ResetChangedItems() #clear the changed items dict + self.save_all_changed_extensions() # uses a different mechanism + + def DeactivateCurrentConfig(self): + #Before a config is saved, some cleanup of current + #config must be done - remove the previous keybindings + winInstances = self.parent.instance_dict.keys() + for instance in winInstances: + instance.RemoveKeybindings() + + def ActivateConfigChanges(self): + "Dynamically apply configuration changes" + winInstances = self.parent.instance_dict.keys() + for instance in winInstances: + instance.ResetColorizer() + instance.ResetFont() + instance.set_notabs_indentwidth() + instance.ApplyKeybindings() + instance.reset_help_menu_entries() + + def Cancel(self): + self.destroy() + + def Ok(self): + self.Apply() + self.destroy() + + def Apply(self): + self.DeactivateCurrentConfig() + self.SaveAllChangedConfigs() + self.ActivateConfigChanges() + + def Help(self): + page = self.tabPages._current_page + view_text(self, title='Help for IDLE preferences', + text=help_common+help_pages.get(page, '')) + + def CreatePageExtensions(self): + """Part of the config dialog used for configuring IDLE extensions. + + This code is generic - it works for any and all IDLE extensions. + + IDLE extensions save their configuration options using idleConf. + This code reads the current configuration using idleConf, supplies a + GUI interface to change the configuration values, and saves the + changes using idleConf. + + Not all changes take effect immediately - some may require restarting IDLE. + This depends on each extension's implementation. + + All values are treated as text, and it is up to the user to supply + reasonable values. The only exception to this are the 'enable*' options, + which are boolean, and can be toggled with a True/False button. + """ + parent = self.parent + frame = self.tabPages.pages['Extensions'].frame + self.ext_defaultCfg = idleConf.defaultCfg['extensions'] + self.ext_userCfg = idleConf.userCfg['extensions'] + self.is_int = self.register(is_int) + self.load_extensions() + # create widgets - a listbox shows all available extensions, with the + # controls for the extension selected in the listbox to the right + self.extension_names = StringVar(self) + frame.rowconfigure(0, weight=1) + frame.columnconfigure(2, weight=1) + self.extension_list = Listbox(frame, listvariable=self.extension_names, + selectmode='browse') + self.extension_list.bind('<>', self.extension_selected) + scroll = Scrollbar(frame, command=self.extension_list.yview) + self.extension_list.yscrollcommand=scroll.set + self.details_frame = LabelFrame(frame, width=250, height=250) + self.extension_list.grid(column=0, row=0, sticky='nws') + scroll.grid(column=1, row=0, sticky='ns') + self.details_frame.grid(column=2, row=0, sticky='nsew', padx=[10, 0]) + frame.configure(padx=10, pady=10) + self.config_frame = {} + self.current_extension = None + + self.outerframe = self # TEMPORARY + self.tabbed_page_set = self.extension_list # TEMPORARY + + # create the frame holding controls for each extension + ext_names = '' + for ext_name in sorted(self.extensions): + self.create_extension_frame(ext_name) + ext_names = ext_names + '{' + ext_name + '} ' + self.extension_names.set(ext_names) + self.extension_list.selection_set(0) + self.extension_selected(None) + + def load_extensions(self): + "Fill self.extensions with data from the default and user configs." + self.extensions = {} + for ext_name in idleConf.GetExtensions(active_only=False): + self.extensions[ext_name] = [] + + for ext_name in self.extensions: + opt_list = sorted(self.ext_defaultCfg.GetOptionList(ext_name)) + + # bring 'enable' options to the beginning of the list + enables = [opt_name for opt_name in opt_list + if opt_name.startswith('enable')] + for opt_name in enables: + opt_list.remove(opt_name) + opt_list = enables + opt_list + + for opt_name in opt_list: + def_str = self.ext_defaultCfg.Get( + ext_name, opt_name, raw=True) + try: + def_obj = {'True':True, 'False':False}[def_str] + opt_type = 'bool' + except KeyError: + try: + def_obj = int(def_str) + opt_type = 'int' + except ValueError: + def_obj = def_str + opt_type = None + try: + value = self.ext_userCfg.Get( + ext_name, opt_name, type=opt_type, raw=True, + default=def_obj) + except ValueError: # Need this until .Get fixed + value = def_obj # bad values overwritten by entry + var = StringVar(self) + var.set(str(value)) + + self.extensions[ext_name].append({'name': opt_name, + 'type': opt_type, + 'default': def_str, + 'value': value, + 'var': var, + }) + + def extension_selected(self, event): + newsel = self.extension_list.curselection() + if newsel: + newsel = self.extension_list.get(newsel) + if newsel is None or newsel != self.current_extension: + if self.current_extension: + self.details_frame.config(text='') + self.config_frame[self.current_extension].grid_forget() + self.current_extension = None + if newsel: + self.details_frame.config(text=newsel) + self.config_frame[newsel].grid(column=0, row=0, sticky='nsew') + self.current_extension = newsel + + def create_extension_frame(self, ext_name): + """Create a frame holding the widgets to configure one extension""" + f = VerticalScrolledFrame(self.details_frame, height=250, width=250) + self.config_frame[ext_name] = f + entry_area = f.interior + # create an entry for each configuration option + for row, opt in enumerate(self.extensions[ext_name]): + # create a row with a label and entry/checkbutton + label = Label(entry_area, text=opt['name']) + label.grid(row=row, column=0, sticky=NW) + var = opt['var'] + if opt['type'] == 'bool': + Checkbutton(entry_area, textvariable=var, variable=var, + onvalue='True', offvalue='False', + indicatoron=FALSE, selectcolor='', width=8 + ).grid(row=row, column=1, sticky=W, padx=7) + elif opt['type'] == 'int': + Entry(entry_area, textvariable=var, validate='key', + validatecommand=(self.is_int, '%P') + ).grid(row=row, column=1, sticky=NSEW, padx=7) + + else: + Entry(entry_area, textvariable=var + ).grid(row=row, column=1, sticky=NSEW, padx=7) + return + + def set_extension_value(self, section, opt): + name = opt['name'] + default = opt['default'] + value = opt['var'].get().strip() or default + opt['var'].set(value) + # if self.defaultCfg.has_section(section): + # Currently, always true; if not, indent to return + if (value == default): + return self.ext_userCfg.RemoveOption(section, name) + # set the option + return self.ext_userCfg.SetOption(section, name, value) + + def save_all_changed_extensions(self): + """Save configuration changes to the user config file.""" + has_changes = False + for ext_name in self.extensions: + options = self.extensions[ext_name] + for opt in options: + if self.set_extension_value(ext_name, opt): + has_changes = True + if has_changes: + self.ext_userCfg.Save() + + +help_common = '''\ +When you click either the Apply or Ok buttons, settings in this +dialog that are different from IDLE's default are saved in +a .idlerc directory in your home directory. Except as noted, +these changes apply to all versions of IDLE installed on this +machine. Some do not take affect until IDLE is restarted. +[Cancel] only cancels changes made since the last save. +''' +help_pages = { + 'Highlighting':''' +Highlighting: +The IDLE Dark color theme is new in October 2015. It can only +be used with older IDLE releases if it is saved as a custom +theme, with a different name. +''' +} + + +def is_int(s): + "Return 's is blank or represents an int'" + if not s: + return True + try: + int(s) + return True + except ValueError: + return False + + +class VerticalScrolledFrame(Frame): + """A pure Tkinter vertically scrollable frame. + + * Use the 'interior' attribute to place widgets inside the scrollable frame + * Construct and pack/place/grid normally + * This frame only allows vertical scrolling + """ + def __init__(self, parent, *args, **kw): + Frame.__init__(self, parent, *args, **kw) + + # create a canvas object and a vertical scrollbar for scrolling it + vscrollbar = Scrollbar(self, orient=VERTICAL) + vscrollbar.pack(fill=Y, side=RIGHT, expand=FALSE) + canvas = Canvas(self, bd=0, highlightthickness=0, + yscrollcommand=vscrollbar.set, width=240) + canvas.pack(side=LEFT, fill=BOTH, expand=TRUE) + vscrollbar.config(command=canvas.yview) + + # reset the view + canvas.xview_moveto(0) + canvas.yview_moveto(0) + + # create a frame inside the canvas which will be scrolled with it + self.interior = interior = Frame(canvas) + interior_id = canvas.create_window(0, 0, window=interior, anchor=NW) + + # track changes to the canvas and frame width and sync them, + # also updating the scrollbar + def _configure_interior(event): + # update the scrollbars to match the size of the inner frame + size = (interior.winfo_reqwidth(), interior.winfo_reqheight()) + canvas.config(scrollregion="0 0 %s %s" % size) + interior.bind('', _configure_interior) + + def _configure_canvas(event): + if interior.winfo_reqwidth() != canvas.winfo_width(): + # update the inner frame's width to fill the canvas + canvas.itemconfigure(interior_id, width=canvas.winfo_width()) + canvas.bind('', _configure_canvas) + + return + + +if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_configdialog', + verbosity=2, exit=False) + from idlelib.idle_test.htest import run + run(ConfigDialog) diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py new file mode 100644 index 0000000..d5e217d --- /dev/null +++ b/Lib/idlelib/debugger.py @@ -0,0 +1,539 @@ +import os +import bdb +from tkinter import * +from idlelib.WindowList import ListedToplevel +from idlelib.ScrolledList import ScrolledList +from idlelib import macosxSupport + + +class Idb(bdb.Bdb): + + def __init__(self, gui): + self.gui = gui + bdb.Bdb.__init__(self) + + def user_line(self, frame): + if self.in_rpc_code(frame): + self.set_step() + return + message = self.__frame2message(frame) + try: + self.gui.interaction(message, frame) + except TclError: # When closing debugger window with [x] in 3.x + pass + + def user_exception(self, frame, info): + if self.in_rpc_code(frame): + self.set_step() + return + message = self.__frame2message(frame) + self.gui.interaction(message, frame, info) + + def in_rpc_code(self, frame): + if frame.f_code.co_filename.count('rpc.py'): + return True + else: + prev_frame = frame.f_back + if prev_frame.f_code.co_filename.count('Debugger.py'): + # (that test will catch both Debugger.py and RemoteDebugger.py) + return False + return self.in_rpc_code(prev_frame) + + def __frame2message(self, frame): + code = frame.f_code + filename = code.co_filename + lineno = frame.f_lineno + basename = os.path.basename(filename) + message = "%s:%s" % (basename, lineno) + if code.co_name != "?": + message = "%s: %s()" % (message, code.co_name) + return message + + +class Debugger: + + vstack = vsource = vlocals = vglobals = None + + def __init__(self, pyshell, idb=None): + if idb is None: + idb = Idb(self) + self.pyshell = pyshell + self.idb = idb + self.frame = None + self.make_gui() + self.interacting = 0 + self.nesting_level = 0 + + def run(self, *args): + # Deal with the scenario where we've already got a program running + # in the debugger and we want to start another. If that is the case, + # our second 'run' was invoked from an event dispatched not from + # the main event loop, but from the nested event loop in 'interaction' + # below. So our stack looks something like this: + # outer main event loop + # run() + # + # callback to debugger's interaction() + # nested event loop + # run() for second command + # + # This kind of nesting of event loops causes all kinds of problems + # (see e.g. issue #24455) especially when dealing with running as a + # subprocess, where there's all kinds of extra stuff happening in + # there - insert a traceback.print_stack() to check it out. + # + # By this point, we've already called restart_subprocess() in + # ScriptBinding. However, we also need to unwind the stack back to + # that outer event loop. To accomplish this, we: + # - return immediately from the nested run() + # - abort_loop ensures the nested event loop will terminate + # - the debugger's interaction routine completes normally + # - the restart_subprocess() will have taken care of stopping + # the running program, which will also let the outer run complete + # + # That leaves us back at the outer main event loop, at which point our + # after event can fire, and we'll come back to this routine with a + # clean stack. + if self.nesting_level > 0: + self.abort_loop() + self.root.after(100, lambda: self.run(*args)) + return + try: + self.interacting = 1 + return self.idb.run(*args) + finally: + self.interacting = 0 + + def close(self, event=None): + try: + self.quit() + except Exception: + pass + if self.interacting: + self.top.bell() + return + if self.stackviewer: + self.stackviewer.close(); self.stackviewer = None + # Clean up pyshell if user clicked debugger control close widget. + # (Causes a harmless extra cycle through close_debugger() if user + # toggled debugger from pyshell Debug menu) + self.pyshell.close_debugger() + # Now close the debugger control window.... + self.top.destroy() + + def make_gui(self): + pyshell = self.pyshell + self.flist = pyshell.flist + self.root = root = pyshell.root + self.top = top = ListedToplevel(root) + self.top.wm_title("Debug Control") + self.top.wm_iconname("Debug") + top.wm_protocol("WM_DELETE_WINDOW", self.close) + self.top.bind("", self.close) + # + self.bframe = bframe = Frame(top) + self.bframe.pack(anchor="w") + self.buttons = bl = [] + # + self.bcont = b = Button(bframe, text="Go", command=self.cont) + bl.append(b) + self.bstep = b = Button(bframe, text="Step", command=self.step) + bl.append(b) + self.bnext = b = Button(bframe, text="Over", command=self.next) + bl.append(b) + self.bret = b = Button(bframe, text="Out", command=self.ret) + bl.append(b) + self.bret = b = Button(bframe, text="Quit", command=self.quit) + bl.append(b) + # + for b in bl: + b.configure(state="disabled") + b.pack(side="left") + # + self.cframe = cframe = Frame(bframe) + self.cframe.pack(side="left") + # + if not self.vstack: + self.__class__.vstack = BooleanVar(top) + self.vstack.set(1) + self.bstack = Checkbutton(cframe, + text="Stack", command=self.show_stack, variable=self.vstack) + self.bstack.grid(row=0, column=0) + if not self.vsource: + self.__class__.vsource = BooleanVar(top) + self.bsource = Checkbutton(cframe, + text="Source", command=self.show_source, variable=self.vsource) + self.bsource.grid(row=0, column=1) + if not self.vlocals: + self.__class__.vlocals = BooleanVar(top) + self.vlocals.set(1) + self.blocals = Checkbutton(cframe, + text="Locals", command=self.show_locals, variable=self.vlocals) + self.blocals.grid(row=1, column=0) + if not self.vglobals: + self.__class__.vglobals = BooleanVar(top) + self.bglobals = Checkbutton(cframe, + text="Globals", command=self.show_globals, variable=self.vglobals) + self.bglobals.grid(row=1, column=1) + # + self.status = Label(top, anchor="w") + self.status.pack(anchor="w") + self.error = Label(top, anchor="w") + self.error.pack(anchor="w", fill="x") + self.errorbg = self.error.cget("background") + # + self.fstack = Frame(top, height=1) + self.fstack.pack(expand=1, fill="both") + self.flocals = Frame(top) + self.flocals.pack(expand=1, fill="both") + self.fglobals = Frame(top, height=1) + self.fglobals.pack(expand=1, fill="both") + # + if self.vstack.get(): + self.show_stack() + if self.vlocals.get(): + self.show_locals() + if self.vglobals.get(): + self.show_globals() + + def interaction(self, message, frame, info=None): + self.frame = frame + self.status.configure(text=message) + # + if info: + type, value, tb = info + try: + m1 = type.__name__ + except AttributeError: + m1 = "%s" % str(type) + if value is not None: + try: + m1 = "%s: %s" % (m1, str(value)) + except: + pass + bg = "yellow" + else: + m1 = "" + tb = None + bg = self.errorbg + self.error.configure(text=m1, background=bg) + # + sv = self.stackviewer + if sv: + stack, i = self.idb.get_stack(self.frame, tb) + sv.load_stack(stack, i) + # + self.show_variables(1) + # + if self.vsource.get(): + self.sync_source_line() + # + for b in self.buttons: + b.configure(state="normal") + # + self.top.wakeup() + # Nested main loop: Tkinter's main loop is not reentrant, so use + # Tcl's vwait facility, which reenters the event loop until an + # event handler sets the variable we're waiting on + self.nesting_level += 1 + self.root.tk.call('vwait', '::idledebugwait') + self.nesting_level -= 1 + # + for b in self.buttons: + b.configure(state="disabled") + self.status.configure(text="") + self.error.configure(text="", background=self.errorbg) + self.frame = None + + def sync_source_line(self): + frame = self.frame + if not frame: + return + filename, lineno = self.__frame2fileline(frame) + if filename[:1] + filename[-1:] != "<>" and os.path.exists(filename): + self.flist.gotofileline(filename, lineno) + + def __frame2fileline(self, frame): + code = frame.f_code + filename = code.co_filename + lineno = frame.f_lineno + return filename, lineno + + def cont(self): + self.idb.set_continue() + self.abort_loop() + + def step(self): + self.idb.set_step() + self.abort_loop() + + def next(self): + self.idb.set_next(self.frame) + self.abort_loop() + + def ret(self): + self.idb.set_return(self.frame) + self.abort_loop() + + def quit(self): + self.idb.set_quit() + self.abort_loop() + + def abort_loop(self): + self.root.tk.call('set', '::idledebugwait', '1') + + stackviewer = None + + def show_stack(self): + if not self.stackviewer and self.vstack.get(): + self.stackviewer = sv = StackViewer(self.fstack, self.flist, self) + if self.frame: + stack, i = self.idb.get_stack(self.frame, None) + sv.load_stack(stack, i) + else: + sv = self.stackviewer + if sv and not self.vstack.get(): + self.stackviewer = None + sv.close() + self.fstack['height'] = 1 + + def show_source(self): + if self.vsource.get(): + self.sync_source_line() + + def show_frame(self, stackitem): + self.frame = stackitem[0] # lineno is stackitem[1] + self.show_variables() + + localsviewer = None + globalsviewer = None + + def show_locals(self): + lv = self.localsviewer + if self.vlocals.get(): + if not lv: + self.localsviewer = NamespaceViewer(self.flocals, "Locals") + else: + if lv: + self.localsviewer = None + lv.close() + self.flocals['height'] = 1 + self.show_variables() + + def show_globals(self): + gv = self.globalsviewer + if self.vglobals.get(): + if not gv: + self.globalsviewer = NamespaceViewer(self.fglobals, "Globals") + else: + if gv: + self.globalsviewer = None + gv.close() + self.fglobals['height'] = 1 + self.show_variables() + + def show_variables(self, force=0): + lv = self.localsviewer + gv = self.globalsviewer + frame = self.frame + if not frame: + ldict = gdict = None + else: + ldict = frame.f_locals + gdict = frame.f_globals + if lv and gv and ldict is gdict: + ldict = None + if lv: + lv.load_dict(ldict, force, self.pyshell.interp.rpcclt) + if gv: + gv.load_dict(gdict, force, self.pyshell.interp.rpcclt) + + def set_breakpoint_here(self, filename, lineno): + self.idb.set_break(filename, lineno) + + def clear_breakpoint_here(self, filename, lineno): + self.idb.clear_break(filename, lineno) + + def clear_file_breaks(self, filename): + self.idb.clear_all_file_breaks(filename) + + def load_breakpoints(self): + "Load PyShellEditorWindow breakpoints into subprocess debugger" + for editwin in self.pyshell.flist.inversedict: + filename = editwin.io.filename + try: + for lineno in editwin.breakpoints: + self.set_breakpoint_here(filename, lineno) + except AttributeError: + continue + +class StackViewer(ScrolledList): + + def __init__(self, master, flist, gui): + if macosxSupport.isAquaTk(): + # At least on with the stock AquaTk version on OSX 10.4 you'll + # get a shaking GUI that eventually kills IDLE if the width + # argument is specified. + ScrolledList.__init__(self, master) + else: + ScrolledList.__init__(self, master, width=80) + self.flist = flist + self.gui = gui + self.stack = [] + + def load_stack(self, stack, index=None): + self.stack = stack + self.clear() + for i in range(len(stack)): + frame, lineno = stack[i] + try: + modname = frame.f_globals["__name__"] + except: + modname = "?" + code = frame.f_code + filename = code.co_filename + funcname = code.co_name + import linecache + sourceline = linecache.getline(filename, lineno) + sourceline = sourceline.strip() + if funcname in ("?", "", None): + item = "%s, line %d: %s" % (modname, lineno, sourceline) + else: + item = "%s.%s(), line %d: %s" % (modname, funcname, + lineno, sourceline) + if i == index: + item = "> " + item + self.append(item) + if index is not None: + self.select(index) + + def popup_event(self, event): + "override base method" + if self.stack: + return ScrolledList.popup_event(self, event) + + def fill_menu(self): + "override base method" + menu = self.menu + menu.add_command(label="Go to source line", + command=self.goto_source_line) + menu.add_command(label="Show stack frame", + command=self.show_stack_frame) + + def on_select(self, index): + "override base method" + if 0 <= index < len(self.stack): + self.gui.show_frame(self.stack[index]) + + def on_double(self, index): + "override base method" + self.show_source(index) + + def goto_source_line(self): + index = self.listbox.index("active") + self.show_source(index) + + def show_stack_frame(self): + index = self.listbox.index("active") + if 0 <= index < len(self.stack): + self.gui.show_frame(self.stack[index]) + + def show_source(self, index): + if not (0 <= index < len(self.stack)): + return + frame, lineno = self.stack[index] + code = frame.f_code + filename = code.co_filename + if os.path.isfile(filename): + edit = self.flist.open(filename) + if edit: + edit.gotoline(lineno) + + +class NamespaceViewer: + + def __init__(self, master, title, dict=None): + width = 0 + height = 40 + if dict: + height = 20*len(dict) # XXX 20 == observed height of Entry widget + self.master = master + self.title = title + import reprlib + self.repr = reprlib.Repr() + self.repr.maxstring = 60 + self.repr.maxother = 60 + self.frame = frame = Frame(master) + self.frame.pack(expand=1, fill="both") + self.label = Label(frame, text=title, borderwidth=2, relief="groove") + self.label.pack(fill="x") + self.vbar = vbar = Scrollbar(frame, name="vbar") + vbar.pack(side="right", fill="y") + self.canvas = canvas = Canvas(frame, + height=min(300, max(40, height)), + scrollregion=(0, 0, width, height)) + canvas.pack(side="left", fill="both", expand=1) + vbar["command"] = canvas.yview + canvas["yscrollcommand"] = vbar.set + self.subframe = subframe = Frame(canvas) + self.sfid = canvas.create_window(0, 0, window=subframe, anchor="nw") + self.load_dict(dict) + + dict = -1 + + def load_dict(self, dict, force=0, rpc_client=None): + if dict is self.dict and not force: + return + subframe = self.subframe + frame = self.frame + for c in list(subframe.children.values()): + c.destroy() + self.dict = None + if not dict: + l = Label(subframe, text="None") + l.grid(row=0, column=0) + else: + #names = sorted(dict) + ### + # Because of (temporary) limitations on the dict_keys type (not yet + # public or pickleable), have the subprocess to send a list of + # keys, not a dict_keys object. sorted() will take a dict_keys + # (no subprocess) or a list. + # + # There is also an obscure bug in sorted(dict) where the + # interpreter gets into a loop requesting non-existing dict[0], + # dict[1], dict[2], etc from the RemoteDebugger.DictProxy. + ### + keys_list = dict.keys() + names = sorted(keys_list) + ### + row = 0 + for name in names: + value = dict[name] + svalue = self.repr.repr(value) # repr(value) + # Strip extra quotes caused by calling repr on the (already) + # repr'd value sent across the RPC interface: + if rpc_client: + svalue = svalue[1:-1] + l = Label(subframe, text=name) + l.grid(row=row, column=0, sticky="nw") + l = Entry(subframe, width=0, borderwidth=0) + l.insert(0, svalue) + l.grid(row=row, column=1, sticky="nw") + row = row+1 + self.dict = dict + # XXX Could we use a callback for the following? + subframe.update_idletasks() # Alas! + width = subframe.winfo_reqwidth() + height = subframe.winfo_reqheight() + canvas = self.canvas + self.canvas["scrollregion"] = (0, 0, width, height) + if height > 300: + canvas["height"] = 300 + frame.pack(expand=1) + else: + canvas["height"] = height + frame.pack(expand=0) + + def close(self): + self.frame.destroy() diff --git a/Lib/idlelib/debugger_r.py b/Lib/idlelib/debugger_r.py new file mode 100644 index 0000000..be2262f --- /dev/null +++ b/Lib/idlelib/debugger_r.py @@ -0,0 +1,388 @@ +"""Support for remote Python debugging. + +Some ASCII art to describe the structure: + + IN PYTHON SUBPROCESS # IN IDLE PROCESS + # + # oid='gui_adapter' + +----------+ # +------------+ +-----+ + | GUIProxy |--remote#call-->| GUIAdapter |--calls-->| GUI | ++-----+--calls-->+----------+ # +------------+ +-----+ +| Idb | # / ++-----+<-calls--+------------+ # +----------+<--calls-/ + | IdbAdapter |<--remote#call--| IdbProxy | + +------------+ # +----------+ + oid='idb_adapter' # + +The purpose of the Proxy and Adapter classes is to translate certain +arguments and return values that cannot be transported through the RPC +barrier, in particular frame and traceback objects. + +""" + +import types +from idlelib import Debugger + +debugging = 0 + +idb_adap_oid = "idb_adapter" +gui_adap_oid = "gui_adapter" + +#======================================= +# +# In the PYTHON subprocess: + +frametable = {} +dicttable = {} +codetable = {} +tracebacktable = {} + +def wrap_frame(frame): + fid = id(frame) + frametable[fid] = frame + return fid + +def wrap_info(info): + "replace info[2], a traceback instance, by its ID" + if info is None: + return None + else: + traceback = info[2] + assert isinstance(traceback, types.TracebackType) + traceback_id = id(traceback) + tracebacktable[traceback_id] = traceback + modified_info = (info[0], info[1], traceback_id) + return modified_info + +class GUIProxy: + + def __init__(self, conn, gui_adap_oid): + self.conn = conn + self.oid = gui_adap_oid + + def interaction(self, message, frame, info=None): + # calls rpc.SocketIO.remotecall() via run.MyHandler instance + # pass frame and traceback object IDs instead of the objects themselves + self.conn.remotecall(self.oid, "interaction", + (message, wrap_frame(frame), wrap_info(info)), + {}) + +class IdbAdapter: + + def __init__(self, idb): + self.idb = idb + + #----------called by an IdbProxy---------- + + def set_step(self): + self.idb.set_step() + + def set_quit(self): + self.idb.set_quit() + + def set_continue(self): + self.idb.set_continue() + + def set_next(self, fid): + frame = frametable[fid] + self.idb.set_next(frame) + + def set_return(self, fid): + frame = frametable[fid] + self.idb.set_return(frame) + + def get_stack(self, fid, tbid): + frame = frametable[fid] + if tbid is None: + tb = None + else: + tb = tracebacktable[tbid] + stack, i = self.idb.get_stack(frame, tb) + stack = [(wrap_frame(frame2), k) for frame2, k in stack] + return stack, i + + def run(self, cmd): + import __main__ + self.idb.run(cmd, __main__.__dict__) + + def set_break(self, filename, lineno): + msg = self.idb.set_break(filename, lineno) + return msg + + def clear_break(self, filename, lineno): + msg = self.idb.clear_break(filename, lineno) + return msg + + def clear_all_file_breaks(self, filename): + msg = self.idb.clear_all_file_breaks(filename) + return msg + + #----------called by a FrameProxy---------- + + def frame_attr(self, fid, name): + frame = frametable[fid] + return getattr(frame, name) + + def frame_globals(self, fid): + frame = frametable[fid] + dict = frame.f_globals + did = id(dict) + dicttable[did] = dict + return did + + def frame_locals(self, fid): + frame = frametable[fid] + dict = frame.f_locals + did = id(dict) + dicttable[did] = dict + return did + + def frame_code(self, fid): + frame = frametable[fid] + code = frame.f_code + cid = id(code) + codetable[cid] = code + return cid + + #----------called by a CodeProxy---------- + + def code_name(self, cid): + code = codetable[cid] + return code.co_name + + def code_filename(self, cid): + code = codetable[cid] + return code.co_filename + + #----------called by a DictProxy---------- + + def dict_keys(self, did): + raise NotImplemented("dict_keys not public or pickleable") +## dict = dicttable[did] +## return dict.keys() + + ### Needed until dict_keys is type is finished and pickealable. + ### Will probably need to extend rpc.py:SocketIO._proxify at that time. + def dict_keys_list(self, did): + dict = dicttable[did] + return list(dict.keys()) + + def dict_item(self, did, key): + dict = dicttable[did] + value = dict[key] + value = repr(value) ### can't pickle module 'builtins' + return value + +#----------end class IdbAdapter---------- + + +def start_debugger(rpchandler, gui_adap_oid): + """Start the debugger and its RPC link in the Python subprocess + + Start the subprocess side of the split debugger and set up that side of the + RPC link by instantiating the GUIProxy, Idb debugger, and IdbAdapter + objects and linking them together. Register the IdbAdapter with the + RPCServer to handle RPC requests from the split debugger GUI via the + IdbProxy. + + """ + gui_proxy = GUIProxy(rpchandler, gui_adap_oid) + idb = Debugger.Idb(gui_proxy) + idb_adap = IdbAdapter(idb) + rpchandler.register(idb_adap_oid, idb_adap) + return idb_adap_oid + + +#======================================= +# +# In the IDLE process: + + +class FrameProxy: + + def __init__(self, conn, fid): + self._conn = conn + self._fid = fid + self._oid = "idb_adapter" + self._dictcache = {} + + def __getattr__(self, name): + if name[:1] == "_": + raise AttributeError(name) + if name == "f_code": + return self._get_f_code() + if name == "f_globals": + return self._get_f_globals() + if name == "f_locals": + return self._get_f_locals() + return self._conn.remotecall(self._oid, "frame_attr", + (self._fid, name), {}) + + def _get_f_code(self): + cid = self._conn.remotecall(self._oid, "frame_code", (self._fid,), {}) + return CodeProxy(self._conn, self._oid, cid) + + def _get_f_globals(self): + did = self._conn.remotecall(self._oid, "frame_globals", + (self._fid,), {}) + return self._get_dict_proxy(did) + + def _get_f_locals(self): + did = self._conn.remotecall(self._oid, "frame_locals", + (self._fid,), {}) + return self._get_dict_proxy(did) + + def _get_dict_proxy(self, did): + if did in self._dictcache: + return self._dictcache[did] + dp = DictProxy(self._conn, self._oid, did) + self._dictcache[did] = dp + return dp + + +class CodeProxy: + + def __init__(self, conn, oid, cid): + self._conn = conn + self._oid = oid + self._cid = cid + + def __getattr__(self, name): + if name == "co_name": + return self._conn.remotecall(self._oid, "code_name", + (self._cid,), {}) + if name == "co_filename": + return self._conn.remotecall(self._oid, "code_filename", + (self._cid,), {}) + + +class DictProxy: + + def __init__(self, conn, oid, did): + self._conn = conn + self._oid = oid + self._did = did + +## def keys(self): +## return self._conn.remotecall(self._oid, "dict_keys", (self._did,), {}) + + # 'temporary' until dict_keys is a pickleable built-in type + def keys(self): + return self._conn.remotecall(self._oid, + "dict_keys_list", (self._did,), {}) + + def __getitem__(self, key): + return self._conn.remotecall(self._oid, "dict_item", + (self._did, key), {}) + + def __getattr__(self, name): + ##print("*** Failed DictProxy.__getattr__:", name) + raise AttributeError(name) + + +class GUIAdapter: + + def __init__(self, conn, gui): + self.conn = conn + self.gui = gui + + def interaction(self, message, fid, modified_info): + ##print("*** Interaction: (%s, %s, %s)" % (message, fid, modified_info)) + frame = FrameProxy(self.conn, fid) + self.gui.interaction(message, frame, modified_info) + + +class IdbProxy: + + def __init__(self, conn, shell, oid): + self.oid = oid + self.conn = conn + self.shell = shell + + def call(self, methodname, *args, **kwargs): + ##print("*** IdbProxy.call %s %s %s" % (methodname, args, kwargs)) + value = self.conn.remotecall(self.oid, methodname, args, kwargs) + ##print("*** IdbProxy.call %s returns %r" % (methodname, value)) + return value + + def run(self, cmd, locals): + # Ignores locals on purpose! + seq = self.conn.asyncqueue(self.oid, "run", (cmd,), {}) + self.shell.interp.active_seq = seq + + def get_stack(self, frame, tbid): + # passing frame and traceback IDs, not the objects themselves + stack, i = self.call("get_stack", frame._fid, tbid) + stack = [(FrameProxy(self.conn, fid), k) for fid, k in stack] + return stack, i + + def set_continue(self): + self.call("set_continue") + + def set_step(self): + self.call("set_step") + + def set_next(self, frame): + self.call("set_next", frame._fid) + + def set_return(self, frame): + self.call("set_return", frame._fid) + + def set_quit(self): + self.call("set_quit") + + def set_break(self, filename, lineno): + msg = self.call("set_break", filename, lineno) + return msg + + def clear_break(self, filename, lineno): + msg = self.call("clear_break", filename, lineno) + return msg + + def clear_all_file_breaks(self, filename): + msg = self.call("clear_all_file_breaks", filename) + return msg + +def start_remote_debugger(rpcclt, pyshell): + """Start the subprocess debugger, initialize the debugger GUI and RPC link + + Request the RPCServer start the Python subprocess debugger and link. Set + up the Idle side of the split debugger by instantiating the IdbProxy, + debugger GUI, and debugger GUIAdapter objects and linking them together. + + Register the GUIAdapter with the RPCClient to handle debugger GUI + interaction requests coming from the subprocess debugger via the GUIProxy. + + The IdbAdapter will pass execution and environment requests coming from the + Idle debugger GUI to the subprocess debugger via the IdbProxy. + + """ + global idb_adap_oid + + idb_adap_oid = rpcclt.remotecall("exec", "start_the_debugger",\ + (gui_adap_oid,), {}) + idb_proxy = IdbProxy(rpcclt, pyshell, idb_adap_oid) + gui = Debugger.Debugger(pyshell, idb_proxy) + gui_adap = GUIAdapter(rpcclt, gui) + rpcclt.register(gui_adap_oid, gui_adap) + return gui + +def close_remote_debugger(rpcclt): + """Shut down subprocess debugger and Idle side of debugger RPC link + + Request that the RPCServer shut down the subprocess debugger and link. + Unregister the GUIAdapter, which will cause a GC on the Idle process + debugger and RPC link objects. (The second reference to the debugger GUI + is deleted in PyShell.close_remote_debugger().) + + """ + close_subprocess_debugger(rpcclt) + rpcclt.unregister(gui_adap_oid) + +def close_subprocess_debugger(rpcclt): + rpcclt.remotecall("exec", "stop_the_debugger", (idb_adap_oid,), {}) + +def restart_subprocess_debugger(rpcclt): + idb_adap_oid_ret = rpcclt.remotecall("exec", "start_the_debugger",\ + (gui_adap_oid,), {}) + assert idb_adap_oid_ret == idb_adap_oid, 'Idb restarted with different oid' diff --git a/Lib/idlelib/debugobj.py b/Lib/idlelib/debugobj.py new file mode 100644 index 0000000..7b57aa4 --- /dev/null +++ b/Lib/idlelib/debugobj.py @@ -0,0 +1,143 @@ +# XXX TO DO: +# - popup menu +# - support partial or total redisplay +# - more doc strings +# - tooltips + +# object browser + +# XXX TO DO: +# - for classes/modules, add "open source" to object browser + +import re + +from idlelib.TreeWidget import TreeItem, TreeNode, ScrolledCanvas + +from reprlib import Repr + +myrepr = Repr() +myrepr.maxstring = 100 +myrepr.maxother = 100 + +class ObjectTreeItem(TreeItem): + def __init__(self, labeltext, object, setfunction=None): + self.labeltext = labeltext + self.object = object + self.setfunction = setfunction + def GetLabelText(self): + return self.labeltext + def GetText(self): + return myrepr.repr(self.object) + def GetIconName(self): + if not self.IsExpandable(): + return "python" + def IsEditable(self): + return self.setfunction is not None + def SetText(self, text): + try: + value = eval(text) + self.setfunction(value) + except: + pass + else: + self.object = value + def IsExpandable(self): + return not not dir(self.object) + def GetSubList(self): + keys = dir(self.object) + sublist = [] + for key in keys: + try: + value = getattr(self.object, key) + except AttributeError: + continue + item = make_objecttreeitem( + str(key) + " =", + value, + lambda value, key=key, object=self.object: + setattr(object, key, value)) + sublist.append(item) + return sublist + +class ClassTreeItem(ObjectTreeItem): + def IsExpandable(self): + return True + def GetSubList(self): + sublist = ObjectTreeItem.GetSubList(self) + if len(self.object.__bases__) == 1: + item = make_objecttreeitem("__bases__[0] =", + self.object.__bases__[0]) + else: + item = make_objecttreeitem("__bases__ =", self.object.__bases__) + sublist.insert(0, item) + return sublist + +class AtomicObjectTreeItem(ObjectTreeItem): + def IsExpandable(self): + return 0 + +class SequenceTreeItem(ObjectTreeItem): + def IsExpandable(self): + return len(self.object) > 0 + def keys(self): + return range(len(self.object)) + def GetSubList(self): + sublist = [] + for key in self.keys(): + try: + value = self.object[key] + except KeyError: + continue + def setfunction(value, key=key, object=self.object): + object[key] = value + item = make_objecttreeitem("%r:" % (key,), value, setfunction) + sublist.append(item) + return sublist + +class DictTreeItem(SequenceTreeItem): + def keys(self): + keys = list(self.object.keys()) + try: + keys.sort() + except: + pass + return keys + +dispatch = { + int: AtomicObjectTreeItem, + float: AtomicObjectTreeItem, + str: AtomicObjectTreeItem, + tuple: SequenceTreeItem, + list: SequenceTreeItem, + dict: DictTreeItem, + type: ClassTreeItem, +} + +def make_objecttreeitem(labeltext, object, setfunction=None): + t = type(object) + if t in dispatch: + c = dispatch[t] + else: + c = ObjectTreeItem + return c(labeltext, object, setfunction) + + +def _object_browser(parent): + import sys + from tkinter import Tk + root = Tk() + root.title("Test ObjectBrowser") + width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) + root.geometry("+%d+%d"%(x, y + 150)) + root.configure(bd=0, bg="yellow") + root.focus_set() + sc = ScrolledCanvas(root, bg="white", highlightthickness=0, takefocus=1) + sc.frame.pack(expand=1, fill="both") + item = make_objecttreeitem("sys", sys) + node = TreeNode(sc.canvas, None, item) + node.update() + root.mainloop() + +if __name__ == '__main__': + from idlelib.idle_test.htest import run + run(_object_browser) diff --git a/Lib/idlelib/debugobj_r.py b/Lib/idlelib/debugobj_r.py new file mode 100644 index 0000000..8031aae --- /dev/null +++ b/Lib/idlelib/debugobj_r.py @@ -0,0 +1,36 @@ +from idlelib import rpc + +def remote_object_tree_item(item): + wrapper = WrappedObjectTreeItem(item) + oid = id(wrapper) + rpc.objecttable[oid] = wrapper + return oid + +class WrappedObjectTreeItem: + # Lives in PYTHON subprocess + + def __init__(self, item): + self.__item = item + + def __getattr__(self, name): + value = getattr(self.__item, name) + return value + + def _GetSubList(self): + sub_list = self.__item._GetSubList() + return list(map(remote_object_tree_item, sub_list)) + +class StubObjectTreeItem: + # Lives in IDLE process + + def __init__(self, sockio, oid): + self.sockio = sockio + self.oid = oid + + def __getattr__(self, name): + value = rpc.MethodProxy(self.sockio, self.oid, name) + return value + + def _GetSubList(self): + sub_list = self.sockio.remotecall(self.oid, "_GetSubList", (), {}) + return [StubObjectTreeItem(self.sockio, oid) for oid in sub_list] diff --git a/Lib/idlelib/delegator.py b/Lib/idlelib/delegator.py new file mode 100644 index 0000000..dc2a1aa --- /dev/null +++ b/Lib/idlelib/delegator.py @@ -0,0 +1,33 @@ +class Delegator: + + def __init__(self, delegate=None): + self.delegate = delegate + self.__cache = set() + # Cache is used to only remove added attributes + # when changing the delegate. + + def __getattr__(self, name): + attr = getattr(self.delegate, name) # May raise AttributeError + setattr(self, name, attr) + self.__cache.add(name) + return attr + + def resetcache(self): + "Removes added attributes while leaving original attributes." + # Function is really about resetting delagator dict + # to original state. Cache is just a means + for key in self.__cache: + try: + delattr(self, key) + except AttributeError: + pass + self.__cache.clear() + + def setdelegate(self, delegate): + "Reset attributes and change delegate." + self.resetcache() + self.delegate = delegate + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_delegator', verbosity=2) diff --git a/Lib/idlelib/dynOptionMenuWidget.py b/Lib/idlelib/dynOptionMenuWidget.py deleted file mode 100644 index 515b4ba..0000000 --- a/Lib/idlelib/dynOptionMenuWidget.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -OptionMenu widget modified to allow dynamic menu reconfiguration -and setting of highlightthickness -""" -import copy -from tkinter import OptionMenu, _setit, StringVar, Button - -class DynOptionMenu(OptionMenu): - """ - unlike OptionMenu, our kwargs can include highlightthickness - """ - def __init__(self, master, variable, value, *values, **kwargs): - # TODO copy value instead of whole dict - kwargsCopy=copy.copy(kwargs) - if 'highlightthickness' in list(kwargs.keys()): - del(kwargs['highlightthickness']) - OptionMenu.__init__(self, master, variable, value, *values, **kwargs) - self.config(highlightthickness=kwargsCopy.get('highlightthickness')) - #self.menu=self['menu'] - self.variable=variable - self.command=kwargs.get('command') - - def SetMenu(self,valueList,value=None): - """ - clear and reload the menu with a new set of options. - valueList - list of new options - value - initial value to set the optionmenu's menubutton to - """ - self['menu'].delete(0,'end') - for item in valueList: - self['menu'].add_command(label=item, - command=_setit(self.variable,item,self.command)) - if value: - self.variable.set(value) - -def _dyn_option_menu(parent): # htest # - from tkinter import Toplevel - - top = Toplevel() - top.title("Tets dynamic option menu") - top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, - parent.winfo_rooty() + 150)) - top.focus_set() - - var = StringVar(top) - var.set("Old option set") #Set the default value - dyn = DynOptionMenu(top,var, "old1","old2","old3","old4") - dyn.pack() - - def update(): - dyn.SetMenu(["new1","new2","new3","new4"], value="new option set") - button = Button(top, text="Change option set", command=update) - button.pack() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_dyn_option_menu) diff --git a/Lib/idlelib/dynoption.py b/Lib/idlelib/dynoption.py new file mode 100644 index 0000000..515b4ba --- /dev/null +++ b/Lib/idlelib/dynoption.py @@ -0,0 +1,57 @@ +""" +OptionMenu widget modified to allow dynamic menu reconfiguration +and setting of highlightthickness +""" +import copy +from tkinter import OptionMenu, _setit, StringVar, Button + +class DynOptionMenu(OptionMenu): + """ + unlike OptionMenu, our kwargs can include highlightthickness + """ + def __init__(self, master, variable, value, *values, **kwargs): + # TODO copy value instead of whole dict + kwargsCopy=copy.copy(kwargs) + if 'highlightthickness' in list(kwargs.keys()): + del(kwargs['highlightthickness']) + OptionMenu.__init__(self, master, variable, value, *values, **kwargs) + self.config(highlightthickness=kwargsCopy.get('highlightthickness')) + #self.menu=self['menu'] + self.variable=variable + self.command=kwargs.get('command') + + def SetMenu(self,valueList,value=None): + """ + clear and reload the menu with a new set of options. + valueList - list of new options + value - initial value to set the optionmenu's menubutton to + """ + self['menu'].delete(0,'end') + for item in valueList: + self['menu'].add_command(label=item, + command=_setit(self.variable,item,self.command)) + if value: + self.variable.set(value) + +def _dyn_option_menu(parent): # htest # + from tkinter import Toplevel + + top = Toplevel() + top.title("Tets dynamic option menu") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + top.focus_set() + + var = StringVar(top) + var.set("Old option set") #Set the default value + dyn = DynOptionMenu(top,var, "old1","old2","old3","old4") + dyn.pack() + + def update(): + dyn.SetMenu(["new1","new2","new3","new4"], value="new option set") + button = Button(top, text="Change option set", command=update) + button.pack() + +if __name__ == '__main__': + from idlelib.idle_test.htest import run + run(_dyn_option_menu) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py new file mode 100644 index 0000000..b5868be --- /dev/null +++ b/Lib/idlelib/editor.py @@ -0,0 +1,1703 @@ +import importlib +import importlib.abc +import importlib.util +import os +import platform +import re +import string +import sys +from tkinter import * +import tkinter.simpledialog as tkSimpleDialog +import tkinter.messagebox as tkMessageBox +import traceback +import webbrowser + +from idlelib.MultiCall import MultiCallCreator +from idlelib import WindowList +from idlelib import SearchDialog +from idlelib import GrepDialog +from idlelib import ReplaceDialog +from idlelib import PyParse +from idlelib.configHandler import idleConf +from idlelib import aboutDialog, textView, configDialog +from idlelib import macosxSupport +from idlelib import help + +# The default tab setting for a Text widget, in average-width characters. +TK_TABWIDTH_DEFAULT = 8 + +_py_version = ' (%s)' % platform.python_version() + +def _sphinx_version(): + "Format sys.version_info to produce the Sphinx version string used to install the chm docs" + major, minor, micro, level, serial = sys.version_info + release = '%s%s' % (major, minor) + release += '%s' % (micro,) + if level == 'candidate': + release += 'rc%s' % (serial,) + elif level != 'final': + release += '%s%s' % (level[0], serial) + return release + + +class HelpDialog(object): + + def __init__(self): + self.parent = None # parent of help window + self.dlg = None # the help window iteself + + def display(self, parent, near=None): + """ Display the help dialog. + + parent - parent widget for the help window + + near - a Toplevel widget (e.g. EditorWindow or PyShell) + to use as a reference for placing the help window + """ + import warnings as w + w.warn("EditorWindow.HelpDialog is no longer used by Idle.\n" + "It will be removed in 3.6 or later.\n" + "It has been replaced by private help.HelpWindow\n", + DeprecationWarning, stacklevel=2) + if self.dlg is None: + self.show_dialog(parent) + if near: + self.nearwindow(near) + + def show_dialog(self, parent): + self.parent = parent + fn=os.path.join(os.path.abspath(os.path.dirname(__file__)),'help.txt') + self.dlg = dlg = textView.view_file(parent,'Help',fn, modal=False) + dlg.bind('', self.destroy, '+') + + def nearwindow(self, near): + # Place the help dialog near the window specified by parent. + # Note - this may not reposition the window in Metacity + # if "/apps/metacity/general/disable_workarounds" is enabled + dlg = self.dlg + geom = (near.winfo_rootx() + 10, near.winfo_rooty() + 10) + dlg.withdraw() + dlg.geometry("=+%d+%d" % geom) + dlg.deiconify() + dlg.lift() + + def destroy(self, ev=None): + self.dlg = None + self.parent = None + +helpDialog = HelpDialog() # singleton instance, no longer used + + +class EditorWindow(object): + from idlelib.Percolator import Percolator + from idlelib.ColorDelegator import ColorDelegator + from idlelib.UndoDelegator import UndoDelegator + from idlelib.IOBinding import IOBinding, filesystemencoding, encoding + from idlelib import Bindings + from tkinter import Toplevel + from idlelib.MultiStatusBar import MultiStatusBar + + help_url = None + + def __init__(self, flist=None, filename=None, key=None, root=None): + if EditorWindow.help_url is None: + dochome = os.path.join(sys.base_prefix, 'Doc', 'index.html') + if sys.platform.count('linux'): + # look for html docs in a couple of standard places + pyver = 'python-docs-' + '%s.%s.%s' % sys.version_info[:3] + if os.path.isdir('/var/www/html/python/'): # "python2" rpm + dochome = '/var/www/html/python/index.html' + else: + basepath = '/usr/share/doc/' # standard location + dochome = os.path.join(basepath, pyver, + 'Doc', 'index.html') + elif sys.platform[:3] == 'win': + chmfile = os.path.join(sys.base_prefix, 'Doc', + 'Python%s.chm' % _sphinx_version()) + if os.path.isfile(chmfile): + dochome = chmfile + elif sys.platform == 'darwin': + # documentation may be stored inside a python framework + dochome = os.path.join(sys.base_prefix, + 'Resources/English.lproj/Documentation/index.html') + dochome = os.path.normpath(dochome) + if os.path.isfile(dochome): + EditorWindow.help_url = dochome + if sys.platform == 'darwin': + # Safari requires real file:-URLs + EditorWindow.help_url = 'file://' + EditorWindow.help_url + else: + EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] + self.flist = flist + root = root or flist.root + self.root = root + try: + sys.ps1 + except AttributeError: + sys.ps1 = '>>> ' + self.menubar = Menu(root) + self.top = top = WindowList.ListedToplevel(root, menu=self.menubar) + if flist: + self.tkinter_vars = flist.vars + #self.top.instance_dict makes flist.inversedict available to + #configDialog.py so it can access all EditorWindow instances + self.top.instance_dict = flist.inversedict + else: + self.tkinter_vars = {} # keys: Tkinter event names + # values: Tkinter variable instances + self.top.instance_dict = {} + self.recent_files_path = os.path.join(idleConf.GetUserCfgDir(), + 'recent-files.lst') + self.text_frame = text_frame = Frame(top) + self.vbar = vbar = Scrollbar(text_frame, name='vbar') + self.width = idleConf.GetOption('main', 'EditorWindow', + 'width', type='int') + text_options = { + 'name': 'text', + 'padx': 5, + 'wrap': 'none', + 'highlightthickness': 0, + 'width': self.width, + 'height': idleConf.GetOption('main', 'EditorWindow', + 'height', type='int')} + if TkVersion >= 8.5: + # Starting with tk 8.5 we have to set the new tabstyle option + # to 'wordprocessor' to achieve the same display of tabs as in + # older tk versions. + text_options['tabstyle'] = 'wordprocessor' + self.text = text = MultiCallCreator(Text)(text_frame, **text_options) + self.top.focused_widget = self.text + + self.createmenubar() + self.apply_bindings() + + self.top.protocol("WM_DELETE_WINDOW", self.close) + self.top.bind("<>", self.close_event) + if macosxSupport.isAquaTk(): + # Command-W on editorwindows doesn't work without this. + text.bind('<>', self.close_event) + # Some OS X systems have only one mouse button, so use + # control-click for popup context menus there. For two + # buttons, AquaTk defines <2> as the right button, not <3>. + text.bind("",self.right_menu_event) + text.bind("<2>", self.right_menu_event) + else: + # Elsewhere, use right-click for popup menus. + text.bind("<3>",self.right_menu_event) + text.bind("<>", self.cut) + text.bind("<>", self.copy) + text.bind("<>", self.paste) + text.bind("<>", self.center_insert_event) + text.bind("<>", self.help_dialog) + text.bind("<>", self.python_docs) + text.bind("<>", self.about_dialog) + text.bind("<>", self.config_dialog) + text.bind("<>", self.open_module) + text.bind("<>", lambda event: "break") + text.bind("<>", self.select_all) + text.bind("<>", self.remove_selection) + text.bind("<>", self.find_event) + text.bind("<>", self.find_again_event) + text.bind("<>", self.find_in_files_event) + text.bind("<>", self.find_selection_event) + text.bind("<>", self.replace_event) + text.bind("<>", self.goto_line_event) + text.bind("<>",self.smart_backspace_event) + text.bind("<>",self.newline_and_indent_event) + text.bind("<>",self.smart_indent_event) + text.bind("<>",self.indent_region_event) + text.bind("<>",self.dedent_region_event) + text.bind("<>",self.comment_region_event) + text.bind("<>",self.uncomment_region_event) + text.bind("<>",self.tabify_region_event) + text.bind("<>",self.untabify_region_event) + text.bind("<>",self.toggle_tabs_event) + text.bind("<>",self.change_indentwidth_event) + text.bind("", self.move_at_edge_if_selection(0)) + text.bind("", self.move_at_edge_if_selection(1)) + text.bind("<>", self.del_word_left) + text.bind("<>", self.del_word_right) + text.bind("<>", self.home_callback) + + if flist: + flist.inversedict[self] = key + if key: + flist.dict[key] = self + text.bind("<>", self.new_callback) + text.bind("<>", self.flist.close_all_callback) + text.bind("<>", self.open_class_browser) + text.bind("<>", self.open_path_browser) + text.bind("<>", self.open_turtle_demo) + + self.set_status_bar() + vbar['command'] = text.yview + vbar.pack(side=RIGHT, fill=Y) + text['yscrollcommand'] = vbar.set + text['font'] = idleConf.GetFont(self.root, 'main', 'EditorWindow') + text_frame.pack(side=LEFT, fill=BOTH, expand=1) + text.pack(side=TOP, fill=BOTH, expand=1) + text.focus_set() + + # usetabs true -> literal tab characters are used by indent and + # dedent cmds, possibly mixed with spaces if + # indentwidth is not a multiple of tabwidth, + # which will cause Tabnanny to nag! + # false -> tab characters are converted to spaces by indent + # and dedent cmds, and ditto TAB keystrokes + # Although use-spaces=0 can be configured manually in config-main.def, + # configuration of tabs v. spaces is not supported in the configuration + # dialog. IDLE promotes the preferred Python indentation: use spaces! + usespaces = idleConf.GetOption('main', 'Indent', + 'use-spaces', type='bool') + self.usetabs = not usespaces + + # tabwidth is the display width of a literal tab character. + # CAUTION: telling Tk to use anything other than its default + # tab setting causes it to use an entirely different tabbing algorithm, + # treating tab stops as fixed distances from the left margin. + # Nobody expects this, so for now tabwidth should never be changed. + self.tabwidth = 8 # must remain 8 until Tk is fixed. + + # indentwidth is the number of screen characters per indent level. + # The recommended Python indentation is four spaces. + self.indentwidth = self.tabwidth + self.set_notabs_indentwidth() + + # If context_use_ps1 is true, parsing searches back for a ps1 line; + # else searches for a popular (if, def, ...) Python stmt. + self.context_use_ps1 = False + + # When searching backwards for a reliable place to begin parsing, + # first start num_context_lines[0] lines back, then + # num_context_lines[1] lines back if that didn't work, and so on. + # The last value should be huge (larger than the # of lines in a + # conceivable file). + # Making the initial values larger slows things down more often. + self.num_context_lines = 50, 500, 5000000 + self.per = per = self.Percolator(text) + self.undo = undo = self.UndoDelegator() + per.insertfilter(undo) + text.undo_block_start = undo.undo_block_start + text.undo_block_stop = undo.undo_block_stop + undo.set_saved_change_hook(self.saved_change_hook) + # IOBinding implements file I/O and printing functionality + self.io = io = self.IOBinding(self) + io.set_filename_change_hook(self.filename_change_hook) + self.good_load = False + self.set_indentation_params(False) + self.color = None # initialized below in self.ResetColorizer + if filename: + if os.path.exists(filename) and not os.path.isdir(filename): + if io.loadfile(filename): + self.good_load = True + is_py_src = self.ispythonsource(filename) + self.set_indentation_params(is_py_src) + else: + io.set_filename(filename) + self.good_load = True + + self.ResetColorizer() + self.saved_change_hook() + self.update_recent_files_list() + self.load_extensions() + menu = self.menudict.get('windows') + if menu: + end = menu.index("end") + if end is None: + end = -1 + if end >= 0: + menu.add_separator() + end = end + 1 + self.wmenu_end = end + WindowList.register_callback(self.postwindowsmenu) + + # Some abstractions so IDLE extensions are cross-IDE + self.askyesno = tkMessageBox.askyesno + self.askinteger = tkSimpleDialog.askinteger + self.showerror = tkMessageBox.showerror + + def _filename_to_unicode(self, filename): + """Return filename as BMP unicode so diplayable in Tk.""" + # Decode bytes to unicode. + if isinstance(filename, bytes): + try: + filename = filename.decode(self.filesystemencoding) + except UnicodeDecodeError: + try: + filename = filename.decode(self.encoding) + except UnicodeDecodeError: + # byte-to-byte conversion + filename = filename.decode('iso8859-1') + # Replace non-BMP char with diamond questionmark. + return re.sub('[\U00010000-\U0010FFFF]', '\ufffd', filename) + + def new_callback(self, event): + dirname, basename = self.io.defaultfilename() + self.flist.new(dirname) + return "break" + + def home_callback(self, event): + if (event.state & 4) != 0 and event.keysym == "Home": + # state&4==Control. If , use the Tk binding. + return + if self.text.index("iomark") and \ + self.text.compare("iomark", "<=", "insert lineend") and \ + self.text.compare("insert linestart", "<=", "iomark"): + # In Shell on input line, go to just after prompt + insertpt = int(self.text.index("iomark").split(".")[1]) + else: + line = self.text.get("insert linestart", "insert lineend") + for insertpt in range(len(line)): + if line[insertpt] not in (' ','\t'): + break + else: + insertpt=len(line) + lineat = int(self.text.index("insert").split('.')[1]) + if insertpt == lineat: + insertpt = 0 + dest = "insert linestart+"+str(insertpt)+"c" + if (event.state&1) == 0: + # shift was not pressed + self.text.tag_remove("sel", "1.0", "end") + else: + if not self.text.index("sel.first"): + # there was no previous selection + self.text.mark_set("my_anchor", "insert") + else: + if self.text.compare(self.text.index("sel.first"), "<", + self.text.index("insert")): + self.text.mark_set("my_anchor", "sel.first") # extend back + else: + self.text.mark_set("my_anchor", "sel.last") # extend forward + first = self.text.index(dest) + last = self.text.index("my_anchor") + if self.text.compare(first,">",last): + first,last = last,first + self.text.tag_remove("sel", "1.0", "end") + self.text.tag_add("sel", first, last) + self.text.mark_set("insert", dest) + self.text.see("insert") + return "break" + + def set_status_bar(self): + self.status_bar = self.MultiStatusBar(self.top) + sep = Frame(self.top, height=1, borderwidth=1, background='grey75') + if sys.platform == "darwin": + # Insert some padding to avoid obscuring some of the statusbar + # by the resize widget. + self.status_bar.set_label('_padding1', ' ', side=RIGHT) + self.status_bar.set_label('column', 'Col: ?', side=RIGHT) + self.status_bar.set_label('line', 'Ln: ?', side=RIGHT) + self.status_bar.pack(side=BOTTOM, fill=X) + sep.pack(side=BOTTOM, fill=X) + self.text.bind("<>", self.set_line_and_column) + self.text.event_add("<>", + "", "") + self.text.after_idle(self.set_line_and_column) + + def set_line_and_column(self, event=None): + line, column = self.text.index(INSERT).split('.') + self.status_bar.set_label('column', 'Col: %s' % column) + self.status_bar.set_label('line', 'Ln: %s' % line) + + menu_specs = [ + ("file", "_File"), + ("edit", "_Edit"), + ("format", "F_ormat"), + ("run", "_Run"), + ("options", "_Options"), + ("windows", "_Window"), + ("help", "_Help"), + ] + + + def createmenubar(self): + mbar = self.menubar + self.menudict = menudict = {} + for name, label in self.menu_specs: + underline, label = prepstr(label) + menudict[name] = menu = Menu(mbar, name=name, tearoff=0) + mbar.add_cascade(label=label, menu=menu, underline=underline) + if macosxSupport.isCarbonTk(): + # Insert the application menu + menudict['application'] = menu = Menu(mbar, name='apple', + tearoff=0) + mbar.add_cascade(label='IDLE', menu=menu) + self.fill_menus() + self.recent_files_menu = Menu(self.menubar, tearoff=0) + self.menudict['file'].insert_cascade(3, label='Recent Files', + underline=0, + menu=self.recent_files_menu) + self.base_helpmenu_length = self.menudict['help'].index(END) + self.reset_help_menu_entries() + + def postwindowsmenu(self): + # Only called when Windows menu exists + menu = self.menudict['windows'] + end = menu.index("end") + if end is None: + end = -1 + if end > self.wmenu_end: + menu.delete(self.wmenu_end+1, end) + WindowList.add_windows_to_menu(menu) + + rmenu = None + + def right_menu_event(self, event): + self.text.mark_set("insert", "@%d,%d" % (event.x, event.y)) + if not self.rmenu: + self.make_rmenu() + rmenu = self.rmenu + self.event = event + iswin = sys.platform[:3] == 'win' + if iswin: + self.text.config(cursor="arrow") + + for item in self.rmenu_specs: + try: + label, eventname, verify_state = item + except ValueError: # see issue1207589 + continue + + if verify_state is None: + continue + state = getattr(self, verify_state)() + rmenu.entryconfigure(label, state=state) + + + rmenu.tk_popup(event.x_root, event.y_root) + if iswin: + self.text.config(cursor="ibeam") + + rmenu_specs = [ + # ("Label", "<>", "statefuncname"), ... + ("Close", "<>", None), # Example + ] + + def make_rmenu(self): + rmenu = Menu(self.text, tearoff=0) + for item in self.rmenu_specs: + label, eventname = item[0], item[1] + if label is not None: + def command(text=self.text, eventname=eventname): + text.event_generate(eventname) + rmenu.add_command(label=label, command=command) + else: + rmenu.add_separator() + self.rmenu = rmenu + + def rmenu_check_cut(self): + return self.rmenu_check_copy() + + def rmenu_check_copy(self): + try: + indx = self.text.index('sel.first') + except TclError: + return 'disabled' + else: + return 'normal' if indx else 'disabled' + + def rmenu_check_paste(self): + try: + self.text.tk.call('tk::GetSelection', self.text, 'CLIPBOARD') + except TclError: + return 'disabled' + else: + return 'normal' + + def about_dialog(self, event=None): + "Handle Help 'About IDLE' event." + # Synchronize with macosxSupport.overrideRootMenu.about_dialog. + aboutDialog.AboutDialog(self.top,'About IDLE') + + def config_dialog(self, event=None): + "Handle Options 'Configure IDLE' event." + # Synchronize with macosxSupport.overrideRootMenu.config_dialog. + configDialog.ConfigDialog(self.top,'Settings') + + def help_dialog(self, event=None): + "Handle Help 'IDLE Help' event." + # Synchronize with macosxSupport.overrideRootMenu.help_dialog. + if self.root: + parent = self.root + else: + parent = self.top + help.show_idlehelp(parent) + + def python_docs(self, event=None): + if sys.platform[:3] == 'win': + try: + os.startfile(self.help_url) + except OSError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) + else: + webbrowser.open(self.help_url) + return "break" + + def cut(self,event): + self.text.event_generate("<>") + return "break" + + def copy(self,event): + if not self.text.tag_ranges("sel"): + # There is no selection, so do nothing and maybe interrupt. + return + self.text.event_generate("<>") + return "break" + + def paste(self,event): + self.text.event_generate("<>") + self.text.see("insert") + return "break" + + def select_all(self, event=None): + self.text.tag_add("sel", "1.0", "end-1c") + self.text.mark_set("insert", "1.0") + self.text.see("insert") + return "break" + + def remove_selection(self, event=None): + self.text.tag_remove("sel", "1.0", "end") + self.text.see("insert") + + def move_at_edge_if_selection(self, edge_index): + """Cursor move begins at start or end of selection + + When a left/right cursor key is pressed create and return to Tkinter a + function which causes a cursor move from the associated edge of the + selection. + + """ + self_text_index = self.text.index + self_text_mark_set = self.text.mark_set + edges_table = ("sel.first+1c", "sel.last-1c") + def move_at_edge(event): + if (event.state & 5) == 0: # no shift(==1) or control(==4) pressed + try: + self_text_index("sel.first") + self_text_mark_set("insert", edges_table[edge_index]) + except TclError: + pass + return move_at_edge + + def del_word_left(self, event): + self.text.event_generate('') + return "break" + + def del_word_right(self, event): + self.text.event_generate('') + return "break" + + def find_event(self, event): + SearchDialog.find(self.text) + return "break" + + def find_again_event(self, event): + SearchDialog.find_again(self.text) + return "break" + + def find_selection_event(self, event): + SearchDialog.find_selection(self.text) + return "break" + + def find_in_files_event(self, event): + GrepDialog.grep(self.text, self.io, self.flist) + return "break" + + def replace_event(self, event): + ReplaceDialog.replace(self.text) + return "break" + + def goto_line_event(self, event): + text = self.text + lineno = tkSimpleDialog.askinteger("Goto", + "Go to line number:",parent=text) + if lineno is None: + return "break" + if lineno <= 0: + text.bell() + return "break" + text.mark_set("insert", "%d.0" % lineno) + text.see("insert") + + def open_module(self, event=None): + # XXX Shouldn't this be in IOBinding? + try: + name = self.text.get("sel.first", "sel.last") + except TclError: + name = "" + else: + name = name.strip() + name = tkSimpleDialog.askstring("Module", + "Enter the name of a Python module\n" + "to search on sys.path and open:", + parent=self.text, initialvalue=name) + if name: + name = name.strip() + if not name: + return + # XXX Ought to insert current file's directory in front of path + try: + spec = importlib.util.find_spec(name) + except (ValueError, ImportError) as msg: + tkMessageBox.showerror("Import error", str(msg), parent=self.text) + return + if spec is None: + tkMessageBox.showerror("Import error", "module not found", + parent=self.text) + return + if not isinstance(spec.loader, importlib.abc.SourceLoader): + tkMessageBox.showerror("Import error", "not a source-based module", + parent=self.text) + return + try: + file_path = spec.loader.get_filename(name) + except AttributeError: + tkMessageBox.showerror("Import error", + "loader does not support get_filename", + parent=self.text) + return + if self.flist: + self.flist.open(file_path) + else: + self.io.loadfile(file_path) + return file_path + + def open_class_browser(self, event=None): + filename = self.io.filename + if not (self.__class__.__name__ == 'PyShellEditorWindow' + and filename): + filename = self.open_module() + if filename is None: + return + head, tail = os.path.split(filename) + base, ext = os.path.splitext(tail) + from idlelib import ClassBrowser + ClassBrowser.ClassBrowser(self.flist, base, [head]) + + def open_path_browser(self, event=None): + from idlelib import PathBrowser + PathBrowser.PathBrowser(self.flist) + + def open_turtle_demo(self, event = None): + import subprocess + + cmd = [sys.executable, + '-c', + 'from turtledemo.__main__ import main; main()'] + subprocess.Popen(cmd, shell=False) + + def gotoline(self, lineno): + if lineno is not None and lineno > 0: + self.text.mark_set("insert", "%d.0" % lineno) + self.text.tag_remove("sel", "1.0", "end") + self.text.tag_add("sel", "insert", "insert +1l") + self.center() + + def ispythonsource(self, filename): + if not filename or os.path.isdir(filename): + return True + base, ext = os.path.splitext(os.path.basename(filename)) + if os.path.normcase(ext) in (".py", ".pyw"): + return True + line = self.text.get('1.0', '1.0 lineend') + return line.startswith('#!') and 'python' in line + + def close_hook(self): + if self.flist: + self.flist.unregister_maybe_terminate(self) + self.flist = None + + def set_close_hook(self, close_hook): + self.close_hook = close_hook + + def filename_change_hook(self): + if self.flist: + self.flist.filename_changed_edit(self) + self.saved_change_hook() + self.top.update_windowlist_registry(self) + self.ResetColorizer() + + def _addcolorizer(self): + if self.color: + return + if self.ispythonsource(self.io.filename): + self.color = self.ColorDelegator() + # can add more colorizers here... + if self.color: + self.per.removefilter(self.undo) + self.per.insertfilter(self.color) + self.per.insertfilter(self.undo) + + def _rmcolorizer(self): + if not self.color: + return + self.color.removecolors() + self.per.removefilter(self.color) + self.color = None + + def ResetColorizer(self): + "Update the color theme" + # Called from self.filename_change_hook and from configDialog.py + self._rmcolorizer() + self._addcolorizer() + theme = idleConf.CurrentTheme() + normal_colors = idleConf.GetHighlight(theme, 'normal') + cursor_color = idleConf.GetHighlight(theme, 'cursor', fgBg='fg') + select_colors = idleConf.GetHighlight(theme, 'hilite') + self.text.config( + foreground=normal_colors['foreground'], + background=normal_colors['background'], + insertbackground=cursor_color, + selectforeground=select_colors['foreground'], + selectbackground=select_colors['background'], + ) + if TkVersion >= 8.5: + self.text.config( + inactiveselectbackground=select_colors['background']) + + IDENTCHARS = string.ascii_letters + string.digits + "_" + + def colorize_syntax_error(self, text, pos): + text.tag_add("ERROR", pos) + char = text.get(pos) + if char and char in self.IDENTCHARS: + text.tag_add("ERROR", pos + " wordstart", pos) + if '\n' == text.get(pos): # error at line end + text.mark_set("insert", pos) + else: + text.mark_set("insert", pos + "+1c") + text.see(pos) + + def ResetFont(self): + "Update the text widgets' font if it is changed" + # Called from configDialog.py + + self.text['font'] = idleConf.GetFont(self.root, 'main','EditorWindow') + + def RemoveKeybindings(self): + "Remove the keybindings before they are changed." + # Called from configDialog.py + self.Bindings.default_keydefs = keydefs = idleConf.GetCurrentKeySet() + for event, keylist in keydefs.items(): + self.text.event_delete(event, *keylist) + for extensionName in self.get_standard_extension_names(): + xkeydefs = idleConf.GetExtensionBindings(extensionName) + if xkeydefs: + for event, keylist in xkeydefs.items(): + self.text.event_delete(event, *keylist) + + def ApplyKeybindings(self): + "Update the keybindings after they are changed" + # Called from configDialog.py + self.Bindings.default_keydefs = keydefs = idleConf.GetCurrentKeySet() + self.apply_bindings() + for extensionName in self.get_standard_extension_names(): + xkeydefs = idleConf.GetExtensionBindings(extensionName) + if xkeydefs: + self.apply_bindings(xkeydefs) + #update menu accelerators + menuEventDict = {} + for menu in self.Bindings.menudefs: + menuEventDict[menu[0]] = {} + for item in menu[1]: + if item: + menuEventDict[menu[0]][prepstr(item[0])[1]] = item[1] + for menubarItem in self.menudict: + menu = self.menudict[menubarItem] + end = menu.index(END) + if end is None: + # Skip empty menus + continue + end += 1 + for index in range(0, end): + if menu.type(index) == 'command': + accel = menu.entrycget(index, 'accelerator') + if accel: + itemName = menu.entrycget(index, 'label') + event = '' + if menubarItem in menuEventDict: + if itemName in menuEventDict[menubarItem]: + event = menuEventDict[menubarItem][itemName] + if event: + accel = get_accelerator(keydefs, event) + menu.entryconfig(index, accelerator=accel) + + def set_notabs_indentwidth(self): + "Update the indentwidth if changed and not using tabs in this window" + # Called from configDialog.py + if not self.usetabs: + self.indentwidth = idleConf.GetOption('main', 'Indent','num-spaces', + type='int') + + def reset_help_menu_entries(self): + "Update the additional help entries on the Help menu" + help_list = idleConf.GetAllExtraHelpSourcesList() + helpmenu = self.menudict['help'] + # first delete the extra help entries, if any + helpmenu_length = helpmenu.index(END) + if helpmenu_length > self.base_helpmenu_length: + helpmenu.delete((self.base_helpmenu_length + 1), helpmenu_length) + # then rebuild them + if help_list: + helpmenu.add_separator() + for entry in help_list: + cmd = self.__extra_help_callback(entry[1]) + helpmenu.add_command(label=entry[0], command=cmd) + # and update the menu dictionary + self.menudict['help'] = helpmenu + + def __extra_help_callback(self, helpfile): + "Create a callback with the helpfile value frozen at definition time" + def display_extra_help(helpfile=helpfile): + if not helpfile.startswith(('www', 'http')): + helpfile = os.path.normpath(helpfile) + if sys.platform[:3] == 'win': + try: + os.startfile(helpfile) + except OSError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) + else: + webbrowser.open(helpfile) + return display_extra_help + + def update_recent_files_list(self, new_file=None): + "Load and update the recent files list and menus" + rf_list = [] + if os.path.exists(self.recent_files_path): + with open(self.recent_files_path, 'r', + encoding='utf_8', errors='replace') as rf_list_file: + rf_list = rf_list_file.readlines() + if new_file: + new_file = os.path.abspath(new_file) + '\n' + if new_file in rf_list: + rf_list.remove(new_file) # move to top + rf_list.insert(0, new_file) + # clean and save the recent files list + bad_paths = [] + for path in rf_list: + if '\0' in path or not os.path.exists(path[0:-1]): + bad_paths.append(path) + rf_list = [path for path in rf_list if path not in bad_paths] + ulchars = "1234567890ABCDEFGHIJK" + rf_list = rf_list[0:len(ulchars)] + try: + with open(self.recent_files_path, 'w', + encoding='utf_8', errors='replace') as rf_file: + rf_file.writelines(rf_list) + except OSError as err: + if not getattr(self.root, "recentfilelist_error_displayed", False): + self.root.recentfilelist_error_displayed = True + tkMessageBox.showwarning(title='IDLE Warning', + message="Cannot update File menu Recent Files list. " + "Your operating system says:\n%s\n" + "Select OK and IDLE will continue without updating." + % self._filename_to_unicode(str(err)), + parent=self.text) + # for each edit window instance, construct the recent files menu + for instance in self.top.instance_dict: + menu = instance.recent_files_menu + menu.delete(0, END) # clear, and rebuild: + for i, file_name in enumerate(rf_list): + file_name = file_name.rstrip() # zap \n + # make unicode string to display non-ASCII chars correctly + ufile_name = self._filename_to_unicode(file_name) + callback = instance.__recent_file_callback(file_name) + menu.add_command(label=ulchars[i] + " " + ufile_name, + command=callback, + underline=0) + + def __recent_file_callback(self, file_name): + def open_recent_file(fn_closure=file_name): + self.io.open(editFile=fn_closure) + return open_recent_file + + def saved_change_hook(self): + short = self.short_title() + long = self.long_title() + if short and long: + title = short + " - " + long + _py_version + elif short: + title = short + elif long: + title = long + else: + title = "Untitled" + icon = short or long or title + if not self.get_saved(): + title = "*%s*" % title + icon = "*%s" % icon + self.top.wm_title(title) + self.top.wm_iconname(icon) + + def get_saved(self): + return self.undo.get_saved() + + def set_saved(self, flag): + self.undo.set_saved(flag) + + def reset_undo(self): + self.undo.reset_undo() + + def short_title(self): + filename = self.io.filename + if filename: + filename = os.path.basename(filename) + else: + filename = "Untitled" + # return unicode string to display non-ASCII chars correctly + return self._filename_to_unicode(filename) + + def long_title(self): + # return unicode string to display non-ASCII chars correctly + return self._filename_to_unicode(self.io.filename or "") + + def center_insert_event(self, event): + self.center() + + def center(self, mark="insert"): + text = self.text + top, bot = self.getwindowlines() + lineno = self.getlineno(mark) + height = bot - top + newtop = max(1, lineno - height//2) + text.yview(float(newtop)) + + def getwindowlines(self): + text = self.text + top = self.getlineno("@0,0") + bot = self.getlineno("@0,65535") + if top == bot and text.winfo_height() == 1: + # Geometry manager hasn't run yet + height = int(text['height']) + bot = top + height - 1 + return top, bot + + def getlineno(self, mark="insert"): + text = self.text + return int(float(text.index(mark))) + + def get_geometry(self): + "Return (width, height, x, y)" + geom = self.top.wm_geometry() + m = re.match(r"(\d+)x(\d+)\+(-?\d+)\+(-?\d+)", geom) + return list(map(int, m.groups())) + + def close_event(self, event): + self.close() + + def maybesave(self): + if self.io: + if not self.get_saved(): + if self.top.state()!='normal': + self.top.deiconify() + self.top.lower() + self.top.lift() + return self.io.maybesave() + + def close(self): + reply = self.maybesave() + if str(reply) != "cancel": + self._close() + return reply + + def _close(self): + if self.io.filename: + self.update_recent_files_list(new_file=self.io.filename) + WindowList.unregister_callback(self.postwindowsmenu) + self.unload_extensions() + self.io.close() + self.io = None + self.undo = None + if self.color: + self.color.close(False) + self.color = None + self.text = None + self.tkinter_vars = None + self.per.close() + self.per = None + self.top.destroy() + if self.close_hook: + # unless override: unregister from flist, terminate if last window + self.close_hook() + + def load_extensions(self): + self.extensions = {} + self.load_standard_extensions() + + def unload_extensions(self): + for ins in list(self.extensions.values()): + if hasattr(ins, "close"): + ins.close() + self.extensions = {} + + def load_standard_extensions(self): + for name in self.get_standard_extension_names(): + try: + self.load_extension(name) + except: + print("Failed to load extension", repr(name)) + traceback.print_exc() + + def get_standard_extension_names(self): + return idleConf.GetExtensions(editor_only=True) + + def load_extension(self, name): + try: + try: + mod = importlib.import_module('.' + name, package=__package__) + except (ImportError, TypeError): + mod = importlib.import_module(name) + except ImportError: + print("\nFailed to import extension: ", name) + raise + cls = getattr(mod, name) + keydefs = idleConf.GetExtensionBindings(name) + if hasattr(cls, "menudefs"): + self.fill_menus(cls.menudefs, keydefs) + ins = cls(self) + self.extensions[name] = ins + if keydefs: + self.apply_bindings(keydefs) + for vevent in keydefs: + methodname = vevent.replace("-", "_") + while methodname[:1] == '<': + methodname = methodname[1:] + while methodname[-1:] == '>': + methodname = methodname[:-1] + methodname = methodname + "_event" + if hasattr(ins, methodname): + self.text.bind(vevent, getattr(ins, methodname)) + + def apply_bindings(self, keydefs=None): + if keydefs is None: + keydefs = self.Bindings.default_keydefs + text = self.text + text.keydefs = keydefs + for event, keylist in keydefs.items(): + if keylist: + text.event_add(event, *keylist) + + def fill_menus(self, menudefs=None, keydefs=None): + """Add appropriate entries to the menus and submenus + + Menus that are absent or None in self.menudict are ignored. + """ + if menudefs is None: + menudefs = self.Bindings.menudefs + if keydefs is None: + keydefs = self.Bindings.default_keydefs + menudict = self.menudict + text = self.text + for mname, entrylist in menudefs: + menu = menudict.get(mname) + if not menu: + continue + for entry in entrylist: + if not entry: + menu.add_separator() + else: + label, eventname = entry + checkbutton = (label[:1] == '!') + if checkbutton: + label = label[1:] + underline, label = prepstr(label) + accelerator = get_accelerator(keydefs, eventname) + def command(text=text, eventname=eventname): + text.event_generate(eventname) + if checkbutton: + var = self.get_var_obj(eventname, BooleanVar) + menu.add_checkbutton(label=label, underline=underline, + command=command, accelerator=accelerator, + variable=var) + else: + menu.add_command(label=label, underline=underline, + command=command, + accelerator=accelerator) + + def getvar(self, name): + var = self.get_var_obj(name) + if var: + value = var.get() + return value + else: + raise NameError(name) + + def setvar(self, name, value, vartype=None): + var = self.get_var_obj(name, vartype) + if var: + var.set(value) + else: + raise NameError(name) + + def get_var_obj(self, name, vartype=None): + var = self.tkinter_vars.get(name) + if not var and vartype: + # create a Tkinter variable object with self.text as master: + self.tkinter_vars[name] = var = vartype(self.text) + return var + + # Tk implementations of "virtual text methods" -- each platform + # reusing IDLE's support code needs to define these for its GUI's + # flavor of widget. + + # Is character at text_index in a Python string? Return 0 for + # "guaranteed no", true for anything else. This info is expensive + # to compute ab initio, but is probably already known by the + # platform's colorizer. + + def is_char_in_string(self, text_index): + if self.color: + # Return true iff colorizer hasn't (re)gotten this far + # yet, or the character is tagged as being in a string + return self.text.tag_prevrange("TODO", text_index) or \ + "STRING" in self.text.tag_names(text_index) + else: + # The colorizer is missing: assume the worst + return 1 + + # If a selection is defined in the text widget, return (start, + # end) as Tkinter text indices, otherwise return (None, None) + def get_selection_indices(self): + try: + first = self.text.index("sel.first") + last = self.text.index("sel.last") + return first, last + except TclError: + return None, None + + # Return the text widget's current view of what a tab stop means + # (equivalent width in spaces). + + def get_tk_tabwidth(self): + current = self.text['tabs'] or TK_TABWIDTH_DEFAULT + return int(current) + + # Set the text widget's current view of what a tab stop means. + + def set_tk_tabwidth(self, newtabwidth): + text = self.text + if self.get_tk_tabwidth() != newtabwidth: + # Set text widget tab width + pixels = text.tk.call("font", "measure", text["font"], + "-displayof", text.master, + "n" * newtabwidth) + text.configure(tabs=pixels) + +### begin autoindent code ### (configuration was moved to beginning of class) + + def set_indentation_params(self, is_py_src, guess=True): + if is_py_src and guess: + i = self.guess_indent() + if 2 <= i <= 8: + self.indentwidth = i + if self.indentwidth != self.tabwidth: + self.usetabs = False + self.set_tk_tabwidth(self.tabwidth) + + def smart_backspace_event(self, event): + text = self.text + first, last = self.get_selection_indices() + if first and last: + text.delete(first, last) + text.mark_set("insert", first) + return "break" + # Delete whitespace left, until hitting a real char or closest + # preceding virtual tab stop. + chars = text.get("insert linestart", "insert") + if chars == '': + if text.compare("insert", ">", "1.0"): + # easy: delete preceding newline + text.delete("insert-1c") + else: + text.bell() # at start of buffer + return "break" + if chars[-1] not in " \t": + # easy: delete preceding real char + text.delete("insert-1c") + return "break" + # Ick. It may require *inserting* spaces if we back up over a + # tab character! This is written to be clear, not fast. + tabwidth = self.tabwidth + have = len(chars.expandtabs(tabwidth)) + assert have > 0 + want = ((have - 1) // self.indentwidth) * self.indentwidth + # Debug prompt is multilined.... + if self.context_use_ps1: + last_line_of_prompt = sys.ps1.split('\n')[-1] + else: + last_line_of_prompt = '' + ncharsdeleted = 0 + while 1: + if chars == last_line_of_prompt: + break + chars = chars[:-1] + ncharsdeleted = ncharsdeleted + 1 + have = len(chars.expandtabs(tabwidth)) + if have <= want or chars[-1] not in " \t": + break + text.undo_block_start() + text.delete("insert-%dc" % ncharsdeleted, "insert") + if have < want: + text.insert("insert", ' ' * (want - have)) + text.undo_block_stop() + return "break" + + def smart_indent_event(self, event): + # if intraline selection: + # delete it + # elif multiline selection: + # do indent-region + # else: + # indent one level + text = self.text + first, last = self.get_selection_indices() + text.undo_block_start() + try: + if first and last: + if index2line(first) != index2line(last): + return self.indent_region_event(event) + text.delete(first, last) + text.mark_set("insert", first) + prefix = text.get("insert linestart", "insert") + raw, effective = classifyws(prefix, self.tabwidth) + if raw == len(prefix): + # only whitespace to the left + self.reindent_to(effective + self.indentwidth) + else: + # tab to the next 'stop' within or to right of line's text: + if self.usetabs: + pad = '\t' + else: + effective = len(prefix.expandtabs(self.tabwidth)) + n = self.indentwidth + pad = ' ' * (n - effective % n) + text.insert("insert", pad) + text.see("insert") + return "break" + finally: + text.undo_block_stop() + + def newline_and_indent_event(self, event): + text = self.text + first, last = self.get_selection_indices() + text.undo_block_start() + try: + if first and last: + text.delete(first, last) + text.mark_set("insert", first) + line = text.get("insert linestart", "insert") + i, n = 0, len(line) + while i < n and line[i] in " \t": + i = i+1 + if i == n: + # the cursor is in or at leading indentation in a continuation + # line; just inject an empty line at the start + text.insert("insert linestart", '\n') + return "break" + indent = line[:i] + # strip whitespace before insert point unless it's in the prompt + i = 0 + last_line_of_prompt = sys.ps1.split('\n')[-1] + while line and line[-1] in " \t" and line != last_line_of_prompt: + line = line[:-1] + i = i+1 + if i: + text.delete("insert - %d chars" % i, "insert") + # strip whitespace after insert point + while text.get("insert") in " \t": + text.delete("insert") + # start new line + text.insert("insert", '\n') + + # adjust indentation for continuations and block + # open/close first need to find the last stmt + lno = index2line(text.index('insert')) + y = PyParse.Parser(self.indentwidth, self.tabwidth) + if not self.context_use_ps1: + for context in self.num_context_lines: + startat = max(lno - context, 1) + startatindex = repr(startat) + ".0" + rawtext = text.get(startatindex, "insert") + y.set_str(rawtext) + bod = y.find_good_parse_start( + self.context_use_ps1, + self._build_char_in_string_func(startatindex)) + if bod is not None or startat == 1: + break + y.set_lo(bod or 0) + else: + r = text.tag_prevrange("console", "insert") + if r: + startatindex = r[1] + else: + startatindex = "1.0" + rawtext = text.get(startatindex, "insert") + y.set_str(rawtext) + y.set_lo(0) + + c = y.get_continuation_type() + if c != PyParse.C_NONE: + # The current stmt hasn't ended yet. + if c == PyParse.C_STRING_FIRST_LINE: + # after the first line of a string; do not indent at all + pass + elif c == PyParse.C_STRING_NEXT_LINES: + # inside a string which started before this line; + # just mimic the current indent + text.insert("insert", indent) + elif c == PyParse.C_BRACKET: + # line up with the first (if any) element of the + # last open bracket structure; else indent one + # level beyond the indent of the line with the + # last open bracket + self.reindent_to(y.compute_bracket_indent()) + elif c == PyParse.C_BACKSLASH: + # if more than one line in this stmt already, just + # mimic the current indent; else if initial line + # has a start on an assignment stmt, indent to + # beyond leftmost =; else to beyond first chunk of + # non-whitespace on initial line + if y.get_num_lines_in_stmt() > 1: + text.insert("insert", indent) + else: + self.reindent_to(y.compute_backslash_indent()) + else: + assert 0, "bogus continuation type %r" % (c,) + return "break" + + # This line starts a brand new stmt; indent relative to + # indentation of initial line of closest preceding + # interesting stmt. + indent = y.get_base_indent_string() + text.insert("insert", indent) + if y.is_block_opener(): + self.smart_indent_event(event) + elif indent and y.is_block_closer(): + self.smart_backspace_event(event) + return "break" + finally: + text.see("insert") + text.undo_block_stop() + + # Our editwin provides an is_char_in_string function that works + # with a Tk text index, but PyParse only knows about offsets into + # a string. This builds a function for PyParse that accepts an + # offset. + + def _build_char_in_string_func(self, startindex): + def inner(offset, _startindex=startindex, + _icis=self.is_char_in_string): + return _icis(_startindex + "+%dc" % offset) + return inner + + def indent_region_event(self, event): + head, tail, chars, lines = self.get_region() + for pos in range(len(lines)): + line = lines[pos] + if line: + raw, effective = classifyws(line, self.tabwidth) + effective = effective + self.indentwidth + lines[pos] = self._make_blanks(effective) + line[raw:] + self.set_region(head, tail, chars, lines) + return "break" + + def dedent_region_event(self, event): + head, tail, chars, lines = self.get_region() + for pos in range(len(lines)): + line = lines[pos] + if line: + raw, effective = classifyws(line, self.tabwidth) + effective = max(effective - self.indentwidth, 0) + lines[pos] = self._make_blanks(effective) + line[raw:] + self.set_region(head, tail, chars, lines) + return "break" + + def comment_region_event(self, event): + head, tail, chars, lines = self.get_region() + for pos in range(len(lines) - 1): + line = lines[pos] + lines[pos] = '##' + line + self.set_region(head, tail, chars, lines) + + def uncomment_region_event(self, event): + head, tail, chars, lines = self.get_region() + for pos in range(len(lines)): + line = lines[pos] + if not line: + continue + if line[:2] == '##': + line = line[2:] + elif line[:1] == '#': + line = line[1:] + lines[pos] = line + self.set_region(head, tail, chars, lines) + + def tabify_region_event(self, event): + head, tail, chars, lines = self.get_region() + tabwidth = self._asktabwidth() + if tabwidth is None: return + for pos in range(len(lines)): + line = lines[pos] + if line: + raw, effective = classifyws(line, tabwidth) + ntabs, nspaces = divmod(effective, tabwidth) + lines[pos] = '\t' * ntabs + ' ' * nspaces + line[raw:] + self.set_region(head, tail, chars, lines) + + def untabify_region_event(self, event): + head, tail, chars, lines = self.get_region() + tabwidth = self._asktabwidth() + if tabwidth is None: return + for pos in range(len(lines)): + lines[pos] = lines[pos].expandtabs(tabwidth) + self.set_region(head, tail, chars, lines) + + def toggle_tabs_event(self, event): + if self.askyesno( + "Toggle tabs", + "Turn tabs " + ("on", "off")[self.usetabs] + + "?\nIndent width " + + ("will be", "remains at")[self.usetabs] + " 8." + + "\n Note: a tab is always 8 columns", + parent=self.text): + self.usetabs = not self.usetabs + # Try to prevent inconsistent indentation. + # User must change indent width manually after using tabs. + self.indentwidth = 8 + return "break" + + # XXX this isn't bound to anything -- see tabwidth comments +## def change_tabwidth_event(self, event): +## new = self._asktabwidth() +## if new != self.tabwidth: +## self.tabwidth = new +## self.set_indentation_params(0, guess=0) +## return "break" + + def change_indentwidth_event(self, event): + new = self.askinteger( + "Indent width", + "New indent width (2-16)\n(Always use 8 when using tabs)", + parent=self.text, + initialvalue=self.indentwidth, + minvalue=2, + maxvalue=16) + if new and new != self.indentwidth and not self.usetabs: + self.indentwidth = new + return "break" + + def get_region(self): + text = self.text + first, last = self.get_selection_indices() + if first and last: + head = text.index(first + " linestart") + tail = text.index(last + "-1c lineend +1c") + else: + head = text.index("insert linestart") + tail = text.index("insert lineend +1c") + chars = text.get(head, tail) + lines = chars.split("\n") + return head, tail, chars, lines + + def set_region(self, head, tail, chars, lines): + text = self.text + newchars = "\n".join(lines) + if newchars == chars: + text.bell() + return + text.tag_remove("sel", "1.0", "end") + text.mark_set("insert", head) + text.undo_block_start() + text.delete(head, tail) + text.insert(head, newchars) + text.undo_block_stop() + text.tag_add("sel", head, "insert") + + # Make string that displays as n leading blanks. + + def _make_blanks(self, n): + if self.usetabs: + ntabs, nspaces = divmod(n, self.tabwidth) + return '\t' * ntabs + ' ' * nspaces + else: + return ' ' * n + + # Delete from beginning of line to insert point, then reinsert + # column logical (meaning use tabs if appropriate) spaces. + + def reindent_to(self, column): + text = self.text + text.undo_block_start() + if text.compare("insert linestart", "!=", "insert"): + text.delete("insert linestart", "insert") + if column: + text.insert("insert", self._make_blanks(column)) + text.undo_block_stop() + + def _asktabwidth(self): + return self.askinteger( + "Tab width", + "Columns per tab? (2-16)", + parent=self.text, + initialvalue=self.indentwidth, + minvalue=2, + maxvalue=16) + + # Guess indentwidth from text content. + # Return guessed indentwidth. This should not be believed unless + # it's in a reasonable range (e.g., it will be 0 if no indented + # blocks are found). + + def guess_indent(self): + opener, indented = IndentSearcher(self.text, self.tabwidth).run() + if opener and indented: + raw, indentsmall = classifyws(opener, self.tabwidth) + raw, indentlarge = classifyws(indented, self.tabwidth) + else: + indentsmall = indentlarge = 0 + return indentlarge - indentsmall + +# "line.col" -> line, as an int +def index2line(index): + return int(float(index)) + +# Look at the leading whitespace in s. +# Return pair (# of leading ws characters, +# effective # of leading blanks after expanding +# tabs to width tabwidth) + +def classifyws(s, tabwidth): + raw = effective = 0 + for ch in s: + if ch == ' ': + raw = raw + 1 + effective = effective + 1 + elif ch == '\t': + raw = raw + 1 + effective = (effective // tabwidth + 1) * tabwidth + else: + break + return raw, effective + +import tokenize +_tokenize = tokenize +del tokenize + +class IndentSearcher(object): + + # .run() chews over the Text widget, looking for a block opener + # and the stmt following it. Returns a pair, + # (line containing block opener, line containing stmt) + # Either or both may be None. + + def __init__(self, text, tabwidth): + self.text = text + self.tabwidth = tabwidth + self.i = self.finished = 0 + self.blkopenline = self.indentedline = None + + def readline(self): + if self.finished: + return "" + i = self.i = self.i + 1 + mark = repr(i) + ".0" + if self.text.compare(mark, ">=", "end"): + return "" + return self.text.get(mark, mark + " lineend+1c") + + def tokeneater(self, type, token, start, end, line, + INDENT=_tokenize.INDENT, + NAME=_tokenize.NAME, + OPENERS=('class', 'def', 'for', 'if', 'try', 'while')): + if self.finished: + pass + elif type == NAME and token in OPENERS: + self.blkopenline = line + elif type == INDENT and self.blkopenline: + self.indentedline = line + self.finished = 1 + + def run(self): + save_tabsize = _tokenize.tabsize + _tokenize.tabsize = self.tabwidth + try: + try: + tokens = _tokenize.generate_tokens(self.readline) + for token in tokens: + self.tokeneater(*token) + except (_tokenize.TokenError, SyntaxError): + # since we cut off the tokenizer early, we can trigger + # spurious errors + pass + finally: + _tokenize.tabsize = save_tabsize + return self.blkopenline, self.indentedline + +### end autoindent code ### + +def prepstr(s): + # Helper to extract the underscore from a string, e.g. + # prepstr("Co_py") returns (2, "Copy"). + i = s.find('_') + if i >= 0: + s = s[:i] + s[i+1:] + return i, s + + +keynames = { + 'bracketleft': '[', + 'bracketright': ']', + 'slash': '/', +} + +def get_accelerator(keydefs, eventname): + keylist = keydefs.get(eventname) + # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5 + # if not keylist: + if (not keylist) or (macosxSupport.isCocoaTk() and eventname in { + "<>", + "<>", + "<>"}): + return "" + s = keylist[0] + s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s) + s = re.sub(r"\b\w+\b", lambda m: keynames.get(m.group(), m.group()), s) + s = re.sub("Key-", "", s) + s = re.sub("Cancel","Ctrl-Break",s) # dscherer@cmu.edu + s = re.sub("Control-", "Ctrl-", s) + s = re.sub("-", "+", s) + s = re.sub("><", " ", s) + s = re.sub("<", "", s) + s = re.sub(">", "", s) + return s + + +def fixwordbreaks(root): + # Make sure that Tk's double-click and next/previous word + # operations use our definition of a word (i.e. an identifier) + tk = root.tk + tk.call('tcl_wordBreakAfter', 'a b', 0) # make sure word.tcl is loaded + tk.call('set', 'tcl_wordchars', '[a-zA-Z0-9_]') + tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') + + +def _editor_window(parent): # htest # + # error if close master window first - timer event, after script + root = parent + fixwordbreaks(root) + if sys.argv[1:]: + filename = sys.argv[1] + else: + filename = None + macosxSupport.setupApp(root, None) + edit = EditorWindow(root=root, filename=filename) + edit.text.bind("<>", edit.close_event) + # Does not stop error, neither does following + # edit.text.bind("<>", edit.close_event) + +if __name__ == '__main__': + from idlelib.idle_test.htest import run + run(_editor_window) diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py new file mode 100644 index 0000000..a9989a8 --- /dev/null +++ b/Lib/idlelib/filelist.py @@ -0,0 +1,129 @@ +import os +from tkinter import * +import tkinter.messagebox as tkMessageBox + + +class FileList: + + # N.B. this import overridden in PyShellFileList. + from idlelib.EditorWindow import EditorWindow + + def __init__(self, root): + self.root = root + self.dict = {} + self.inversedict = {} + self.vars = {} # For EditorWindow.getrawvar (shared Tcl variables) + + def open(self, filename, action=None): + assert filename + filename = self.canonize(filename) + if os.path.isdir(filename): + # This can happen when bad filename is passed on command line: + tkMessageBox.showerror( + "File Error", + "%r is a directory." % (filename,), + master=self.root) + return None + key = os.path.normcase(filename) + if key in self.dict: + edit = self.dict[key] + edit.top.wakeup() + return edit + if action: + # Don't create window, perform 'action', e.g. open in same window + return action(filename) + else: + edit = self.EditorWindow(self, filename, key) + if edit.good_load: + return edit + else: + edit._close() + return None + + def gotofileline(self, filename, lineno=None): + edit = self.open(filename) + if edit is not None and lineno is not None: + edit.gotoline(lineno) + + def new(self, filename=None): + return self.EditorWindow(self, filename) + + def close_all_callback(self, *args, **kwds): + for edit in list(self.inversedict): + reply = edit.close() + if reply == "cancel": + break + return "break" + + def unregister_maybe_terminate(self, edit): + try: + key = self.inversedict[edit] + except KeyError: + print("Don't know this EditorWindow object. (close)") + return + if key: + del self.dict[key] + del self.inversedict[edit] + if not self.inversedict: + self.root.quit() + + def filename_changed_edit(self, edit): + edit.saved_change_hook() + try: + key = self.inversedict[edit] + except KeyError: + print("Don't know this EditorWindow object. (rename)") + return + filename = edit.io.filename + if not filename: + if key: + del self.dict[key] + self.inversedict[edit] = None + return + filename = self.canonize(filename) + newkey = os.path.normcase(filename) + if newkey == key: + return + if newkey in self.dict: + conflict = self.dict[newkey] + self.inversedict[conflict] = None + tkMessageBox.showerror( + "Name Conflict", + "You now have multiple edit windows open for %r" % (filename,), + master=self.root) + self.dict[newkey] = edit + self.inversedict[edit] = newkey + if key: + try: + del self.dict[key] + except KeyError: + pass + + def canonize(self, filename): + if not os.path.isabs(filename): + try: + pwd = os.getcwd() + except OSError: + pass + else: + filename = os.path.join(pwd, filename) + return os.path.normpath(filename) + + +def _test(): + from idlelib.EditorWindow import fixwordbreaks + import sys + root = Tk() + fixwordbreaks(root) + root.withdraw() + flist = FileList(root) + if sys.argv[1:]: + for filename in sys.argv[1:]: + flist.open(filename) + else: + flist.new() + if flist.inversedict: + root.mainloop() + +if __name__ == '__main__': + _test() diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py new file mode 100644 index 0000000..721b231 --- /dev/null +++ b/Lib/idlelib/grep.py @@ -0,0 +1,158 @@ +import os +import fnmatch +import re # for htest +import sys +from tkinter import StringVar, BooleanVar, Checkbutton # for GrepDialog +from tkinter import Tk, Text, Button, SEL, END # for htest +from idlelib import SearchEngine +from idlelib.SearchDialogBase import SearchDialogBase +# Importing OutputWindow fails due to import loop +# EditorWindow -> GrepDialop -> OutputWindow -> EditorWindow + +def grep(text, io=None, flist=None): + root = text._root() + engine = SearchEngine.get(root) + if not hasattr(engine, "_grepdialog"): + engine._grepdialog = GrepDialog(root, engine, flist) + dialog = engine._grepdialog + searchphrase = text.get("sel.first", "sel.last") + dialog.open(text, searchphrase, io) + +class GrepDialog(SearchDialogBase): + + title = "Find in Files Dialog" + icon = "Grep" + needwrapbutton = 0 + + def __init__(self, root, engine, flist): + SearchDialogBase.__init__(self, root, engine) + self.flist = flist + self.globvar = StringVar(root) + self.recvar = BooleanVar(root) + + def open(self, text, searchphrase, io=None): + SearchDialogBase.open(self, text, searchphrase) + if io: + path = io.filename or "" + else: + path = "" + dir, base = os.path.split(path) + head, tail = os.path.splitext(base) + if not tail: + tail = ".py" + self.globvar.set(os.path.join(dir, "*" + tail)) + + def create_entries(self): + SearchDialogBase.create_entries(self) + self.globent = self.make_entry("In files:", self.globvar)[0] + + def create_other_buttons(self): + f = self.make_frame()[0] + + btn = Checkbutton(f, anchor="w", + variable=self.recvar, + text="Recurse down subdirectories") + btn.pack(side="top", fill="both") + btn.select() + + def create_command_buttons(self): + SearchDialogBase.create_command_buttons(self) + self.make_button("Search Files", self.default_command, 1) + + def default_command(self, event=None): + prog = self.engine.getprog() + if not prog: + return + path = self.globvar.get() + if not path: + self.top.bell() + return + from idlelib.OutputWindow import OutputWindow # leave here! + save = sys.stdout + try: + sys.stdout = OutputWindow(self.flist) + self.grep_it(prog, path) + finally: + sys.stdout = save + + def grep_it(self, prog, path): + dir, base = os.path.split(path) + list = self.findfiles(dir, base, self.recvar.get()) + list.sort() + self.close() + pat = self.engine.getpat() + print("Searching %r in %s ..." % (pat, path)) + hits = 0 + try: + for fn in list: + try: + with open(fn, errors='replace') as f: + for lineno, line in enumerate(f, 1): + if line[-1:] == '\n': + line = line[:-1] + if prog.search(line): + sys.stdout.write("%s: %s: %s\n" % + (fn, lineno, line)) + hits += 1 + except OSError as msg: + print(msg) + print(("Hits found: %s\n" + "(Hint: right-click to open locations.)" + % hits) if hits else "No hits.") + except AttributeError: + # Tk window has been closed, OutputWindow.text = None, + # so in OW.write, OW.text.insert fails. + pass + + def findfiles(self, dir, base, rec): + try: + names = os.listdir(dir or os.curdir) + except OSError as msg: + print(msg) + return [] + list = [] + subdirs = [] + for name in names: + fn = os.path.join(dir, name) + if os.path.isdir(fn): + subdirs.append(fn) + else: + if fnmatch.fnmatch(name, base): + list.append(fn) + if rec: + for subdir in subdirs: + list.extend(self.findfiles(subdir, base, rec)) + return list + + def close(self, event=None): + if self.top: + self.top.grab_release() + self.top.withdraw() + + +def _grep_dialog(parent): # htest # + from idlelib.PyShell import PyShellFileList + root = Tk() + root.title("Test GrepDialog") + width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) + root.geometry("+%d+%d"%(x, y + 150)) + + flist = PyShellFileList(root) + text = Text(root, height=5) + text.pack() + + def show_grep_dialog(): + text.tag_add(SEL, "1.0", END) + grep(text, flist=flist) + text.tag_remove(SEL, "1.0", END) + + button = Button(root, text="Show GrepDialog", command=show_grep_dialog) + button.pack() + root.mainloop() + +if __name__ == "__main__": + import unittest + unittest.main('idlelib.idle_test.test_grep', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(_grep_dialog) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py new file mode 100644 index 0000000..3112e6a --- /dev/null +++ b/Lib/idlelib/help_about.py @@ -0,0 +1,149 @@ +"""About Dialog for IDLE + +""" + +import os +from sys import version +from tkinter import * +from idlelib import textView + +class AboutDialog(Toplevel): + """Modal about dialog for idle + + """ + def __init__(self, parent, title, _htest=False): + """ + _htest - bool, change box location when running htest + """ + Toplevel.__init__(self, parent) + self.configure(borderwidth=5) + # place dialog below parent if running htest + self.geometry("+%d+%d" % ( + parent.winfo_rootx()+30, + parent.winfo_rooty()+(30 if not _htest else 100))) + self.bg = "#707070" + self.fg = "#ffffff" + self.CreateWidgets() + self.resizable(height=FALSE, width=FALSE) + self.title(title) + self.transient(parent) + self.grab_set() + self.protocol("WM_DELETE_WINDOW", self.Ok) + self.parent = parent + self.buttonOk.focus_set() + self.bind('',self.Ok) #dismiss dialog + self.bind('',self.Ok) #dismiss dialog + self.wait_window() + + def CreateWidgets(self): + release = version[:version.index(' ')] + frameMain = Frame(self, borderwidth=2, relief=SUNKEN) + frameButtons = Frame(self) + frameButtons.pack(side=BOTTOM, fill=X) + frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) + self.buttonOk = Button(frameButtons, text='Close', + command=self.Ok) + self.buttonOk.pack(padx=5, pady=5) + #self.picture = Image('photo', data=self.pictureData) + frameBg = Frame(frameMain, bg=self.bg) + frameBg.pack(expand=TRUE, fill=BOTH) + labelTitle = Label(frameBg, text='IDLE', fg=self.fg, bg=self.bg, + font=('courier', 24, 'bold')) + labelTitle.grid(row=0, column=0, sticky=W, padx=10, pady=10) + #labelPicture = Label(frameBg, text='[picture]') + #image=self.picture, bg=self.bg) + #labelPicture.grid(row=1, column=1, sticky=W, rowspan=2, + # padx=0, pady=3) + byline = "Python's Integrated DeveLopment Environment" + 5*'\n' + labelDesc = Label(frameBg, text=byline, justify=LEFT, + fg=self.fg, bg=self.bg) + labelDesc.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) + labelEmail = Label(frameBg, text='email: idle-dev@python.org', + justify=LEFT, fg=self.fg, bg=self.bg) + labelEmail.grid(row=6, column=0, columnspan=2, + sticky=W, padx=10, pady=0) + labelWWW = Label(frameBg, text='https://docs.python.org/' + + version[:3] + '/library/idle.html', + justify=LEFT, fg=self.fg, bg=self.bg) + labelWWW.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) + Frame(frameBg, borderwidth=1, relief=SUNKEN, + height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, + columnspan=3, padx=5, pady=5) + labelPythonVer = Label(frameBg, text='Python version: ' + + release, fg=self.fg, bg=self.bg) + labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) + tkVer = self.tk.call('info', 'patchlevel') + labelTkVer = Label(frameBg, text='Tk version: '+ + tkVer, fg=self.fg, bg=self.bg) + labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) + py_button_f = Frame(frameBg, bg=self.bg) + py_button_f.grid(row=10, column=0, columnspan=2, sticky=NSEW) + buttonLicense = Button(py_button_f, text='License', width=8, + highlightbackground=self.bg, + command=self.ShowLicense) + buttonLicense.pack(side=LEFT, padx=10, pady=10) + buttonCopyright = Button(py_button_f, text='Copyright', width=8, + highlightbackground=self.bg, + command=self.ShowCopyright) + buttonCopyright.pack(side=LEFT, padx=10, pady=10) + buttonCredits = Button(py_button_f, text='Credits', width=8, + highlightbackground=self.bg, + command=self.ShowPythonCredits) + buttonCredits.pack(side=LEFT, padx=10, pady=10) + Frame(frameBg, borderwidth=1, relief=SUNKEN, + height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, + columnspan=3, padx=5, pady=5) + idle_v = Label(frameBg, text='IDLE version: ' + release, + fg=self.fg, bg=self.bg) + idle_v.grid(row=12, column=0, sticky=W, padx=10, pady=0) + idle_button_f = Frame(frameBg, bg=self.bg) + idle_button_f.grid(row=13, column=0, columnspan=3, sticky=NSEW) + idle_about_b = Button(idle_button_f, text='README', width=8, + highlightbackground=self.bg, + command=self.ShowIDLEAbout) + idle_about_b.pack(side=LEFT, padx=10, pady=10) + idle_news_b = Button(idle_button_f, text='NEWS', width=8, + highlightbackground=self.bg, + command=self.ShowIDLENEWS) + idle_news_b.pack(side=LEFT, padx=10, pady=10) + idle_credits_b = Button(idle_button_f, text='Credits', width=8, + highlightbackground=self.bg, + command=self.ShowIDLECredits) + idle_credits_b.pack(side=LEFT, padx=10, pady=10) + + # License, et all, are of type _sitebuiltins._Printer + def ShowLicense(self): + self.display_printer_text('About - License', license) + + def ShowCopyright(self): + self.display_printer_text('About - Copyright', copyright) + + def ShowPythonCredits(self): + self.display_printer_text('About - Python Credits', credits) + + # Encode CREDITS.txt to utf-8 for proper version of Loewis. + # Specify others as ascii until need utf-8, so catch errors. + def ShowIDLECredits(self): + self.display_file_text('About - Credits', 'CREDITS.txt', 'utf-8') + + def ShowIDLEAbout(self): + self.display_file_text('About - Readme', 'README.txt', 'ascii') + + def ShowIDLENEWS(self): + self.display_file_text('About - NEWS', 'NEWS.txt', 'ascii') + + def display_printer_text(self, title, printer): + printer._Printer__setup() + text = '\n'.join(printer._Printer__lines) + textView.view_text(self, title, text) + + def display_file_text(self, title, filename, encoding=None): + fn = os.path.join(os.path.abspath(os.path.dirname(__file__)), filename) + textView.view_file(self, title, fn, encoding) + + def Ok(self, event=None): + self.destroy() + +if __name__ == '__main__': + from idlelib.idle_test.htest import run + run(AboutDialog) diff --git a/Lib/idlelib/history.py b/Lib/idlelib/history.py new file mode 100644 index 0000000..078af29 --- /dev/null +++ b/Lib/idlelib/history.py @@ -0,0 +1,104 @@ +"Implement Idle Shell history mechanism with History class" + +from idlelib.configHandler import idleConf + +class History: + ''' Implement Idle Shell history mechanism. + + store - Store source statement (called from PyShell.resetoutput). + fetch - Fetch stored statement matching prefix already entered. + history_next - Bound to <> event (default Alt-N). + history_prev - Bound to <> event (default Alt-P). + ''' + def __init__(self, text): + '''Initialize data attributes and bind event methods. + + .text - Idle wrapper of tk Text widget, with .bell(). + .history - source statements, possibly with multiple lines. + .prefix - source already entered at prompt; filters history list. + .pointer - index into history. + .cyclic - wrap around history list (or not). + ''' + self.text = text + self.history = [] + self.prefix = None + self.pointer = None + self.cyclic = idleConf.GetOption("main", "History", "cyclic", 1, "bool") + text.bind("<>", self.history_prev) + text.bind("<>", self.history_next) + + def history_next(self, event): + "Fetch later statement; start with ealiest if cyclic." + self.fetch(reverse=False) + return "break" + + def history_prev(self, event): + "Fetch earlier statement; start with most recent." + self.fetch(reverse=True) + return "break" + + def fetch(self, reverse): + '''Fetch statememt and replace current line in text widget. + + Set prefix and pointer as needed for successive fetches. + Reset them to None, None when returning to the start line. + Sound bell when return to start line or cannot leave a line + because cyclic is False. + ''' + nhist = len(self.history) + pointer = self.pointer + prefix = self.prefix + if pointer is not None and prefix is not None: + if self.text.compare("insert", "!=", "end-1c") or \ + self.text.get("iomark", "end-1c") != self.history[pointer]: + pointer = prefix = None + self.text.mark_set("insert", "end-1c") # != after cursor move + if pointer is None or prefix is None: + prefix = self.text.get("iomark", "end-1c") + if reverse: + pointer = nhist # will be decremented + else: + if self.cyclic: + pointer = -1 # will be incremented + else: # abort history_next + self.text.bell() + return + nprefix = len(prefix) + while 1: + pointer += -1 if reverse else 1 + if pointer < 0 or pointer >= nhist: + self.text.bell() + if not self.cyclic and pointer < 0: # abort history_prev + return + else: + if self.text.get("iomark", "end-1c") != prefix: + self.text.delete("iomark", "end-1c") + self.text.insert("iomark", prefix) + pointer = prefix = None + break + item = self.history[pointer] + if item[:nprefix] == prefix and len(item) > nprefix: + self.text.delete("iomark", "end-1c") + self.text.insert("iomark", item) + break + self.text.see("insert") + self.text.tag_remove("sel", "1.0", "end") + self.pointer = pointer + self.prefix = prefix + + def store(self, source): + "Store Shell input statement into history list." + source = source.strip() + if len(source) > 2: + # avoid duplicates + try: + self.history.remove(source) + except ValueError: + pass + self.history.append(source) + self.pointer = None + self.prefix = None + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_idlehistory', verbosity=2, exit=False) diff --git a/Lib/idlelib/hyperparser.py b/Lib/idlelib/hyperparser.py new file mode 100644 index 0000000..77cb057 --- /dev/null +++ b/Lib/idlelib/hyperparser.py @@ -0,0 +1,313 @@ +"""Provide advanced parsing abilities for ParenMatch and other extensions. + +HyperParser uses PyParser. PyParser mostly gives information on the +proper indentation of code. HyperParser gives additional information on +the structure of code. +""" + +import string +from keyword import iskeyword +from idlelib import PyParse + + +# all ASCII chars that may be in an identifier +_ASCII_ID_CHARS = frozenset(string.ascii_letters + string.digits + "_") +# all ASCII chars that may be the first char of an identifier +_ASCII_ID_FIRST_CHARS = frozenset(string.ascii_letters + "_") + +# lookup table for whether 7-bit ASCII chars are valid in a Python identifier +_IS_ASCII_ID_CHAR = [(chr(x) in _ASCII_ID_CHARS) for x in range(128)] +# lookup table for whether 7-bit ASCII chars are valid as the first +# char in a Python identifier +_IS_ASCII_ID_FIRST_CHAR = \ + [(chr(x) in _ASCII_ID_FIRST_CHARS) for x in range(128)] + + +class HyperParser: + def __init__(self, editwin, index): + "To initialize, analyze the surroundings of the given index." + + self.editwin = editwin + self.text = text = editwin.text + + parser = PyParse.Parser(editwin.indentwidth, editwin.tabwidth) + + def index2line(index): + return int(float(index)) + lno = index2line(text.index(index)) + + if not editwin.context_use_ps1: + for context in editwin.num_context_lines: + startat = max(lno - context, 1) + startatindex = repr(startat) + ".0" + stopatindex = "%d.end" % lno + # We add the newline because PyParse requires a newline + # at end. We add a space so that index won't be at end + # of line, so that its status will be the same as the + # char before it, if should. + parser.set_str(text.get(startatindex, stopatindex)+' \n') + bod = parser.find_good_parse_start( + editwin._build_char_in_string_func(startatindex)) + if bod is not None or startat == 1: + break + parser.set_lo(bod or 0) + else: + r = text.tag_prevrange("console", index) + if r: + startatindex = r[1] + else: + startatindex = "1.0" + stopatindex = "%d.end" % lno + # We add the newline because PyParse requires it. We add a + # space so that index won't be at end of line, so that its + # status will be the same as the char before it, if should. + parser.set_str(text.get(startatindex, stopatindex)+' \n') + parser.set_lo(0) + + # We want what the parser has, minus the last newline and space. + self.rawtext = parser.str[:-2] + # Parser.str apparently preserves the statement we are in, so + # that stopatindex can be used to synchronize the string with + # the text box indices. + self.stopatindex = stopatindex + self.bracketing = parser.get_last_stmt_bracketing() + # find which pairs of bracketing are openers. These always + # correspond to a character of rawtext. + self.isopener = [i>0 and self.bracketing[i][1] > + self.bracketing[i-1][1] + for i in range(len(self.bracketing))] + + self.set_index(index) + + def set_index(self, index): + """Set the index to which the functions relate. + + The index must be in the same statement. + """ + indexinrawtext = (len(self.rawtext) - + len(self.text.get(index, self.stopatindex))) + if indexinrawtext < 0: + raise ValueError("Index %s precedes the analyzed statement" + % index) + self.indexinrawtext = indexinrawtext + # find the rightmost bracket to which index belongs + self.indexbracket = 0 + while (self.indexbracket < len(self.bracketing)-1 and + self.bracketing[self.indexbracket+1][0] < self.indexinrawtext): + self.indexbracket += 1 + if (self.indexbracket < len(self.bracketing)-1 and + self.bracketing[self.indexbracket+1][0] == self.indexinrawtext and + not self.isopener[self.indexbracket+1]): + self.indexbracket += 1 + + def is_in_string(self): + """Is the index given to the HyperParser in a string?""" + # The bracket to which we belong should be an opener. + # If it's an opener, it has to have a character. + return (self.isopener[self.indexbracket] and + self.rawtext[self.bracketing[self.indexbracket][0]] + in ('"', "'")) + + def is_in_code(self): + """Is the index given to the HyperParser in normal code?""" + return (not self.isopener[self.indexbracket] or + self.rawtext[self.bracketing[self.indexbracket][0]] + not in ('#', '"', "'")) + + def get_surrounding_brackets(self, openers='([{', mustclose=False): + """Return bracket indexes or None. + + If the index given to the HyperParser is surrounded by a + bracket defined in openers (or at least has one before it), + return the indices of the opening bracket and the closing + bracket (or the end of line, whichever comes first). + + If it is not surrounded by brackets, or the end of line comes + before the closing bracket and mustclose is True, returns None. + """ + + bracketinglevel = self.bracketing[self.indexbracket][1] + before = self.indexbracket + while (not self.isopener[before] or + self.rawtext[self.bracketing[before][0]] not in openers or + self.bracketing[before][1] > bracketinglevel): + before -= 1 + if before < 0: + return None + bracketinglevel = min(bracketinglevel, self.bracketing[before][1]) + after = self.indexbracket + 1 + while (after < len(self.bracketing) and + self.bracketing[after][1] >= bracketinglevel): + after += 1 + + beforeindex = self.text.index("%s-%dc" % + (self.stopatindex, len(self.rawtext)-self.bracketing[before][0])) + if (after >= len(self.bracketing) or + self.bracketing[after][0] > len(self.rawtext)): + if mustclose: + return None + afterindex = self.stopatindex + else: + # We are after a real char, so it is a ')' and we give the + # index before it. + afterindex = self.text.index( + "%s-%dc" % (self.stopatindex, + len(self.rawtext)-(self.bracketing[after][0]-1))) + + return beforeindex, afterindex + + # the set of built-in identifiers which are also keywords, + # i.e. keyword.iskeyword() returns True for them + _ID_KEYWORDS = frozenset({"True", "False", "None"}) + + @classmethod + def _eat_identifier(cls, str, limit, pos): + """Given a string and pos, return the number of chars in the + identifier which ends at pos, or 0 if there is no such one. + + This ignores non-identifier eywords are not identifiers. + """ + is_ascii_id_char = _IS_ASCII_ID_CHAR + + # Start at the end (pos) and work backwards. + i = pos + + # Go backwards as long as the characters are valid ASCII + # identifier characters. This is an optimization, since it + # is faster in the common case where most of the characters + # are ASCII. + while i > limit and ( + ord(str[i - 1]) < 128 and + is_ascii_id_char[ord(str[i - 1])] + ): + i -= 1 + + # If the above loop ended due to reaching a non-ASCII + # character, continue going backwards using the most generic + # test for whether a string contains only valid identifier + # characters. + if i > limit and ord(str[i - 1]) >= 128: + while i - 4 >= limit and ('a' + str[i - 4:pos]).isidentifier(): + i -= 4 + if i - 2 >= limit and ('a' + str[i - 2:pos]).isidentifier(): + i -= 2 + if i - 1 >= limit and ('a' + str[i - 1:pos]).isidentifier(): + i -= 1 + + # The identifier candidate starts here. If it isn't a valid + # identifier, don't eat anything. At this point that is only + # possible if the first character isn't a valid first + # character for an identifier. + if not str[i:pos].isidentifier(): + return 0 + elif i < pos: + # All characters in str[i:pos] are valid ASCII identifier + # characters, so it is enough to check that the first is + # valid as the first character of an identifier. + if not _IS_ASCII_ID_FIRST_CHAR[ord(str[i])]: + return 0 + + # All keywords are valid identifiers, but should not be + # considered identifiers here, except for True, False and None. + if i < pos and ( + iskeyword(str[i:pos]) and + str[i:pos] not in cls._ID_KEYWORDS + ): + return 0 + + return pos - i + + # This string includes all chars that may be in a white space + _whitespace_chars = " \t\n\\" + + def get_expression(self): + """Return a string with the Python expression which ends at the + given index, which is empty if there is no real one. + """ + if not self.is_in_code(): + raise ValueError("get_expression should only be called" + "if index is inside a code.") + + rawtext = self.rawtext + bracketing = self.bracketing + + brck_index = self.indexbracket + brck_limit = bracketing[brck_index][0] + pos = self.indexinrawtext + + last_identifier_pos = pos + postdot_phase = True + + while 1: + # Eat whitespaces, comments, and if postdot_phase is False - a dot + while 1: + if pos>brck_limit and rawtext[pos-1] in self._whitespace_chars: + # Eat a whitespace + pos -= 1 + elif (not postdot_phase and + pos > brck_limit and rawtext[pos-1] == '.'): + # Eat a dot + pos -= 1 + postdot_phase = True + # The next line will fail if we are *inside* a comment, + # but we shouldn't be. + elif (pos == brck_limit and brck_index > 0 and + rawtext[bracketing[brck_index-1][0]] == '#'): + # Eat a comment + brck_index -= 2 + brck_limit = bracketing[brck_index][0] + pos = bracketing[brck_index+1][0] + else: + # If we didn't eat anything, quit. + break + + if not postdot_phase: + # We didn't find a dot, so the expression end at the + # last identifier pos. + break + + ret = self._eat_identifier(rawtext, brck_limit, pos) + if ret: + # There is an identifier to eat + pos = pos - ret + last_identifier_pos = pos + # Now, to continue the search, we must find a dot. + postdot_phase = False + # (the loop continues now) + + elif pos == brck_limit: + # We are at a bracketing limit. If it is a closing + # bracket, eat the bracket, otherwise, stop the search. + level = bracketing[brck_index][1] + while brck_index > 0 and bracketing[brck_index-1][1] > level: + brck_index -= 1 + if bracketing[brck_index][0] == brck_limit: + # We were not at the end of a closing bracket + break + pos = bracketing[brck_index][0] + brck_index -= 1 + brck_limit = bracketing[brck_index][0] + last_identifier_pos = pos + if rawtext[pos] in "([": + # [] and () may be used after an identifier, so we + # continue. postdot_phase is True, so we don't allow a dot. + pass + else: + # We can't continue after other types of brackets + if rawtext[pos] in "'\"": + # Scan a string prefix + while pos > 0 and rawtext[pos - 1] in "rRbBuU": + pos -= 1 + last_identifier_pos = pos + break + + else: + # We've found an operator or something. + break + + return rawtext[last_identifier_pos:self.indexinrawtext] + + +if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_hyperparser', verbosity=2) diff --git a/Lib/idlelib/idle_test/test_formatparagraph.py b/Lib/idlelib/idle_test/test_formatparagraph.py deleted file mode 100644 index f6039e6..0000000 --- a/Lib/idlelib/idle_test/test_formatparagraph.py +++ /dev/null @@ -1,377 +0,0 @@ -# Test the functions and main class method of FormatParagraph.py -import unittest -from idlelib import FormatParagraph as fp -from idlelib.EditorWindow import EditorWindow -from tkinter import Tk, Text -from test.support import requires - - -class Is_Get_Test(unittest.TestCase): - """Test the is_ and get_ functions""" - test_comment = '# This is a comment' - test_nocomment = 'This is not a comment' - trailingws_comment = '# This is a comment ' - leadingws_comment = ' # This is a comment' - leadingws_nocomment = ' This is not a comment' - - def test_is_all_white(self): - self.assertTrue(fp.is_all_white('')) - self.assertTrue(fp.is_all_white('\t\n\r\f\v')) - self.assertFalse(fp.is_all_white(self.test_comment)) - - def test_get_indent(self): - Equal = self.assertEqual - Equal(fp.get_indent(self.test_comment), '') - Equal(fp.get_indent(self.trailingws_comment), '') - Equal(fp.get_indent(self.leadingws_comment), ' ') - Equal(fp.get_indent(self.leadingws_nocomment), ' ') - - def test_get_comment_header(self): - Equal = self.assertEqual - # Test comment strings - Equal(fp.get_comment_header(self.test_comment), '#') - Equal(fp.get_comment_header(self.trailingws_comment), '#') - Equal(fp.get_comment_header(self.leadingws_comment), ' #') - # Test non-comment strings - Equal(fp.get_comment_header(self.leadingws_nocomment), ' ') - Equal(fp.get_comment_header(self.test_nocomment), '') - - -class FindTest(unittest.TestCase): - """Test the find_paragraph function in FormatParagraph. - - Using the runcase() function, find_paragraph() is called with 'mark' set at - multiple indexes before and inside the test paragraph. - - It appears that code with the same indentation as a quoted string is grouped - as part of the same paragraph, which is probably incorrect behavior. - """ - - @classmethod - def setUpClass(cls): - from idlelib.idle_test.mock_tk import Text - cls.text = Text() - - def runcase(self, inserttext, stopline, expected): - # Check that find_paragraph returns the expected paragraph when - # the mark index is set to beginning, middle, end of each line - # up to but not including the stop line - text = self.text - text.insert('1.0', inserttext) - for line in range(1, stopline): - linelength = int(text.index("%d.end" % line).split('.')[1]) - for col in (0, linelength//2, linelength): - tempindex = "%d.%d" % (line, col) - self.assertEqual(fp.find_paragraph(text, tempindex), expected) - text.delete('1.0', 'end') - - def test_find_comment(self): - comment = ( - "# Comment block with no blank lines before\n" - "# Comment line\n" - "\n") - self.runcase(comment, 3, ('1.0', '3.0', '#', comment[0:58])) - - comment = ( - "\n" - "# Comment block with whitespace line before and after\n" - "# Comment line\n" - "\n") - self.runcase(comment, 4, ('2.0', '4.0', '#', comment[1:70])) - - comment = ( - "\n" - " # Indented comment block with whitespace before and after\n" - " # Comment line\n" - "\n") - self.runcase(comment, 4, ('2.0', '4.0', ' #', comment[1:82])) - - comment = ( - "\n" - "# Single line comment\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:23])) - - comment = ( - "\n" - " # Single line comment with leading whitespace\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:51])) - - comment = ( - "\n" - "# Comment immediately followed by code\n" - "x = 42\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:40])) - - comment = ( - "\n" - " # Indented comment immediately followed by code\n" - "x = 42\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:53])) - - comment = ( - "\n" - "# Comment immediately followed by indented code\n" - " x = 42\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:49])) - - def test_find_paragraph(self): - teststring = ( - '"""String with no blank lines before\n' - 'String line\n' - '"""\n' - '\n') - self.runcase(teststring, 4, ('1.0', '4.0', '', teststring[0:53])) - - teststring = ( - "\n" - '"""String with whitespace line before and after\n' - 'String line.\n' - '"""\n' - '\n') - self.runcase(teststring, 5, ('2.0', '5.0', '', teststring[1:66])) - - teststring = ( - '\n' - ' """Indented string with whitespace before and after\n' - ' Comment string.\n' - ' """\n' - '\n') - self.runcase(teststring, 5, ('2.0', '5.0', ' ', teststring[1:85])) - - teststring = ( - '\n' - '"""Single line string."""\n' - '\n') - self.runcase(teststring, 3, ('2.0', '3.0', '', teststring[1:27])) - - teststring = ( - '\n' - ' """Single line string with leading whitespace."""\n' - '\n') - self.runcase(teststring, 3, ('2.0', '3.0', ' ', teststring[1:55])) - - -class ReformatFunctionTest(unittest.TestCase): - """Test the reformat_paragraph function without the editor window.""" - - def test_reformat_paragrah(self): - Equal = self.assertEqual - reform = fp.reformat_paragraph - hw = "O hello world" - Equal(reform(' ', 1), ' ') - Equal(reform("Hello world", 20), "Hello world") - - # Test without leading newline - Equal(reform(hw, 1), "O\nhello\nworld") - Equal(reform(hw, 6), "O\nhello\nworld") - Equal(reform(hw, 7), "O hello\nworld") - Equal(reform(hw, 12), "O hello\nworld") - Equal(reform(hw, 13), "O hello world") - - # Test with leading newline - hw = "\nO hello world" - Equal(reform(hw, 1), "\nO\nhello\nworld") - Equal(reform(hw, 6), "\nO\nhello\nworld") - Equal(reform(hw, 7), "\nO hello\nworld") - Equal(reform(hw, 12), "\nO hello\nworld") - Equal(reform(hw, 13), "\nO hello world") - - -class ReformatCommentTest(unittest.TestCase): - """Test the reformat_comment function without the editor window.""" - - def test_reformat_comment(self): - Equal = self.assertEqual - - # reformat_comment formats to a minimum of 20 characters - test_string = ( - " \"\"\"this is a test of a reformat for a triple quoted string" - " will it reformat to less than 70 characters for me?\"\"\"") - result = fp.reformat_comment(test_string, 70, " ") - expected = ( - " \"\"\"this is a test of a reformat for a triple quoted string will it\n" - " reformat to less than 70 characters for me?\"\"\"") - Equal(result, expected) - - test_comment = ( - "# this is a test of a reformat for a triple quoted string will " - "it reformat to less than 70 characters for me?") - result = fp.reformat_comment(test_comment, 70, "#") - expected = ( - "# this is a test of a reformat for a triple quoted string will it\n" - "# reformat to less than 70 characters for me?") - Equal(result, expected) - - -class FormatClassTest(unittest.TestCase): - def test_init_close(self): - instance = fp.FormatParagraph('editor') - self.assertEqual(instance.editwin, 'editor') - instance.close() - self.assertEqual(instance.editwin, None) - - -# For testing format_paragraph_event, Initialize FormatParagraph with -# a mock Editor with .text and .get_selection_indices. The text must -# be a Text wrapper that adds two methods - -# A real EditorWindow creates unneeded, time-consuming baggage and -# sometimes emits shutdown warnings like this: -# "warning: callback failed in WindowList -# : invalid command name ".55131368.windows". -# Calling EditorWindow._close in tearDownClass prevents this but causes -# other problems (windows left open). - -class TextWrapper: - def __init__(self, master): - self.text = Text(master=master) - def __getattr__(self, name): - return getattr(self.text, name) - def undo_block_start(self): pass - def undo_block_stop(self): pass - -class Editor: - def __init__(self, root): - self.text = TextWrapper(root) - get_selection_indices = EditorWindow. get_selection_indices - -class FormatEventTest(unittest.TestCase): - """Test the formatting of text inside a Text widget. - - This is done with FormatParagraph.format.paragraph_event, - which calls functions in the module as appropriate. - """ - test_string = ( - " '''this is a test of a reformat for a triple " - "quoted string will it reformat to less than 70 " - "characters for me?'''\n") - multiline_test_string = ( - " '''The first line is under the max width.\n" - " The second line's length is way over the max width. It goes " - "on and on until it is over 100 characters long.\n" - " Same thing with the third line. It is also way over the max " - "width, but FormatParagraph will fix it.\n" - " '''\n") - multiline_test_comment = ( - "# The first line is under the max width.\n" - "# The second line's length is way over the max width. It goes on " - "and on until it is over 100 characters long.\n" - "# Same thing with the third line. It is also way over the max " - "width, but FormatParagraph will fix it.\n" - "# The fourth line is short like the first line.") - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - editor = Editor(root=cls.root) - cls.text = editor.text.text # Test code does not need the wrapper. - cls.formatter = fp.FormatParagraph(editor).format_paragraph_event - # Sets the insert mark just after the re-wrapped and inserted text. - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - del cls.text - del cls.formatter - - def test_short_line(self): - self.text.insert('1.0', "Short line\n") - self.formatter("Dummy") - self.assertEqual(self.text.get('1.0', 'insert'), "Short line\n" ) - self.text.delete('1.0', 'end') - - def test_long_line(self): - text = self.text - - # Set cursor ('insert' mark) to '1.0', within text. - text.insert('1.0', self.test_string) - text.mark_set('insert', '1.0') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - # find function includes \n - expected = ( -" '''this is a test of a reformat for a triple quoted string will it\n" -" reformat to less than 70 characters for me?'''\n") # yes - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - # Select from 1.11 to line end. - text.insert('1.0', self.test_string) - text.tag_add('sel', '1.11', '1.end') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - # selection excludes \n - expected = ( -" '''this is a test of a reformat for a triple quoted string will it reformat\n" -" to less than 70 characters for me?'''") # no - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - def test_multiple_lines(self): - text = self.text - # Select 2 long lines. - text.insert('1.0', self.multiline_test_string) - text.tag_add('sel', '2.0', '4.0') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('2.0', 'insert') - expected = ( -" The second line's length is way over the max width. It goes on and\n" -" on until it is over 100 characters long. Same thing with the third\n" -" line. It is also way over the max width, but FormatParagraph will\n" -" fix it.\n") - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - def test_comment_block(self): - text = self.text - - # Set cursor ('insert') to '1.0', within block. - text.insert('1.0', self.multiline_test_comment) - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - expected = ( -"# The first line is under the max width. The second line's length is\n" -"# way over the max width. It goes on and on until it is over 100\n" -"# characters long. Same thing with the third line. It is also way over\n" -"# the max width, but FormatParagraph will fix it. The fourth line is\n" -"# short like the first line.\n") - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - # Select line 2, verify line 1 unaffected. - text.insert('1.0', self.multiline_test_comment) - text.tag_add('sel', '2.0', '3.0') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - expected = ( -"# The first line is under the max width.\n" -"# The second line's length is way over the max width. It goes on and\n" -"# on until it is over 100 characters long.\n") - self.assertEqual(result, expected) - text.delete('1.0', 'end') - -# The following block worked with EditorWindow but fails with the mock. -# Lines 2 and 3 get pasted together even though the previous block left -# the previous line alone. More investigation is needed. -## # Select lines 3 and 4 -## text.insert('1.0', self.multiline_test_comment) -## text.tag_add('sel', '3.0', '5.0') -## self.formatter('ParameterDoesNothing') -## result = text.get('3.0', 'insert') -## expected = ( -##"# Same thing with the third line. It is also way over the max width,\n" -##"# but FormatParagraph will fix it. The fourth line is short like the\n" -##"# first line.\n") -## self.assertEqual(result, expected) -## text.delete('1.0', 'end') - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_history.py b/Lib/idlelib/idle_test/test_history.py new file mode 100644 index 0000000..d7c3d70 --- /dev/null +++ b/Lib/idlelib/idle_test/test_history.py @@ -0,0 +1,167 @@ +import unittest +from test.support import requires + +import tkinter as tk +from tkinter import Text as tkText +from idlelib.idle_test.mock_tk import Text as mkText +from idlelib.IdleHistory import History +from idlelib.configHandler import idleConf + +line1 = 'a = 7' +line2 = 'b = a' + +class StoreTest(unittest.TestCase): + '''Tests History.__init__ and History.store with mock Text''' + + @classmethod + def setUpClass(cls): + cls.text = mkText() + cls.history = History(cls.text) + + def tearDown(self): + self.text.delete('1.0', 'end') + self.history.history = [] + + def test_init(self): + self.assertIs(self.history.text, self.text) + self.assertEqual(self.history.history, []) + self.assertIsNone(self.history.prefix) + self.assertIsNone(self.history.pointer) + self.assertEqual(self.history.cyclic, + idleConf.GetOption("main", "History", "cyclic", 1, "bool")) + + def test_store_short(self): + self.history.store('a') + self.assertEqual(self.history.history, []) + self.history.store(' a ') + self.assertEqual(self.history.history, []) + + def test_store_dup(self): + self.history.store(line1) + self.assertEqual(self.history.history, [line1]) + self.history.store(line2) + self.assertEqual(self.history.history, [line1, line2]) + self.history.store(line1) + self.assertEqual(self.history.history, [line2, line1]) + + def test_store_reset(self): + self.history.prefix = line1 + self.history.pointer = 0 + self.history.store(line2) + self.assertIsNone(self.history.prefix) + self.assertIsNone(self.history.pointer) + + +class TextWrapper: + def __init__(self, master): + self.text = tkText(master=master) + self._bell = False + def __getattr__(self, name): + return getattr(self.text, name) + def bell(self): + self._bell = True + +class FetchTest(unittest.TestCase): + '''Test History.fetch with wrapped tk.Text. + ''' + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = tk.Tk() + + def setUp(self): + self.text = text = TextWrapper(self.root) + text.insert('1.0', ">>> ") + text.mark_set('iomark', '1.4') + text.mark_gravity('iomark', 'left') + self.history = History(text) + self.history.history = [line1, line2] + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def fetch_test(self, reverse, line, prefix, index, *, bell=False): + # Perform one fetch as invoked by Alt-N or Alt-P + # Test the result. The line test is the most important. + # The last two are diagnostic of fetch internals. + History = self.history + History.fetch(reverse) + + Equal = self.assertEqual + Equal(self.text.get('iomark', 'end-1c'), line) + Equal(self.text._bell, bell) + if bell: + self.text._bell = False + Equal(History.prefix, prefix) + Equal(History.pointer, index) + Equal(self.text.compare("insert", '==', "end-1c"), 1) + + def test_fetch_prev_cyclic(self): + prefix = '' + test = self.fetch_test + test(True, line2, prefix, 1) + test(True, line1, prefix, 0) + test(True, prefix, None, None, bell=True) + + def test_fetch_next_cyclic(self): + prefix = '' + test = self.fetch_test + test(False, line1, prefix, 0) + test(False, line2, prefix, 1) + test(False, prefix, None, None, bell=True) + + # Prefix 'a' tests skip line2, which starts with 'b' + def test_fetch_prev_prefix(self): + prefix = 'a' + self.text.insert('iomark', prefix) + self.fetch_test(True, line1, prefix, 0) + self.fetch_test(True, prefix, None, None, bell=True) + + def test_fetch_next_prefix(self): + prefix = 'a' + self.text.insert('iomark', prefix) + self.fetch_test(False, line1, prefix, 0) + self.fetch_test(False, prefix, None, None, bell=True) + + def test_fetch_prev_noncyclic(self): + prefix = '' + self.history.cyclic = False + test = self.fetch_test + test(True, line2, prefix, 1) + test(True, line1, prefix, 0) + test(True, line1, prefix, 0, bell=True) + + def test_fetch_next_noncyclic(self): + prefix = '' + self.history.cyclic = False + test = self.fetch_test + test(False, prefix, None, None, bell=True) + test(True, line2, prefix, 1) + test(False, prefix, None, None, bell=True) + test(False, prefix, None, None, bell=True) + + def test_fetch_cursor_move(self): + # Move cursor after fetch + self.history.fetch(reverse=True) # initialization + self.text.mark_set('insert', 'iomark') + self.fetch_test(True, line2, None, None, bell=True) + + def test_fetch_edit(self): + # Edit after fetch + self.history.fetch(reverse=True) # initialization + self.text.delete('iomark', 'insert', ) + self.text.insert('iomark', 'a =') + self.fetch_test(True, line1, 'a =', 0) # prefix is reset + + def test_history_prev_next(self): + # Minimally test functions bound to events + self.history.history_prev('dummy event') + self.assertEqual(self.history.pointer, 1) + self.history.history_next('dummy event') + self.assertEqual(self.history.pointer, None) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_idlehistory.py b/Lib/idlelib/idle_test/test_idlehistory.py deleted file mode 100644 index d7c3d70..0000000 --- a/Lib/idlelib/idle_test/test_idlehistory.py +++ /dev/null @@ -1,167 +0,0 @@ -import unittest -from test.support import requires - -import tkinter as tk -from tkinter import Text as tkText -from idlelib.idle_test.mock_tk import Text as mkText -from idlelib.IdleHistory import History -from idlelib.configHandler import idleConf - -line1 = 'a = 7' -line2 = 'b = a' - -class StoreTest(unittest.TestCase): - '''Tests History.__init__ and History.store with mock Text''' - - @classmethod - def setUpClass(cls): - cls.text = mkText() - cls.history = History(cls.text) - - def tearDown(self): - self.text.delete('1.0', 'end') - self.history.history = [] - - def test_init(self): - self.assertIs(self.history.text, self.text) - self.assertEqual(self.history.history, []) - self.assertIsNone(self.history.prefix) - self.assertIsNone(self.history.pointer) - self.assertEqual(self.history.cyclic, - idleConf.GetOption("main", "History", "cyclic", 1, "bool")) - - def test_store_short(self): - self.history.store('a') - self.assertEqual(self.history.history, []) - self.history.store(' a ') - self.assertEqual(self.history.history, []) - - def test_store_dup(self): - self.history.store(line1) - self.assertEqual(self.history.history, [line1]) - self.history.store(line2) - self.assertEqual(self.history.history, [line1, line2]) - self.history.store(line1) - self.assertEqual(self.history.history, [line2, line1]) - - def test_store_reset(self): - self.history.prefix = line1 - self.history.pointer = 0 - self.history.store(line2) - self.assertIsNone(self.history.prefix) - self.assertIsNone(self.history.pointer) - - -class TextWrapper: - def __init__(self, master): - self.text = tkText(master=master) - self._bell = False - def __getattr__(self, name): - return getattr(self.text, name) - def bell(self): - self._bell = True - -class FetchTest(unittest.TestCase): - '''Test History.fetch with wrapped tk.Text. - ''' - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = tk.Tk() - - def setUp(self): - self.text = text = TextWrapper(self.root) - text.insert('1.0', ">>> ") - text.mark_set('iomark', '1.4') - text.mark_gravity('iomark', 'left') - self.history = History(text) - self.history.history = [line1, line2] - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - def fetch_test(self, reverse, line, prefix, index, *, bell=False): - # Perform one fetch as invoked by Alt-N or Alt-P - # Test the result. The line test is the most important. - # The last two are diagnostic of fetch internals. - History = self.history - History.fetch(reverse) - - Equal = self.assertEqual - Equal(self.text.get('iomark', 'end-1c'), line) - Equal(self.text._bell, bell) - if bell: - self.text._bell = False - Equal(History.prefix, prefix) - Equal(History.pointer, index) - Equal(self.text.compare("insert", '==', "end-1c"), 1) - - def test_fetch_prev_cyclic(self): - prefix = '' - test = self.fetch_test - test(True, line2, prefix, 1) - test(True, line1, prefix, 0) - test(True, prefix, None, None, bell=True) - - def test_fetch_next_cyclic(self): - prefix = '' - test = self.fetch_test - test(False, line1, prefix, 0) - test(False, line2, prefix, 1) - test(False, prefix, None, None, bell=True) - - # Prefix 'a' tests skip line2, which starts with 'b' - def test_fetch_prev_prefix(self): - prefix = 'a' - self.text.insert('iomark', prefix) - self.fetch_test(True, line1, prefix, 0) - self.fetch_test(True, prefix, None, None, bell=True) - - def test_fetch_next_prefix(self): - prefix = 'a' - self.text.insert('iomark', prefix) - self.fetch_test(False, line1, prefix, 0) - self.fetch_test(False, prefix, None, None, bell=True) - - def test_fetch_prev_noncyclic(self): - prefix = '' - self.history.cyclic = False - test = self.fetch_test - test(True, line2, prefix, 1) - test(True, line1, prefix, 0) - test(True, line1, prefix, 0, bell=True) - - def test_fetch_next_noncyclic(self): - prefix = '' - self.history.cyclic = False - test = self.fetch_test - test(False, prefix, None, None, bell=True) - test(True, line2, prefix, 1) - test(False, prefix, None, None, bell=True) - test(False, prefix, None, None, bell=True) - - def test_fetch_cursor_move(self): - # Move cursor after fetch - self.history.fetch(reverse=True) # initialization - self.text.mark_set('insert', 'iomark') - self.fetch_test(True, line2, None, None, bell=True) - - def test_fetch_edit(self): - # Edit after fetch - self.history.fetch(reverse=True) # initialization - self.text.delete('iomark', 'insert', ) - self.text.insert('iomark', 'a =') - self.fetch_test(True, line1, 'a =', 0) # prefix is reset - - def test_history_prev_next(self): - # Minimally test functions bound to events - self.history.history_prev('dummy event') - self.assertEqual(self.history.pointer, 1) - self.history.history_next('dummy event') - self.assertEqual(self.history.pointer, None) - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_io.py b/Lib/idlelib/idle_test/test_io.py deleted file mode 100644 index e0e3b98..0000000 --- a/Lib/idlelib/idle_test/test_io.py +++ /dev/null @@ -1,233 +0,0 @@ -import unittest -import io -from idlelib.PyShell import PseudoInputFile, PseudoOutputFile - - -class S(str): - def __str__(self): - return '%s:str' % type(self).__name__ - def __unicode__(self): - return '%s:unicode' % type(self).__name__ - def __len__(self): - return 3 - def __iter__(self): - return iter('abc') - def __getitem__(self, *args): - return '%s:item' % type(self).__name__ - def __getslice__(self, *args): - return '%s:slice' % type(self).__name__ - -class MockShell: - def __init__(self): - self.reset() - - def write(self, *args): - self.written.append(args) - - def readline(self): - return self.lines.pop() - - def close(self): - pass - - def reset(self): - self.written = [] - - def push(self, lines): - self.lines = list(lines)[::-1] - - -class PseudeOutputFilesTest(unittest.TestCase): - def test_misc(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) - self.assertIsNone(f.newlines) - self.assertEqual(f.name, '') - self.assertFalse(f.closed) - self.assertTrue(f.isatty()) - self.assertFalse(f.readable()) - self.assertTrue(f.writable()) - self.assertFalse(f.seekable()) - - def test_unsupported(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertRaises(OSError, f.fileno) - self.assertRaises(OSError, f.tell) - self.assertRaises(OSError, f.seek, 0) - self.assertRaises(OSError, f.read, 0) - self.assertRaises(OSError, f.readline, 0) - - def test_write(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - f.write('test') - self.assertEqual(shell.written, [('test', 'stdout')]) - shell.reset() - f.write('t\xe8st') - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - shell.reset() - - f.write(S('t\xe8st')) - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - self.assertEqual(type(shell.written[0][0]), str) - shell.reset() - - self.assertRaises(TypeError, f.write) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.write, b'test') - self.assertRaises(TypeError, f.write, 123) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.write, 'test', 'spam') - self.assertEqual(shell.written, []) - - def test_writelines(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - f.writelines([]) - self.assertEqual(shell.written, []) - shell.reset() - f.writelines(['one\n', 'two']) - self.assertEqual(shell.written, - [('one\n', 'stdout'), ('two', 'stdout')]) - shell.reset() - f.writelines(['on\xe8\n', 'tw\xf2']) - self.assertEqual(shell.written, - [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')]) - shell.reset() - - f.writelines([S('t\xe8st')]) - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - self.assertEqual(type(shell.written[0][0]), str) - shell.reset() - - self.assertRaises(TypeError, f.writelines) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, 123) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, [b'test']) - self.assertRaises(TypeError, f.writelines, [123]) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, [], []) - self.assertEqual(shell.written, []) - - def test_close(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertFalse(f.closed) - f.write('test') - f.close() - self.assertTrue(f.closed) - self.assertRaises(ValueError, f.write, 'x') - self.assertEqual(shell.written, [('test', 'stdout')]) - f.close() - self.assertRaises(TypeError, f.close, 1) - - -class PseudeInputFilesTest(unittest.TestCase): - def test_misc(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) - self.assertIsNone(f.newlines) - self.assertEqual(f.name, '') - self.assertFalse(f.closed) - self.assertTrue(f.isatty()) - self.assertTrue(f.readable()) - self.assertFalse(f.writable()) - self.assertFalse(f.seekable()) - - def test_unsupported(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - self.assertRaises(OSError, f.fileno) - self.assertRaises(OSError, f.tell) - self.assertRaises(OSError, f.seek, 0) - self.assertRaises(OSError, f.write, 'x') - self.assertRaises(OSError, f.writelines, ['x']) - - def test_read(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(), 'one\ntwo\n') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(-1), 'one\ntwo\n') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(None), 'one\ntwo\n') - shell.push(['one\n', 'two\n', 'three\n', '']) - self.assertEqual(f.read(2), 'on') - self.assertEqual(f.read(3), 'e\nt') - self.assertEqual(f.read(10), 'wo\nthree\n') - - shell.push(['one\n', 'two\n']) - self.assertEqual(f.read(0), '') - self.assertRaises(TypeError, f.read, 1.5) - self.assertRaises(TypeError, f.read, '1') - self.assertRaises(TypeError, f.read, 1, 1) - - def test_readline(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', 'three\n', 'four\n']) - self.assertEqual(f.readline(), 'one\n') - self.assertEqual(f.readline(-1), 'two\n') - self.assertEqual(f.readline(None), 'three\n') - shell.push(['one\ntwo\n']) - self.assertEqual(f.readline(), 'one\n') - self.assertEqual(f.readline(), 'two\n') - shell.push(['one', 'two', 'three']) - self.assertEqual(f.readline(), 'one') - self.assertEqual(f.readline(), 'two') - shell.push(['one\n', 'two\n', 'three\n']) - self.assertEqual(f.readline(2), 'on') - self.assertEqual(f.readline(1), 'e') - self.assertEqual(f.readline(1), '\n') - self.assertEqual(f.readline(10), 'two\n') - - shell.push(['one\n', 'two\n']) - self.assertEqual(f.readline(0), '') - self.assertRaises(TypeError, f.readlines, 1.5) - self.assertRaises(TypeError, f.readlines, '1') - self.assertRaises(TypeError, f.readlines, 1, 1) - - def test_readlines(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(-1), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(None), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(0), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(3), ['one\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(4), ['one\n', 'two\n']) - - shell.push(['one\n', 'two\n', '']) - self.assertRaises(TypeError, f.readlines, 1.5) - self.assertRaises(TypeError, f.readlines, '1') - self.assertRaises(TypeError, f.readlines, 1, 1) - - def test_close(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertFalse(f.closed) - self.assertEqual(f.readline(), 'one\n') - f.close() - self.assertFalse(f.closed) - self.assertEqual(f.readline(), 'two\n') - self.assertRaises(TypeError, f.close, 1) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/idlelib/idle_test/test_iomenu.py b/Lib/idlelib/idle_test/test_iomenu.py new file mode 100644 index 0000000..e0e3b98 --- /dev/null +++ b/Lib/idlelib/idle_test/test_iomenu.py @@ -0,0 +1,233 @@ +import unittest +import io +from idlelib.PyShell import PseudoInputFile, PseudoOutputFile + + +class S(str): + def __str__(self): + return '%s:str' % type(self).__name__ + def __unicode__(self): + return '%s:unicode' % type(self).__name__ + def __len__(self): + return 3 + def __iter__(self): + return iter('abc') + def __getitem__(self, *args): + return '%s:item' % type(self).__name__ + def __getslice__(self, *args): + return '%s:slice' % type(self).__name__ + +class MockShell: + def __init__(self): + self.reset() + + def write(self, *args): + self.written.append(args) + + def readline(self): + return self.lines.pop() + + def close(self): + pass + + def reset(self): + self.written = [] + + def push(self, lines): + self.lines = list(lines)[::-1] + + +class PseudeOutputFilesTest(unittest.TestCase): + def test_misc(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.encoding, 'utf-8') + self.assertIsNone(f.errors) + self.assertIsNone(f.newlines) + self.assertEqual(f.name, '') + self.assertFalse(f.closed) + self.assertTrue(f.isatty()) + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertFalse(f.seekable()) + + def test_unsupported(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertRaises(OSError, f.fileno) + self.assertRaises(OSError, f.tell) + self.assertRaises(OSError, f.seek, 0) + self.assertRaises(OSError, f.read, 0) + self.assertRaises(OSError, f.readline, 0) + + def test_write(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + f.write('test') + self.assertEqual(shell.written, [('test', 'stdout')]) + shell.reset() + f.write('t\xe8st') + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + shell.reset() + + f.write(S('t\xe8st')) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + + self.assertRaises(TypeError, f.write) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, b'test') + self.assertRaises(TypeError, f.write, 123) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, 'test', 'spam') + self.assertEqual(shell.written, []) + + def test_writelines(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + f.writelines([]) + self.assertEqual(shell.written, []) + shell.reset() + f.writelines(['one\n', 'two']) + self.assertEqual(shell.written, + [('one\n', 'stdout'), ('two', 'stdout')]) + shell.reset() + f.writelines(['on\xe8\n', 'tw\xf2']) + self.assertEqual(shell.written, + [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')]) + shell.reset() + + f.writelines([S('t\xe8st')]) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + + self.assertRaises(TypeError, f.writelines) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, 123) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, [b'test']) + self.assertRaises(TypeError, f.writelines, [123]) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, [], []) + self.assertEqual(shell.written, []) + + def test_close(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertFalse(f.closed) + f.write('test') + f.close() + self.assertTrue(f.closed) + self.assertRaises(ValueError, f.write, 'x') + self.assertEqual(shell.written, [('test', 'stdout')]) + f.close() + self.assertRaises(TypeError, f.close, 1) + + +class PseudeInputFilesTest(unittest.TestCase): + def test_misc(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.encoding, 'utf-8') + self.assertIsNone(f.errors) + self.assertIsNone(f.newlines) + self.assertEqual(f.name, '') + self.assertFalse(f.closed) + self.assertTrue(f.isatty()) + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertFalse(f.seekable()) + + def test_unsupported(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + self.assertRaises(OSError, f.fileno) + self.assertRaises(OSError, f.tell) + self.assertRaises(OSError, f.seek, 0) + self.assertRaises(OSError, f.write, 'x') + self.assertRaises(OSError, f.writelines, ['x']) + + def test_read(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(), 'one\ntwo\n') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(-1), 'one\ntwo\n') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(None), 'one\ntwo\n') + shell.push(['one\n', 'two\n', 'three\n', '']) + self.assertEqual(f.read(2), 'on') + self.assertEqual(f.read(3), 'e\nt') + self.assertEqual(f.read(10), 'wo\nthree\n') + + shell.push(['one\n', 'two\n']) + self.assertEqual(f.read(0), '') + self.assertRaises(TypeError, f.read, 1.5) + self.assertRaises(TypeError, f.read, '1') + self.assertRaises(TypeError, f.read, 1, 1) + + def test_readline(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', 'three\n', 'four\n']) + self.assertEqual(f.readline(), 'one\n') + self.assertEqual(f.readline(-1), 'two\n') + self.assertEqual(f.readline(None), 'three\n') + shell.push(['one\ntwo\n']) + self.assertEqual(f.readline(), 'one\n') + self.assertEqual(f.readline(), 'two\n') + shell.push(['one', 'two', 'three']) + self.assertEqual(f.readline(), 'one') + self.assertEqual(f.readline(), 'two') + shell.push(['one\n', 'two\n', 'three\n']) + self.assertEqual(f.readline(2), 'on') + self.assertEqual(f.readline(1), 'e') + self.assertEqual(f.readline(1), '\n') + self.assertEqual(f.readline(10), 'two\n') + + shell.push(['one\n', 'two\n']) + self.assertEqual(f.readline(0), '') + self.assertRaises(TypeError, f.readlines, 1.5) + self.assertRaises(TypeError, f.readlines, '1') + self.assertRaises(TypeError, f.readlines, 1, 1) + + def test_readlines(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(-1), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(None), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(0), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(3), ['one\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(4), ['one\n', 'two\n']) + + shell.push(['one\n', 'two\n', '']) + self.assertRaises(TypeError, f.readlines, 1.5) + self.assertRaises(TypeError, f.readlines, '1') + self.assertRaises(TypeError, f.readlines, 1, 1) + + def test_close(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertFalse(f.closed) + self.assertEqual(f.readline(), 'one\n') + f.close() + self.assertFalse(f.closed) + self.assertEqual(f.readline(), 'two\n') + self.assertRaises(TypeError, f.close, 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/idlelib/idle_test/test_paragraph.py b/Lib/idlelib/idle_test/test_paragraph.py new file mode 100644 index 0000000..f6039e6 --- /dev/null +++ b/Lib/idlelib/idle_test/test_paragraph.py @@ -0,0 +1,377 @@ +# Test the functions and main class method of FormatParagraph.py +import unittest +from idlelib import FormatParagraph as fp +from idlelib.EditorWindow import EditorWindow +from tkinter import Tk, Text +from test.support import requires + + +class Is_Get_Test(unittest.TestCase): + """Test the is_ and get_ functions""" + test_comment = '# This is a comment' + test_nocomment = 'This is not a comment' + trailingws_comment = '# This is a comment ' + leadingws_comment = ' # This is a comment' + leadingws_nocomment = ' This is not a comment' + + def test_is_all_white(self): + self.assertTrue(fp.is_all_white('')) + self.assertTrue(fp.is_all_white('\t\n\r\f\v')) + self.assertFalse(fp.is_all_white(self.test_comment)) + + def test_get_indent(self): + Equal = self.assertEqual + Equal(fp.get_indent(self.test_comment), '') + Equal(fp.get_indent(self.trailingws_comment), '') + Equal(fp.get_indent(self.leadingws_comment), ' ') + Equal(fp.get_indent(self.leadingws_nocomment), ' ') + + def test_get_comment_header(self): + Equal = self.assertEqual + # Test comment strings + Equal(fp.get_comment_header(self.test_comment), '#') + Equal(fp.get_comment_header(self.trailingws_comment), '#') + Equal(fp.get_comment_header(self.leadingws_comment), ' #') + # Test non-comment strings + Equal(fp.get_comment_header(self.leadingws_nocomment), ' ') + Equal(fp.get_comment_header(self.test_nocomment), '') + + +class FindTest(unittest.TestCase): + """Test the find_paragraph function in FormatParagraph. + + Using the runcase() function, find_paragraph() is called with 'mark' set at + multiple indexes before and inside the test paragraph. + + It appears that code with the same indentation as a quoted string is grouped + as part of the same paragraph, which is probably incorrect behavior. + """ + + @classmethod + def setUpClass(cls): + from idlelib.idle_test.mock_tk import Text + cls.text = Text() + + def runcase(self, inserttext, stopline, expected): + # Check that find_paragraph returns the expected paragraph when + # the mark index is set to beginning, middle, end of each line + # up to but not including the stop line + text = self.text + text.insert('1.0', inserttext) + for line in range(1, stopline): + linelength = int(text.index("%d.end" % line).split('.')[1]) + for col in (0, linelength//2, linelength): + tempindex = "%d.%d" % (line, col) + self.assertEqual(fp.find_paragraph(text, tempindex), expected) + text.delete('1.0', 'end') + + def test_find_comment(self): + comment = ( + "# Comment block with no blank lines before\n" + "# Comment line\n" + "\n") + self.runcase(comment, 3, ('1.0', '3.0', '#', comment[0:58])) + + comment = ( + "\n" + "# Comment block with whitespace line before and after\n" + "# Comment line\n" + "\n") + self.runcase(comment, 4, ('2.0', '4.0', '#', comment[1:70])) + + comment = ( + "\n" + " # Indented comment block with whitespace before and after\n" + " # Comment line\n" + "\n") + self.runcase(comment, 4, ('2.0', '4.0', ' #', comment[1:82])) + + comment = ( + "\n" + "# Single line comment\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:23])) + + comment = ( + "\n" + " # Single line comment with leading whitespace\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:51])) + + comment = ( + "\n" + "# Comment immediately followed by code\n" + "x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:40])) + + comment = ( + "\n" + " # Indented comment immediately followed by code\n" + "x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:53])) + + comment = ( + "\n" + "# Comment immediately followed by indented code\n" + " x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:49])) + + def test_find_paragraph(self): + teststring = ( + '"""String with no blank lines before\n' + 'String line\n' + '"""\n' + '\n') + self.runcase(teststring, 4, ('1.0', '4.0', '', teststring[0:53])) + + teststring = ( + "\n" + '"""String with whitespace line before and after\n' + 'String line.\n' + '"""\n' + '\n') + self.runcase(teststring, 5, ('2.0', '5.0', '', teststring[1:66])) + + teststring = ( + '\n' + ' """Indented string with whitespace before and after\n' + ' Comment string.\n' + ' """\n' + '\n') + self.runcase(teststring, 5, ('2.0', '5.0', ' ', teststring[1:85])) + + teststring = ( + '\n' + '"""Single line string."""\n' + '\n') + self.runcase(teststring, 3, ('2.0', '3.0', '', teststring[1:27])) + + teststring = ( + '\n' + ' """Single line string with leading whitespace."""\n' + '\n') + self.runcase(teststring, 3, ('2.0', '3.0', ' ', teststring[1:55])) + + +class ReformatFunctionTest(unittest.TestCase): + """Test the reformat_paragraph function without the editor window.""" + + def test_reformat_paragrah(self): + Equal = self.assertEqual + reform = fp.reformat_paragraph + hw = "O hello world" + Equal(reform(' ', 1), ' ') + Equal(reform("Hello world", 20), "Hello world") + + # Test without leading newline + Equal(reform(hw, 1), "O\nhello\nworld") + Equal(reform(hw, 6), "O\nhello\nworld") + Equal(reform(hw, 7), "O hello\nworld") + Equal(reform(hw, 12), "O hello\nworld") + Equal(reform(hw, 13), "O hello world") + + # Test with leading newline + hw = "\nO hello world" + Equal(reform(hw, 1), "\nO\nhello\nworld") + Equal(reform(hw, 6), "\nO\nhello\nworld") + Equal(reform(hw, 7), "\nO hello\nworld") + Equal(reform(hw, 12), "\nO hello\nworld") + Equal(reform(hw, 13), "\nO hello world") + + +class ReformatCommentTest(unittest.TestCase): + """Test the reformat_comment function without the editor window.""" + + def test_reformat_comment(self): + Equal = self.assertEqual + + # reformat_comment formats to a minimum of 20 characters + test_string = ( + " \"\"\"this is a test of a reformat for a triple quoted string" + " will it reformat to less than 70 characters for me?\"\"\"") + result = fp.reformat_comment(test_string, 70, " ") + expected = ( + " \"\"\"this is a test of a reformat for a triple quoted string will it\n" + " reformat to less than 70 characters for me?\"\"\"") + Equal(result, expected) + + test_comment = ( + "# this is a test of a reformat for a triple quoted string will " + "it reformat to less than 70 characters for me?") + result = fp.reformat_comment(test_comment, 70, "#") + expected = ( + "# this is a test of a reformat for a triple quoted string will it\n" + "# reformat to less than 70 characters for me?") + Equal(result, expected) + + +class FormatClassTest(unittest.TestCase): + def test_init_close(self): + instance = fp.FormatParagraph('editor') + self.assertEqual(instance.editwin, 'editor') + instance.close() + self.assertEqual(instance.editwin, None) + + +# For testing format_paragraph_event, Initialize FormatParagraph with +# a mock Editor with .text and .get_selection_indices. The text must +# be a Text wrapper that adds two methods + +# A real EditorWindow creates unneeded, time-consuming baggage and +# sometimes emits shutdown warnings like this: +# "warning: callback failed in WindowList +# : invalid command name ".55131368.windows". +# Calling EditorWindow._close in tearDownClass prevents this but causes +# other problems (windows left open). + +class TextWrapper: + def __init__(self, master): + self.text = Text(master=master) + def __getattr__(self, name): + return getattr(self.text, name) + def undo_block_start(self): pass + def undo_block_stop(self): pass + +class Editor: + def __init__(self, root): + self.text = TextWrapper(root) + get_selection_indices = EditorWindow. get_selection_indices + +class FormatEventTest(unittest.TestCase): + """Test the formatting of text inside a Text widget. + + This is done with FormatParagraph.format.paragraph_event, + which calls functions in the module as appropriate. + """ + test_string = ( + " '''this is a test of a reformat for a triple " + "quoted string will it reformat to less than 70 " + "characters for me?'''\n") + multiline_test_string = ( + " '''The first line is under the max width.\n" + " The second line's length is way over the max width. It goes " + "on and on until it is over 100 characters long.\n" + " Same thing with the third line. It is also way over the max " + "width, but FormatParagraph will fix it.\n" + " '''\n") + multiline_test_comment = ( + "# The first line is under the max width.\n" + "# The second line's length is way over the max width. It goes on " + "and on until it is over 100 characters long.\n" + "# Same thing with the third line. It is also way over the max " + "width, but FormatParagraph will fix it.\n" + "# The fourth line is short like the first line.") + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + editor = Editor(root=cls.root) + cls.text = editor.text.text # Test code does not need the wrapper. + cls.formatter = fp.FormatParagraph(editor).format_paragraph_event + # Sets the insert mark just after the re-wrapped and inserted text. + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + del cls.text + del cls.formatter + + def test_short_line(self): + self.text.insert('1.0', "Short line\n") + self.formatter("Dummy") + self.assertEqual(self.text.get('1.0', 'insert'), "Short line\n" ) + self.text.delete('1.0', 'end') + + def test_long_line(self): + text = self.text + + # Set cursor ('insert' mark) to '1.0', within text. + text.insert('1.0', self.test_string) + text.mark_set('insert', '1.0') + self.formatter('ParameterDoesNothing', limit=70) + result = text.get('1.0', 'insert') + # find function includes \n + expected = ( +" '''this is a test of a reformat for a triple quoted string will it\n" +" reformat to less than 70 characters for me?'''\n") # yes + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + # Select from 1.11 to line end. + text.insert('1.0', self.test_string) + text.tag_add('sel', '1.11', '1.end') + self.formatter('ParameterDoesNothing', limit=70) + result = text.get('1.0', 'insert') + # selection excludes \n + expected = ( +" '''this is a test of a reformat for a triple quoted string will it reformat\n" +" to less than 70 characters for me?'''") # no + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + def test_multiple_lines(self): + text = self.text + # Select 2 long lines. + text.insert('1.0', self.multiline_test_string) + text.tag_add('sel', '2.0', '4.0') + self.formatter('ParameterDoesNothing', limit=70) + result = text.get('2.0', 'insert') + expected = ( +" The second line's length is way over the max width. It goes on and\n" +" on until it is over 100 characters long. Same thing with the third\n" +" line. It is also way over the max width, but FormatParagraph will\n" +" fix it.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + def test_comment_block(self): + text = self.text + + # Set cursor ('insert') to '1.0', within block. + text.insert('1.0', self.multiline_test_comment) + self.formatter('ParameterDoesNothing', limit=70) + result = text.get('1.0', 'insert') + expected = ( +"# The first line is under the max width. The second line's length is\n" +"# way over the max width. It goes on and on until it is over 100\n" +"# characters long. Same thing with the third line. It is also way over\n" +"# the max width, but FormatParagraph will fix it. The fourth line is\n" +"# short like the first line.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + # Select line 2, verify line 1 unaffected. + text.insert('1.0', self.multiline_test_comment) + text.tag_add('sel', '2.0', '3.0') + self.formatter('ParameterDoesNothing', limit=70) + result = text.get('1.0', 'insert') + expected = ( +"# The first line is under the max width.\n" +"# The second line's length is way over the max width. It goes on and\n" +"# on until it is over 100 characters long.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + +# The following block worked with EditorWindow but fails with the mock. +# Lines 2 and 3 get pasted together even though the previous block left +# the previous line alone. More investigation is needed. +## # Select lines 3 and 4 +## text.insert('1.0', self.multiline_test_comment) +## text.tag_add('sel', '3.0', '5.0') +## self.formatter('ParameterDoesNothing') +## result = text.get('3.0', 'insert') +## expected = ( +##"# Same thing with the third line. It is also way over the max width,\n" +##"# but FormatParagraph will fix it. The fourth line is short like the\n" +##"# first line.\n") +## self.assertEqual(result, expected) +## text.delete('1.0', 'end') + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_redirector.py b/Lib/idlelib/idle_test/test_redirector.py new file mode 100644 index 0000000..6440561 --- /dev/null +++ b/Lib/idlelib/idle_test/test_redirector.py @@ -0,0 +1,122 @@ +"""Unittest for idlelib.WidgetRedirector + +100% coverage +""" +from test.support import requires +import unittest +from idlelib.idle_test.mock_idle import Func +from tkinter import Tk, Text, TclError +from idlelib.WidgetRedirector import WidgetRedirector + + +class InitCloseTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.tk = Tk() + cls.text = Text(cls.tk) + + @classmethod + def tearDownClass(cls): + cls.text.destroy() + cls.tk.destroy() + del cls.text, cls.tk + + def test_init(self): + redir = WidgetRedirector(self.text) + self.assertEqual(redir.widget, self.text) + self.assertEqual(redir.tk, self.text.tk) + self.assertRaises(TclError, WidgetRedirector, self.text) + redir.close() # restore self.tk, self.text + + def test_close(self): + redir = WidgetRedirector(self.text) + redir.register('insert', Func) + redir.close() + self.assertEqual(redir._operations, {}) + self.assertFalse(hasattr(self.text, 'widget')) + + +class WidgetRedirectorTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.tk = Tk() + cls.text = Text(cls.tk) + + @classmethod + def tearDownClass(cls): + cls.text.destroy() + cls.tk.destroy() + del cls.text, cls.tk + + def setUp(self): + self.redir = WidgetRedirector(self.text) + self.func = Func() + self.orig_insert = self.redir.register('insert', self.func) + self.text.insert('insert', 'asdf') # leaves self.text empty + + def tearDown(self): + self.text.delete('1.0', 'end') + self.redir.close() + + def test_repr(self): # partly for 100% coverage + self.assertIn('Redirector', repr(self.redir)) + self.assertIn('Original', repr(self.orig_insert)) + + def test_register(self): + self.assertEqual(self.text.get('1.0', 'end'), '\n') + self.assertEqual(self.func.args, ('insert', 'asdf')) + self.assertIn('insert', self.redir._operations) + self.assertIn('insert', self.text.__dict__) + self.assertEqual(self.text.insert, self.func) + + def test_original_command(self): + self.assertEqual(self.orig_insert.operation, 'insert') + self.assertEqual(self.orig_insert.tk_call, self.text.tk.call) + self.orig_insert('insert', 'asdf') + self.assertEqual(self.text.get('1.0', 'end'), 'asdf\n') + + def test_unregister(self): + self.assertIsNone(self.redir.unregister('invalid operation name')) + self.assertEqual(self.redir.unregister('insert'), self.func) + self.assertNotIn('insert', self.redir._operations) + self.assertNotIn('insert', self.text.__dict__) + + def test_unregister_no_attribute(self): + del self.text.insert + self.assertEqual(self.redir.unregister('insert'), self.func) + + def test_dispatch_intercept(self): + self.func.__init__(True) + self.assertTrue(self.redir.dispatch('insert', False)) + self.assertFalse(self.func.args[0]) + + def test_dispatch_bypass(self): + self.orig_insert('insert', 'asdf') + # tk.call returns '' where Python would return None + self.assertEqual(self.redir.dispatch('delete', '1.0', 'end'), '') + self.assertEqual(self.text.get('1.0', 'end'), '\n') + + def test_dispatch_error(self): + self.func.__init__(TclError()) + self.assertEqual(self.redir.dispatch('insert', False), '') + self.assertEqual(self.redir.dispatch('invalid'), '') + + def test_command_dispatch(self): + # Test that .__init__ causes redirection of tk calls + # through redir.dispatch + self.tk.call(self.text._w, 'insert', 'hello') + self.assertEqual(self.func.args, ('hello',)) + self.assertEqual(self.text.get('1.0', 'end'), '\n') + # Ensure that called through redir .dispatch and not through + # self.text.insert by having mock raise TclError. + self.func.__init__(TclError()) + self.assertEqual(self.tk.call(self.text._w, 'insert', 'boo'), '') + + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_replace.py b/Lib/idlelib/idle_test/test_replace.py new file mode 100644 index 0000000..09669f8 --- /dev/null +++ b/Lib/idlelib/idle_test/test_replace.py @@ -0,0 +1,292 @@ +"""Unittest for idlelib.ReplaceDialog""" +from test.support import requires +requires('gui') + +import unittest +from unittest.mock import Mock +from tkinter import Tk, Text +from idlelib.idle_test.mock_tk import Mbox +import idlelib.SearchEngine as se +import idlelib.ReplaceDialog as rd + +orig_mbox = se.tkMessageBox +showerror = Mbox.showerror + + +class ReplaceDialogTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.root = Tk() + cls.root.withdraw() + se.tkMessageBox = Mbox + cls.engine = se.SearchEngine(cls.root) + cls.dialog = rd.ReplaceDialog(cls.root, cls.engine) + cls.dialog.ok = Mock() + cls.text = Text(cls.root) + cls.text.undo_block_start = Mock() + cls.text.undo_block_stop = Mock() + cls.dialog.text = cls.text + + @classmethod + def tearDownClass(cls): + se.tkMessageBox = orig_mbox + cls.root.destroy() + del cls.text, cls.dialog, cls.engine, cls.root + + def setUp(self): + self.text.insert('insert', 'This is a sample sTring') + + def tearDown(self): + self.engine.patvar.set('') + self.dialog.replvar.set('') + self.engine.wordvar.set(False) + self.engine.casevar.set(False) + self.engine.revar.set(False) + self.engine.wrapvar.set(True) + self.engine.backvar.set(False) + showerror.title = '' + showerror.message = '' + self.text.delete('1.0', 'end') + + def test_replace_simple(self): + # Test replace function with all options at default setting. + # Wrap around - True + # Regular Expression - False + # Match case - False + # Match word - False + # Direction - Forwards + text = self.text + equal = self.assertEqual + pv = self.engine.patvar + rv = self.dialog.replvar + replace = self.dialog.replace_it + + # test accessor method + self.engine.setpat('asdf') + equal(self.engine.getpat(), pv.get()) + + # text found and replaced + pv.set('a') + rv.set('asdf') + self.dialog.open(self.text) + replace() + equal(text.get('1.8', '1.12'), 'asdf') + + # dont "match word" case + text.mark_set('insert', '1.0') + pv.set('is') + rv.set('hello') + replace() + equal(text.get('1.2', '1.7'), 'hello') + + # dont "match case" case + pv.set('string') + rv.set('world') + replace() + equal(text.get('1.23', '1.28'), 'world') + + # without "regular expression" case + text.mark_set('insert', 'end') + text.insert('insert', '\nline42:') + before_text = text.get('1.0', 'end') + pv.set('[a-z][\d]+') + replace() + after_text = text.get('1.0', 'end') + equal(before_text, after_text) + + # test with wrap around selected and complete a cycle + text.mark_set('insert', '1.9') + pv.set('i') + rv.set('j') + replace() + equal(text.get('1.8'), 'i') + equal(text.get('2.1'), 'j') + replace() + equal(text.get('2.1'), 'j') + equal(text.get('1.8'), 'j') + before_text = text.get('1.0', 'end') + replace() + after_text = text.get('1.0', 'end') + equal(before_text, after_text) + + # text not found + before_text = text.get('1.0', 'end') + pv.set('foobar') + replace() + after_text = text.get('1.0', 'end') + equal(before_text, after_text) + + # test access method + self.dialog.find_it(0) + + def test_replace_wrap_around(self): + text = self.text + equal = self.assertEqual + pv = self.engine.patvar + rv = self.dialog.replvar + replace = self.dialog.replace_it + self.engine.wrapvar.set(False) + + # replace candidate found both after and before 'insert' + text.mark_set('insert', '1.4') + pv.set('i') + rv.set('j') + replace() + equal(text.get('1.2'), 'i') + equal(text.get('1.5'), 'j') + replace() + equal(text.get('1.2'), 'i') + equal(text.get('1.20'), 'j') + replace() + equal(text.get('1.2'), 'i') + + # replace candidate found only before 'insert' + text.mark_set('insert', '1.8') + pv.set('is') + before_text = text.get('1.0', 'end') + replace() + after_text = text.get('1.0', 'end') + equal(before_text, after_text) + + def test_replace_whole_word(self): + text = self.text + equal = self.assertEqual + pv = self.engine.patvar + rv = self.dialog.replvar + replace = self.dialog.replace_it + self.engine.wordvar.set(True) + + pv.set('is') + rv.set('hello') + replace() + equal(text.get('1.0', '1.4'), 'This') + equal(text.get('1.5', '1.10'), 'hello') + + def test_replace_match_case(self): + equal = self.assertEqual + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace = self.dialog.replace_it + self.engine.casevar.set(True) + + before_text = self.text.get('1.0', 'end') + pv.set('this') + rv.set('that') + replace() + after_text = self.text.get('1.0', 'end') + equal(before_text, after_text) + + pv.set('This') + replace() + equal(text.get('1.0', '1.4'), 'that') + + def test_replace_regex(self): + equal = self.assertEqual + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace = self.dialog.replace_it + self.engine.revar.set(True) + + before_text = text.get('1.0', 'end') + pv.set('[a-z][\d]+') + rv.set('hello') + replace() + after_text = text.get('1.0', 'end') + equal(before_text, after_text) + + text.insert('insert', '\nline42') + replace() + equal(text.get('2.0', '2.8'), 'linhello') + + pv.set('') + replace() + self.assertIn('error', showerror.title) + self.assertIn('Empty', showerror.message) + + pv.set('[\d') + replace() + self.assertIn('error', showerror.title) + self.assertIn('Pattern', showerror.message) + + showerror.title = '' + showerror.message = '' + pv.set('[a]') + rv.set('test\\') + replace() + self.assertIn('error', showerror.title) + self.assertIn('Invalid Replace Expression', showerror.message) + + # test access method + self.engine.setcookedpat("\'") + equal(pv.get(), "\\'") + + def test_replace_backwards(self): + equal = self.assertEqual + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace = self.dialog.replace_it + self.engine.backvar.set(True) + + text.insert('insert', '\nis as ') + + pv.set('is') + rv.set('was') + replace() + equal(text.get('1.2', '1.4'), 'is') + equal(text.get('2.0', '2.3'), 'was') + replace() + equal(text.get('1.5', '1.8'), 'was') + replace() + equal(text.get('1.2', '1.5'), 'was') + + def test_replace_all(self): + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace_all = self.dialog.replace_all + + text.insert('insert', '\n') + text.insert('insert', text.get('1.0', 'end')*100) + pv.set('is') + rv.set('was') + replace_all() + self.assertNotIn('is', text.get('1.0', 'end')) + + self.engine.revar.set(True) + pv.set('') + replace_all() + self.assertIn('error', showerror.title) + self.assertIn('Empty', showerror.message) + + pv.set('[s][T]') + rv.set('\\') + replace_all() + + self.engine.revar.set(False) + pv.set('text which is not present') + rv.set('foobar') + replace_all() + + def test_default_command(self): + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace_find = self.dialog.default_command + equal = self.assertEqual + + pv.set('This') + rv.set('was') + replace_find() + equal(text.get('sel.first', 'sel.last'), 'was') + + self.engine.revar.set(True) + pv.set('') + replace_find() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_replacedialog.py b/Lib/idlelib/idle_test/test_replacedialog.py deleted file mode 100644 index 09669f8..0000000 --- a/Lib/idlelib/idle_test/test_replacedialog.py +++ /dev/null @@ -1,292 +0,0 @@ -"""Unittest for idlelib.ReplaceDialog""" -from test.support import requires -requires('gui') - -import unittest -from unittest.mock import Mock -from tkinter import Tk, Text -from idlelib.idle_test.mock_tk import Mbox -import idlelib.SearchEngine as se -import idlelib.ReplaceDialog as rd - -orig_mbox = se.tkMessageBox -showerror = Mbox.showerror - - -class ReplaceDialogTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.root = Tk() - cls.root.withdraw() - se.tkMessageBox = Mbox - cls.engine = se.SearchEngine(cls.root) - cls.dialog = rd.ReplaceDialog(cls.root, cls.engine) - cls.dialog.ok = Mock() - cls.text = Text(cls.root) - cls.text.undo_block_start = Mock() - cls.text.undo_block_stop = Mock() - cls.dialog.text = cls.text - - @classmethod - def tearDownClass(cls): - se.tkMessageBox = orig_mbox - cls.root.destroy() - del cls.text, cls.dialog, cls.engine, cls.root - - def setUp(self): - self.text.insert('insert', 'This is a sample sTring') - - def tearDown(self): - self.engine.patvar.set('') - self.dialog.replvar.set('') - self.engine.wordvar.set(False) - self.engine.casevar.set(False) - self.engine.revar.set(False) - self.engine.wrapvar.set(True) - self.engine.backvar.set(False) - showerror.title = '' - showerror.message = '' - self.text.delete('1.0', 'end') - - def test_replace_simple(self): - # Test replace function with all options at default setting. - # Wrap around - True - # Regular Expression - False - # Match case - False - # Match word - False - # Direction - Forwards - text = self.text - equal = self.assertEqual - pv = self.engine.patvar - rv = self.dialog.replvar - replace = self.dialog.replace_it - - # test accessor method - self.engine.setpat('asdf') - equal(self.engine.getpat(), pv.get()) - - # text found and replaced - pv.set('a') - rv.set('asdf') - self.dialog.open(self.text) - replace() - equal(text.get('1.8', '1.12'), 'asdf') - - # dont "match word" case - text.mark_set('insert', '1.0') - pv.set('is') - rv.set('hello') - replace() - equal(text.get('1.2', '1.7'), 'hello') - - # dont "match case" case - pv.set('string') - rv.set('world') - replace() - equal(text.get('1.23', '1.28'), 'world') - - # without "regular expression" case - text.mark_set('insert', 'end') - text.insert('insert', '\nline42:') - before_text = text.get('1.0', 'end') - pv.set('[a-z][\d]+') - replace() - after_text = text.get('1.0', 'end') - equal(before_text, after_text) - - # test with wrap around selected and complete a cycle - text.mark_set('insert', '1.9') - pv.set('i') - rv.set('j') - replace() - equal(text.get('1.8'), 'i') - equal(text.get('2.1'), 'j') - replace() - equal(text.get('2.1'), 'j') - equal(text.get('1.8'), 'j') - before_text = text.get('1.0', 'end') - replace() - after_text = text.get('1.0', 'end') - equal(before_text, after_text) - - # text not found - before_text = text.get('1.0', 'end') - pv.set('foobar') - replace() - after_text = text.get('1.0', 'end') - equal(before_text, after_text) - - # test access method - self.dialog.find_it(0) - - def test_replace_wrap_around(self): - text = self.text - equal = self.assertEqual - pv = self.engine.patvar - rv = self.dialog.replvar - replace = self.dialog.replace_it - self.engine.wrapvar.set(False) - - # replace candidate found both after and before 'insert' - text.mark_set('insert', '1.4') - pv.set('i') - rv.set('j') - replace() - equal(text.get('1.2'), 'i') - equal(text.get('1.5'), 'j') - replace() - equal(text.get('1.2'), 'i') - equal(text.get('1.20'), 'j') - replace() - equal(text.get('1.2'), 'i') - - # replace candidate found only before 'insert' - text.mark_set('insert', '1.8') - pv.set('is') - before_text = text.get('1.0', 'end') - replace() - after_text = text.get('1.0', 'end') - equal(before_text, after_text) - - def test_replace_whole_word(self): - text = self.text - equal = self.assertEqual - pv = self.engine.patvar - rv = self.dialog.replvar - replace = self.dialog.replace_it - self.engine.wordvar.set(True) - - pv.set('is') - rv.set('hello') - replace() - equal(text.get('1.0', '1.4'), 'This') - equal(text.get('1.5', '1.10'), 'hello') - - def test_replace_match_case(self): - equal = self.assertEqual - text = self.text - pv = self.engine.patvar - rv = self.dialog.replvar - replace = self.dialog.replace_it - self.engine.casevar.set(True) - - before_text = self.text.get('1.0', 'end') - pv.set('this') - rv.set('that') - replace() - after_text = self.text.get('1.0', 'end') - equal(before_text, after_text) - - pv.set('This') - replace() - equal(text.get('1.0', '1.4'), 'that') - - def test_replace_regex(self): - equal = self.assertEqual - text = self.text - pv = self.engine.patvar - rv = self.dialog.replvar - replace = self.dialog.replace_it - self.engine.revar.set(True) - - before_text = text.get('1.0', 'end') - pv.set('[a-z][\d]+') - rv.set('hello') - replace() - after_text = text.get('1.0', 'end') - equal(before_text, after_text) - - text.insert('insert', '\nline42') - replace() - equal(text.get('2.0', '2.8'), 'linhello') - - pv.set('') - replace() - self.assertIn('error', showerror.title) - self.assertIn('Empty', showerror.message) - - pv.set('[\d') - replace() - self.assertIn('error', showerror.title) - self.assertIn('Pattern', showerror.message) - - showerror.title = '' - showerror.message = '' - pv.set('[a]') - rv.set('test\\') - replace() - self.assertIn('error', showerror.title) - self.assertIn('Invalid Replace Expression', showerror.message) - - # test access method - self.engine.setcookedpat("\'") - equal(pv.get(), "\\'") - - def test_replace_backwards(self): - equal = self.assertEqual - text = self.text - pv = self.engine.patvar - rv = self.dialog.replvar - replace = self.dialog.replace_it - self.engine.backvar.set(True) - - text.insert('insert', '\nis as ') - - pv.set('is') - rv.set('was') - replace() - equal(text.get('1.2', '1.4'), 'is') - equal(text.get('2.0', '2.3'), 'was') - replace() - equal(text.get('1.5', '1.8'), 'was') - replace() - equal(text.get('1.2', '1.5'), 'was') - - def test_replace_all(self): - text = self.text - pv = self.engine.patvar - rv = self.dialog.replvar - replace_all = self.dialog.replace_all - - text.insert('insert', '\n') - text.insert('insert', text.get('1.0', 'end')*100) - pv.set('is') - rv.set('was') - replace_all() - self.assertNotIn('is', text.get('1.0', 'end')) - - self.engine.revar.set(True) - pv.set('') - replace_all() - self.assertIn('error', showerror.title) - self.assertIn('Empty', showerror.message) - - pv.set('[s][T]') - rv.set('\\') - replace_all() - - self.engine.revar.set(False) - pv.set('text which is not present') - rv.set('foobar') - replace_all() - - def test_default_command(self): - text = self.text - pv = self.engine.patvar - rv = self.dialog.replvar - replace_find = self.dialog.default_command - equal = self.assertEqual - - pv.set('This') - rv.set('was') - replace_find() - equal(text.get('sel.first', 'sel.last'), 'was') - - self.engine.revar.set(True) - pv.set('') - replace_find() - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_search.py b/Lib/idlelib/idle_test/test_search.py new file mode 100644 index 0000000..190c866 --- /dev/null +++ b/Lib/idlelib/idle_test/test_search.py @@ -0,0 +1,80 @@ +"""Test SearchDialog class in SearchDialogue.py""" + +# Does not currently test the event handler wrappers. +# A usage test should simulate clicks and check hilighting. +# Tests need to be coordinated with SearchDialogBase tests +# to avoid duplication. + +from test.support import requires +requires('gui') + +import unittest +import tkinter as tk +from tkinter import BooleanVar +import idlelib.SearchEngine as se +import idlelib.SearchDialog as sd + + +class SearchDialogTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.root = tk.Tk() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def setUp(self): + self.engine = se.SearchEngine(self.root) + self.dialog = sd.SearchDialog(self.root, self.engine) + self.text = tk.Text(self.root) + self.text.insert('1.0', 'Hello World!') + + def test_find_again(self): + # Search for various expressions + text = self.text + + self.engine.setpat('') + self.assertFalse(self.dialog.find_again(text)) + + self.engine.setpat('Hello') + self.assertTrue(self.dialog.find_again(text)) + + self.engine.setpat('Goodbye') + self.assertFalse(self.dialog.find_again(text)) + + self.engine.setpat('World!') + self.assertTrue(self.dialog.find_again(text)) + + self.engine.setpat('Hello World!') + self.assertTrue(self.dialog.find_again(text)) + + # Regular expression + self.engine.revar = BooleanVar(self.root, True) + self.engine.setpat('W[aeiouy]r') + self.assertTrue(self.dialog.find_again(text)) + + def test_find_selection(self): + # Select some text and make sure it's found + text = self.text + # Add additional line to find + self.text.insert('2.0', 'Hello World!') + + text.tag_add('sel', '1.0', '1.4') # Select 'Hello' + self.assertTrue(self.dialog.find_selection(text)) + + text.tag_remove('sel', '1.0', 'end') + text.tag_add('sel', '1.6', '1.11') # Select 'World!' + self.assertTrue(self.dialog.find_selection(text)) + + text.tag_remove('sel', '1.0', 'end') + text.tag_add('sel', '1.0', '1.11') # Select 'Hello World!' + self.assertTrue(self.dialog.find_selection(text)) + + # Remove additional line + text.delete('2.0', 'end') + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_searchbase.py b/Lib/idlelib/idle_test/test_searchbase.py new file mode 100644 index 0000000..8036b91 --- /dev/null +++ b/Lib/idlelib/idle_test/test_searchbase.py @@ -0,0 +1,165 @@ +'''Unittests for idlelib/SearchDialogBase.py + +Coverage: 99%. The only thing not covered is inconsequential -- +testing skipping of suite when self.needwrapbutton is false. + +''' +import unittest +from test.support import requires +from tkinter import Tk, Toplevel, Frame ##, BooleanVar, StringVar +from idlelib import SearchEngine as se +from idlelib import SearchDialogBase as sdb +from idlelib.idle_test.mock_idle import Func +## from idlelib.idle_test.mock_tk import Var + +# The ## imports above & following could help make some tests gui-free. +# However, they currently make radiobutton tests fail. +##def setUpModule(): +## # Replace tk objects used to initialize se.SearchEngine. +## se.BooleanVar = Var +## se.StringVar = Var +## +##def tearDownModule(): +## se.BooleanVar = BooleanVar +## se.StringVar = StringVar + +class SearchDialogBaseTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def setUp(self): + self.engine = se.SearchEngine(self.root) # None also seems to work + self.dialog = sdb.SearchDialogBase(root=self.root, engine=self.engine) + + def tearDown(self): + self.dialog.close() + + def test_open_and_close(self): + # open calls create_widgets, which needs default_command + self.dialog.default_command = None + + # Since text parameter of .open is not used in base class, + # pass dummy 'text' instead of tk.Text(). + self.dialog.open('text') + self.assertEqual(self.dialog.top.state(), 'normal') + self.dialog.close() + self.assertEqual(self.dialog.top.state(), 'withdrawn') + + self.dialog.open('text', searchphrase="hello") + self.assertEqual(self.dialog.ent.get(), 'hello') + self.dialog.close() + + def test_create_widgets(self): + self.dialog.create_entries = Func() + self.dialog.create_option_buttons = Func() + self.dialog.create_other_buttons = Func() + self.dialog.create_command_buttons = Func() + + self.dialog.default_command = None + self.dialog.create_widgets() + + self.assertTrue(self.dialog.create_entries.called) + self.assertTrue(self.dialog.create_option_buttons.called) + self.assertTrue(self.dialog.create_other_buttons.called) + self.assertTrue(self.dialog.create_command_buttons.called) + + def test_make_entry(self): + equal = self.assertEqual + self.dialog.row = 0 + self.dialog.top = Toplevel(self.root) + entry, label = self.dialog.make_entry("Test:", 'hello') + equal(label['text'], 'Test:') + + self.assertIn(entry.get(), 'hello') + egi = entry.grid_info() + equal(int(egi['row']), 0) + equal(int(egi['column']), 1) + equal(int(egi['rowspan']), 1) + equal(int(egi['columnspan']), 1) + equal(self.dialog.row, 1) + + def test_create_entries(self): + self.dialog.row = 0 + self.engine.setpat('hello') + self.dialog.create_entries() + self.assertIn(self.dialog.ent.get(), 'hello') + + def test_make_frame(self): + self.dialog.row = 0 + self.dialog.top = Toplevel(self.root) + frame, label = self.dialog.make_frame() + self.assertEqual(label, '') + self.assertIsInstance(frame, Frame) + + frame, label = self.dialog.make_frame('testlabel') + self.assertEqual(label['text'], 'testlabel') + self.assertIsInstance(frame, Frame) + + def btn_test_setup(self, meth): + self.dialog.top = Toplevel(self.root) + self.dialog.row = 0 + return meth() + + def test_create_option_buttons(self): + e = self.engine + for state in (0, 1): + for var in (e.revar, e.casevar, e.wordvar, e.wrapvar): + var.set(state) + frame, options = self.btn_test_setup( + self.dialog.create_option_buttons) + for spec, button in zip (options, frame.pack_slaves()): + var, label = spec + self.assertEqual(button['text'], label) + self.assertEqual(var.get(), state) + if state == 1: + button.deselect() + else: + button.select() + self.assertEqual(var.get(), 1 - state) + + def test_create_other_buttons(self): + for state in (False, True): + var = self.engine.backvar + var.set(state) + frame, others = self.btn_test_setup( + self.dialog.create_other_buttons) + buttons = frame.pack_slaves() + for spec, button in zip(others, buttons): + val, label = spec + self.assertEqual(button['text'], label) + if val == state: + # hit other button, then this one + # indexes depend on button order + self.assertEqual(var.get(), state) + buttons[val].select() + self.assertEqual(var.get(), 1 - state) + buttons[1-val].select() + self.assertEqual(var.get(), state) + + def test_make_button(self): + self.dialog.top = Toplevel(self.root) + self.dialog.buttonframe = Frame(self.dialog.top) + btn = self.dialog.make_button('Test', self.dialog.close) + self.assertEqual(btn['text'], 'Test') + + def test_create_command_buttons(self): + self.dialog.create_command_buttons() + # Look for close button command in buttonframe + closebuttoncommand = '' + for child in self.dialog.buttonframe.winfo_children(): + if child['text'] == 'close': + closebuttoncommand = child['command'] + self.assertIn('close', closebuttoncommand) + + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_searchdialog.py b/Lib/idlelib/idle_test/test_searchdialog.py deleted file mode 100644 index 190c866..0000000 --- a/Lib/idlelib/idle_test/test_searchdialog.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Test SearchDialog class in SearchDialogue.py""" - -# Does not currently test the event handler wrappers. -# A usage test should simulate clicks and check hilighting. -# Tests need to be coordinated with SearchDialogBase tests -# to avoid duplication. - -from test.support import requires -requires('gui') - -import unittest -import tkinter as tk -from tkinter import BooleanVar -import idlelib.SearchEngine as se -import idlelib.SearchDialog as sd - - -class SearchDialogTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.root = tk.Tk() - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - def setUp(self): - self.engine = se.SearchEngine(self.root) - self.dialog = sd.SearchDialog(self.root, self.engine) - self.text = tk.Text(self.root) - self.text.insert('1.0', 'Hello World!') - - def test_find_again(self): - # Search for various expressions - text = self.text - - self.engine.setpat('') - self.assertFalse(self.dialog.find_again(text)) - - self.engine.setpat('Hello') - self.assertTrue(self.dialog.find_again(text)) - - self.engine.setpat('Goodbye') - self.assertFalse(self.dialog.find_again(text)) - - self.engine.setpat('World!') - self.assertTrue(self.dialog.find_again(text)) - - self.engine.setpat('Hello World!') - self.assertTrue(self.dialog.find_again(text)) - - # Regular expression - self.engine.revar = BooleanVar(self.root, True) - self.engine.setpat('W[aeiouy]r') - self.assertTrue(self.dialog.find_again(text)) - - def test_find_selection(self): - # Select some text and make sure it's found - text = self.text - # Add additional line to find - self.text.insert('2.0', 'Hello World!') - - text.tag_add('sel', '1.0', '1.4') # Select 'Hello' - self.assertTrue(self.dialog.find_selection(text)) - - text.tag_remove('sel', '1.0', 'end') - text.tag_add('sel', '1.6', '1.11') # Select 'World!' - self.assertTrue(self.dialog.find_selection(text)) - - text.tag_remove('sel', '1.0', 'end') - text.tag_add('sel', '1.0', '1.11') # Select 'Hello World!' - self.assertTrue(self.dialog.find_selection(text)) - - # Remove additional line - text.delete('2.0', 'end') - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_searchdialogbase.py b/Lib/idlelib/idle_test/test_searchdialogbase.py deleted file mode 100644 index 8036b91..0000000 --- a/Lib/idlelib/idle_test/test_searchdialogbase.py +++ /dev/null @@ -1,165 +0,0 @@ -'''Unittests for idlelib/SearchDialogBase.py - -Coverage: 99%. The only thing not covered is inconsequential -- -testing skipping of suite when self.needwrapbutton is false. - -''' -import unittest -from test.support import requires -from tkinter import Tk, Toplevel, Frame ##, BooleanVar, StringVar -from idlelib import SearchEngine as se -from idlelib import SearchDialogBase as sdb -from idlelib.idle_test.mock_idle import Func -## from idlelib.idle_test.mock_tk import Var - -# The ## imports above & following could help make some tests gui-free. -# However, they currently make radiobutton tests fail. -##def setUpModule(): -## # Replace tk objects used to initialize se.SearchEngine. -## se.BooleanVar = Var -## se.StringVar = Var -## -##def tearDownModule(): -## se.BooleanVar = BooleanVar -## se.StringVar = StringVar - -class SearchDialogBaseTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - def setUp(self): - self.engine = se.SearchEngine(self.root) # None also seems to work - self.dialog = sdb.SearchDialogBase(root=self.root, engine=self.engine) - - def tearDown(self): - self.dialog.close() - - def test_open_and_close(self): - # open calls create_widgets, which needs default_command - self.dialog.default_command = None - - # Since text parameter of .open is not used in base class, - # pass dummy 'text' instead of tk.Text(). - self.dialog.open('text') - self.assertEqual(self.dialog.top.state(), 'normal') - self.dialog.close() - self.assertEqual(self.dialog.top.state(), 'withdrawn') - - self.dialog.open('text', searchphrase="hello") - self.assertEqual(self.dialog.ent.get(), 'hello') - self.dialog.close() - - def test_create_widgets(self): - self.dialog.create_entries = Func() - self.dialog.create_option_buttons = Func() - self.dialog.create_other_buttons = Func() - self.dialog.create_command_buttons = Func() - - self.dialog.default_command = None - self.dialog.create_widgets() - - self.assertTrue(self.dialog.create_entries.called) - self.assertTrue(self.dialog.create_option_buttons.called) - self.assertTrue(self.dialog.create_other_buttons.called) - self.assertTrue(self.dialog.create_command_buttons.called) - - def test_make_entry(self): - equal = self.assertEqual - self.dialog.row = 0 - self.dialog.top = Toplevel(self.root) - entry, label = self.dialog.make_entry("Test:", 'hello') - equal(label['text'], 'Test:') - - self.assertIn(entry.get(), 'hello') - egi = entry.grid_info() - equal(int(egi['row']), 0) - equal(int(egi['column']), 1) - equal(int(egi['rowspan']), 1) - equal(int(egi['columnspan']), 1) - equal(self.dialog.row, 1) - - def test_create_entries(self): - self.dialog.row = 0 - self.engine.setpat('hello') - self.dialog.create_entries() - self.assertIn(self.dialog.ent.get(), 'hello') - - def test_make_frame(self): - self.dialog.row = 0 - self.dialog.top = Toplevel(self.root) - frame, label = self.dialog.make_frame() - self.assertEqual(label, '') - self.assertIsInstance(frame, Frame) - - frame, label = self.dialog.make_frame('testlabel') - self.assertEqual(label['text'], 'testlabel') - self.assertIsInstance(frame, Frame) - - def btn_test_setup(self, meth): - self.dialog.top = Toplevel(self.root) - self.dialog.row = 0 - return meth() - - def test_create_option_buttons(self): - e = self.engine - for state in (0, 1): - for var in (e.revar, e.casevar, e.wordvar, e.wrapvar): - var.set(state) - frame, options = self.btn_test_setup( - self.dialog.create_option_buttons) - for spec, button in zip (options, frame.pack_slaves()): - var, label = spec - self.assertEqual(button['text'], label) - self.assertEqual(var.get(), state) - if state == 1: - button.deselect() - else: - button.select() - self.assertEqual(var.get(), 1 - state) - - def test_create_other_buttons(self): - for state in (False, True): - var = self.engine.backvar - var.set(state) - frame, others = self.btn_test_setup( - self.dialog.create_other_buttons) - buttons = frame.pack_slaves() - for spec, button in zip(others, buttons): - val, label = spec - self.assertEqual(button['text'], label) - if val == state: - # hit other button, then this one - # indexes depend on button order - self.assertEqual(var.get(), state) - buttons[val].select() - self.assertEqual(var.get(), 1 - state) - buttons[1-val].select() - self.assertEqual(var.get(), state) - - def test_make_button(self): - self.dialog.top = Toplevel(self.root) - self.dialog.buttonframe = Frame(self.dialog.top) - btn = self.dialog.make_button('Test', self.dialog.close) - self.assertEqual(btn['text'], 'Test') - - def test_create_command_buttons(self): - self.dialog.create_command_buttons() - # Look for close button command in buttonframe - closebuttoncommand = '' - for child in self.dialog.buttonframe.winfo_children(): - if child['text'] == 'close': - closebuttoncommand = child['command'] - self.assertIn('close', closebuttoncommand) - - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_undo.py b/Lib/idlelib/idle_test/test_undo.py new file mode 100644 index 0000000..2657984 --- /dev/null +++ b/Lib/idlelib/idle_test/test_undo.py @@ -0,0 +1,134 @@ +"""Unittest for UndoDelegator in idlelib.UndoDelegator. + +Coverage about 80% (retest). +""" +from test.support import requires +requires('gui') + +import unittest +from unittest.mock import Mock +from tkinter import Text, Tk +from idlelib.UndoDelegator import UndoDelegator +from idlelib.Percolator import Percolator + + +class UndoDelegatorTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.root = Tk() + cls.text = Text(cls.root) + cls.percolator = Percolator(cls.text) + + @classmethod + def tearDownClass(cls): + cls.percolator.redir.close() + cls.root.destroy() + del cls.percolator, cls.text, cls.root + + def setUp(self): + self.delegator = UndoDelegator() + self.percolator.insertfilter(self.delegator) + self.delegator.bell = Mock(wraps=self.delegator.bell) + + def tearDown(self): + self.percolator.removefilter(self.delegator) + self.text.delete('1.0', 'end') + self.delegator.resetcache() + + def test_undo_event(self): + text = self.text + + text.insert('insert', 'foobar') + text.insert('insert', 'h') + text.event_generate('<>') + self.assertEqual(text.get('1.0', 'end'), '\n') + + text.insert('insert', 'foo') + text.insert('insert', 'bar') + text.delete('1.2', '1.4') + text.insert('insert', 'hello') + text.event_generate('<>') + self.assertEqual(text.get('1.0', '1.4'), 'foar') + text.event_generate('<>') + self.assertEqual(text.get('1.0', '1.6'), 'foobar') + text.event_generate('<>') + self.assertEqual(text.get('1.0', '1.3'), 'foo') + text.event_generate('<>') + self.delegator.undo_event('event') + self.assertTrue(self.delegator.bell.called) + + def test_redo_event(self): + text = self.text + + text.insert('insert', 'foo') + text.insert('insert', 'bar') + text.delete('1.0', '1.3') + text.event_generate('<>') + text.event_generate('<>') + self.assertEqual(text.get('1.0', '1.3'), 'bar') + text.event_generate('<>') + self.assertTrue(self.delegator.bell.called) + + def test_dump_event(self): + """ + Dump_event cannot be tested directly without changing + environment variables. So, test statements in dump_event + indirectly + """ + text = self.text + d = self.delegator + + text.insert('insert', 'foo') + text.insert('insert', 'bar') + text.delete('1.2', '1.4') + self.assertTupleEqual((d.pointer, d.can_merge), (3, True)) + text.event_generate('<>') + self.assertTupleEqual((d.pointer, d.can_merge), (2, False)) + + def test_get_set_saved(self): + # test the getter method get_saved + # test the setter method set_saved + # indirectly test check_saved + d = self.delegator + + self.assertTrue(d.get_saved()) + self.text.insert('insert', 'a') + self.assertFalse(d.get_saved()) + d.saved_change_hook = Mock() + + d.set_saved(True) + self.assertEqual(d.pointer, d.saved) + self.assertTrue(d.saved_change_hook.called) + + d.set_saved(False) + self.assertEqual(d.saved, -1) + self.assertTrue(d.saved_change_hook.called) + + def test_undo_start_stop(self): + # test the undo_block_start and undo_block_stop methods + text = self.text + + text.insert('insert', 'foo') + self.delegator.undo_block_start() + text.insert('insert', 'bar') + text.insert('insert', 'bar') + self.delegator.undo_block_stop() + self.assertEqual(text.get('1.0', '1.3'), 'foo') + + # test another code path + self.delegator.undo_block_start() + text.insert('insert', 'bar') + self.delegator.undo_block_stop() + self.assertEqual(text.get('1.0', '1.3'), 'foo') + + def test_addcmd(self): + text = self.text + # when number of undo operations exceeds max_undo + self.delegator.max_undo = max_undo = 10 + for i in range(max_undo + 10): + text.insert('insert', 'foo') + self.assertLessEqual(len(self.delegator.undolist), max_undo) + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_undodelegator.py b/Lib/idlelib/idle_test/test_undodelegator.py deleted file mode 100644 index 2657984..0000000 --- a/Lib/idlelib/idle_test/test_undodelegator.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Unittest for UndoDelegator in idlelib.UndoDelegator. - -Coverage about 80% (retest). -""" -from test.support import requires -requires('gui') - -import unittest -from unittest.mock import Mock -from tkinter import Text, Tk -from idlelib.UndoDelegator import UndoDelegator -from idlelib.Percolator import Percolator - - -class UndoDelegatorTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.root = Tk() - cls.text = Text(cls.root) - cls.percolator = Percolator(cls.text) - - @classmethod - def tearDownClass(cls): - cls.percolator.redir.close() - cls.root.destroy() - del cls.percolator, cls.text, cls.root - - def setUp(self): - self.delegator = UndoDelegator() - self.percolator.insertfilter(self.delegator) - self.delegator.bell = Mock(wraps=self.delegator.bell) - - def tearDown(self): - self.percolator.removefilter(self.delegator) - self.text.delete('1.0', 'end') - self.delegator.resetcache() - - def test_undo_event(self): - text = self.text - - text.insert('insert', 'foobar') - text.insert('insert', 'h') - text.event_generate('<>') - self.assertEqual(text.get('1.0', 'end'), '\n') - - text.insert('insert', 'foo') - text.insert('insert', 'bar') - text.delete('1.2', '1.4') - text.insert('insert', 'hello') - text.event_generate('<>') - self.assertEqual(text.get('1.0', '1.4'), 'foar') - text.event_generate('<>') - self.assertEqual(text.get('1.0', '1.6'), 'foobar') - text.event_generate('<>') - self.assertEqual(text.get('1.0', '1.3'), 'foo') - text.event_generate('<>') - self.delegator.undo_event('event') - self.assertTrue(self.delegator.bell.called) - - def test_redo_event(self): - text = self.text - - text.insert('insert', 'foo') - text.insert('insert', 'bar') - text.delete('1.0', '1.3') - text.event_generate('<>') - text.event_generate('<>') - self.assertEqual(text.get('1.0', '1.3'), 'bar') - text.event_generate('<>') - self.assertTrue(self.delegator.bell.called) - - def test_dump_event(self): - """ - Dump_event cannot be tested directly without changing - environment variables. So, test statements in dump_event - indirectly - """ - text = self.text - d = self.delegator - - text.insert('insert', 'foo') - text.insert('insert', 'bar') - text.delete('1.2', '1.4') - self.assertTupleEqual((d.pointer, d.can_merge), (3, True)) - text.event_generate('<>') - self.assertTupleEqual((d.pointer, d.can_merge), (2, False)) - - def test_get_set_saved(self): - # test the getter method get_saved - # test the setter method set_saved - # indirectly test check_saved - d = self.delegator - - self.assertTrue(d.get_saved()) - self.text.insert('insert', 'a') - self.assertFalse(d.get_saved()) - d.saved_change_hook = Mock() - - d.set_saved(True) - self.assertEqual(d.pointer, d.saved) - self.assertTrue(d.saved_change_hook.called) - - d.set_saved(False) - self.assertEqual(d.saved, -1) - self.assertTrue(d.saved_change_hook.called) - - def test_undo_start_stop(self): - # test the undo_block_start and undo_block_stop methods - text = self.text - - text.insert('insert', 'foo') - self.delegator.undo_block_start() - text.insert('insert', 'bar') - text.insert('insert', 'bar') - self.delegator.undo_block_stop() - self.assertEqual(text.get('1.0', '1.3'), 'foo') - - # test another code path - self.delegator.undo_block_start() - text.insert('insert', 'bar') - self.delegator.undo_block_stop() - self.assertEqual(text.get('1.0', '1.3'), 'foo') - - def test_addcmd(self): - text = self.text - # when number of undo operations exceeds max_undo - self.delegator.max_undo = max_undo = 10 - for i in range(max_undo + 10): - text.insert('insert', 'foo') - self.assertLessEqual(len(self.delegator.undolist), max_undo) - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_widgetredir.py b/Lib/idlelib/idle_test/test_widgetredir.py deleted file mode 100644 index 6440561..0000000 --- a/Lib/idlelib/idle_test/test_widgetredir.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Unittest for idlelib.WidgetRedirector - -100% coverage -""" -from test.support import requires -import unittest -from idlelib.idle_test.mock_idle import Func -from tkinter import Tk, Text, TclError -from idlelib.WidgetRedirector import WidgetRedirector - - -class InitCloseTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.tk = Tk() - cls.text = Text(cls.tk) - - @classmethod - def tearDownClass(cls): - cls.text.destroy() - cls.tk.destroy() - del cls.text, cls.tk - - def test_init(self): - redir = WidgetRedirector(self.text) - self.assertEqual(redir.widget, self.text) - self.assertEqual(redir.tk, self.text.tk) - self.assertRaises(TclError, WidgetRedirector, self.text) - redir.close() # restore self.tk, self.text - - def test_close(self): - redir = WidgetRedirector(self.text) - redir.register('insert', Func) - redir.close() - self.assertEqual(redir._operations, {}) - self.assertFalse(hasattr(self.text, 'widget')) - - -class WidgetRedirectorTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.tk = Tk() - cls.text = Text(cls.tk) - - @classmethod - def tearDownClass(cls): - cls.text.destroy() - cls.tk.destroy() - del cls.text, cls.tk - - def setUp(self): - self.redir = WidgetRedirector(self.text) - self.func = Func() - self.orig_insert = self.redir.register('insert', self.func) - self.text.insert('insert', 'asdf') # leaves self.text empty - - def tearDown(self): - self.text.delete('1.0', 'end') - self.redir.close() - - def test_repr(self): # partly for 100% coverage - self.assertIn('Redirector', repr(self.redir)) - self.assertIn('Original', repr(self.orig_insert)) - - def test_register(self): - self.assertEqual(self.text.get('1.0', 'end'), '\n') - self.assertEqual(self.func.args, ('insert', 'asdf')) - self.assertIn('insert', self.redir._operations) - self.assertIn('insert', self.text.__dict__) - self.assertEqual(self.text.insert, self.func) - - def test_original_command(self): - self.assertEqual(self.orig_insert.operation, 'insert') - self.assertEqual(self.orig_insert.tk_call, self.text.tk.call) - self.orig_insert('insert', 'asdf') - self.assertEqual(self.text.get('1.0', 'end'), 'asdf\n') - - def test_unregister(self): - self.assertIsNone(self.redir.unregister('invalid operation name')) - self.assertEqual(self.redir.unregister('insert'), self.func) - self.assertNotIn('insert', self.redir._operations) - self.assertNotIn('insert', self.text.__dict__) - - def test_unregister_no_attribute(self): - del self.text.insert - self.assertEqual(self.redir.unregister('insert'), self.func) - - def test_dispatch_intercept(self): - self.func.__init__(True) - self.assertTrue(self.redir.dispatch('insert', False)) - self.assertFalse(self.func.args[0]) - - def test_dispatch_bypass(self): - self.orig_insert('insert', 'asdf') - # tk.call returns '' where Python would return None - self.assertEqual(self.redir.dispatch('delete', '1.0', 'end'), '') - self.assertEqual(self.text.get('1.0', 'end'), '\n') - - def test_dispatch_error(self): - self.func.__init__(TclError()) - self.assertEqual(self.redir.dispatch('insert', False), '') - self.assertEqual(self.redir.dispatch('invalid'), '') - - def test_command_dispatch(self): - # Test that .__init__ causes redirection of tk calls - # through redir.dispatch - self.tk.call(self.text._w, 'insert', 'hello') - self.assertEqual(self.func.args, ('hello',)) - self.assertEqual(self.text.get('1.0', 'end'), '\n') - # Ensure that called through redir .dispatch and not through - # self.text.insert by having mock raise TclError. - self.func.__init__(TclError()) - self.assertEqual(self.tk.call(self.text._w, 'insert', 'boo'), '') - - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py new file mode 100644 index 0000000..84f39a2 --- /dev/null +++ b/Lib/idlelib/iomenu.py @@ -0,0 +1,565 @@ +import codecs +from codecs import BOM_UTF8 +import os +import re +import shlex +import sys +import tempfile + +import tkinter.filedialog as tkFileDialog +import tkinter.messagebox as tkMessageBox +from tkinter.simpledialog import askstring + +from idlelib.configHandler import idleConf + + +# Try setting the locale, so that we can find out +# what encoding to use +try: + import locale + locale.setlocale(locale.LC_CTYPE, "") +except (ImportError, locale.Error): + pass + +# Encoding for file names +filesystemencoding = sys.getfilesystemencoding() ### currently unused + +locale_encoding = 'ascii' +if sys.platform == 'win32': + # On Windows, we could use "mbcs". However, to give the user + # a portable encoding name, we need to find the code page + try: + locale_encoding = locale.getdefaultlocale()[1] + codecs.lookup(locale_encoding) + except LookupError: + pass +else: + try: + # Different things can fail here: the locale module may not be + # loaded, it may not offer nl_langinfo, or CODESET, or the + # resulting codeset may be unknown to Python. We ignore all + # these problems, falling back to ASCII + locale_encoding = locale.nl_langinfo(locale.CODESET) + if locale_encoding is None or locale_encoding is '': + # situation occurs on Mac OS X + locale_encoding = 'ascii' + codecs.lookup(locale_encoding) + except (NameError, AttributeError, LookupError): + # Try getdefaultlocale: it parses environment variables, + # which may give a clue. Unfortunately, getdefaultlocale has + # bugs that can cause ValueError. + try: + locale_encoding = locale.getdefaultlocale()[1] + if locale_encoding is None or locale_encoding is '': + # situation occurs on Mac OS X + locale_encoding = 'ascii' + codecs.lookup(locale_encoding) + except (ValueError, LookupError): + pass + +locale_encoding = locale_encoding.lower() + +encoding = locale_encoding ### KBK 07Sep07 This is used all over IDLE, check! + ### 'encoding' is used below in encode(), check! + +coding_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) + +def coding_spec(data): + """Return the encoding declaration according to PEP 263. + + When checking encoded data, only the first two lines should be passed + in to avoid a UnicodeDecodeError if the rest of the data is not unicode. + The first two lines would contain the encoding specification. + + Raise a LookupError if the encoding is declared but unknown. + """ + if isinstance(data, bytes): + # This encoding might be wrong. However, the coding + # spec must be ASCII-only, so any non-ASCII characters + # around here will be ignored. Decoding to Latin-1 should + # never fail (except for memory outage) + lines = data.decode('iso-8859-1') + else: + lines = data + # consider only the first two lines + if '\n' in lines: + lst = lines.split('\n', 2)[:2] + elif '\r' in lines: + lst = lines.split('\r', 2)[:2] + else: + lst = [lines] + for line in lst: + match = coding_re.match(line) + if match is not None: + break + if not blank_re.match(line): + return None + else: + return None + name = match.group(1) + try: + codecs.lookup(name) + except LookupError: + # The standard encoding error does not indicate the encoding + raise LookupError("Unknown encoding: "+name) + return name + + +class IOBinding: + + def __init__(self, editwin): + self.editwin = editwin + self.text = editwin.text + self.__id_open = self.text.bind("<>", self.open) + self.__id_save = self.text.bind("<>", self.save) + self.__id_saveas = self.text.bind("<>", + self.save_as) + self.__id_savecopy = self.text.bind("<>", + self.save_a_copy) + self.fileencoding = None + self.__id_print = self.text.bind("<>", self.print_window) + + def close(self): + # Undo command bindings + self.text.unbind("<>", self.__id_open) + self.text.unbind("<>", self.__id_save) + self.text.unbind("<>",self.__id_saveas) + self.text.unbind("<>", self.__id_savecopy) + self.text.unbind("<>", self.__id_print) + # Break cycles + self.editwin = None + self.text = None + self.filename_change_hook = None + + def get_saved(self): + return self.editwin.get_saved() + + def set_saved(self, flag): + self.editwin.set_saved(flag) + + def reset_undo(self): + self.editwin.reset_undo() + + filename_change_hook = None + + def set_filename_change_hook(self, hook): + self.filename_change_hook = hook + + filename = None + dirname = None + + def set_filename(self, filename): + if filename and os.path.isdir(filename): + self.filename = None + self.dirname = filename + else: + self.filename = filename + self.dirname = None + self.set_saved(1) + if self.filename_change_hook: + self.filename_change_hook() + + def open(self, event=None, editFile=None): + flist = self.editwin.flist + # Save in case parent window is closed (ie, during askopenfile()). + if flist: + if not editFile: + filename = self.askopenfile() + else: + filename=editFile + if filename: + # If editFile is valid and already open, flist.open will + # shift focus to its existing window. + # If the current window exists and is a fresh unnamed, + # unmodified editor window (not an interpreter shell), + # pass self.loadfile to flist.open so it will load the file + # in the current window (if the file is not already open) + # instead of a new window. + if (self.editwin and + not getattr(self.editwin, 'interp', None) and + not self.filename and + self.get_saved()): + flist.open(filename, self.loadfile) + else: + flist.open(filename) + else: + if self.text: + self.text.focus_set() + return "break" + + # Code for use outside IDLE: + if self.get_saved(): + reply = self.maybesave() + if reply == "cancel": + self.text.focus_set() + return "break" + if not editFile: + filename = self.askopenfile() + else: + filename=editFile + if filename: + self.loadfile(filename) + else: + self.text.focus_set() + return "break" + + eol = r"(\r\n)|\n|\r" # \r\n (Windows), \n (UNIX), or \r (Mac) + eol_re = re.compile(eol) + eol_convention = os.linesep # default + + def loadfile(self, filename): + try: + # open the file in binary mode so that we can handle + # end-of-line convention ourselves. + with open(filename, 'rb') as f: + two_lines = f.readline() + f.readline() + f.seek(0) + bytes = f.read() + except OSError as msg: + tkMessageBox.showerror("I/O Error", str(msg), parent=self.text) + return False + chars, converted = self._decode(two_lines, bytes) + if chars is None: + tkMessageBox.showerror("Decoding Error", + "File %s\nFailed to Decode" % filename, + parent=self.text) + return False + # We now convert all end-of-lines to '\n's + firsteol = self.eol_re.search(chars) + if firsteol: + self.eol_convention = firsteol.group(0) + chars = self.eol_re.sub(r"\n", chars) + self.text.delete("1.0", "end") + self.set_filename(None) + self.text.insert("1.0", chars) + self.reset_undo() + self.set_filename(filename) + if converted: + # We need to save the conversion results first + # before being able to execute the code + self.set_saved(False) + self.text.mark_set("insert", "1.0") + self.text.yview("insert") + self.updaterecentfileslist(filename) + return True + + def _decode(self, two_lines, bytes): + "Create a Unicode string." + chars = None + # Check presence of a UTF-8 signature first + if bytes.startswith(BOM_UTF8): + try: + chars = bytes[3:].decode("utf-8") + except UnicodeDecodeError: + # has UTF-8 signature, but fails to decode... + return None, False + else: + # Indicates that this file originally had a BOM + self.fileencoding = 'BOM' + return chars, False + # Next look for coding specification + try: + enc = coding_spec(two_lines) + except LookupError as name: + tkMessageBox.showerror( + title="Error loading the file", + message="The encoding '%s' is not known to this Python "\ + "installation. The file may not display correctly" % name, + parent = self.text) + enc = None + except UnicodeDecodeError: + return None, False + if enc: + try: + chars = str(bytes, enc) + self.fileencoding = enc + return chars, False + except UnicodeDecodeError: + pass + # Try ascii: + try: + chars = str(bytes, 'ascii') + self.fileencoding = None + return chars, False + except UnicodeDecodeError: + pass + # Try utf-8: + try: + chars = str(bytes, 'utf-8') + self.fileencoding = 'utf-8' + return chars, False + except UnicodeDecodeError: + pass + # Finally, try the locale's encoding. This is deprecated; + # the user should declare a non-ASCII encoding + try: + # Wait for the editor window to appear + self.editwin.text.update() + enc = askstring( + "Specify file encoding", + "The file's encoding is invalid for Python 3.x.\n" + "IDLE will convert it to UTF-8.\n" + "What is the current encoding of the file?", + initialvalue = locale_encoding, + parent = self.editwin.text) + + if enc: + chars = str(bytes, enc) + self.fileencoding = None + return chars, True + except (UnicodeDecodeError, LookupError): + pass + return None, False # None on failure + + def maybesave(self): + if self.get_saved(): + return "yes" + message = "Do you want to save %s before closing?" % ( + self.filename or "this untitled document") + confirm = tkMessageBox.askyesnocancel( + title="Save On Close", + message=message, + default=tkMessageBox.YES, + parent=self.text) + if confirm: + reply = "yes" + self.save(None) + if not self.get_saved(): + reply = "cancel" + elif confirm is None: + reply = "cancel" + else: + reply = "no" + self.text.focus_set() + return reply + + def save(self, event): + if not self.filename: + self.save_as(event) + else: + if self.writefile(self.filename): + self.set_saved(True) + try: + self.editwin.store_file_breaks() + except AttributeError: # may be a PyShell + pass + self.text.focus_set() + return "break" + + def save_as(self, event): + filename = self.asksavefile() + if filename: + if self.writefile(filename): + self.set_filename(filename) + self.set_saved(1) + try: + self.editwin.store_file_breaks() + except AttributeError: + pass + self.text.focus_set() + self.updaterecentfileslist(filename) + return "break" + + def save_a_copy(self, event): + filename = self.asksavefile() + if filename: + self.writefile(filename) + self.text.focus_set() + self.updaterecentfileslist(filename) + return "break" + + def writefile(self, filename): + self.fixlastline() + text = self.text.get("1.0", "end-1c") + if self.eol_convention != "\n": + text = text.replace("\n", self.eol_convention) + chars = self.encode(text) + try: + with open(filename, "wb") as f: + f.write(chars) + return True + except OSError as msg: + tkMessageBox.showerror("I/O Error", str(msg), + parent=self.text) + return False + + def encode(self, chars): + if isinstance(chars, bytes): + # This is either plain ASCII, or Tk was returning mixed-encoding + # text to us. Don't try to guess further. + return chars + # Preserve a BOM that might have been present on opening + if self.fileencoding == 'BOM': + return BOM_UTF8 + chars.encode("utf-8") + # See whether there is anything non-ASCII in it. + # If not, no need to figure out the encoding. + try: + return chars.encode('ascii') + except UnicodeError: + pass + # Check if there is an encoding declared + try: + # a string, let coding_spec slice it to the first two lines + enc = coding_spec(chars) + failed = None + except LookupError as msg: + failed = msg + enc = None + else: + if not enc: + # PEP 3120: default source encoding is UTF-8 + enc = 'utf-8' + if enc: + try: + return chars.encode(enc) + except UnicodeError: + failed = "Invalid encoding '%s'" % enc + tkMessageBox.showerror( + "I/O Error", + "%s.\nSaving as UTF-8" % failed, + parent = self.text) + # Fallback: save as UTF-8, with BOM - ignoring the incorrect + # declared encoding + return BOM_UTF8 + chars.encode("utf-8") + + def fixlastline(self): + c = self.text.get("end-2c") + if c != '\n': + self.text.insert("end-1c", "\n") + + def print_window(self, event): + confirm = tkMessageBox.askokcancel( + title="Print", + message="Print to Default Printer", + default=tkMessageBox.OK, + parent=self.text) + if not confirm: + self.text.focus_set() + return "break" + tempfilename = None + saved = self.get_saved() + if saved: + filename = self.filename + # shell undo is reset after every prompt, looks saved, probably isn't + if not saved or filename is None: + (tfd, tempfilename) = tempfile.mkstemp(prefix='IDLE_tmp_') + filename = tempfilename + os.close(tfd) + if not self.writefile(tempfilename): + os.unlink(tempfilename) + return "break" + platform = os.name + printPlatform = True + if platform == 'posix': #posix platform + command = idleConf.GetOption('main','General', + 'print-command-posix') + command = command + " 2>&1" + elif platform == 'nt': #win32 platform + command = idleConf.GetOption('main','General','print-command-win') + else: #no printing for this platform + printPlatform = False + if printPlatform: #we can try to print for this platform + command = command % shlex.quote(filename) + pipe = os.popen(command, "r") + # things can get ugly on NT if there is no printer available. + output = pipe.read().strip() + status = pipe.close() + if status: + output = "Printing failed (exit status 0x%x)\n" % \ + status + output + if output: + output = "Printing command: %s\n" % repr(command) + output + tkMessageBox.showerror("Print status", output, parent=self.text) + else: #no printing for this platform + message = "Printing is not enabled for this platform: %s" % platform + tkMessageBox.showinfo("Print status", message, parent=self.text) + if tempfilename: + os.unlink(tempfilename) + return "break" + + opendialog = None + savedialog = None + + filetypes = [ + ("Python files", "*.py *.pyw", "TEXT"), + ("Text files", "*.txt", "TEXT"), + ("All files", "*"), + ] + + defaultextension = '.py' if sys.platform == 'darwin' else '' + + def askopenfile(self): + dir, base = self.defaultfilename("open") + if not self.opendialog: + self.opendialog = tkFileDialog.Open(parent=self.text, + filetypes=self.filetypes) + filename = self.opendialog.show(initialdir=dir, initialfile=base) + return filename + + def defaultfilename(self, mode="open"): + if self.filename: + return os.path.split(self.filename) + elif self.dirname: + return self.dirname, "" + else: + try: + pwd = os.getcwd() + except OSError: + pwd = "" + return pwd, "" + + def asksavefile(self): + dir, base = self.defaultfilename("save") + if not self.savedialog: + self.savedialog = tkFileDialog.SaveAs( + parent=self.text, + filetypes=self.filetypes, + defaultextension=self.defaultextension) + filename = self.savedialog.show(initialdir=dir, initialfile=base) + return filename + + def updaterecentfileslist(self,filename): + "Update recent file list on all editor windows" + if self.editwin.flist: + self.editwin.update_recent_files_list(filename) + +def _io_binding(parent): # htest # + from tkinter import Toplevel, Text + + root = Toplevel(parent) + root.title("Test IOBinding") + width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) + root.geometry("+%d+%d"%(x, y + 150)) + class MyEditWin: + def __init__(self, text): + self.text = text + self.flist = None + self.text.bind("", self.open) + self.text.bind('', self.print) + self.text.bind("", self.save) + self.text.bind("", self.saveas) + self.text.bind('', self.savecopy) + def get_saved(self): return 0 + def set_saved(self, flag): pass + def reset_undo(self): pass + def open(self, event): + self.text.event_generate("<>") + def print(self, event): + self.text.event_generate("<>") + def save(self, event): + self.text.event_generate("<>") + def saveas(self, event): + self.text.event_generate("<>") + def savecopy(self, event): + self.text.event_generate("<>") + + text = Text(root) + text.pack() + text.focus_set() + editwin = MyEditWin(text) + IOBinding(editwin) + +if __name__ == "__main__": + from idlelib.idle_test.htest import run + run(_io_binding) diff --git a/Lib/idlelib/keybindingDialog.py b/Lib/idlelib/keybindingDialog.py deleted file mode 100644 index e6438bf..0000000 --- a/Lib/idlelib/keybindingDialog.py +++ /dev/null @@ -1,266 +0,0 @@ -""" -Dialog for building Tkinter accelerator key bindings -""" -from tkinter import * -import tkinter.messagebox as tkMessageBox -import string -import sys - -class GetKeysDialog(Toplevel): - def __init__(self,parent,title,action,currentKeySequences,_htest=False): - """ - action - string, the name of the virtual event these keys will be - mapped to - currentKeys - list, a list of all key sequence lists currently mapped - to virtual events, for overlap checking - _htest - bool, change box location when running htest - """ - Toplevel.__init__(self, parent) - self.configure(borderwidth=5) - self.resizable(height=FALSE,width=FALSE) - self.title(title) - self.transient(parent) - self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Cancel) - self.parent = parent - self.action=action - self.currentKeySequences=currentKeySequences - self.result='' - self.keyString=StringVar(self) - self.keyString.set('') - self.SetModifiersForPlatform() # set self.modifiers, self.modifier_label - self.modifier_vars = [] - for modifier in self.modifiers: - variable = StringVar(self) - variable.set('') - self.modifier_vars.append(variable) - self.advanced = False - self.CreateWidgets() - self.LoadFinalKeyList() - self.withdraw() #hide while setting geometry - self.update_idletasks() - self.geometry( - "+%d+%d" % ( - parent.winfo_rootx() + - (parent.winfo_width()/2 - self.winfo_reqwidth()/2), - parent.winfo_rooty() + - ((parent.winfo_height()/2 - self.winfo_reqheight()/2) - if not _htest else 150) - ) ) #centre dialog over parent (or below htest box) - self.deiconify() #geometry set, unhide - self.wait_window() - - def CreateWidgets(self): - frameMain = Frame(self,borderwidth=2,relief=SUNKEN) - frameMain.pack(side=TOP,expand=TRUE,fill=BOTH) - frameButtons=Frame(self) - frameButtons.pack(side=BOTTOM,fill=X) - self.buttonOK = Button(frameButtons,text='OK', - width=8,command=self.OK) - self.buttonOK.grid(row=0,column=0,padx=5,pady=5) - self.buttonCancel = Button(frameButtons,text='Cancel', - width=8,command=self.Cancel) - self.buttonCancel.grid(row=0,column=1,padx=5,pady=5) - self.frameKeySeqBasic = Frame(frameMain) - self.frameKeySeqAdvanced = Frame(frameMain) - self.frameControlsBasic = Frame(frameMain) - self.frameHelpAdvanced = Frame(frameMain) - self.frameKeySeqAdvanced.grid(row=0,column=0,sticky=NSEW,padx=5,pady=5) - self.frameKeySeqBasic.grid(row=0,column=0,sticky=NSEW,padx=5,pady=5) - self.frameKeySeqBasic.lift() - self.frameHelpAdvanced.grid(row=1,column=0,sticky=NSEW,padx=5) - self.frameControlsBasic.grid(row=1,column=0,sticky=NSEW,padx=5) - self.frameControlsBasic.lift() - self.buttonLevel = Button(frameMain,command=self.ToggleLevel, - text='Advanced Key Binding Entry >>') - self.buttonLevel.grid(row=2,column=0,stick=EW,padx=5,pady=5) - labelTitleBasic = Label(self.frameKeySeqBasic, - text="New keys for '"+self.action+"' :") - labelTitleBasic.pack(anchor=W) - labelKeysBasic = Label(self.frameKeySeqBasic,justify=LEFT, - textvariable=self.keyString,relief=GROOVE,borderwidth=2) - labelKeysBasic.pack(ipadx=5,ipady=5,fill=X) - self.modifier_checkbuttons = {} - column = 0 - for modifier, variable in zip(self.modifiers, self.modifier_vars): - label = self.modifier_label.get(modifier, modifier) - check=Checkbutton(self.frameControlsBasic, - command=self.BuildKeyString, - text=label,variable=variable,onvalue=modifier,offvalue='') - check.grid(row=0,column=column,padx=2,sticky=W) - self.modifier_checkbuttons[modifier] = check - column += 1 - labelFnAdvice=Label(self.frameControlsBasic,justify=LEFT, - text=\ - "Select the desired modifier keys\n"+ - "above, and the final key from the\n"+ - "list on the right.\n\n" + - "Use upper case Symbols when using\n" + - "the Shift modifier. (Letters will be\n" + - "converted automatically.)") - labelFnAdvice.grid(row=1,column=0,columnspan=4,padx=2,sticky=W) - self.listKeysFinal=Listbox(self.frameControlsBasic,width=15,height=10, - selectmode=SINGLE) - self.listKeysFinal.bind('',self.FinalKeySelected) - self.listKeysFinal.grid(row=0,column=4,rowspan=4,sticky=NS) - scrollKeysFinal=Scrollbar(self.frameControlsBasic,orient=VERTICAL, - command=self.listKeysFinal.yview) - self.listKeysFinal.config(yscrollcommand=scrollKeysFinal.set) - scrollKeysFinal.grid(row=0,column=5,rowspan=4,sticky=NS) - self.buttonClear=Button(self.frameControlsBasic, - text='Clear Keys',command=self.ClearKeySeq) - self.buttonClear.grid(row=2,column=0,columnspan=4) - labelTitleAdvanced = Label(self.frameKeySeqAdvanced,justify=LEFT, - text="Enter new binding(s) for '"+self.action+"' :\n"+ - "(These bindings will not be checked for validity!)") - labelTitleAdvanced.pack(anchor=W) - self.entryKeysAdvanced=Entry(self.frameKeySeqAdvanced, - textvariable=self.keyString) - self.entryKeysAdvanced.pack(fill=X) - labelHelpAdvanced=Label(self.frameHelpAdvanced,justify=LEFT, - text="Key bindings are specified using Tkinter keysyms as\n"+ - "in these samples: , , ,\n" - ", , .\n" - "Upper case is used when the Shift modifier is present!\n\n" + - "'Emacs style' multi-keystroke bindings are specified as\n" + - "follows: , where the first key\n" + - "is the 'do-nothing' keybinding.\n\n" + - "Multiple separate bindings for one action should be\n"+ - "separated by a space, eg., ." ) - labelHelpAdvanced.grid(row=0,column=0,sticky=NSEW) - - def SetModifiersForPlatform(self): - """Determine list of names of key modifiers for this platform. - - The names are used to build Tk bindings -- it doesn't matter if the - keyboard has these keys, it matters if Tk understands them. The - order is also important: key binding equality depends on it, so - config-keys.def must use the same ordering. - """ - if sys.platform == "darwin": - self.modifiers = ['Shift', 'Control', 'Option', 'Command'] - else: - self.modifiers = ['Control', 'Alt', 'Shift'] - self.modifier_label = {'Control': 'Ctrl'} # short name - - def ToggleLevel(self): - if self.buttonLevel.cget('text')[:8]=='Advanced': - self.ClearKeySeq() - self.buttonLevel.config(text='<< Basic Key Binding Entry') - self.frameKeySeqAdvanced.lift() - self.frameHelpAdvanced.lift() - self.entryKeysAdvanced.focus_set() - self.advanced = True - else: - self.ClearKeySeq() - self.buttonLevel.config(text='Advanced Key Binding Entry >>') - self.frameKeySeqBasic.lift() - self.frameControlsBasic.lift() - self.advanced = False - - def FinalKeySelected(self,event): - self.BuildKeyString() - - def BuildKeyString(self): - keyList = modifiers = self.GetModifiers() - finalKey = self.listKeysFinal.get(ANCHOR) - if finalKey: - finalKey = self.TranslateKey(finalKey, modifiers) - keyList.append(finalKey) - self.keyString.set('<' + '-'.join(keyList) + '>') - - def GetModifiers(self): - modList = [variable.get() for variable in self.modifier_vars] - return [mod for mod in modList if mod] - - def ClearKeySeq(self): - self.listKeysFinal.select_clear(0,END) - self.listKeysFinal.yview(MOVETO, '0.0') - for variable in self.modifier_vars: - variable.set('') - self.keyString.set('') - - def LoadFinalKeyList(self): - #these tuples are also available for use in validity checks - self.functionKeys=('F1','F2','F2','F4','F5','F6','F7','F8','F9', - 'F10','F11','F12') - self.alphanumKeys=tuple(string.ascii_lowercase+string.digits) - self.punctuationKeys=tuple('~!@#%^&*()_-+={}[]|;:,.<>/?') - self.whitespaceKeys=('Tab','Space','Return') - self.editKeys=('BackSpace','Delete','Insert') - self.moveKeys=('Home','End','Page Up','Page Down','Left Arrow', - 'Right Arrow','Up Arrow','Down Arrow') - #make a tuple of most of the useful common 'final' keys - keys=(self.alphanumKeys+self.punctuationKeys+self.functionKeys+ - self.whitespaceKeys+self.editKeys+self.moveKeys) - self.listKeysFinal.insert(END, *keys) - - def TranslateKey(self, key, modifiers): - "Translate from keycap symbol to the Tkinter keysym" - translateDict = {'Space':'space', - '~':'asciitilde','!':'exclam','@':'at','#':'numbersign', - '%':'percent','^':'asciicircum','&':'ampersand','*':'asterisk', - '(':'parenleft',')':'parenright','_':'underscore','-':'minus', - '+':'plus','=':'equal','{':'braceleft','}':'braceright', - '[':'bracketleft',']':'bracketright','|':'bar',';':'semicolon', - ':':'colon',',':'comma','.':'period','<':'less','>':'greater', - '/':'slash','?':'question','Page Up':'Prior','Page Down':'Next', - 'Left Arrow':'Left','Right Arrow':'Right','Up Arrow':'Up', - 'Down Arrow': 'Down', 'Tab':'Tab'} - if key in translateDict: - key = translateDict[key] - if 'Shift' in modifiers and key in string.ascii_lowercase: - key = key.upper() - key = 'Key-' + key - return key - - def OK(self, event=None): - if self.advanced or self.KeysOK(): # doesn't check advanced string yet - self.result=self.keyString.get() - self.destroy() - - def Cancel(self, event=None): - self.result='' - self.destroy() - - def KeysOK(self): - '''Validity check on user's 'basic' keybinding selection. - - Doesn't check the string produced by the advanced dialog because - 'modifiers' isn't set. - - ''' - keys = self.keyString.get() - keys.strip() - finalKey = self.listKeysFinal.get(ANCHOR) - modifiers = self.GetModifiers() - # create a key sequence list for overlap check: - keySequence = keys.split() - keysOK = False - title = 'Key Sequence Error' - if not keys: - tkMessageBox.showerror(title=title, parent=self, - message='No keys specified.') - elif not keys.endswith('>'): - tkMessageBox.showerror(title=title, parent=self, - message='Missing the final Key') - elif (not modifiers - and finalKey not in self.functionKeys + self.moveKeys): - tkMessageBox.showerror(title=title, parent=self, - message='No modifier key(s) specified.') - elif (modifiers == ['Shift']) \ - and (finalKey not in - self.functionKeys + self.moveKeys + ('Tab', 'Space')): - msg = 'The shift modifier by itself may not be used with'\ - ' this key symbol.' - tkMessageBox.showerror(title=title, parent=self, message=msg) - elif keySequence in self.currentKeySequences: - msg = 'This key combination is already in use.' - tkMessageBox.showerror(title=title, parent=self, message=msg) - else: - keysOK = True - return keysOK - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(GetKeysDialog) diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py new file mode 100644 index 0000000..268426f --- /dev/null +++ b/Lib/idlelib/macosx.py @@ -0,0 +1,239 @@ +""" +A number of functions that enhance IDLE on Mac OSX. +""" +import sys +import tkinter +import warnings + +def runningAsOSXApp(): + warnings.warn("runningAsOSXApp() is deprecated, use isAquaTk()", + DeprecationWarning, stacklevel=2) + return isAquaTk() + +def isCarbonAquaTk(root): + warnings.warn("isCarbonAquaTk(root) is deprecated, use isCarbonTk()", + DeprecationWarning, stacklevel=2) + return isCarbonTk() + +_tk_type = None + +def _initializeTkVariantTests(root): + """ + Initializes OS X Tk variant values for + isAquaTk(), isCarbonTk(), isCocoaTk(), and isXQuartz(). + """ + global _tk_type + if sys.platform == 'darwin': + ws = root.tk.call('tk', 'windowingsystem') + if 'x11' in ws: + _tk_type = "xquartz" + elif 'aqua' not in ws: + _tk_type = "other" + elif 'AppKit' in root.tk.call('winfo', 'server', '.'): + _tk_type = "cocoa" + else: + _tk_type = "carbon" + else: + _tk_type = "other" + +def isAquaTk(): + """ + Returns True if IDLE is using a native OS X Tk (Cocoa or Carbon). + """ + assert _tk_type is not None + return _tk_type == "cocoa" or _tk_type == "carbon" + +def isCarbonTk(): + """ + Returns True if IDLE is using a Carbon Aqua Tk (instead of the + newer Cocoa Aqua Tk). + """ + assert _tk_type is not None + return _tk_type == "carbon" + +def isCocoaTk(): + """ + Returns True if IDLE is using a Cocoa Aqua Tk. + """ + assert _tk_type is not None + return _tk_type == "cocoa" + +def isXQuartz(): + """ + Returns True if IDLE is using an OS X X11 Tk. + """ + assert _tk_type is not None + return _tk_type == "xquartz" + +def tkVersionWarning(root): + """ + Returns a string warning message if the Tk version in use appears to + be one known to cause problems with IDLE. + 1. Apple Cocoa-based Tk 8.5.7 shipped with Mac OS X 10.6 is unusable. + 2. Apple Cocoa-based Tk 8.5.9 in OS X 10.7 and 10.8 is better but + can still crash unexpectedly. + """ + + if isCocoaTk(): + patchlevel = root.tk.call('info', 'patchlevel') + if patchlevel not in ('8.5.7', '8.5.9'): + return False + return (r"WARNING: The version of Tcl/Tk ({0}) in use may" + r" be unstable.\n" + r"Visit http://www.python.org/download/mac/tcltk/" + r" for current information.".format(patchlevel)) + else: + return False + +def addOpenEventSupport(root, flist): + """ + This ensures that the application will respond to open AppleEvents, which + makes is feasible to use IDLE as the default application for python files. + """ + def doOpenFile(*args): + for fn in args: + flist.open(fn) + + # The command below is a hook in aquatk that is called whenever the app + # receives a file open event. The callback can have multiple arguments, + # one for every file that should be opened. + root.createcommand("::tk::mac::OpenDocument", doOpenFile) + +def hideTkConsole(root): + try: + root.tk.call('console', 'hide') + except tkinter.TclError: + # Some versions of the Tk framework don't have a console object + pass + +def overrideRootMenu(root, flist): + """ + Replace the Tk root menu by something that is more appropriate for + IDLE with an Aqua Tk. + """ + # The menu that is attached to the Tk root (".") is also used by AquaTk for + # all windows that don't specify a menu of their own. The default menubar + # contains a number of menus, none of which are appropriate for IDLE. The + # Most annoying of those is an 'About Tck/Tk...' menu in the application + # menu. + # + # This function replaces the default menubar by a mostly empty one, it + # should only contain the correct application menu and the window menu. + # + # Due to a (mis-)feature of TkAqua the user will also see an empty Help + # menu. + from tkinter import Menu + from idlelib import Bindings + from idlelib import WindowList + + closeItem = Bindings.menudefs[0][1][-2] + + # Remove the last 3 items of the file menu: a separator, close window and + # quit. Close window will be reinserted just above the save item, where + # it should be according to the HIG. Quit is in the application menu. + del Bindings.menudefs[0][1][-3:] + Bindings.menudefs[0][1].insert(6, closeItem) + + # Remove the 'About' entry from the help menu, it is in the application + # menu + del Bindings.menudefs[-1][1][0:2] + # Remove the 'Configure Idle' entry from the options menu, it is in the + # application menu as 'Preferences' + del Bindings.menudefs[-2][1][0] + menubar = Menu(root) + root.configure(menu=menubar) + menudict = {} + + menudict['windows'] = menu = Menu(menubar, name='windows', tearoff=0) + menubar.add_cascade(label='Window', menu=menu, underline=0) + + def postwindowsmenu(menu=menu): + end = menu.index('end') + if end is None: + end = -1 + + if end > 0: + menu.delete(0, end) + WindowList.add_windows_to_menu(menu) + WindowList.register_callback(postwindowsmenu) + + def about_dialog(event=None): + "Handle Help 'About IDLE' event." + # Synchronize with EditorWindow.EditorWindow.about_dialog. + from idlelib import aboutDialog + aboutDialog.AboutDialog(root, 'About IDLE') + + def config_dialog(event=None): + "Handle Options 'Configure IDLE' event." + # Synchronize with EditorWindow.EditorWindow.config_dialog. + from idlelib import configDialog + + # Ensure that the root object has an instance_dict attribute, + # mirrors code in EditorWindow (although that sets the attribute + # on an EditorWindow instance that is then passed as the first + # argument to ConfigDialog) + root.instance_dict = flist.inversedict + configDialog.ConfigDialog(root, 'Settings') + + def help_dialog(event=None): + "Handle Help 'IDLE Help' event." + # Synchronize with EditorWindow.EditorWindow.help_dialog. + from idlelib import help + help.show_idlehelp(root) + + root.bind('<>', about_dialog) + root.bind('<>', config_dialog) + root.createcommand('::tk::mac::ShowPreferences', config_dialog) + if flist: + root.bind('<>', flist.close_all_callback) + + # The binding above doesn't reliably work on all versions of Tk + # on MacOSX. Adding command definition below does seem to do the + # right thing for now. + root.createcommand('exit', flist.close_all_callback) + + if isCarbonTk(): + # for Carbon AquaTk, replace the default Tk apple menu + menudict['application'] = menu = Menu(menubar, name='apple', + tearoff=0) + menubar.add_cascade(label='IDLE', menu=menu) + Bindings.menudefs.insert(0, + ('application', [ + ('About IDLE', '<>'), + None, + ])) + tkversion = root.tk.eval('info patchlevel') + if tuple(map(int, tkversion.split('.'))) < (8, 4, 14): + # for earlier AquaTk versions, supply a Preferences menu item + Bindings.menudefs[0][1].append( + ('_Preferences....', '<>'), + ) + if isCocoaTk(): + # replace default About dialog with About IDLE one + root.createcommand('tkAboutDialog', about_dialog) + # replace default "Help" item in Help menu + root.createcommand('::tk::mac::ShowHelp', help_dialog) + # remove redundant "IDLE Help" from menu + del Bindings.menudefs[-1][1][0] + +def setupApp(root, flist): + """ + Perform initial OS X customizations if needed. + Called from PyShell.main() after initial calls to Tk() + + There are currently three major versions of Tk in use on OS X: + 1. Aqua Cocoa Tk (native default since OS X 10.6) + 2. Aqua Carbon Tk (original native, 32-bit only, deprecated) + 3. X11 (supported by some third-party distributors, deprecated) + There are various differences among the three that affect IDLE + behavior, primarily with menus, mouse key events, and accelerators. + Some one-time customizations are performed here. + Others are dynamically tested throughout idlelib by calls to the + isAquaTk(), isCarbonTk(), isCocoaTk(), isXQuartz() functions which + are initialized here as well. + """ + _initializeTkVariantTests(root) + if isAquaTk(): + hideTkConsole(root) + overrideRootMenu(root, flist) + addOpenEventSupport(root, flist) diff --git a/Lib/idlelib/macosxSupport.py b/Lib/idlelib/macosxSupport.py deleted file mode 100644 index 268426f..0000000 --- a/Lib/idlelib/macosxSupport.py +++ /dev/null @@ -1,239 +0,0 @@ -""" -A number of functions that enhance IDLE on Mac OSX. -""" -import sys -import tkinter -import warnings - -def runningAsOSXApp(): - warnings.warn("runningAsOSXApp() is deprecated, use isAquaTk()", - DeprecationWarning, stacklevel=2) - return isAquaTk() - -def isCarbonAquaTk(root): - warnings.warn("isCarbonAquaTk(root) is deprecated, use isCarbonTk()", - DeprecationWarning, stacklevel=2) - return isCarbonTk() - -_tk_type = None - -def _initializeTkVariantTests(root): - """ - Initializes OS X Tk variant values for - isAquaTk(), isCarbonTk(), isCocoaTk(), and isXQuartz(). - """ - global _tk_type - if sys.platform == 'darwin': - ws = root.tk.call('tk', 'windowingsystem') - if 'x11' in ws: - _tk_type = "xquartz" - elif 'aqua' not in ws: - _tk_type = "other" - elif 'AppKit' in root.tk.call('winfo', 'server', '.'): - _tk_type = "cocoa" - else: - _tk_type = "carbon" - else: - _tk_type = "other" - -def isAquaTk(): - """ - Returns True if IDLE is using a native OS X Tk (Cocoa or Carbon). - """ - assert _tk_type is not None - return _tk_type == "cocoa" or _tk_type == "carbon" - -def isCarbonTk(): - """ - Returns True if IDLE is using a Carbon Aqua Tk (instead of the - newer Cocoa Aqua Tk). - """ - assert _tk_type is not None - return _tk_type == "carbon" - -def isCocoaTk(): - """ - Returns True if IDLE is using a Cocoa Aqua Tk. - """ - assert _tk_type is not None - return _tk_type == "cocoa" - -def isXQuartz(): - """ - Returns True if IDLE is using an OS X X11 Tk. - """ - assert _tk_type is not None - return _tk_type == "xquartz" - -def tkVersionWarning(root): - """ - Returns a string warning message if the Tk version in use appears to - be one known to cause problems with IDLE. - 1. Apple Cocoa-based Tk 8.5.7 shipped with Mac OS X 10.6 is unusable. - 2. Apple Cocoa-based Tk 8.5.9 in OS X 10.7 and 10.8 is better but - can still crash unexpectedly. - """ - - if isCocoaTk(): - patchlevel = root.tk.call('info', 'patchlevel') - if patchlevel not in ('8.5.7', '8.5.9'): - return False - return (r"WARNING: The version of Tcl/Tk ({0}) in use may" - r" be unstable.\n" - r"Visit http://www.python.org/download/mac/tcltk/" - r" for current information.".format(patchlevel)) - else: - return False - -def addOpenEventSupport(root, flist): - """ - This ensures that the application will respond to open AppleEvents, which - makes is feasible to use IDLE as the default application for python files. - """ - def doOpenFile(*args): - for fn in args: - flist.open(fn) - - # The command below is a hook in aquatk that is called whenever the app - # receives a file open event. The callback can have multiple arguments, - # one for every file that should be opened. - root.createcommand("::tk::mac::OpenDocument", doOpenFile) - -def hideTkConsole(root): - try: - root.tk.call('console', 'hide') - except tkinter.TclError: - # Some versions of the Tk framework don't have a console object - pass - -def overrideRootMenu(root, flist): - """ - Replace the Tk root menu by something that is more appropriate for - IDLE with an Aqua Tk. - """ - # The menu that is attached to the Tk root (".") is also used by AquaTk for - # all windows that don't specify a menu of their own. The default menubar - # contains a number of menus, none of which are appropriate for IDLE. The - # Most annoying of those is an 'About Tck/Tk...' menu in the application - # menu. - # - # This function replaces the default menubar by a mostly empty one, it - # should only contain the correct application menu and the window menu. - # - # Due to a (mis-)feature of TkAqua the user will also see an empty Help - # menu. - from tkinter import Menu - from idlelib import Bindings - from idlelib import WindowList - - closeItem = Bindings.menudefs[0][1][-2] - - # Remove the last 3 items of the file menu: a separator, close window and - # quit. Close window will be reinserted just above the save item, where - # it should be according to the HIG. Quit is in the application menu. - del Bindings.menudefs[0][1][-3:] - Bindings.menudefs[0][1].insert(6, closeItem) - - # Remove the 'About' entry from the help menu, it is in the application - # menu - del Bindings.menudefs[-1][1][0:2] - # Remove the 'Configure Idle' entry from the options menu, it is in the - # application menu as 'Preferences' - del Bindings.menudefs[-2][1][0] - menubar = Menu(root) - root.configure(menu=menubar) - menudict = {} - - menudict['windows'] = menu = Menu(menubar, name='windows', tearoff=0) - menubar.add_cascade(label='Window', menu=menu, underline=0) - - def postwindowsmenu(menu=menu): - end = menu.index('end') - if end is None: - end = -1 - - if end > 0: - menu.delete(0, end) - WindowList.add_windows_to_menu(menu) - WindowList.register_callback(postwindowsmenu) - - def about_dialog(event=None): - "Handle Help 'About IDLE' event." - # Synchronize with EditorWindow.EditorWindow.about_dialog. - from idlelib import aboutDialog - aboutDialog.AboutDialog(root, 'About IDLE') - - def config_dialog(event=None): - "Handle Options 'Configure IDLE' event." - # Synchronize with EditorWindow.EditorWindow.config_dialog. - from idlelib import configDialog - - # Ensure that the root object has an instance_dict attribute, - # mirrors code in EditorWindow (although that sets the attribute - # on an EditorWindow instance that is then passed as the first - # argument to ConfigDialog) - root.instance_dict = flist.inversedict - configDialog.ConfigDialog(root, 'Settings') - - def help_dialog(event=None): - "Handle Help 'IDLE Help' event." - # Synchronize with EditorWindow.EditorWindow.help_dialog. - from idlelib import help - help.show_idlehelp(root) - - root.bind('<>', about_dialog) - root.bind('<>', config_dialog) - root.createcommand('::tk::mac::ShowPreferences', config_dialog) - if flist: - root.bind('<>', flist.close_all_callback) - - # The binding above doesn't reliably work on all versions of Tk - # on MacOSX. Adding command definition below does seem to do the - # right thing for now. - root.createcommand('exit', flist.close_all_callback) - - if isCarbonTk(): - # for Carbon AquaTk, replace the default Tk apple menu - menudict['application'] = menu = Menu(menubar, name='apple', - tearoff=0) - menubar.add_cascade(label='IDLE', menu=menu) - Bindings.menudefs.insert(0, - ('application', [ - ('About IDLE', '<>'), - None, - ])) - tkversion = root.tk.eval('info patchlevel') - if tuple(map(int, tkversion.split('.'))) < (8, 4, 14): - # for earlier AquaTk versions, supply a Preferences menu item - Bindings.menudefs[0][1].append( - ('_Preferences....', '<>'), - ) - if isCocoaTk(): - # replace default About dialog with About IDLE one - root.createcommand('tkAboutDialog', about_dialog) - # replace default "Help" item in Help menu - root.createcommand('::tk::mac::ShowHelp', help_dialog) - # remove redundant "IDLE Help" from menu - del Bindings.menudefs[-1][1][0] - -def setupApp(root, flist): - """ - Perform initial OS X customizations if needed. - Called from PyShell.main() after initial calls to Tk() - - There are currently three major versions of Tk in use on OS X: - 1. Aqua Cocoa Tk (native default since OS X 10.6) - 2. Aqua Carbon Tk (original native, 32-bit only, deprecated) - 3. X11 (supported by some third-party distributors, deprecated) - There are various differences among the three that affect IDLE - behavior, primarily with menus, mouse key events, and accelerators. - Some one-time customizations are performed here. - Others are dynamically tested throughout idlelib by calls to the - isAquaTk(), isCarbonTk(), isCocoaTk(), isXQuartz() functions which - are initialized here as well. - """ - _initializeTkVariantTests(root) - if isAquaTk(): - hideTkConsole(root) - overrideRootMenu(root, flist) - addOpenEventSupport(root, flist) diff --git a/Lib/idlelib/mainmenu.py b/Lib/idlelib/mainmenu.py new file mode 100644 index 0000000..ab25ff1 --- /dev/null +++ b/Lib/idlelib/mainmenu.py @@ -0,0 +1,94 @@ +"""Define the menu contents, hotkeys, and event bindings. + +There is additional configuration information in the EditorWindow class (and +subclasses): the menus are created there based on the menu_specs (class) +variable, and menus not created are silently skipped in the code here. This +makes it possible, for example, to define a Debug menu which is only present in +the PythonShell window, and a Format menu which is only present in the Editor +windows. + +""" +from importlib.util import find_spec + +from idlelib.configHandler import idleConf + +# Warning: menudefs is altered in macosxSupport.overrideRootMenu() +# after it is determined that an OS X Aqua Tk is in use, +# which cannot be done until after Tk() is first called. +# Do not alter the 'file', 'options', or 'help' cascades here +# without altering overrideRootMenu() as well. +# TODO: Make this more robust + +menudefs = [ + # underscore prefixes character to underscore + ('file', [ + ('_New File', '<>'), + ('_Open...', '<>'), + ('Open _Module...', '<>'), + ('Class _Browser', '<>'), + ('_Path Browser', '<>'), + None, + ('_Save', '<>'), + ('Save _As...', '<>'), + ('Save Cop_y As...', '<>'), + None, + ('Prin_t Window', '<>'), + None, + ('_Close', '<>'), + ('E_xit', '<>'), + ]), + ('edit', [ + ('_Undo', '<>'), + ('_Redo', '<>'), + None, + ('Cu_t', '<>'), + ('_Copy', '<>'), + ('_Paste', '<>'), + ('Select _All', '<>'), + None, + ('_Find...', '<>'), + ('Find A_gain', '<>'), + ('Find _Selection', '<>'), + ('Find in Files...', '<>'), + ('R_eplace...', '<>'), + ('Go to _Line', '<>'), + ]), +('format', [ + ('_Indent Region', '<>'), + ('_Dedent Region', '<>'), + ('Comment _Out Region', '<>'), + ('U_ncomment Region', '<>'), + ('Tabify Region', '<>'), + ('Untabify Region', '<>'), + ('Toggle Tabs', '<>'), + ('New Indent Width', '<>'), + ]), + ('run', [ + ('Python Shell', '<>'), + ]), + ('shell', [ + ('_View Last Restart', '<>'), + ('_Restart Shell', '<>'), + ]), + ('debug', [ + ('_Go to File/Line', '<>'), + ('!_Debugger', '<>'), + ('_Stack Viewer', '<>'), + ('!_Auto-open Stack Viewer', '<>'), + ]), + ('options', [ + ('Configure _IDLE', '<>'), + None, + ]), + ('help', [ + ('_About IDLE', '<>'), + None, + ('_IDLE Help', '<>'), + ('Python _Docs', '<>'), + ]), +] + +if find_spec('turtledemo'): + menudefs[-1][1].append(('Turtle Demo', '<>')) + +default_keydefs = idleConf.GetCurrentKeySet() diff --git a/Lib/idlelib/multicall.py b/Lib/idlelib/multicall.py new file mode 100644 index 0000000..251a84d --- /dev/null +++ b/Lib/idlelib/multicall.py @@ -0,0 +1,446 @@ +""" +MultiCall - a class which inherits its methods from a Tkinter widget (Text, for +example), but enables multiple calls of functions per virtual event - all +matching events will be called, not only the most specific one. This is done +by wrapping the event functions - event_add, event_delete and event_info. +MultiCall recognizes only a subset of legal event sequences. Sequences which +are not recognized are treated by the original Tk handling mechanism. A +more-specific event will be called before a less-specific event. + +The recognized sequences are complete one-event sequences (no emacs-style +Ctrl-X Ctrl-C, no shortcuts like <3>), for all types of events. +Key/Button Press/Release events can have modifiers. +The recognized modifiers are Shift, Control, Option and Command for Mac, and +Control, Alt, Shift, Meta/M for other platforms. + +For all events which were handled by MultiCall, a new member is added to the +event instance passed to the binded functions - mc_type. This is one of the +event type constants defined in this module (such as MC_KEYPRESS). +For Key/Button events (which are handled by MultiCall and may receive +modifiers), another member is added - mc_state. This member gives the state +of the recognized modifiers, as a combination of the modifier constants +also defined in this module (for example, MC_SHIFT). +Using these members is absolutely portable. + +The order by which events are called is defined by these rules: +1. A more-specific event will be called before a less-specific event. +2. A recently-binded event will be called before a previously-binded event, + unless this conflicts with the first rule. +Each function will be called at most once for each event. +""" + +import sys +import re +import tkinter + +# the event type constants, which define the meaning of mc_type +MC_KEYPRESS=0; MC_KEYRELEASE=1; MC_BUTTONPRESS=2; MC_BUTTONRELEASE=3; +MC_ACTIVATE=4; MC_CIRCULATE=5; MC_COLORMAP=6; MC_CONFIGURE=7; +MC_DEACTIVATE=8; MC_DESTROY=9; MC_ENTER=10; MC_EXPOSE=11; MC_FOCUSIN=12; +MC_FOCUSOUT=13; MC_GRAVITY=14; MC_LEAVE=15; MC_MAP=16; MC_MOTION=17; +MC_MOUSEWHEEL=18; MC_PROPERTY=19; MC_REPARENT=20; MC_UNMAP=21; MC_VISIBILITY=22; +# the modifier state constants, which define the meaning of mc_state +MC_SHIFT = 1<<0; MC_CONTROL = 1<<2; MC_ALT = 1<<3; MC_META = 1<<5 +MC_OPTION = 1<<6; MC_COMMAND = 1<<7 + +# define the list of modifiers, to be used in complex event types. +if sys.platform == "darwin": + _modifiers = (("Shift",), ("Control",), ("Option",), ("Command",)) + _modifier_masks = (MC_SHIFT, MC_CONTROL, MC_OPTION, MC_COMMAND) +else: + _modifiers = (("Control",), ("Alt",), ("Shift",), ("Meta", "M")) + _modifier_masks = (MC_CONTROL, MC_ALT, MC_SHIFT, MC_META) + +# a dictionary to map a modifier name into its number +_modifier_names = dict([(name, number) + for number in range(len(_modifiers)) + for name in _modifiers[number]]) + +# In 3.4, if no shell window is ever open, the underlying Tk widget is +# destroyed before .__del__ methods here are called. The following +# is used to selectively ignore shutdown exceptions to avoid +# 'Exception ignored' messages. See http://bugs.python.org/issue20167 +APPLICATION_GONE = "application has been destroyed" + +# A binder is a class which binds functions to one type of event. It has two +# methods: bind and unbind, which get a function and a parsed sequence, as +# returned by _parse_sequence(). There are two types of binders: +# _SimpleBinder handles event types with no modifiers and no detail. +# No Python functions are called when no events are binded. +# _ComplexBinder handles event types with modifiers and a detail. +# A Python function is called each time an event is generated. + +class _SimpleBinder: + def __init__(self, type, widget, widgetinst): + self.type = type + self.sequence = '<'+_types[type][0]+'>' + self.widget = widget + self.widgetinst = widgetinst + self.bindedfuncs = [] + self.handlerid = None + + def bind(self, triplet, func): + if not self.handlerid: + def handler(event, l = self.bindedfuncs, mc_type = self.type): + event.mc_type = mc_type + wascalled = {} + for i in range(len(l)-1, -1, -1): + func = l[i] + if func not in wascalled: + wascalled[func] = True + r = func(event) + if r: + return r + self.handlerid = self.widget.bind(self.widgetinst, + self.sequence, handler) + self.bindedfuncs.append(func) + + def unbind(self, triplet, func): + self.bindedfuncs.remove(func) + if not self.bindedfuncs: + self.widget.unbind(self.widgetinst, self.sequence, self.handlerid) + self.handlerid = None + + def __del__(self): + if self.handlerid: + try: + self.widget.unbind(self.widgetinst, self.sequence, + self.handlerid) + except tkinter.TclError as e: + if not APPLICATION_GONE in e.args[0]: + raise + +# An int in range(1 << len(_modifiers)) represents a combination of modifiers +# (if the least significent bit is on, _modifiers[0] is on, and so on). +# _state_subsets gives for each combination of modifiers, or *state*, +# a list of the states which are a subset of it. This list is ordered by the +# number of modifiers is the state - the most specific state comes first. +_states = range(1 << len(_modifiers)) +_state_names = [''.join(m[0]+'-' + for i, m in enumerate(_modifiers) + if (1 << i) & s) + for s in _states] + +def expand_substates(states): + '''For each item of states return a list containing all combinations of + that item with individual bits reset, sorted by the number of set bits. + ''' + def nbits(n): + "number of bits set in n base 2" + nb = 0 + while n: + n, rem = divmod(n, 2) + nb += rem + return nb + statelist = [] + for state in states: + substates = list(set(state & x for x in states)) + substates.sort(key=nbits, reverse=True) + statelist.append(substates) + return statelist + +_state_subsets = expand_substates(_states) + +# _state_codes gives for each state, the portable code to be passed as mc_state +_state_codes = [] +for s in _states: + r = 0 + for i in range(len(_modifiers)): + if (1 << i) & s: + r |= _modifier_masks[i] + _state_codes.append(r) + +class _ComplexBinder: + # This class binds many functions, and only unbinds them when it is deleted. + # self.handlerids is the list of seqs and ids of binded handler functions. + # The binded functions sit in a dictionary of lists of lists, which maps + # a detail (or None) and a state into a list of functions. + # When a new detail is discovered, handlers for all the possible states + # are binded. + + def __create_handler(self, lists, mc_type, mc_state): + def handler(event, lists = lists, + mc_type = mc_type, mc_state = mc_state, + ishandlerrunning = self.ishandlerrunning, + doafterhandler = self.doafterhandler): + ishandlerrunning[:] = [True] + event.mc_type = mc_type + event.mc_state = mc_state + wascalled = {} + r = None + for l in lists: + for i in range(len(l)-1, -1, -1): + func = l[i] + if func not in wascalled: + wascalled[func] = True + r = l[i](event) + if r: + break + if r: + break + ishandlerrunning[:] = [] + # Call all functions in doafterhandler and remove them from list + for f in doafterhandler: + f() + doafterhandler[:] = [] + if r: + return r + return handler + + def __init__(self, type, widget, widgetinst): + self.type = type + self.typename = _types[type][0] + self.widget = widget + self.widgetinst = widgetinst + self.bindedfuncs = {None: [[] for s in _states]} + self.handlerids = [] + # we don't want to change the lists of functions while a handler is + # running - it will mess up the loop and anyway, we usually want the + # change to happen from the next event. So we have a list of functions + # for the handler to run after it finishes calling the binded functions. + # It calls them only once. + # ishandlerrunning is a list. An empty one means no, otherwise - yes. + # this is done so that it would be mutable. + self.ishandlerrunning = [] + self.doafterhandler = [] + for s in _states: + lists = [self.bindedfuncs[None][i] for i in _state_subsets[s]] + handler = self.__create_handler(lists, type, _state_codes[s]) + seq = '<'+_state_names[s]+self.typename+'>' + self.handlerids.append((seq, self.widget.bind(self.widgetinst, + seq, handler))) + + def bind(self, triplet, func): + if triplet[2] not in self.bindedfuncs: + self.bindedfuncs[triplet[2]] = [[] for s in _states] + for s in _states: + lists = [ self.bindedfuncs[detail][i] + for detail in (triplet[2], None) + for i in _state_subsets[s] ] + handler = self.__create_handler(lists, self.type, + _state_codes[s]) + seq = "<%s%s-%s>"% (_state_names[s], self.typename, triplet[2]) + self.handlerids.append((seq, self.widget.bind(self.widgetinst, + seq, handler))) + doit = lambda: self.bindedfuncs[triplet[2]][triplet[0]].append(func) + if not self.ishandlerrunning: + doit() + else: + self.doafterhandler.append(doit) + + def unbind(self, triplet, func): + doit = lambda: self.bindedfuncs[triplet[2]][triplet[0]].remove(func) + if not self.ishandlerrunning: + doit() + else: + self.doafterhandler.append(doit) + + def __del__(self): + for seq, id in self.handlerids: + try: + self.widget.unbind(self.widgetinst, seq, id) + except tkinter.TclError as e: + if not APPLICATION_GONE in e.args[0]: + raise + +# define the list of event types to be handled by MultiEvent. the order is +# compatible with the definition of event type constants. +_types = ( + ("KeyPress", "Key"), ("KeyRelease",), ("ButtonPress", "Button"), + ("ButtonRelease",), ("Activate",), ("Circulate",), ("Colormap",), + ("Configure",), ("Deactivate",), ("Destroy",), ("Enter",), ("Expose",), + ("FocusIn",), ("FocusOut",), ("Gravity",), ("Leave",), ("Map",), + ("Motion",), ("MouseWheel",), ("Property",), ("Reparent",), ("Unmap",), + ("Visibility",), +) + +# which binder should be used for every event type? +_binder_classes = (_ComplexBinder,) * 4 + (_SimpleBinder,) * (len(_types)-4) + +# A dictionary to map a type name into its number +_type_names = dict([(name, number) + for number in range(len(_types)) + for name in _types[number]]) + +_keysym_re = re.compile(r"^\w+$") +_button_re = re.compile(r"^[1-5]$") +def _parse_sequence(sequence): + """Get a string which should describe an event sequence. If it is + successfully parsed as one, return a tuple containing the state (as an int), + the event type (as an index of _types), and the detail - None if none, or a + string if there is one. If the parsing is unsuccessful, return None. + """ + if not sequence or sequence[0] != '<' or sequence[-1] != '>': + return None + words = sequence[1:-1].split('-') + modifiers = 0 + while words and words[0] in _modifier_names: + modifiers |= 1 << _modifier_names[words[0]] + del words[0] + if words and words[0] in _type_names: + type = _type_names[words[0]] + del words[0] + else: + return None + if _binder_classes[type] is _SimpleBinder: + if modifiers or words: + return None + else: + detail = None + else: + # _ComplexBinder + if type in [_type_names[s] for s in ("KeyPress", "KeyRelease")]: + type_re = _keysym_re + else: + type_re = _button_re + + if not words: + detail = None + elif len(words) == 1 and type_re.match(words[0]): + detail = words[0] + else: + return None + + return modifiers, type, detail + +def _triplet_to_sequence(triplet): + if triplet[2]: + return '<'+_state_names[triplet[0]]+_types[triplet[1]][0]+'-'+ \ + triplet[2]+'>' + else: + return '<'+_state_names[triplet[0]]+_types[triplet[1]][0]+'>' + +_multicall_dict = {} +def MultiCallCreator(widget): + """Return a MultiCall class which inherits its methods from the + given widget class (for example, Tkinter.Text). This is used + instead of a templating mechanism. + """ + if widget in _multicall_dict: + return _multicall_dict[widget] + + class MultiCall (widget): + assert issubclass(widget, tkinter.Misc) + + def __init__(self, *args, **kwargs): + widget.__init__(self, *args, **kwargs) + # a dictionary which maps a virtual event to a tuple with: + # 0. the function binded + # 1. a list of triplets - the sequences it is binded to + self.__eventinfo = {} + self.__binders = [_binder_classes[i](i, widget, self) + for i in range(len(_types))] + + def bind(self, sequence=None, func=None, add=None): + #print("bind(%s, %s, %s)" % (sequence, func, add), + # file=sys.__stderr__) + if type(sequence) is str and len(sequence) > 2 and \ + sequence[:2] == "<<" and sequence[-2:] == ">>": + if sequence in self.__eventinfo: + ei = self.__eventinfo[sequence] + if ei[0] is not None: + for triplet in ei[1]: + self.__binders[triplet[1]].unbind(triplet, ei[0]) + ei[0] = func + if ei[0] is not None: + for triplet in ei[1]: + self.__binders[triplet[1]].bind(triplet, func) + else: + self.__eventinfo[sequence] = [func, []] + return widget.bind(self, sequence, func, add) + + def unbind(self, sequence, funcid=None): + if type(sequence) is str and len(sequence) > 2 and \ + sequence[:2] == "<<" and sequence[-2:] == ">>" and \ + sequence in self.__eventinfo: + func, triplets = self.__eventinfo[sequence] + if func is not None: + for triplet in triplets: + self.__binders[triplet[1]].unbind(triplet, func) + self.__eventinfo[sequence][0] = None + return widget.unbind(self, sequence, funcid) + + def event_add(self, virtual, *sequences): + #print("event_add(%s, %s)" % (repr(virtual), repr(sequences)), + # file=sys.__stderr__) + if virtual not in self.__eventinfo: + self.__eventinfo[virtual] = [None, []] + + func, triplets = self.__eventinfo[virtual] + for seq in sequences: + triplet = _parse_sequence(seq) + if triplet is None: + #print("Tkinter event_add(%s)" % seq, file=sys.__stderr__) + widget.event_add(self, virtual, seq) + else: + if func is not None: + self.__binders[triplet[1]].bind(triplet, func) + triplets.append(triplet) + + def event_delete(self, virtual, *sequences): + if virtual not in self.__eventinfo: + return + func, triplets = self.__eventinfo[virtual] + for seq in sequences: + triplet = _parse_sequence(seq) + if triplet is None: + #print("Tkinter event_delete: %s" % seq, file=sys.__stderr__) + widget.event_delete(self, virtual, seq) + else: + if func is not None: + self.__binders[triplet[1]].unbind(triplet, func) + triplets.remove(triplet) + + def event_info(self, virtual=None): + if virtual is None or virtual not in self.__eventinfo: + return widget.event_info(self, virtual) + else: + return tuple(map(_triplet_to_sequence, + self.__eventinfo[virtual][1])) + \ + widget.event_info(self, virtual) + + def __del__(self): + for virtual in self.__eventinfo: + func, triplets = self.__eventinfo[virtual] + if func: + for triplet in triplets: + try: + self.__binders[triplet[1]].unbind(triplet, func) + except tkinter.TclError as e: + if not APPLICATION_GONE in e.args[0]: + raise + + _multicall_dict[widget] = MultiCall + return MultiCall + + +def _multi_call(parent): + root = tkinter.Tk() + root.title("Test MultiCall") + width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) + root.geometry("+%d+%d"%(x, y + 150)) + text = MultiCallCreator(tkinter.Text)(root) + text.pack() + def bindseq(seq, n=[0]): + def handler(event): + print(seq) + text.bind("<>"%n[0], handler) + text.event_add("<>"%n[0], seq) + n[0] += 1 + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + bindseq("") + root.mainloop() + +if __name__ == "__main__": + from idlelib.idle_test.htest import run + run(_multi_call) diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py new file mode 100644 index 0000000..e614f9b --- /dev/null +++ b/Lib/idlelib/outwin.py @@ -0,0 +1,144 @@ +from tkinter import * +from idlelib.EditorWindow import EditorWindow +import re +import tkinter.messagebox as tkMessageBox +from idlelib import IOBinding + +class OutputWindow(EditorWindow): + + """An editor window that can serve as an output file. + + Also the future base class for the Python shell window. + This class has no input facilities. + """ + + def __init__(self, *args): + EditorWindow.__init__(self, *args) + self.text.bind("<>", self.goto_file_line) + + # Customize EditorWindow + + def ispythonsource(self, filename): + # No colorization needed + return 0 + + def short_title(self): + return "Output" + + def maybesave(self): + # Override base class method -- don't ask any questions + if self.get_saved(): + return "yes" + else: + return "no" + + # Act as output file + + def write(self, s, tags=(), mark="insert"): + if isinstance(s, (bytes, bytes)): + s = s.decode(IOBinding.encoding, "replace") + self.text.insert(mark, s, tags) + self.text.see(mark) + self.text.update() + return len(s) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def flush(self): + pass + + # Our own right-button menu + + rmenu_specs = [ + ("Cut", "<>", "rmenu_check_cut"), + ("Copy", "<>", "rmenu_check_copy"), + ("Paste", "<>", "rmenu_check_paste"), + (None, None, None), + ("Go to file/line", "<>", None), + ] + + file_line_pats = [ + # order of patterns matters + r'file "([^"]*)", line (\d+)', + r'([^\s]+)$(\d+)$', + r'^(\s*\S.*?):\s*(\d+):', # Win filename, maybe starting with spaces + r'([^\s]+):\s*(\d+):', # filename or path, ltrim + r'^\s*(\S.*?):\s*(\d+):', # Win abs path with embedded spaces, ltrim + ] + + file_line_progs = None + + def goto_file_line(self, event=None): + if self.file_line_progs is None: + l = [] + for pat in self.file_line_pats: + l.append(re.compile(pat, re.IGNORECASE)) + self.file_line_progs = l + # x, y = self.event.x, self.event.y + # self.text.mark_set("insert", "@%d,%d" % (x, y)) + line = self.text.get("insert linestart", "insert lineend") + result = self._file_line_helper(line) + if not result: + # Try the previous line. This is handy e.g. in tracebacks, + # where you tend to right-click on the displayed source line + line = self.text.get("insert -1line linestart", + "insert -1line lineend") + result = self._file_line_helper(line) + if not result: + tkMessageBox.showerror( + "No special line", + "The line you point at doesn't look like " + "a valid file name followed by a line number.", + parent=self.text) + return + filename, lineno = result + edit = self.flist.open(filename) + edit.gotoline(lineno) + + def _file_line_helper(self, line): + for prog in self.file_line_progs: + match = prog.search(line) + if match: + filename, lineno = match.group(1, 2) + try: + f = open(filename, "r") + f.close() + break + except OSError: + continue + else: + return None + try: + return filename, int(lineno) + except TypeError: + return None + +# These classes are currently not used but might come in handy + +class OnDemandOutputWindow: + + tagdefs = { + # XXX Should use IdlePrefs.ColorPrefs + "stdout": {"foreground": "blue"}, + "stderr": {"foreground": "#007700"}, + } + + def __init__(self, flist): + self.flist = flist + self.owin = None + + def write(self, s, tags, mark): + if not self.owin: + self.setup() + self.owin.write(s, tags, mark) + + def setup(self): + self.owin = owin = OutputWindow(self.flist) + text = owin.text + for tag, cnf in self.tagdefs.items(): + if cnf: + text.tag_configure(tag, **cnf) + text.tag_raise('sel') + self.write = self.owin.write diff --git a/Lib/idlelib/paragraph.py b/Lib/idlelib/paragraph.py new file mode 100644 index 0000000..7a9d185 --- /dev/null +++ b/Lib/idlelib/paragraph.py @@ -0,0 +1,195 @@ +"""Extension to format a paragraph or selection to a max width. + +Does basic, standard text formatting, and also understands Python +comment blocks. Thus, for editing Python source code, this +extension is really only suitable for reformatting these comment +blocks or triple-quoted strings. + +Known problems with comment reformatting: +* If there is a selection marked, and the first line of the + selection is not complete, the block will probably not be detected + as comments, and will have the normal "text formatting" rules + applied. +* If a comment block has leading whitespace that mixes tabs and + spaces, they will not be considered part of the same block. +* Fancy comments, like this bulleted list, aren't handled :-) +""" + +import re +from idlelib.configHandler import idleConf + +class FormatParagraph: + + menudefs = [ + ('format', [ # /s/edit/format dscherer@cmu.edu + ('Format Paragraph', '<>'), + ]) + ] + + def __init__(self, editwin): + self.editwin = editwin + + def close(self): + self.editwin = None + + def format_paragraph_event(self, event, limit=None): + """Formats paragraph to a max width specified in idleConf. + + If text is selected, format_paragraph_event will start breaking lines + at the max width, starting from the beginning selection. + + If no text is selected, format_paragraph_event uses the current + cursor location to determine the paragraph (lines of text surrounded + by blank lines) and formats it. + + The length limit parameter is for testing with a known value. + """ + if limit is None: + # The default length limit is that defined by pep8 + limit = idleConf.GetOption( + 'extensions', 'FormatParagraph', 'max-width', + type='int', default=72) + text = self.editwin.text + first, last = self.editwin.get_selection_indices() + if first and last: + data = text.get(first, last) + comment_header = get_comment_header(data) + else: + first, last, comment_header, data = \ + find_paragraph(text, text.index("insert")) + if comment_header: + newdata = reformat_comment(data, limit, comment_header) + else: + newdata = reformat_paragraph(data, limit) + text.tag_remove("sel", "1.0", "end") + + if newdata != data: + text.mark_set("insert", first) + text.undo_block_start() + text.delete(first, last) + text.insert(first, newdata) + text.undo_block_stop() + else: + text.mark_set("insert", last) + text.see("insert") + return "break" + +def find_paragraph(text, mark): + """Returns the start/stop indices enclosing the paragraph that mark is in. + + Also returns the comment format string, if any, and paragraph of text + between the start/stop indices. + """ + lineno, col = map(int, mark.split(".")) + line = text.get("%d.0" % lineno, "%d.end" % lineno) + + # Look for start of next paragraph if the index passed in is a blank line + while text.compare("%d.0" % lineno, "<", "end") and is_all_white(line): + lineno = lineno + 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + first_lineno = lineno + comment_header = get_comment_header(line) + comment_header_len = len(comment_header) + + # Once start line found, search for end of paragraph (a blank line) + while get_comment_header(line)==comment_header and \ + not is_all_white(line[comment_header_len:]): + lineno = lineno + 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + last = "%d.0" % lineno + + # Search back to beginning of paragraph (first blank line before) + lineno = first_lineno - 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + while lineno > 0 and \ + get_comment_header(line)==comment_header and \ + not is_all_white(line[comment_header_len:]): + lineno = lineno - 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + first = "%d.0" % (lineno+1) + + return first, last, comment_header, text.get(first, last) + +# This should perhaps be replaced with textwrap.wrap +def reformat_paragraph(data, limit): + """Return data reformatted to specified width (limit).""" + lines = data.split("\n") + i = 0 + n = len(lines) + while i < n and is_all_white(lines[i]): + i = i+1 + if i >= n: + return data + indent1 = get_indent(lines[i]) + if i+1 < n and not is_all_white(lines[i+1]): + indent2 = get_indent(lines[i+1]) + else: + indent2 = indent1 + new = lines[:i] + partial = indent1 + while i < n and not is_all_white(lines[i]): + # XXX Should take double space after period (etc.) into account + words = re.split("(\s+)", lines[i]) + for j in range(0, len(words), 2): + word = words[j] + if not word: + continu