Many changes from the upstream repo (https://github.com/python/typing).

This syncs to rev 7b43ada77821d23e55e3a4b35f6055a59b9e1ad7 there.

Summary:

- Add typing.DefaultDict (as a generic variant of collections.defaultdict).

- Use collections.Reversible if it exists (only relevant for Python 3.6).

- Revamped generic class behavior to conform to updated PEP 484.

- Improve speed of Generic.__new__.

- Make sure __init__ is called for new Generic instances. Fix issue #26391.

- Refactor async support to be compatible with 3.2, 3.3, 3.4.

- Remove 'io' and 're' from __all__ (they still exist, just not
  included by "import *"). Fix issue #26234.

- Change @overload -- you can now use it outside stubs (you still
  cannot call the decorated function though).

1 files changed, 242 insertions, 152 deletions

diff --git a/Lib/typing.py b/Lib/typing.py
index 823f9be..de2a462 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1,7 +1,3 @@
-# TODO nits:
-# Get rid of asserts that are the caller's fault.
-# Docstrings (e.g. ABCs).
-
 import abc
 from abc import abstractmethod, abstractproperty
 import collections
@@ -56,6 +52,7 @@ __all__ = [
 
     # Concrete collection types.
     'Dict',
+    'DefaultDict',
     'List',
     'Set',
     'NamedTuple',  # Not really a type.
@@ -68,12 +65,12 @@ __all__ = [
     'no_type_check',
     'no_type_check_decorator',
     'overload',
-
-    # Submodules.
-    'io',
-    're',
 ]
 
+# The pseudo-submodules 're' and 'io' are part of the public
+# namespace, but excluded from __all__ because they might stomp on
+# legitimate imports of those modules.
+
 
 def _qualname(x):
     if sys.version_info[:2] >= (3, 3):
@@ -117,8 +114,8 @@ class TypingMeta(type):
         """
         return self
 
-    def _has_type_var(self):
-        return False
+    def _get_type_vars(self, tvars):
+        pass
 
     def __repr__(self):
         return '%s.%s' % (self.__module__, _qualname(self))
@@ -214,8 +211,8 @@ class _TypeAlias:
         someone tries to subclass a type alias (not a good idea).
         """
         if (len(args) == 3 and
-            isinstance(args[0], str) and
-            isinstance(args[1], tuple)):
+                isinstance(args[0], str) and
+                isinstance(args[1], tuple)):
             # Close enough.
             raise TypeError("A type alias cannot be subclassed")
         return object.__new__(cls)
@@ -271,8 +268,16 @@ class _TypeAlias:
             return issubclass(cls, self.impl_type)
 
 
-def _has_type_var(t):
-    return t is not None and isinstance(t, TypingMeta) and t._has_type_var()
+def _get_type_vars(types, tvars):
+    for t in types:
+        if isinstance(t, TypingMeta):
+            t._get_type_vars(tvars)
+
+
+def _type_vars(types):
+    tvars = []
+    _get_type_vars(types, tvars)
+    return tuple(tvars)
 
 
 def _eval_type(t, globalns, localns):
@@ -376,7 +381,7 @@ class TypeVar(TypingMeta, metaclass=TypingMeta, _root=True):
     At runtime, isinstance(x, T) will raise TypeError.  However,
     issubclass(C, T) is true for any class C, and issubclass(str, A)
     and issubclass(bytes, A) are true, and issubclass(int, A) is
-    false.
+    false.  (TODO: Why is this needed?  This may change.  See #136.)
 
     Type variables may be marked covariant or contravariant by passing
     covariant=True or contravariant=True.  See PEP 484 for more
@@ -410,8 +415,9 @@ class TypeVar(TypingMeta, metaclass=TypingMeta, _root=True):
             self.__bound__ = None
         return self
 
-    def _has_type_var(self):
-        return True
+    def _get_type_vars(self, tvars):
+        if self not in tvars:
+            tvars.append(self)
 
     def __repr__(self):
         if self.__covariant__:
@@ -448,7 +454,6 @@ VT_co = TypeVar('VT_co', covariant=True)  # Value type covariant containers.
 T_contra = TypeVar('T_contra', contravariant=True)  # Ditto contravariant.
 
 # A useful type variable with constraints.  This represents string types.
-# TODO: What about bytearray, memoryview?
 AnyStr = TypeVar('AnyStr', bytes, str)
 
 
@@ -514,12 +519,9 @@ class UnionMeta(TypingMeta):
             return self.__class__(self.__name__, self.__bases__, {},
                                   p, _root=True)
 
-    def _has_type_var(self):
+    def _get_type_vars(self, tvars):
         if self.__union_params__:
-            for t in self.__union_params__:
-                if _has_type_var(t):
-                    return True
-        return False
+            _get_type_vars(self.__union_params__, tvars)
 
     def __repr__(self):
         r = super().__repr__()
@@ -656,12 +658,9 @@ class TupleMeta(TypingMeta):
         self.__tuple_use_ellipsis__ = use_ellipsis
         return self
 
-    def _has_type_var(self):
+    def _get_type_vars(self, tvars):
         if self.__tuple_params__:
-            for t in self.__tuple_params__:
-                if _has_type_var(t):
-                    return True
-        return False
+            _get_type_vars(self.__tuple_params__, tvars)
 
     def _eval_type(self, globalns, localns):
         tp = self.__tuple_params__
@@ -769,12 +768,9 @@ class CallableMeta(TypingMeta):
         self.__result__ = result
         return self
 
-    def _has_type_var(self):
+    def _get_type_vars(self, tvars):
         if self.__args__:
-            for t in self.__args__:
-                if _has_type_var(t):
-                    return True
-        return _has_type_var(self.__result__)
+            _get_type_vars(self.__args__, tvars)
 
     def _eval_type(self, globalns, localns):
         if self.__args__ is None and self.__result__ is None:
@@ -878,76 +874,106 @@ def _geqv(a, b):
     return _gorg(a) is _gorg(b)
 
 
-class GenericMeta(TypingMeta, abc.ABCMeta):
-    """Metaclass for generic types."""
+def _next_in_mro(cls):
+    """Helper for Generic.__new__.
+
+    Returns the class after the last occurrence of Generic or
+    Generic[...] in cls.__mro__.
+    """
+    next_in_mro = object
+    # Look for the last occurrence of Generic or Generic[...].
+    for i, c in enumerate(cls.__mro__[:-1]):
+        if isinstance(c, GenericMeta) and _gorg(c) is Generic:
+            next_in_mro = cls.__mro__[i+1]
+    return next_in_mro
 
-    # TODO: Constrain more how Generic is used; only a few
-    # standard patterns should be allowed.
 
-    # TODO: Use a more precise rule than matching __name__ to decide
-    # whether two classes are the same.  Also, save the formal
-    # parameters.  (These things are related!  A solution lies in
-    # using origin.)
+class GenericMeta(TypingMeta, abc.ABCMeta):
+    """Metaclass for generic types."""
 
     __extra__ = None
 
     def __new__(cls, name, bases, namespace,
-                parameters=None, origin=None, extra=None):
-        if parameters is None:
-            # Extract parameters from direct base classes.  Only
-            # direct bases are considered and only those that are
-            # themselves generic, and parameterized with type
-            # variables.  Don't use bases like Any, Union, Tuple,
-            # Callable or type variables.
-            params = None
+                tvars=None, args=None, origin=None, extra=None):
+        self = super().__new__(cls, name, bases, namespace, _root=True)
+
+        if tvars is not None:
+            # Called from __getitem__() below.
+            assert origin is not None
+            assert all(isinstance(t, TypeVar) for t in tvars), tvars
+        else:
+            # Called from class statement.
+            assert tvars is None, tvars
+            assert args is None, args
+            assert origin is None, origin
+
+            # Get the full set of tvars from the bases.
+            tvars = _type_vars(bases)
+            # Look for Generic[T1, ..., Tn].
+            # If found, tvars must be a subset of it.
+            # If not found, tvars is it.
+            # Also check for and reject plain Generic,
+            # and reject multiple Generic[...].
+            gvars = None
             for base in bases:
-                if isinstance(base, TypingMeta):
-                    if not isinstance(base, GenericMeta):
+                if base is Generic:
+                    raise TypeError("Cannot inherit from plain Generic")
+                if (isinstance(base, GenericMeta) and
+                        base.__origin__ is Generic):
+                    if gvars is not None:
                         raise TypeError(
-                            "You cannot inherit from magic class %s" %
-                            repr(base))
-                    if base.__parameters__ is None:
-                        continue  # The base is unparameterized.
-                    for bp in base.__parameters__:
-                        if _has_type_var(bp) and not isinstance(bp, TypeVar):
-                            raise TypeError(
-                                "Cannot inherit from a generic class "
-                                "parameterized with "
-                                "non-type-variable %s" % bp)
-                        if params is None:
-                            params = []
-                        if bp not in params:
-                            params.append(bp)
-            if params is not None:
-                parameters = tuple(params)
-        self = super().__new__(cls, name, bases, namespace, _root=True)
-        self.__parameters__ = parameters
+                            "Cannot inherit from Generic[...] multiple types.")
+                    gvars = base.__parameters__
+            if gvars is None:
+                gvars = tvars
+            else:
+                tvarset = set(tvars)
+                gvarset = set(gvars)
+                if not tvarset <= gvarset:
+                    raise TypeError(
+                        "Some type variables (%s) "
+                        "are not listed in Generic[%s]" %
+                        (", ".join(str(t) for t in tvars if t not in gvarset),
+                         ", ".join(str(g) for g in gvars)))
+                tvars = gvars
+
+        self.__parameters__ = tvars
+        self.__args__ = args
+        self.__origin__ = origin
         if extra is not None:
             self.__extra__ = extra
         # Else __extra__ is inherited, eventually from the
         # (meta-)class default above.
-        self.__origin__ = origin
+        # Speed hack (https://github.com/python/typing/issues/196).
+        self.__next_in_mro__ = _next_in_mro(self)
         return self
 
-    def _has_type_var(self):
-        if self.__parameters__:
-            for t in self.__parameters__:
-                if _has_type_var(t):
-                    return True
-        return False
+    def _get_type_vars(self, tvars):
+        if self.__origin__ and self.__parameters__:
+            _get_type_vars(self.__parameters__, tvars)
 
     def __repr__(self):
-        r = super().__repr__()
-        if self.__parameters__ is not None:
+        if self.__origin__ is not None:
+            r = repr(self.__origin__)
+        else:
+            r = super().__repr__()
+        if self.__args__:
             r += '[%s]' % (
+                ', '.join(_type_repr(p) for p in self.__args__))
+        if self.__parameters__:
+            r += '<%s>' % (
                 ', '.join(_type_repr(p) for p in self.__parameters__))
         return r
 
     def __eq__(self, other):
         if not isinstance(other, GenericMeta):
             return NotImplemented
-        return (_geqv(self, other) and
-                self.__parameters__ == other.__parameters__)
+        if self.__origin__ is not None:
+            return (self.__origin__ is other.__origin__ and
+                    self.__args__ == other.__args__ and
+                    self.__parameters__ == other.__parameters__)
+        else:
+            return self is other
 
     def __hash__(self):
         return hash((self.__name__, self.__parameters__))
@@ -956,37 +982,45 @@ class GenericMeta(TypingMeta, abc.ABCMeta):
         if not isinstance(params, tuple):
             params = (params,)
         if not params:
-            raise TypeError("Cannot have empty parameter list")
+            raise TypeError(
+                "Parameter list to %s[...] cannot be empty" % _qualname(self))
         msg = "Parameters to generic types must be types."
         params = tuple(_type_check(p, msg) for p in params)
-        if self.__parameters__ is None:
-            for p in params:
-                if not isinstance(p, TypeVar):
-                    raise TypeError("Initial parameters must be "
-                                    "type variables; got %s" % p)
+        if self is Generic:
+            # Generic can only be subscripted with unique type variables.
+            if not all(isinstance(p, TypeVar) for p in params):
+                raise TypeError(
+                    "Parameters to Generic[...] must all be type variables")
             if len(set(params)) != len(params):
                 raise TypeError(
-                    "All type variables in Generic[...] must be distinct.")
+                    "Parameters to Generic[...] must all be unique")
+            tvars = params
+            args = None
+        elif self is _Protocol:
+            # _Protocol is internal, don't check anything.
+            tvars = params
+            args = None
+        elif self.__origin__ in (Generic, _Protocol):
+            # Can't subscript Generic[...] or _Protocol[...].
+            raise TypeError("Cannot subscript already-subscripted %s" %
+                            repr(self))
         else:
-            if len(params) != len(self.__parameters__):
-                raise TypeError("Cannot change parameter count from %d to %d" %
-                                (len(self.__parameters__), len(params)))
-            for new, old in zip(params, self.__parameters__):
-                if isinstance(old, TypeVar):
-                    if not old.__constraints__:
-                        # Substituting for an unconstrained TypeVar is OK.
-                        continue
-                    if issubclass(new, Union[old.__constraints__]):
-                        # Specializing a constrained type variable is OK.
-                        continue
-                if not issubclass(new, old):
-                    raise TypeError(
-                        "Cannot substitute %s for %s in %s" %
-                        (_type_repr(new), _type_repr(old), self))
-
-        return self.__class__(self.__name__, (self,) + self.__bases__,
+            # Subscripting a regular Generic subclass.
+            if not self.__parameters__:
+                raise TypeError("%s is not a generic class" % repr(self))
+            alen = len(params)
+            elen = len(self.__parameters__)
+            if alen != elen:
+                raise TypeError(
+                    "Too %s parameters for %s; actual %s, expected %s" %
+                    ("many" if alen > elen else "few", repr(self), alen, elen))
+            tvars = _type_vars(params)
+            args = params
+        return self.__class__(self.__name__,
+                              (self,) + self.__bases__,
                               dict(self.__dict__),
-                              parameters=params,
+                              tvars=tvars,
+                              args=args,
                               origin=self,
                               extra=self.__extra__)
 
@@ -1006,10 +1040,10 @@ class GenericMeta(TypingMeta, abc.ABCMeta):
             # C[X] is a subclass of C[Y] iff X is a subclass of Y.
             origin = self.__origin__
             if origin is not None and origin is cls.__origin__:
-                assert len(self.__parameters__) == len(origin.__parameters__)
-                assert len(cls.__parameters__) == len(origin.__parameters__)
-                for p_self, p_cls, p_origin in zip(self.__parameters__,
-                                                   cls.__parameters__,
+                assert len(self.__args__) == len(origin.__parameters__)
+                assert len(cls.__args__) == len(origin.__parameters__)
+                for p_self, p_cls, p_origin in zip(self.__args__,
+                                                   cls.__args__,
                                                    origin.__parameters__):
                     if isinstance(p_origin, TypeVar):
                         if p_origin.__covariant__:
@@ -1039,6 +1073,10 @@ class GenericMeta(TypingMeta, abc.ABCMeta):
         return issubclass(cls, self.__extra__)
 
 
+# Prevent checks for Generic to crash when defining Generic.
+Generic = None
+
+
 class Generic(metaclass=GenericMeta):
     """Abstract base class for generic types.
 
@@ -1053,29 +1091,23 @@ class Generic(metaclass=GenericMeta):
 
     This class can then be used as follows::
 
-      def lookup_name(mapping: Mapping, key: KT, default: VT) -> VT:
+      def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT:
           try:
               return mapping[key]
           except KeyError:
               return default
-
-    For clarity the type variables may be redefined, e.g.::
-
-      X = TypeVar('X')
-      Y = TypeVar('Y')
-      def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y:
-          # Same body as above.
     """
 
     __slots__ = ()
 
     def __new__(cls, *args, **kwds):
-        next_in_mro = object
-        # Look for the last occurrence of Generic or Generic[...].
-        for i, c in enumerate(cls.__mro__[:-1]):
-            if isinstance(c, GenericMeta) and _gorg(c) is Generic:
-                next_in_mro = cls.__mro__[i+1]
-        return next_in_mro.__new__(_gorg(cls))
+        if cls.__origin__ is None:
+            return cls.__next_in_mro__.__new__(cls)
+        else:
+            origin = _gorg(cls)
+            obj = cls.__next_in_mro__.__new__(origin)
+            obj.__init__(*args, **kwds)
+            return obj
 
 
 def cast(typ, val):
@@ -1093,9 +1125,7 @@ def _get_defaults(func):
     """Internal helper to extract the default arguments, by name."""
     code = func.__code__
     pos_count = code.co_argcount
-    kw_count = code.co_kwonlyargcount
     arg_names = code.co_varnames
-    kwarg_names = arg_names[pos_count:pos_count + kw_count]
     arg_names = arg_names[:pos_count]
     defaults = func.__defaults__ or ()
     kwdefaults = func.__kwdefaults__
@@ -1148,7 +1178,6 @@ def get_type_hints(obj, globalns=None, localns=None):
     return hints
 
 
-# TODO: Also support this as a class decorator.
 def no_type_check(arg):
     """Decorator to indicate that annotations are not type hints.
 
@@ -1183,8 +1212,42 @@ def no_type_check_decorator(decorator):
     return wrapped_decorator
 
 
+def _overload_dummy(*args, **kwds):
+    """Helper for @overload to raise when called."""
+    raise NotImplementedError(
+        "You should not call an overloaded function. "
+        "A series of @overload-decorated functions "
+        "outside a stub module should always be followed "
+        "by an implementation that is not @overload-ed.")
+
+
 def overload(func):
-    raise RuntimeError("Overloading is only supported in library stubs")
+    """Decorator for overloaded functions/methods.
+
+    In a stub file, place two or more stub definitions for the same
+    function in a row, each decorated with @overload.  For example:
+
+      @overload
+      def utf8(value: None) -> None: ...
+      @overload
+      def utf8(value: bytes) -> bytes: ...
+      @overload
+      def utf8(value: str) -> bytes: ...
+
+    In a non-stub file (i.e. a regular .py file), do the same but
+    follow it with an implementation.  The implementation should *not*
+    be decorated with @overload.  For example:
+
+      @overload
+      def utf8(value: None) -> None: ...
+      @overload
+      def utf8(value: bytes) -> bytes: ...
+      @overload
+      def utf8(value: str) -> bytes: ...
+      def utf8(value):
+          # implementation goes here
+    """
+    return _overload_dummy
 
 
 class _ProtocolMeta(GenericMeta):
@@ -1232,14 +1295,16 @@ class _ProtocolMeta(GenericMeta):
                         break
                 else:
                     if (not attr.startswith('_abc_') and
-                        attr != '__abstractmethods__' and
-                        attr != '_is_protocol' and
-                        attr != '__dict__' and
-                        attr != '__slots__' and
-                        attr != '_get_protocol_attrs' and
-                        attr != '__parameters__' and
-                        attr != '__origin__' and
-                        attr != '__module__'):
+                            attr != '__abstractmethods__' and
+                            attr != '_is_protocol' and
+                            attr != '__dict__' and
+                            attr != '__args__' and
+                            attr != '__slots__' and
+                            attr != '_get_protocol_attrs' and
+                            attr != '__next_in_mro__' and
+                            attr != '__parameters__' and
+                            attr != '__origin__' and
+                            attr != '__module__'):
                         attrs.add(attr)
 
         return attrs
@@ -1264,16 +1329,25 @@ class _Protocol(metaclass=_ProtocolMeta):
 Hashable = collections_abc.Hashable  # Not generic.
 
 
-class Awaitable(Generic[T_co], extra=collections_abc.Awaitable):
-    __slots__ = ()
+if hasattr(collections_abc, 'Awaitable'):
+    class Awaitable(Generic[T_co], extra=collections_abc.Awaitable):
+        __slots__ = ()
+else:
+    Awaitable = None
 
 
-class AsyncIterable(Generic[T_co], extra=collections_abc.AsyncIterable):
-    __slots__ = ()
+if hasattr(collections_abc, 'AsyncIterable'):
 
+    class AsyncIterable(Generic[T_co], extra=collections_abc.AsyncIterable):
+        __slots__ = ()
 
-class AsyncIterator(AsyncIterable[T_co], extra=collections_abc.AsyncIterator):
-    __slots__ = ()
+    class AsyncIterator(AsyncIterable[T_co],
+                        extra=collections_abc.AsyncIterator):
+        __slots__ = ()
+
+else:
+    AsyncIterable = None
+    AsyncIterator = None
 
 
 class Iterable(Generic[T_co], extra=collections_abc.Iterable):
@@ -1332,12 +1406,16 @@ class SupportsRound(_Protocol[T_co]):
         pass
 
 
-class Reversible(_Protocol[T_co]):
-    __slots__ = ()
+if hasattr(collections_abc, 'Reversible'):
+    class Reversible(Iterable[T_co], extra=collections_abc.Reversible):
+        __slots__ = ()
+else:
+    class Reversible(_Protocol[T_co]):
+        __slots__ = ()
 
-    @abstractmethod
-    def __reversed__(self) -> 'Iterator[T_co]':
-        pass
+        @abstractmethod
+        def __reversed__(self) -> 'Iterator[T_co]':
+            pass
 
 
 Sized = collections_abc.Sized  # Not generic.
@@ -1360,7 +1438,7 @@ class MutableSet(AbstractSet[T], extra=collections_abc.MutableSet):
 
 
 # NOTE: Only the value type is covariant.
-class Mapping(Sized, Iterable[KT], Container[KT], Generic[VT_co],
+class Mapping(Sized, Iterable[KT], Container[KT], Generic[KT, VT_co],
               extra=collections_abc.Mapping):
     pass
 
@@ -1368,10 +1446,14 @@ class Mapping(Sized, Iterable[KT], Container[KT], Generic[VT_co],
 class MutableMapping(Mapping[KT, VT], extra=collections_abc.MutableMapping):
     pass
 
-
-class Sequence(Sized, Iterable[T_co], Container[T_co],
+if hasattr(collections_abc, 'Reversible'):
+    class Sequence(Sized, Reversible[T_co], Container[T_co],
                extra=collections_abc.Sequence):
-    pass
+        pass
+else:
+    class Sequence(Sized, Iterable[T_co], Container[T_co],
+                   extra=collections_abc.Sequence):
+        pass
 
 
 class MutableSequence(Sequence[T], extra=collections_abc.MutableSequence):
@@ -1436,8 +1518,9 @@ class KeysView(MappingView[KT], AbstractSet[KT],
     pass
 
 
-# TODO: Enable Set[Tuple[KT, VT_co]] instead of Generic[KT, VT_co].
-class ItemsView(MappingView, Generic[KT, VT_co],
+class ItemsView(MappingView[Tuple[KT, VT_co]],
+                Set[Tuple[KT, VT_co]],
+                Generic[KT, VT_co],
                 extra=collections_abc.ItemsView):
     pass
 
@@ -1454,6 +1537,13 @@ class Dict(dict, MutableMapping[KT, VT]):
                             "use dict() instead")
         return dict.__new__(cls, *args, **kwds)
 
+class DefaultDict(collections.defaultdict, MutableMapping[KT, VT]):
+
+    def __new__(cls, *args, **kwds):
+        if _geqv(cls, DefaultDict):
+            raise TypeError("Type DefaultDict cannot be instantiated; "
+                            "use collections.defaultdict() instead")
+        return collections.defaultdict.__new__(cls, *args, **kwds)
 
 # Determine what base class to use for Generator.
 if hasattr(collections_abc, 'Generator'):