From e8c2f72b94ae5dfba50c0f2e2c06b7ee975c8acb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 30 Apr 2022 08:22:46 +0300 Subject: bpo-43224: Implement substitution of unpacked TypeVarTuple in C (GH-31828) Co-authored-by: Matthew Rahtz --- Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init.h | 1 + Lib/test/test_typing.py | 63 ++++++++++------- Lib/typing.py | 10 ++- Objects/genericaliasobject.c | 118 +++++++++++++++++++++++++++---- 5 files changed, 152 insertions(+), 41 deletions(-) diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cc94662..28fffa9 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -202,6 +202,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__truediv__) STRUCT_FOR_ID(__trunc__) STRUCT_FOR_ID(__typing_subst__) + STRUCT_FOR_ID(__typing_unpacked__) STRUCT_FOR_ID(__warningregistry__) STRUCT_FOR_ID(__weakref__) STRUCT_FOR_ID(__xor__) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 9a3a9d0..941badf 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -825,6 +825,7 @@ extern "C" { INIT_ID(__truediv__), \ INIT_ID(__trunc__), \ INIT_ID(__typing_subst__), \ + INIT_ID(__typing_unpacked__), \ INIT_ID(__warningregistry__), \ INIT_ID(__weakref__), \ INIT_ID(__xor__), \ diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 412d117..88be285 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -754,89 +754,89 @@ class GenericAliasSubstitutionTests(BaseTestCase): tests = [ # Alias # Args # Expected result ('C[*Ts]', '[()]', 'C[()]'), - ('tuple[*Ts]', '[()]', 'TypeError'), # Should be tuple[()] + ('tuple[*Ts]', '[()]', 'tuple[()]'), ('Tuple[*Ts]', '[()]', 'Tuple[()]'), ('C[*Ts]', '[int]', 'C[int]'), - ('tuple[*Ts]', '[int]', 'tuple[(int,),]'), # Should be tuple[int] + ('tuple[*Ts]', '[int]', 'tuple[int]'), ('Tuple[*Ts]', '[int]', 'Tuple[int]'), ('C[*Ts]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts]', '[int, str]', 'TypeError'), # Should be tuple[int, str] + ('tuple[*Ts]', '[int, str]', 'tuple[int, str]'), ('Tuple[*Ts]', '[int, str]', 'Tuple[int, str]'), ('C[*Ts]', '[*tuple_type[int]]', 'C[*tuple_type[int]]'), # Should be C[int] - ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[(*tuple_type[int],),]'), # Should be tuple[int] + ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[*tuple_type[int]]'), # Should be tuple[int] ('Tuple[*Ts]', '[*tuple_type[int]]', 'Tuple[*tuple_type[int]]'), # Should be Tuple[int] ('C[*Ts]', '[*tuple_type[*Ts]]', 'C[*tuple_type[*Ts]]'), # Should be C[*Ts] - ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[(*tuple_type[*Ts],),]'), # Should be tuple[*Ts] + ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[*tuple_type[*Ts]]'), # Should be tuple[*Ts] ('Tuple[*Ts]', '[*tuple_type[*Ts]]', 'Tuple[*tuple_type[*Ts]]'), # Should be Tuple[*Ts] ('C[*Ts]', '[*tuple_type[int, str]]', 'C[*tuple_type[int, str]]'), # Should be C[int, str] - ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[(*tuple_type[int, str],),]'), # Should be tuple[int, str] + ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[*tuple_type[int, str]]'), # Should be tuple[int, str] ('Tuple[*Ts]', '[*tuple_type[int, str]]', 'Tuple[*tuple_type[int, str]]'), # Should be Tuple[int, str] ('C[*Ts]', '[tuple_type[int, ...]]', 'C[tuple_type[int, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[(tuple_type[int, ...],),]'), # Should be tuple[tuple_type[int, ...]] + ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[tuple_type[int, ...]]'), ('Tuple[*Ts]', '[tuple_type[int, ...]]', 'Tuple[tuple_type[int, ...]]'), ('C[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'C[tuple_type[int, ...], tuple_type[str, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'TypeError'), # Should be tuple[tuple_type[int, ...], tuple_type[str, ...]] + ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'tuple[tuple_type[int, ...], tuple_type[str, ...]]'), ('Tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'Tuple[tuple_type[int, ...], tuple_type[str, ...]]'), ('C[*Ts]', '[*tuple_type[int, ...]]', 'C[*tuple_type[int, ...]]'), - ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[(*tuple_type[int, ...],),]'), # Should be tuple[*tuple_type[int, ...]] + ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[*tuple_type[int, ...]]'), ('Tuple[*Ts]', '[*tuple_type[int, ...]]', 'Tuple[*tuple_type[int, ...]]'), # Technically, multiple unpackings are forbidden by PEP 646, but we # choose to be less restrictive at runtime, to allow folks room # to experiment. So all three of these should be valid. ('C[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'C[*tuple_type[int, ...], *tuple_type[str, ...]]'), - # Should be tuple[*tuple_type[int, ...], *tuple_type[str, ...]], to match the other two. - ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), + ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), ('Tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'Tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), ('C[*Ts]', '[*Ts]', 'C[*Ts]'), - ('tuple[*Ts]', '[*Ts]', 'tuple[(*Ts,),]'), # Should be tuple[*Ts] + ('tuple[*Ts]', '[*Ts]', 'tuple[*Ts]'), ('Tuple[*Ts]', '[*Ts]', 'Tuple[*Ts]'), ('C[*Ts]', '[T, *Ts]', 'C[T, *Ts]'), - ('tuple[*Ts]', '[T, *Ts]', 'TypeError'), # Should be tuple[T, *Ts] + ('tuple[*Ts]', '[T, *Ts]', 'tuple[T, *Ts]'), ('Tuple[*Ts]', '[T, *Ts]', 'Tuple[T, *Ts]'), ('C[*Ts]', '[*Ts, T]', 'C[*Ts, T]'), - ('tuple[*Ts]', '[*Ts, T]', 'TypeError'), # Should be tuple[*Ts, T] + ('tuple[*Ts]', '[*Ts, T]', 'tuple[*Ts, T]'), ('Tuple[*Ts]', '[*Ts, T]', 'Tuple[*Ts, T]'), ('C[T, *Ts]', '[int]', 'C[int]'), - ('tuple[T, *Ts]', '[int]', 'TypeError'), # Should be tuple[int] + ('tuple[T, *Ts]', '[int]', 'tuple[int]'), ('Tuple[T, *Ts]', '[int]', 'Tuple[int]'), ('C[T, *Ts]', '[int, str]', 'C[int, str]'), - ('tuple[T, *Ts]', '[int, str]', 'tuple[int, (str,)]'), # Should be tuple[int, str] + ('tuple[T, *Ts]', '[int, str]', 'tuple[int, str]'), ('Tuple[T, *Ts]', '[int, str]', 'Tuple[int, str]'), ('C[T, *Ts]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[T, *Ts]', '[int, str, bool]', 'TypeError'), # Should be tuple[int, str, bool] + ('tuple[T, *Ts]', '[int, str, bool]', 'tuple[int, str, bool]'), ('Tuple[T, *Ts]', '[int, str, bool]', 'Tuple[int, str, bool]'), ('C[T, *Ts]', '[*tuple[int, ...]]', 'C[*tuple[int, ...]]'), # Should be C[int, *tuple[int, ...]] ('C[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto - ('tuple[T, *Ts]', '[*tuple_type[int, ...]]', 'TypeError'), # Should be tuple[int, *tuple[int, ...]] - ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Ditto - ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto + ('tuple[T, *Ts]', '[*tuple[int, ...]]', 'tuple[*tuple[int, ...]]'), # Should be tuple[int, *tuple[int, ...]] + ('tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *Tuple[int, ...]] + ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Should be Tuple[int, *tuple[int, ...]] + ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *Tuple[int, ...]] ('C[*Ts, T]', '[int]', 'C[int]'), - ('tuple[*Ts, T]', '[int]', 'TypeError'), # Should be tuple[int] + ('tuple[*Ts, T]', '[int]', 'tuple[int]'), ('Tuple[*Ts, T]', '[int]', 'Tuple[int]'), ('C[*Ts, T]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts, T]', '[int, str]', 'tuple[(int,), str]'), # Should be tuple[int, str] + ('tuple[*Ts, T]', '[int, str]', 'tuple[int, str]'), ('Tuple[*Ts, T]', '[int, str]', 'Tuple[int, str]'), ('C[*Ts, T]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[*Ts, T]', '[int, str, bool]', 'TypeError'), # Should be tuple[int, str, bool] + ('tuple[*Ts, T]', '[int, str, bool]', 'tuple[int, str, bool]'), ('Tuple[*Ts, T]', '[int, str, bool]', 'Tuple[int, str, bool]'), ('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), @@ -945,7 +945,7 @@ class TypeVarTupleTests(BaseTestCase): T2 = TypeVar('T2') class G(Generic[Unpack[Ts]]): pass - for A in G, Tuple: + for A in G, Tuple, tuple: B = A[Unpack[Ts]] self.assertEqual(B[()], A[()]) self.assertEqual(B[float], A[float]) @@ -984,6 +984,21 @@ class TypeVarTupleTests(BaseTestCase): T2 = TypeVar('T2') class G(Generic[Unpack[Ts]]): pass + for A in G, Tuple, tuple: + B = A[Ts] + with self.assertRaises(TypeError): + B[int, str] + + C = A[T, T2] + with self.assertRaises(TypeError): + C[Unpack[Ts]] + + def test_repr_is_correct(self): + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T2 = TypeVar('T2') + class G(Generic[Unpack[Ts]]): pass + for A in G, Tuple: B = A[T, Unpack[Ts], str, T2] with self.assertRaises(TypeError): diff --git a/Lib/typing.py b/Lib/typing.py index b250f29..35deb32 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1019,7 +1019,7 @@ class TypeVarTuple(_Final, _Immutable, _PickleUsingNameMixin, _root=True): return self.__name__ def __typing_subst__(self, arg): - raise AssertionError + raise TypeError("Substitution of bare TypeVarTuple is not supported") class ParamSpecArgs(_Final, _Immutable, _root=True): @@ -1686,11 +1686,15 @@ class _UnpackGenericAlias(_GenericAlias, _root=True): return '*' + repr(self.__args__[0]) def __getitem__(self, args): - if (len(self.__parameters__) == 1 and - isinstance(self.__parameters__[0], TypeVarTuple)): + if self.__typing_unpacked__(): return args return super().__getitem__(args) + def __typing_unpacked__(self): + # If x is Unpack[tuple[...]], __parameters__ will be empty. + return bool(self.__parameters__ and + isinstance(self.__parameters__[0], TypeVarTuple)) + class Generic: """Abstract base class for generic types. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 7059c40..7b68991 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -190,6 +190,23 @@ tuple_add(PyObject *self, Py_ssize_t len, PyObject *item) return 0; } +static Py_ssize_t +tuple_extend(PyObject **dst, Py_ssize_t dstindex, + PyObject **src, Py_ssize_t count) +{ + assert(count >= 0); + if (_PyTuple_Resize(dst, PyTuple_GET_SIZE(*dst) + count - 1) != 0) { + return -1; + } + assert(dstindex + count <= PyTuple_GET_SIZE(*dst)); + for (Py_ssize_t i = 0; i < count; ++i) { + PyObject *item = src[i]; + Py_INCREF(item); + PyTuple_SET_ITEM(*dst, dstindex + i, item); + } + return dstindex + count; +} + PyObject * _Py_make_parameters(PyObject *args) { @@ -251,7 +268,8 @@ _Py_make_parameters(PyObject *args) If obj doesn't have a __parameters__ attribute or that's not a non-empty tuple, return a new reference to obj. */ static PyObject * -subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) +subs_tvars(PyObject *obj, PyObject *params, + PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam) { PyObject *subparams; if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) { @@ -265,14 +283,27 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) Py_DECREF(subparams); return NULL; } - for (Py_ssize_t i = 0; i < nsubargs; ++i) { + for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) { PyObject *arg = PyTuple_GET_ITEM(subparams, i); Py_ssize_t iparam = tuple_index(params, nparams, arg); - if (iparam >= 0) { - arg = argitems[iparam]; + if (iparam == varparam) { + j = tuple_extend(&subargs, j, + argitems + iparam, nargs - nparams + 1); + if (j < 0) { + return NULL; + } + } + else { + if (iparam >= 0) { + if (iparam > varparam) { + iparam += nargs - nsubargs; + } + arg = argitems[iparam]; + } + Py_INCREF(arg); + PyTuple_SET_ITEM(subargs, j, arg); + j++; } - Py_INCREF(arg); - PyTuple_SET_ITEM(subargs, i, arg); } obj = PyObject_GetItem(obj, subargs); @@ -286,6 +317,23 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) return obj; } +static int +_is_unpacked_typevartuple(PyObject *arg) +{ + PyObject *meth; + int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked__), &meth); + if (res > 0) { + PyObject *tmp = PyObject_CallNoArgs(meth); + Py_DECREF(meth); + if (tmp == NULL) { + return -1; + } + res = PyObject_IsTrue(tmp); + Py_DECREF(tmp); + } + return res; +} + PyObject * _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item) { @@ -298,11 +346,27 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje int is_tuple = PyTuple_Check(item); Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; - if (nitems != nparams) { - return PyErr_Format(PyExc_TypeError, - "Too %s arguments for %R", - nitems > nparams ? "many" : "few", - self); + Py_ssize_t varparam = 0; + for (; varparam < nparams; varparam++) { + PyObject *param = PyTuple_GET_ITEM(parameters, varparam); + if (Py_TYPE(param)->tp_iter) { // TypeVarTuple + break; + } + } + if (varparam < nparams) { + if (nitems < nparams - 1) { + return PyErr_Format(PyExc_TypeError, + "Too few arguments for %R", + self); + } + } + else { + if (nitems != nparams) { + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %R", + nitems > nparams ? "many" : "few", + self); + } } /* Replace all type variables (specified by parameters) with corresponding values specified by argitems. @@ -315,8 +379,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (newargs == NULL) { return NULL; } - for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); + int unpack = _is_unpacked_typevartuple(arg); + if (unpack < 0) { + Py_DECREF(newargs); + return NULL; + } PyObject *subst; if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) { Py_DECREF(newargs); @@ -325,17 +394,38 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); assert(iparam >= 0); + if (iparam == varparam) { + Py_DECREF(subst); + Py_DECREF(newargs); + PyErr_SetString(PyExc_TypeError, + "Substitution of bare TypeVarTuple is not supported"); + return NULL; + } + if (iparam > varparam) { + iparam += nitems - nparams; + } arg = PyObject_CallOneArg(subst, argitems[iparam]); Py_DECREF(subst); } else { - arg = subs_tvars(arg, parameters, argitems); + arg = subs_tvars(arg, parameters, argitems, nitems, varparam); } if (arg == NULL) { Py_DECREF(newargs); return NULL; } - PyTuple_SET_ITEM(newargs, iarg, arg); + if (unpack) { + jarg = tuple_extend(&newargs, jarg, + &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); + Py_DECREF(arg); + if (jarg < 0) { + return NULL; + } + } + else { + PyTuple_SET_ITEM(newargs, jarg, arg); + jarg++; + } } return newargs; -- cgit v0.12