From 9af34c935185eca497617a216d141c72ffaeae9c Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Fri, 16 Jul 2021 18:43:02 +0300 Subject: bpo-20201: variadic arguments support for AC (GH-18609) Implement support for `*args` in AC, and port `print()` to use it. --- Include/modsupport.h | 11 +- Lib/test/clinic.test | 220 +++++++++++++++++++++ Lib/test/test_call.py | 2 +- .../2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst | 1 + Python/bltinmodule.c | 80 ++++---- Python/clinic/bltinmodule.c.h | 76 ++++++- Python/getargs.c | 160 +++++++++++++++ Tools/clinic/clinic.py | 214 ++++++++++++++------ 8 files changed, 664 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst diff --git a/Include/modsupport.h b/Include/modsupport.h index f009d58..7d37b49 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -50,6 +50,7 @@ PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...); PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); +#define ANY_VARARGS(n) (n == PY_SSIZE_T_MAX) #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyArg_UnpackStack( PyObject *const *args, @@ -73,7 +74,7 @@ PyAPI_FUNC(void) _PyArg_BadArgument(const char *, const char *, const char *, Py PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, Py_ssize_t, Py_ssize_t); #define _PyArg_CheckPositional(funcname, nargs, min, max) \ - (((min) <= (nargs) && (nargs) <= (max)) \ + ((!ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) #endif @@ -127,6 +128,14 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( struct _PyArg_Parser *parser, int minpos, int maxpos, int minkw, PyObject **buf); + +PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject *kwnames, + struct _PyArg_Parser *parser, + int minpos, int maxpos, int minkw, + int vararg, PyObject **buf); + #define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ (minpos) <= (nargs) && (nargs) <= (maxpos) && args != NULL) ? (args) : \ diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 8224317..d2934b3 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3304,3 +3304,223 @@ test_preprocessor_guarded_else(PyObject *module, PyObject *Py_UNUSED(ignored)) #define TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF #endif /* !defined(TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF) */ /*[clinic end generated code: output=3804bb18d454038c input=3fc80c9989d2f2e1]*/ + +/*[clinic input] +test_vararg_and_posonly + + + a: object + *args: object + / + +[clinic start generated code]*/ + +PyDoc_STRVAR(test_vararg_and_posonly__doc__, +"test_vararg_and_posonly($module, a, /, *args)\n" +"--\n" +"\n"); + +#define TEST_VARARG_AND_POSONLY_METHODDEF \ + {"test_vararg_and_posonly", (PyCFunction)(void(*)(void))test_vararg_and_posonly, METH_FASTCALL, test_vararg_and_posonly__doc__}, + +static PyObject * +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("test_vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + __clinic_args = PyTuple_New(nargs - 1); + for (Py_ssize_t i = 0; i < nargs - 1; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, args[1 + i]); + } + return_value = test_vararg_and_posonly_impl(module, a, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static PyObject * +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=ada613d2d87c9341 input=08dc2bf7afbf1613]*/ + +/*[clinic input] +test_vararg + + + a: object + *args: object + +[clinic start generated code]*/ + +PyDoc_STRVAR(test_vararg__doc__, +"test_vararg($module, /, a, *args)\n" +"--\n" +"\n"); + +#define TEST_VARARG_METHODDEF \ + {"test_vararg", (PyCFunction)(void(*)(void))test_vararg, METH_FASTCALL|METH_KEYWORDS, test_vararg__doc__}, + +static PyObject * +test_vararg_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "test_vararg", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + return_value = test_vararg_impl(module, a, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static PyObject * +test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=f721025731c3bfe8 input=81d33815ad1bae6e]*/ + +/*[clinic input] +test_vararg_with_default + + + a: object + *args: object + b: bool = False + +[clinic start generated code]*/ + +PyDoc_STRVAR(test_vararg_with_default__doc__, +"test_vararg_with_default($module, /, a, *args, b=False)\n" +"--\n" +"\n"); + +#define TEST_VARARG_WITH_DEFAULT_METHODDEF \ + {"test_vararg_with_default", (PyCFunction)(void(*)(void))test_vararg_with_default, METH_FASTCALL|METH_KEYWORDS, test_vararg_with_default__doc__}, + +static PyObject * +test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b); + +static PyObject * +test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "test_vararg_with_default", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *__clinic_args = NULL; + int b = 0; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = PyObject_IsTrue(args[2]); + if (b < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = test_vararg_with_default_impl(module, a, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static PyObject * +test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b) +/*[clinic end generated code: output=63b34d3241c52fda input=6e110b54acd9b22d]*/ + +/*[clinic input] +test_vararg_with_only_defaults + + + *args: object + b: bool = False + c: object = ' ' + +[clinic start generated code]*/ + +PyDoc_STRVAR(test_vararg_with_only_defaults__doc__, +"test_vararg_with_only_defaults($module, /, *args, b=False, c=\' \')\n" +"--\n" +"\n"); + +#define TEST_VARARG_WITH_ONLY_DEFAULTS_METHODDEF \ + {"test_vararg_with_only_defaults", (PyCFunction)(void(*)(void))test_vararg_with_only_defaults, METH_FASTCALL|METH_KEYWORDS, test_vararg_with_only_defaults__doc__}, + +static PyObject * +test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, + PyObject *c); + +static PyObject * +test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "test_vararg_with_only_defaults", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *__clinic_args = NULL; + int b = 0; + PyObject *c = " "; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + b = PyObject_IsTrue(args[1]); + if (b < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[2]; +skip_optional_kwonly: + return_value = test_vararg_with_only_defaults_impl(module, __clinic_args, b, c); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static PyObject * +test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, + PyObject *c) +/*[clinic end generated code: output=dc29ce6ebc2ec10c input=fa56a709a035666e]*/ diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index c929ca8..1d27452 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -129,7 +129,7 @@ class CFunctionCallsErrorMessages(unittest.TestCase): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"^print\(\) takes at most 4 keyword arguments \(5 given\)$" + msg = r"'foo' is an invalid keyword argument for print\(\)$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) diff --git a/Misc/NEWS.d/next/Tools-Demos/2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst b/Misc/NEWS.d/next/Tools-Demos/2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst new file mode 100644 index 0000000..c64c548 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2020-02-25-18-22-09.bpo-20291.AyrDiZ.rst @@ -0,0 +1 @@ +Added support for variadic positional parameters in Argument Clinic. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 66c5fba..bfe21ad 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1952,24 +1952,31 @@ builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp, return PyNumber_Power(base, exp, mod); } +/*[clinic input] +print as builtin_print + + *args: object + sep: object(c_default="Py_None") = ' ' + string inserted between values, default a space. + end: object(c_default="Py_None") = '\n' + string appended after the last value, default a newline. + file: object = None + a file-like object (stream); defaults to the current sys.stdout. + flush: bool = False + whether to forcibly flush the stream. + +Prints the values to a stream, or to sys.stdout by default. + +[clinic start generated code]*/ -/* AC: cannot convert yet, waiting for *args support */ static PyObject * -builtin_print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +builtin_print_impl(PyObject *module, PyObject *args, PyObject *sep, + PyObject *end, PyObject *file, int flush) +/*[clinic end generated code: output=3cfc0940f5bc237b input=c143c575d24fe665]*/ { - static const char * const _keywords[] = {"sep", "end", "file", "flush", 0}; - static struct _PyArg_Parser _parser = {"|OOOp:print", _keywords, 0}; - PyObject *sep = NULL, *end = NULL, *file = NULL; - int flush = 0; int i, err; - if (kwnames != NULL && - !_PyArg_ParseStackAndKeywords(args + nargs, 0, kwnames, &_parser, - &sep, &end, &file, &flush)) { - return NULL; - } - - if (file == NULL || file == Py_None) { + if (file == Py_None) { file = _PySys_GetObjectId(&PyId_stdout); if (file == NULL) { PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); @@ -1977,8 +1984,9 @@ builtin_print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject } /* sys.stdout may be None when FILE* stdout isn't connected */ - if (file == Py_None) + if (file == Py_None) { Py_RETURN_NONE; + } } if (sep == Py_None) { @@ -2000,48 +2008,45 @@ builtin_print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject return NULL; } - for (i = 0; i < nargs; i++) { + for (i = 0; i < PyTuple_GET_SIZE(args); i++) { if (i > 0) { - if (sep == NULL) + if (sep == NULL) { err = PyFile_WriteString(" ", file); - else - err = PyFile_WriteObject(sep, file, - Py_PRINT_RAW); - if (err) + } + else { + err = PyFile_WriteObject(sep, file, Py_PRINT_RAW); + } + if (err) { return NULL; + } } - err = PyFile_WriteObject(args[i], file, Py_PRINT_RAW); - if (err) + err = PyFile_WriteObject(PyTuple_GET_ITEM(args, i), file, Py_PRINT_RAW); + if (err) { return NULL; + } } - if (end == NULL) + if (end == NULL) { err = PyFile_WriteString("\n", file); - else + } + else { err = PyFile_WriteObject(end, file, Py_PRINT_RAW); - if (err) + } + if (err) { return NULL; + } if (flush) { PyObject *tmp = _PyObject_CallMethodIdNoArgs(file, &PyId_flush); - if (tmp == NULL) + if (tmp == NULL) { return NULL; + } Py_DECREF(tmp); } Py_RETURN_NONE; } -PyDoc_STRVAR(print_doc, -"print(value, ..., sep=' ', end='\\n', file=sys.stdout, flush=False)\n\ -\n\ -Prints the values to a stream, or to sys.stdout by default.\n\ -Optional keyword arguments:\n\ -file: a file-like object (stream); defaults to the current sys.stdout.\n\ -sep: string inserted between values, default a space.\n\ -end: string appended after the last value, default a newline.\n\ -flush: whether to forcibly flush the stream."); - /*[clinic input] input as builtin_input @@ -2644,7 +2649,6 @@ builtin_issubclass_impl(PyObject *module, PyObject *cls, return PyBool_FromLong(retval); } - typedef struct { PyObject_HEAD Py_ssize_t tuplesize; @@ -2955,7 +2959,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF - {"print", (PyCFunction)(void(*)(void))builtin_print, METH_FASTCALL | METH_KEYWORDS, print_doc}, + BUILTIN_PRINT_METHODDEF BUILTIN_REPR_METHODDEF BUILTIN_ROUND_METHODDEF BUILTIN_SETATTR_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 4ea5876..1fade99 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -674,6 +674,80 @@ exit: return return_value; } +PyDoc_STRVAR(builtin_print__doc__, +"print($module, /, *args, sep=\' \', end=\'\\n\', file=None, flush=False)\n" +"--\n" +"\n" +"Prints the values to a stream, or to sys.stdout by default.\n" +"\n" +" sep\n" +" string inserted between values, default a space.\n" +" end\n" +" string appended after the last value, default a newline.\n" +" file\n" +" a file-like object (stream); defaults to the current sys.stdout.\n" +" flush\n" +" whether to forcibly flush the stream."); + +#define BUILTIN_PRINT_METHODDEF \ + {"print", (PyCFunction)(void(*)(void))builtin_print, METH_FASTCALL|METH_KEYWORDS, builtin_print__doc__}, + +static PyObject * +builtin_print_impl(PyObject *module, PyObject *args, PyObject *sep, + PyObject *end, PyObject *file, int flush); + +static PyObject * +builtin_print(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"sep", "end", "file", "flush", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "print", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *__clinic_args = NULL; + PyObject *sep = Py_None; + PyObject *end = Py_None; + PyObject *file = Py_None; + int flush = 0; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + sep = args[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[2]) { + end = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[3]) { + file = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + flush = PyObject_IsTrue(args[4]); + if (flush < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = builtin_print_impl(module, __clinic_args, sep, end, file, flush); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(builtin_input__doc__, "input($module, prompt=None, /)\n" "--\n" @@ -877,4 +951,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e1d8057298b5de61 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=77ace832b3fb38e0 input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index d5e0835..330f2b4 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2465,6 +2465,166 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, return buf; } +PyObject * const * +_PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject *kwnames, + struct _PyArg_Parser *parser, + int minpos, int maxpos, int minkw, + int vararg, PyObject **buf) +{ + PyObject *kwtuple; + PyObject *keyword; + Py_ssize_t varargssize = 0; + int i, posonly, minposonly, maxargs; + int reqlimit = minkw ? maxpos + minkw : minpos; + Py_ssize_t nkwargs; + PyObject *current_arg; + PyObject * const *kwstack = NULL; + + assert(kwargs == NULL || PyDict_Check(kwargs)); + assert(kwargs == NULL || kwnames == NULL); + + if (parser == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + if (kwnames != NULL && !PyTuple_Check(kwnames)) { + PyErr_BadInternalCall(); + return NULL; + } + + if (args == NULL && nargs == 0) { + args = buf; + } + + if (!parser_init(parser)) { + return NULL; + } + + kwtuple = parser->kwtuple; + posonly = parser->pos; + minposonly = Py_MIN(posonly, minpos); + maxargs = posonly + (int)PyTuple_GET_SIZE(kwtuple); + if (kwargs != NULL) { + nkwargs = PyDict_GET_SIZE(kwargs); + } + else if (kwnames != NULL) { + nkwargs = PyTuple_GET_SIZE(kwnames); + kwstack = args + nargs; + } + else { + nkwargs = 0; + } + if (nargs < minposonly) { + PyErr_Format(PyExc_TypeError, + "%.200s%s takes %s %d positional argument%s" + " (%zd given)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + minposonly < maxpos ? "at least" : "exactly", + minposonly, + minposonly == 1 ? "" : "s", + nargs); + return NULL; + } + + /* create varargs tuple */ + varargssize = nargs - maxpos; + if (varargssize < 0) { + varargssize = 0; + } + buf[vararg] = PyTuple_New(varargssize); + if (!buf[vararg]) { + return NULL; + } + + /* copy tuple args */ + for (i = 0; i < nargs; i++) { + if (i >= vararg) { + Py_INCREF(args[i]); + PyTuple_SET_ITEM(buf[vararg], i - vararg, args[i]); + continue; + } + else { + buf[i] = args[i]; + } + } + + /* copy keyword args using kwtuple to drive process */ + for (i = Py_MAX((int)nargs, posonly) - varargssize; i < maxargs; i++) { + if (nkwargs) { + keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); + if (kwargs != NULL) { + current_arg = PyDict_GetItemWithError(kwargs, keyword); + if (!current_arg && PyErr_Occurred()) { + goto exit; + } + } + else { + current_arg = find_keyword(kwnames, kwstack, keyword); + } + } + else { + current_arg = NULL; + } + + buf[i + vararg + 1] = current_arg; + + if (current_arg) { + --nkwargs; + } + else if (i < minpos || (maxpos <= i && i < reqlimit)) { + /* Less arguments than required */ + keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); + PyErr_Format(PyExc_TypeError, "%.200s%s missing required " + "argument '%U' (pos %d)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + keyword, i+1); + goto exit; + } + } + + if (nkwargs > 0) { + Py_ssize_t j; + /* make sure there are no extraneous keyword arguments */ + j = 0; + while (1) { + int match; + if (kwargs != NULL) { + if (!PyDict_Next(kwargs, &j, &keyword, NULL)) + break; + } + else { + if (j >= PyTuple_GET_SIZE(kwnames)) + break; + keyword = PyTuple_GET_ITEM(kwnames, j); + j++; + } + + match = PySequence_Contains(kwtuple, keyword); + if (match <= 0) { + if (!match) { + PyErr_Format(PyExc_TypeError, + "'%S' is an invalid keyword " + "argument for %.200s%s", + keyword, + (parser->fname == NULL) ? "this function" : parser->fname, + (parser->fname == NULL) ? "" : "()"); + } + goto exit; + } + } + } + + return buf; + +exit: + Py_XDECREF(buf[vararg]); + return NULL; +} + static const char * skipitem(const char **p_format, va_list *p_va, int flags) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index b287916..b52ac3b 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -44,6 +44,9 @@ NoneType = type(None) version = '1' NoneType = type(None) +NO_VARARG = "PY_SSIZE_T_MAX" +CLINIC_PREFIX = "__clinic_" +CLINIC_PREFIXED_ARGS = {"args"} class Unspecified: def __repr__(self): @@ -644,14 +647,21 @@ class CLanguage(Language): new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) - pos_only = min_pos = max_pos = min_kw_only = 0 + vararg = NO_VARARG + pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 for i, p in enumerate(parameters, 1): - if p.is_keyword_only(): + if p.is_keyword_only() or vararg != NO_VARARG: assert not p.is_positional_only() if not p.is_optional(): min_kw_only = i - max_pos + elif p.is_vararg(): + if vararg != NO_VARARG: + fail("Too many var args") + pseudo_args += 1 + vararg = i - 1 else: - max_pos = i + if vararg == NO_VARARG: + max_pos = i if p.is_positional_only(): pos_only = i if not p.is_optional(): @@ -834,7 +844,7 @@ class CLanguage(Language): parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') - elif not requires_defining_class and pos_only == len(parameters): + elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: if not new_or_init: # positional-only, but no option groups # we only need one call to _PyArg_ParseStack @@ -852,15 +862,44 @@ class CLanguage(Language): nargs = 'PyTuple_GET_SIZE(args)' argname_fmt = 'PyTuple_GET_ITEM(args, %d)' + + left_args = "{} - {}".format(nargs, max_pos) + max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos parser_code = [normalize_snippet(""" - if (!_PyArg_CheckPositional("{name}", %s, %d, %d)) {{ + if (!_PyArg_CheckPositional("{name}", %s, %d, %s)) {{ goto exit; }} - """ % (nargs, min_pos, max_pos), indent=4)] + """ % (nargs, min_pos, max_args), indent=4)] + has_optional = False for i, p in enumerate(parameters): displayname = p.get_displayname(i+1) - parsearg = p.converter.parse_arg(argname_fmt % i, displayname) + argname = argname_fmt % i + + if p.is_vararg(): + if not new_or_init: + parser_code.append(normalize_snippet(""" + %s = PyTuple_New(%s); + for (Py_ssize_t i = 0; i < %s; ++i) {{ + PyTuple_SET_ITEM(%s, i, args[%d + i]); + }} + """ % ( + p.converter.parser_name, + left_args, + left_args, + p.converter.parser_name, + max_pos + ), indent=4)) + else: + parser_code.append(normalize_snippet(""" + %s = PyTuple_GetSlice(%d, -1); + """ % ( + p.converter.parser_name, + max_pos + ), indent=4)) + continue + + parsearg = p.converter.parse_arg(argname, displayname) if parsearg is None: #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) parser_code = None @@ -896,6 +935,19 @@ class CLanguage(Language): else: has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters)) + if vararg == NO_VARARG: + args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only + ) + else: + args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only, + vararg + ) if not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" parser_prototype = parser_prototype_fastcall_keywords @@ -906,13 +958,14 @@ class CLanguage(Language): PyObject *argsbuf[%s]; """ % len(converters)) if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (min_pos + min_kw_only) + pre_buffer = "0" if vararg != NO_VARARG else "nargs" + declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (pre_buffer, min_pos + min_kw_only) parser_code = [normalize_snippet(""" - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, %d, %d, %d, argsbuf); + args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); if (!args) {{ goto exit; }} - """ % (min_pos, max_pos, min_kw_only), indent=4)] + """ % args_declaration, indent=4)] else: # positional-or-keyword arguments flags = "METH_VARARGS|METH_KEYWORDS" @@ -928,11 +981,12 @@ class CLanguage(Language): if has_optional_kw: declarations += "\nPy_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (min_pos + min_kw_only) parser_code = [normalize_snippet(""" - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %d, %d, %d, argsbuf); + fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); if (!fastargs) {{ goto exit; }} - """ % (min_pos, max_pos, min_kw_only), indent=4)] + """ % args_declaration, indent=4)] + if requires_defining_class: flags = 'METH_METHOD|' + flags parser_prototype = parser_prototype_def_class @@ -969,6 +1023,8 @@ class CLanguage(Language): else: label = 'skip_optional_kwonly' first_opt = max_pos + min_kw_only + if vararg != NO_VARARG: + first_opt += 1 if i == first_opt: add_label = label parser_code.append(normalize_snippet(""" @@ -1249,6 +1305,9 @@ class CLanguage(Language): if (i != -1) and (p.default is not unspecified): first_optional = min(first_optional, i) + if p.is_vararg(): + data.cleanup.append("Py_XDECREF({});".format(c.parser_name)) + # insert group variable group = p.group if last_group != group: @@ -2307,8 +2366,11 @@ class Parameter: def is_positional_only(self): return self.kind == inspect.Parameter.POSITIONAL_ONLY + def is_vararg(self): + return self.kind == inspect.Parameter.VAR_POSITIONAL + def is_optional(self): - return (self.default is not unspecified) + return not self.is_vararg() and (self.default is not unspecified) def copy(self, **overrides): kwargs = { @@ -2523,7 +2585,7 @@ class CConverter(metaclass=CConverterAutoRegister): def _render_self(self, parameter, data): self.parameter = parameter - name = self.name + name = self.parser_name # impl_arguments s = ("&" if self.impl_by_reference else "") + name @@ -2541,7 +2603,7 @@ class CConverter(metaclass=CConverterAutoRegister): name = self.name # declarations - d = self.declaration() + d = self.declaration(in_parser=True) data.declarations.append(d) # initializers @@ -2555,7 +2617,9 @@ class CConverter(metaclass=CConverterAutoRegister): data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) # keywords - if parameter.is_positional_only(): + if parameter.is_vararg(): + pass + elif parameter.is_positional_only(): data.keywords.append('') else: data.keywords.append(parameter.name) @@ -2587,7 +2651,7 @@ class CConverter(metaclass=CConverterAutoRegister): """Computes the name of the associated "length" variable.""" if not self.length: return None - return self.name + "_length" + return self.parser_name + "_length" # Why is this one broken out separately? # For "positional-only" function parsing, @@ -2613,7 +2677,7 @@ class CConverter(metaclass=CConverterAutoRegister): # All the functions after here are intended as extension points. # - def simple_declaration(self, by_reference=False): + def simple_declaration(self, by_reference=False, *, in_parser=False): """ Computes the basic declaration of the variable. Used in computing the prototype declaration and the @@ -2624,14 +2688,18 @@ class CConverter(metaclass=CConverterAutoRegister): prototype.append(" ") if by_reference: prototype.append('*') - prototype.append(self.name) + if in_parser: + name = self.parser_name + else: + name = self.name + prototype.append(name) return "".join(prototype) - def declaration(self): + def declaration(self, *, in_parser=False): """ The C statement to declare this variable. """ - declaration = [self.simple_declaration()] + declaration = [self.simple_declaration(in_parser=True)] default = self.c_default if not default and self.parameter.group: default = self.c_ignored_default @@ -2683,7 +2751,7 @@ class CConverter(metaclass=CConverterAutoRegister): if (!{converter}({argname}, &{paramname})) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, converter=self.converter) if self.format_unit == 'O!': cast = '(%s)' % self.type if self.type != 'PyObject *' else '' @@ -2695,7 +2763,7 @@ class CConverter(metaclass=CConverterAutoRegister): goto exit; }}}} {paramname} = {cast}{argname}; - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname, typecheck=typecheck, typename=typename, cast=cast) return """ @@ -2704,19 +2772,25 @@ class CConverter(metaclass=CConverterAutoRegister): goto exit; }}}} {paramname} = {cast}{argname}; - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, subclass_of=self.subclass_of, cast=cast, displayname=displayname) if self.format_unit == 'O': cast = '(%s)' % self.type if self.type != 'PyObject *' else '' return """ {paramname} = {cast}{argname}; - """.format(argname=argname, paramname=self.name, cast=cast) + """.format(argname=argname, paramname=self.parser_name, cast=cast) return None def set_template_dict(self, template_dict): pass + @property + def parser_name(self): + if self.name in CLINIC_PREFIXED_ARGS: # bpo-39741 + return CLINIC_PREFIX + self.name + else: + return self.name type_checks = { '&PyLong_Type': ('PyLong_Check', 'int'), @@ -2755,14 +2829,14 @@ class bool_converter(CConverter): if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) elif self.format_unit == 'p': return """ {paramname} = PyObject_IsTrue({argname}); if ({paramname} < 0) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) class defining_class_converter(CConverter): @@ -2812,7 +2886,7 @@ class char_converter(CConverter): _PyArg_BadArgument("{{name}}", {displayname}, "a byte string of length 1", {argname}); goto exit; }}}} - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) return super().parse_arg(argname, displayname) @@ -2850,7 +2924,7 @@ class unsigned_char_converter(CConverter): {paramname} = (unsigned char) ival; }}}} }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) elif self.format_unit == 'B': return """ {{{{ @@ -2862,7 +2936,7 @@ class unsigned_char_converter(CConverter): {paramname} = (unsigned char) ival; }}}} }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) class byte_converter(unsigned_char_converter): pass @@ -2895,7 +2969,7 @@ class short_converter(CConverter): {paramname} = (short) ival; }}}} }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) class unsigned_short_converter(CConverter): @@ -2916,7 +2990,7 @@ class unsigned_short_converter(CConverter): if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) @add_legacy_c_converter('C', accept={str}) @@ -2941,7 +3015,7 @@ class int_converter(CConverter): if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) elif self.format_unit == 'C': return """ if (!PyUnicode_Check({argname})) {{{{ @@ -2956,7 +3030,7 @@ class int_converter(CConverter): goto exit; }}}} {paramname} = PyUnicode_READ_CHAR({argname}, 0); - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) return super().parse_arg(argname, displayname) @@ -2978,7 +3052,7 @@ class unsigned_int_converter(CConverter): if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) class long_converter(CConverter): @@ -2994,7 +3068,7 @@ class long_converter(CConverter): if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) class unsigned_long_converter(CConverter): @@ -3016,7 +3090,7 @@ class unsigned_long_converter(CConverter): goto exit; }}}} {paramname} = PyLong_AsUnsignedLongMask({argname}); - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) return super().parse_arg(argname, displayname) @@ -3033,7 +3107,7 @@ class long_long_converter(CConverter): if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) class unsigned_long_long_converter(CConverter): @@ -3055,7 +3129,7 @@ class unsigned_long_long_converter(CConverter): goto exit; }}}} {paramname} = PyLong_AsUnsignedLongLongMask({argname}); - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) return super().parse_arg(argname, displayname) @@ -3087,7 +3161,7 @@ class Py_ssize_t_converter(CConverter): }}}} {paramname} = ival; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) @@ -3114,7 +3188,7 @@ class size_t_converter(CConverter): if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) @@ -3150,7 +3224,7 @@ class float_converter(CConverter): goto exit; }}}} }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) class double_converter(CConverter): @@ -3172,7 +3246,7 @@ class double_converter(CConverter): goto exit; }}}} }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) @@ -3189,7 +3263,7 @@ class Py_complex_converter(CConverter): if (PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """.format(argname=argname, paramname=self.parser_name) return super().parse_arg(argname, displayname) @@ -3274,7 +3348,7 @@ class str_converter(CConverter): PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) if self.format_unit == 'z': return """ @@ -3296,7 +3370,7 @@ class str_converter(CConverter): _PyArg_BadArgument("{{name}}", {displayname}, "str or None", {argname}); goto exit; }}}} - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) return super().parse_arg(argname, displayname) @@ -3361,7 +3435,7 @@ class PyBytesObject_converter(CConverter): goto exit; }}}} {paramname} = ({type}){argname}; - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, type=self.type, displayname=displayname) return super().parse_arg(argname, displayname) @@ -3378,7 +3452,7 @@ class PyByteArrayObject_converter(CConverter): goto exit; }}}} {paramname} = ({type}){argname}; - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, type=self.type, displayname=displayname) return super().parse_arg(argname, displayname) @@ -3398,7 +3472,7 @@ class unicode_converter(CConverter): goto exit; }}}} {paramname} = {argname}; - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) return super().parse_arg(argname, displayname) @@ -3514,7 +3588,7 @@ class Py_buffer_converter(CConverter): _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); goto exit; }}}} - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) elif self.format_unit == 's*': return """ @@ -3535,7 +3609,7 @@ class Py_buffer_converter(CConverter): goto exit; }}}} }}}} - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) elif self.format_unit == 'w*': return """ @@ -3548,7 +3622,7 @@ class Py_buffer_converter(CConverter): _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); goto exit; }}}} - """.format(argname=argname, paramname=self.name, + """.format(argname=argname, paramname=self.parser_name, displayname=displayname) return super().parse_arg(argname, displayname) @@ -4042,7 +4116,7 @@ class DSLParser: def directive_dump(self, name): self.block.output.append(self.clinic.get_destination(name).dump()) - def directive_print(self, *args): + def directive_printout(self, *args): self.block.output.append(' '.join(args)) self.block.output.append('\n') @@ -4436,10 +4510,15 @@ class DSLParser: fail("Function " + self.function.name + " has an invalid parameter declaration (comma?):\n\t" + line) if function_args.defaults or function_args.kw_defaults: fail("Function " + self.function.name + " has an invalid parameter declaration (default value?):\n\t" + line) - if function_args.vararg or function_args.kwarg: - fail("Function " + self.function.name + " has an invalid parameter declaration (*args? **kwargs?):\n\t" + line) + if function_args.kwarg: + fail("Function " + self.function.name + " has an invalid parameter declaration (**kwargs?):\n\t" + line) - parameter = function_args.args[0] + if function_args.vararg: + is_vararg = True + parameter = function_args.vararg + else: + is_vararg = False + parameter = function_args.args[0] parameter_name = parameter.arg name, legacy, kwargs = self.parse_converter(parameter.annotation) @@ -4447,10 +4526,17 @@ class DSLParser: if not default: if self.parameter_state == self.ps_optional: fail("Can't have a parameter without a default (" + repr(parameter_name) + ")\nafter a parameter with a default!") - value = unspecified + if is_vararg: + value = NULL + kwargs.setdefault('c_default', "NULL") + else: + value = unspecified if 'py_default' in kwargs: fail("You can't specify py_default without specifying a default value!") else: + if is_vararg: + fail("Vararg can't take a default value!") + if self.parameter_state == self.ps_required: self.parameter_state = self.ps_optional default = default.strip() @@ -4570,7 +4656,12 @@ class DSLParser: # but the parameter object gets the python name converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) - kind = inspect.Parameter.KEYWORD_ONLY if self.keyword_only else inspect.Parameter.POSITIONAL_OR_KEYWORD + if is_vararg: + kind = inspect.Parameter.VAR_POSITIONAL + elif self.keyword_only: + kind = inspect.Parameter.KEYWORD_ONLY + else: + kind = inspect.Parameter.POSITIONAL_OR_KEYWORD if isinstance(converter, self_converter): if len(self.function.parameters) == 1: @@ -4664,6 +4755,8 @@ class DSLParser: fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") # fixup preceding parameters for p in self.function.parameters.values(): + if p.is_vararg(): + continue if (p.kind != inspect.Parameter.POSITIONAL_OR_KEYWORD and not isinstance(p.converter, self_converter)): fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") p.kind = inspect.Parameter.POSITIONAL_ONLY @@ -4858,10 +4951,13 @@ class DSLParser: # calling the class to construct a new instance. p_add('$') + if p.is_vararg(): + p_add("*") + name = p.converter.signature_name or p.name p_add(name) - if p.converter.is_optional(): + if not p.is_vararg() and p.converter.is_optional(): p_add('=') value = p.converter.py_default if not value: -- cgit v0.12