diff options
| -rw-r--r-- | Doc/c-api/arg.rst | 2 | ||||
| -rw-r--r-- | Doc/howto/clinic.rst | 269 | ||||
| -rw-r--r-- | Lib/inspect.py | 95 | ||||
| -rw-r--r-- | Lib/test/test_inspect.py | 5 | ||||
| -rw-r--r-- | Misc/NEWS | 3 | ||||
| -rw-r--r-- | Modules/_cursesmodule.c | 4 | ||||
| -rw-r--r-- | Modules/_dbmmodule.c | 6 | ||||
| -rw-r--r-- | Modules/_opcode.c | 4 | ||||
| -rw-r--r-- | Modules/_testcapimodule.c | 4 | ||||
| -rw-r--r-- | Modules/posixmodule.c | 4 | ||||
| -rw-r--r-- | Modules/zlibmodule.c | 4 | ||||
| -rwxr-xr-x | Tools/clinic/clinic.py | 248 | ||||
| -rw-r--r-- | Tools/clinic/clinic_test.py | 23 |
13 files changed, 478 insertions, 193 deletions
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index b4efbf0..ee38e18 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -294,6 +294,8 @@ Other objects the object pointer is stored. If the Python object does not have the required type, :exc:`TypeError` is raised. +.. _o_ampersand: + ``O&`` (object) [*converter*, *anything*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 0df8d44..96f84b0 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -127,6 +127,12 @@ convert a function to work with it. Let's dive in! margin, with no line wider than 80 characters. (Argument Clinic will preserve indents inside the docstring.) + If the old docstring had a first line that looked like a function + signature, throw that line away. (The docstring doesn't need it + anymore--when you use ``help()`` on your builtin in the future, + the first line will be built automatically based on the function's + signature.) + Sample:: /*[clinic input] @@ -196,6 +202,10 @@ convert a function to work with it. Let's dive in! name_of_parameter: converter = default_value + Argument Clinic's support for "default values" is quite sophisticated; + please see :ref:`the section below on default values <default_values>` + for more information. + Add a blank line below the parameters. What's a "converter"? It establishes both the type @@ -513,16 +523,6 @@ The base function would now be named ``pickler_dumper()``, and the impl function would now be named ``pickler_dumper_impl()``. -The NULL default value ----------------------- - -For string and object parameters, you can set them to ``None`` to indicate -that there is no default. However, that means the C variable will be -initialized to ``Py_None``. For convenience's sakes, there's a special -value called ``NULL`` for just this case: from Python's perspective it -behaves like a default value of ``None``, but the C variable is initialized -with ``NULL``. - Converting functions using PyArg_UnpackTuple -------------------------------------------- @@ -654,35 +654,70 @@ the same converters. All arguments to Argument Clinic converters are keyword-only. All Argument Clinic converters accept the following arguments: -``py_default`` - The default value for this parameter when defined in Python. - Specifically, the value that will be used in the ``inspect.Signature`` - string. - If a default value is specified for the parameter, defaults to - ``repr(default)``, else defaults to ``None``. - Specified as a string. - -``c_default`` - The default value for this parameter when defined in C. - Specifically, this will be the initializer for the variable declared - in the "parse function". - Specified as a string. - -``required`` - If a parameter takes a default value, Argument Clinic infers that the - parameter is optional. However, you may want a parameter to take a - default value in C, but not behave in Python as if the parameter is - optional. Passing in ``required=True`` to a converter tells Argument - Clinic that this parameter is not optional, even if it has a default - value. - - (The need for ``required`` may be obviated by ``c_default``, which is - newer but arguably a better solution.) - -``annotation`` - The annotation value for this parameter. Not currently supported, - because PEP 8 mandates that the Python library may not use - annotations. + ``c_default`` + The default value for this parameter when defined in C. + Specifically, this will be the initializer for the variable declared + in the "parse function". See :ref:`the section on default values <default_values>` + for how to use this. + Specified as a string. + + ``annotation`` + The annotation value for this parameter. Not currently supported, + because PEP 8 mandates that the Python library may not use + annotations. + +In addition, some converters accept additional arguments. Here is a list +of these arguments, along with their meanings: + + ``bitwise`` + Only supported for unsigned integers. The native integer value of this + Python argument will be written to the parameter without any range checking, + even for negative values. + + ``converter`` + Only supported by the ``object`` converter. Specifies the name of a + :ref:`C "converter function" <o_ampersand>` + to use to convert this object to a native type. + + ``encoding`` + Only supported for strings. Specifies the encoding to use when converting + this string from a Python str (Unicode) value into a C ``char *`` value. + + ``length`` + Only supported for strings. If true, requests that the length of the + string be passed in to the impl function, just after the string parameter, + in a parameter named ``<parameter_name>_length``. + + ``nullable`` + Only supported for strings. If true, this parameter may also be set to + ``None``, in which case the C parameter will be set to ``NULL``. + + ``subclass_of`` + Only supported for the ``object`` converter. Requires that the Python + value be a subclass of a Python type, as expressed in C. + + ``types`` + Only supported for the ``object`` (and ``self``) converter. Specifies + the C type that will be used to declare the variable. Default value is + ``"PyObject *"``. + + ``types`` + A string containing a list of Python types (and possibly pseudo-types); + this restricts the allowable Python argument to values of these types. + (This is not a general-purpose facility; as a rule it only supports + specific lists of types as shown in the legacy converter table.) + + ``zeroes`` + Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are + permitted inside the value. + +Please note, not every possible combination of arguments will work. +Often these arguments are implemented internally by specific ``PyArg_ParseTuple`` +*format units*, with specific behavior. For example, currently you cannot +call ``str`` and pass in ``zeroes=True`` without also specifying an ``encoding``; +although it's perfectly reasonable to think this would work, these semantics don't +map to any existing format unit. So Argument Clinic doesn't support it. (Or, at +least, not yet.) Below is a table showing the mapping of legacy converters into real Argument Clinic converters. On the left is the legacy converter, @@ -720,9 +755,9 @@ on the right is the text you'd replace it with. ``'u'`` ``Py_UNICODE`` ``'U'`` ``unicode`` ``'w*'`` ``Py_buffer(types='bytearray rwbuffer')`` -``'y#'`` ``str(type='bytes', length=True)`` +``'y#'`` ``str(types='bytes', length=True)`` ``'Y'`` ``PyByteArrayObject`` -``'y'`` ``str(type='bytes')`` +``'y'`` ``str(types='bytes')`` ``'y*'`` ``Py_buffer`` ``'Z#'`` ``Py_UNICODE(nullable=True, length=True)`` ``'z#'`` ``str(nullable=True, length=True)`` @@ -789,6 +824,90 @@ This restriction doesn't seem unreasonable; CPython itself always passes in stat hard-coded encoding strings for parameters whose format units start with ``e``. +.. _default_values: + +Parameter default values +------------------------ + +Default values for parameters can be any of a number of values. +At their simplest, they can be string, int, or float literals:: + + foo: str = "abc" + bar: int = 123 + bat: float = 45.6 + +They can also use any of Python's built-in constants:: + + yep: bool = True + nope: bool = False + nada: object = None + +There's also special support for a default value of ``NULL``, and +for simple expressions, documented in the following sections. + + +The ``NULL`` default value +-------------------------- + +For string and object parameters, you can set them to ``None`` to indicate +that there's no default. However, that means the C variable will be +initialized to ``Py_None``. For convenience's sakes, there's a special +value called ``NULL`` for just this reason: from Python's perspective it +behaves like a default value of ``None``, but the C variable is initialized +with ``NULL``. + +Expressions specified as default values +--------------------------------------- + +The default value for a parameter can be more than just a literal value. +It can be an entire expression, using math operators and looking up attributes +on objects. However, this support isn't exactly simple, because of some +non-obvious semantics. + +Consider the following example:: + + foo: Py_ssize_t = sys.maxsize - 1 + +``sys.maxsize`` can have different values on different platforms. Therefore +Argument Clinic can't simply evaluate that expression locally and hard-code it +in C. So it stores the default in such a way that it will get evaluated at +runtime, when the user asks for the function's signature. + +What namespace is available when the expression is evaluated? It's evaluated +in the context of the module the builtin came from. So, if your module has an +attribute called "``max_widgets``", you may simply use it:: + + foo: Py_ssize_t = max_widgets + +If the symbol isn't found in the current module, it fails over to looking in +``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you +don't know in advance what modules the user will load into their interpreter, +it's best to restrict yourself to modules that are preloaded by Python itself.) + +Evaluating default values only at runtime means Argument Clinic can't compute +the correct equivalent C default value. So you need to tell it explicitly. +When you use an expression, you must also specify the equivalent expression +in C, using the ``c_default`` parameter to the converter:: + + foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1 + +Another complication: Argument Clinic can't know in advance whether or not the +expression you supply is valid. It parses it to make sure it looks legal, but +it can't *actually* know. You must be very careful when using expressions to +specify values that are guaranteed to be valid at runtime! + +Finally, because expressions must be representable as static C values, there +are many restrictions on legal expressions. Here's a list of Python features +you're not permitted to use: + +* Function calls. +* Inline if statements (``3 if foo else 5``). +* Automatic sequence unpacking (``*[1, 2, 3]``). +* List/set/dict comprehensions and generator expressions. +* Tuple/list/set/dict literals. + + + Using a return converter ------------------------ @@ -1096,7 +1215,73 @@ any arguments. You can still use a self converter, a return converter, and specify a ``type`` argument to the object converter for ``METH_O``. -Using Argument Clinic in Python files +The #ifdef trick +---------------------------------------------- + +If you're converting a function that isn't available on all platforms, +there's a trick you can use to make life a little easier. The existing +code probably looks like this:: + + #ifdef HAVE_FUNCTIONNAME + static module_functionname(...) + { + ... + } + #endif /* HAVE_FUNCTIONNAME */ + +And then in the ``PyMethodDef`` structure at the bottom you'll have:: + + #ifdef HAVE_FUNCTIONNAME + {'functionname', ... }, + #endif /* HAVE_FUNCTIONNAME */ + +In this scenario, you should change the code to look like the following:: + + #ifdef HAVE_FUNCTIONNAME + /*[clinic input] + module.functionname + ... + [clinic start generated code]*/ + static module_functionname(...) + { + ... + } + #endif /* HAVE_FUNCTIONNAME */ + +Run Argument Clinic on the code in this state, then refresh the file in +your editor. Now you'll have the generated code, including the #define +for the ``PyMethodDef``, like so:: + + #ifdef HAVE_FUNCTIONNAME + /*[clinic input] + ... + [clinic start generated code]*/ + ... + #define MODULE_FUNCTIONNAME \ + {'functionname', ... }, + ... + /*[clinic end generated code: checksum=...]*/ + static module_functionname(...) + { + ... + } + #endif /* HAVE_FUNCTIONNAME */ + +Change the #endif at the bottom as follows:: + + #else + #define MODULE_FUNCTIONNAME + #endif /* HAVE_FUNCTIONNAME */ + +Now you can remove the #ifdefs around the ``PyMethodDef`` structure +at the end, and replace those three lines with ``MODULE_FUNCTIONNAME``. +If the function is available, the macro turns into the ``PyMethodDef`` +static value, including the trailing comma; if the function isn't +available, the macro turns into nothing. Perfect! + +(This is the preferred approach for optional functions; in the future, +Argument Clinic may generate the entire ``PyMethodDef`` structure.) + ------------------------------------- It's actually possible to use Argument Clinic to preprocess Python files. diff --git a/Lib/inspect.py b/Lib/inspect.py index c9d10dc..d6bd8cd 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1954,6 +1954,8 @@ class Signature: if not s: return None + Parameter = cls._parameter_cls + if s.endswith("/)"): kind = Parameter.POSITIONAL_ONLY s = s[:-2] + ')' @@ -1969,55 +1971,74 @@ class Signature: if not isinstance(module, ast.Module): return None - # ast.FunctionDef f = module.body[0] parameters = [] empty = Parameter.empty invalid = object() - def parse_attribute(node): - if not isinstance(node.ctx, ast.Load): - return None - - value = node.value - o = parse_node(value) - if o is invalid: - return invalid - - if isinstance(value, ast.Name): - name = o - if name not in sys.modules: - return invalid - o = sys.modules[name] - - return getattr(o, node.attr, invalid) - - def parse_node(node): - if isinstance(node, ast.arg): - if node.annotation != None: - raise ValueError("Annotations are not currently supported") - return node.arg - if isinstance(node, ast.Num): - return node.n - if isinstance(node, ast.Str): - return node.s - if isinstance(node, ast.NameConstant): - return node.value - if isinstance(node, ast.Attribute): - return parse_attribute(node) - if isinstance(node, ast.Name): + module = None + module_dict = {} + module_name = getattr(func, '__module__', None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + module_dict = module.__dict__ + sys_module_dict = sys.modules + + def parse_name(node): + assert isinstance(node, ast.arg) + if node.annotation != None: + raise ValueError("Annotations are not currently supported") + return node.arg + + def wrap_value(s): + try: + value = eval(s, module_dict) + except NameError: + try: + value = eval(s, sys_module_dict) + except NameError: + raise RuntimeError() + + if isinstance(value, str): + return ast.Str(value) + if isinstance(value, (int, float)): + return ast.Num(value) + if isinstance(value, bytes): + return ast.Bytes(value) + if value in (True, False, None): + return ast.NameConstant(value) + raise RuntimeError() + + class RewriteSymbolics(ast.NodeTransformer): + def visit_Attribute(self, node): + a = [] + n = node + while isinstance(n, ast.Attribute): + a.append(n.attr) + n = n.value + if not isinstance(n, ast.Name): + raise RuntimeError() + a.append(n.id) + value = ".".join(reversed(a)) + return wrap_value(value) + + def visit_Name(self, node): if not isinstance(node.ctx, ast.Load): - return invalid - return node.id - return invalid + raise ValueError() + return wrap_value(node.id) def p(name_node, default_node, default=empty): - name = parse_node(name_node) + name = parse_name(name_node) if name is invalid: return None if default_node: - o = parse_node(default_node) + try: + default_node = RewriteSymbolics().visit(default_node) + o = ast.literal_eval(default_node) + except ValueError: + o = invalid if o is invalid: return None default = o if o is not invalid else default diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 9dc5475..1bfe724 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1601,12 +1601,15 @@ class TestSignatureObject(unittest.TestCase): self.assertTrue(isinstance(signature, inspect.Signature)) def p(name): return signature.parameters[name].default self.assertEqual(p('s'), 'avocado') + self.assertEqual(p('b'), b'bytes') self.assertEqual(p('d'), 3.14) self.assertEqual(p('i'), 35) - self.assertEqual(p('c'), sys.maxsize) self.assertEqual(p('n'), None) self.assertEqual(p('t'), True) self.assertEqual(p('f'), False) + self.assertEqual(p('local'), 3) + self.assertEqual(p('sys'), sys.maxsize) + self.assertEqual(p('exp'), sys.maxsize - 1) def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): @@ -88,6 +88,9 @@ Tests Tools/Demos ----------- +- Issue #20226: Argument Clinic now permits simple expressions + (e.g. "sys.maxsize - 1") as default values for parameters. + - Issue #19936: Added executable bits or shebang lines to Python scripts which requires them. Disable executable bits and shebang lines in test and benchmark files in order to prevent using a random system python, and in diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 2a70337..9eaf09c 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -618,7 +618,7 @@ curses_window_addch(PyObject *self, PyObject *args) int group_right_1 = 0; long attr = 0; - switch (PyTuple_Size(args)) { + switch (PyTuple_GET_SIZE(args)) { case 1: if (!PyArg_ParseTuple(args, "O:addch", &ch)) return NULL; @@ -650,7 +650,7 @@ curses_window_addch(PyObject *self, PyObject *args) static PyObject * curses_window_addch_impl(PyObject *self, int group_left_1, int x, int y, PyObject *ch, int group_right_1, long attr) -/*[clinic end generated code: checksum=44ed958b891cde91205e584c766e048f3999714f]*/ +/*[clinic end generated code: checksum=b073327add8197b6ba7fb96c87062422c8312954]*/ { PyCursesWindowObject *cwself = (PyCursesWindowObject *)self; int coordinates_group = group_left_1; diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index c9e16e3..d027e1c 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -297,7 +297,7 @@ dbm_dbm_get(PyObject *self, PyObject *args) int group_right_1 = 0; PyObject *default_value = NULL; - switch (PyTuple_Size(args)) { + switch (PyTuple_GET_SIZE(args)) { case 1: if (!PyArg_ParseTuple(args, "s#:get", &key, &key_length)) return NULL; @@ -318,7 +318,7 @@ dbm_dbm_get(PyObject *self, PyObject *args) static PyObject * dbm_dbm_get_impl(dbmobject *dp, const char *key, Py_ssize_clean_t key_length, int group_right_1, PyObject *default_value) -/*[clinic end generated code: checksum=28cf8928811bde51e535d67ae98ea039d79df717]*/ +/*[clinic end generated code: checksum=2c3209571267017f1b9abbd19e1b521849fd5d4a]*/ { datum dbm_key, val; @@ -450,7 +450,7 @@ dbm.open as dbmopen flags: str="r" How to open the file. "r" for reading, "w" for writing, etc. - mode: int(doc_default="0o666") = 0o666 + mode: int(py_default="0o666") = 0o666 If creating a new file, the mode bits for the new file (e.g. os.O_RDWR). diff --git a/Modules/_opcode.c b/Modules/_opcode.c index fe7f8ab..2e84699 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -39,7 +39,7 @@ _opcode_stack_effect(PyModuleDef *module, PyObject *args) int oparg = 0; int _return_value; - switch (PyTuple_Size(args)) { + switch (PyTuple_GET_SIZE(args)) { case 1: if (!PyArg_ParseTuple(args, "i:stack_effect", &opcode)) return NULL; @@ -64,7 +64,7 @@ exit: static int _opcode_stack_effect_impl(PyModuleDef *module, int opcode, int group_right_1, int oparg) -/*[clinic end generated code: checksum=e880e62dc7b0de73403026eaf4f8074aa106358b]*/ +/*[clinic end generated code: checksum=47e76ec27523da4ab192713642d32482cd743aa4]*/ { int effect; if (HAS_ARG(opcode)) { diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b8fe8d5..4f4c69e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2870,7 +2870,7 @@ PyDoc_STRVAR(docstring_with_signature_and_extra_newlines, ); PyDoc_STRVAR(docstring_with_signature_with_defaults, -"docstring_with_signature_with_defaults(s='avocado', d=3.14, i=35, c=sys.maxsize, n=None, t=True, f=False)\n" +"docstring_with_signature_with_defaults(s='avocado', b=b'bytes', d=3.14, i=35, n=None, t=True, f=False, local=the_number_three, sys=sys.maxsize, exp=sys.maxsize - 1)\n" "\n" "\n" "\n" @@ -3317,6 +3317,8 @@ PyInit__testcapi(void) Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); + PyModule_AddIntConstant(m, "the_number_three", 3); + TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError); PyModule_AddObject(m, "error", TestError); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f9dc735..0a3b250 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2401,7 +2401,7 @@ class dir_fd_converter(CConverter): /*[clinic input] -os.stat -> object(doc_default='stat_result') +os.stat path : path_t(allow_fd=True) Path to be examined; can be string, bytes, or open-file-descriptor int. @@ -2523,7 +2523,7 @@ posix_lstat(PyObject *self, PyObject *args, PyObject *kwargs) #define OS_ACCESS_DIR_FD_CONVERTER dir_fd_unavailable #endif /*[clinic input] -os.access -> object(doc_default='True if granted, False otherwise') +os.access path: path_t(allow_fd=True) Path to be tested; can be string, bytes, or open-file-descriptor int. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 575dbd2..59fc620 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -202,7 +202,7 @@ zlib_compress(PyModuleDef *module, PyObject *args) int group_right_1 = 0; int level = 0; - switch (PyTuple_Size(args)) { + switch (PyTuple_GET_SIZE(args)) { case 1: if (!PyArg_ParseTuple(args, "y*:compress", &bytes)) return NULL; @@ -227,7 +227,7 @@ zlib_compress(PyModuleDef *module, PyObject *args) static PyObject * zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int group_right_1, int level) -/*[clinic end generated code: checksum=2c59af563a4595c5ecea4011701f482ae350aa5f]*/ +/*[clinic end generated code: checksum=66c4d16d0b8b9dd423648d9ef00d6a89d3363665]*/ { PyObject *ReturnVal = NULL; Byte *input, *output = NULL; diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 23e0b93..cdbe70a 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -55,6 +55,13 @@ class Null: NULL = Null() +class Unknown: + def __repr__(self): + return '<Unknown>' + +unknown = Unknown() + + def _text_accumulator(): text = [] def output(): @@ -197,7 +204,7 @@ def version_splitter(s): accumulator = [] def flush(): if not accumulator: - raise ValueError('Malformed version string: ' + repr(s)) + raise ValueError('Unsupported version string: ' + repr(s)) version.append(int(''.join(accumulator))) accumulator.clear() @@ -596,7 +603,7 @@ static {impl_return_type} count_min = sys.maxsize count_max = -1 - add("switch (PyTuple_Size(args)) {{\n") + add("switch (PyTuple_GET_SIZE(args)) {{\n") for subset in permute_optional_groups(left, required, right): count = len(subset) count_min = min(count_min, count) @@ -1069,6 +1076,7 @@ class Clinic: self.filename = filename self.modules = collections.OrderedDict() self.classes = collections.OrderedDict() + self.functions = [] global clinic clinic = self @@ -1343,29 +1351,7 @@ class Parameter: def is_keyword_only(self): return self.kind == inspect.Parameter.KEYWORD_ONLY -py_special_values = { - NULL: "None", -} -def py_repr(o): - special = py_special_values.get(o) - if special: - return special - return repr(o) - - -c_special_values = { - NULL: "NULL", - None: "Py_None", -} - -def c_repr(o): - special = c_special_values.get(o) - if special: - return special - if isinstance(o, str): - return '"' + quoted_for_c_string(o) + '"' - return repr(o) def add_c_converter(f, name=None): if not name: @@ -1407,8 +1393,7 @@ class CConverter(metaclass=CConverterAutoRegister): """ For the init function, self, name, function, and default must be keyword-or-positional parameters. All other - parameters (including "required" and "doc_default") - must be keyword-only. + parameters must be keyword-only. """ # The C type to use for this variable. @@ -1418,23 +1403,23 @@ class CConverter(metaclass=CConverterAutoRegister): # The Python default value for this parameter, as a Python value. # Or the magic value "unspecified" if there is no default. + # Or the magic value "unknown" if this value is a cannot be evaluated + # at Argument-Clinic-preprocessing time (but is presumed to be valid + # at runtime). default = unspecified # If not None, default must be isinstance() of this type. # (You can also specify a tuple of types.) default_type = None - # "default" as it should appear in the documentation, as a string. - # Or None if there is no default. - doc_default = None - - # "default" converted into a str for rendering into Python code. - py_default = None - # "default" converted into a C value, as a string. # Or None if there is no default. c_default = None + # "default" converted into a Python value, as a string. + # Or None if there is no default. + py_default = None + # The default value used to initialize the C variable when # there is no default, but not specifying a default may # result in an "uninitialized variable" warning. This can @@ -1485,12 +1470,12 @@ class CConverter(metaclass=CConverterAutoRegister): # Only used by format units ending with '#'. length = False - def __init__(self, name, function, default=unspecified, *, doc_default=None, c_default=None, py_default=None, required=False, annotation=unspecified, **kwargs): + def __init__(self, name, function, default=unspecified, *, c_default=None, py_default=None, annotation=unspecified, **kwargs): self.function = function self.name = name if default is not unspecified: - if self.default_type and not isinstance(default, self.default_type): + if self.default_type and not isinstance(default, (self.default_type, Unknown)): if isinstance(self.default_type, type): types_str = self.default_type.__name__ else: @@ -1498,23 +1483,19 @@ class CConverter(metaclass=CConverterAutoRegister): fail("{}: default value {!r} for field {} is not of type {}".format( self.__class__.__name__, default, name, types_str)) self.default = default - self.py_default = py_default if py_default is not None else py_repr(default) - self.doc_default = doc_default if doc_default is not None else self.py_default - self.c_default = c_default if c_default is not None else c_repr(default) - else: - self.py_default = py_default - self.doc_default = doc_default - self.c_default = c_default + + self.c_default = c_default + self.py_default = py_default + if annotation != unspecified: fail("The 'annotation' parameter is not currently permitted.") - self.required = required self.converter_init(**kwargs) def converter_init(self): pass def is_optional(self): - return (self.default is not unspecified) and (not self.required) + return (self.default is not unspecified) def render(self, parameter, data): """ @@ -1655,8 +1636,9 @@ class bool_converter(CConverter): c_ignored_default = '0' def converter_init(self): - self.default = bool(self.default) - self.c_default = str(int(self.default)) + if self.default is not unspecified: + self.default = bool(self.default) + self.c_default = str(int(self.default)) class char_converter(CConverter): type = 'char' @@ -1665,7 +1647,7 @@ class char_converter(CConverter): c_ignored_default = "'\0'" def converter_init(self): - if len(self.default) != 1: + if isinstance(self.default, str) and (len(self.default) != 1): fail("char_converter: illegal default value " + repr(self.default)) @@ -1797,8 +1779,8 @@ class object_converter(CConverter): @add_legacy_c_converter('s#', length=True) -@add_legacy_c_converter('y', type="bytes") -@add_legacy_c_converter('y#', type="bytes", length=True) +@add_legacy_c_converter('y', types="bytes") +@add_legacy_c_converter('y#', types="bytes", length=True) @add_legacy_c_converter('z', nullable=True) @add_legacy_c_converter('z#', nullable=True, length=True) class str_converter(CConverter): @@ -1993,8 +1975,8 @@ class CReturnConverter(metaclass=CReturnConverterAutoRegister): # Or the magic value "unspecified" if there is no default. default = None - def __init__(self, *, doc_default=None, **kwargs): - self.doc_default = doc_default + def __init__(self, *, py_default=None, **kwargs): + self.py_default = py_default try: self.return_converter_init(**kwargs) except TypeError as e: @@ -2212,6 +2194,7 @@ class DSLParser: self.indent = IndentStack() self.kind = CALLABLE self.coexist = False + self.parameter_continuation = '' def directive_version(self, required): global version @@ -2244,15 +2227,18 @@ class DSLParser: self.block.signatures.append(c) def at_classmethod(self): - assert self.kind is CALLABLE + if self.kind is not CALLABLE: + fail("Can't set @classmethod, function is not a normal callable") self.kind = CLASS_METHOD def at_staticmethod(self): - assert self.kind is CALLABLE + if self.kind is not CALLABLE: + fail("Can't set @staticmethod, function is not a normal callable") self.kind = STATIC_METHOD def at_coexist(self): - assert self.coexist == False + if self.coexist: + fail("Called @coexist twice!") self.coexist = True @@ -2503,6 +2489,7 @@ class DSLParser: if not self.indent.infer(line): return self.next(self.state_function_docstring, line) + self.parameter_continuation = '' return self.next(self.state_parameter, line) @@ -2516,6 +2503,10 @@ class DSLParser: p.group = -p.group def state_parameter(self, line): + if self.parameter_continuation: + line = self.parameter_continuation + ' ' + line.lstrip() + self.parameter_continuation = '' + if self.ignore_line(line): return @@ -2529,6 +2520,11 @@ class DSLParser: # we indented, must be to new parameter docstring column return self.next(self.state_parameter_docstring_start, line) + line = line.rstrip() + if line.endswith('\\'): + self.parameter_continuation = line[:-1] + return + line = line.lstrip() if line in ('*', '/', '[', ']'): @@ -2547,48 +2543,123 @@ class DSLParser: else: fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")") - ast_input = "def x({}): pass".format(line) + base, equals, default = line.rpartition('=') + if not equals: + base = default + default = None module = None try: + ast_input = "def x({}): pass".format(base) module = ast.parse(ast_input) except SyntaxError: - pass + try: + default = None + ast_input = "def x({}): pass".format(line) + module = ast.parse(ast_input) + except SyntaxError: + pass if not module: fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) function_args = module.body[0].args parameter = function_args.args[0] - py_default = None - parameter_name = parameter.arg name, legacy, kwargs = self.parse_converter(parameter.annotation) - if function_args.defaults: - expr = function_args.defaults[0] - # mild hack: explicitly support NULL as a default value - if isinstance(expr, ast.Name) and expr.id == 'NULL': - value = NULL - elif isinstance(expr, ast.Attribute): + if not default: + value = unspecified + if 'py_default' in kwargs: + fail("You can't specify py_default without specifying a default value!") + else: + default = default.strip() + ast_input = "x = {}".format(default) + try: + module = ast.parse(ast_input) + + # blacklist of disallowed ast nodes + class DetectBadNodes(ast.NodeVisitor): + bad = False + def bad_node(self, node): + self.bad = True + + # inline function call + visit_Call = bad_node + # inline if statement ("x = 3 if y else z") + visit_IfExp = bad_node + + # comprehensions and generator expressions + visit_ListComp = visit_SetComp = bad_node + visit_DictComp = visit_GeneratorExp = bad_node + + # literals for advanced types + visit_Dict = visit_Set = bad_node + visit_List = visit_Tuple = bad_node + + # "starred": "a = [1, 2, 3]; *a" + visit_Starred = bad_node + + # allow ellipsis, for now + # visit_Ellipsis = bad_node + + blacklist = DetectBadNodes() + blacklist.visit(module) + if blacklist.bad: + fail("Unsupported expression as default value: " + repr(default)) + + expr = module.body[0].value + # mild hack: explicitly support NULL as a default value + if isinstance(expr, ast.Name) and expr.id == 'NULL': + value = NULL + py_default = 'None' + c_default = "NULL" + elif (isinstance(expr, ast.BinOp) or + (isinstance(expr, ast.UnaryOp) and not isinstance(expr.operand, ast.Num))): + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail("When you specify an expression (" + repr(default) + ") as your default value,\nyou MUST specify a valid c_default.") + py_default = default + value = unknown + elif isinstance(expr, ast.Attribute): + a = [] + n = expr + while isinstance(n, ast.Attribute): + a.append(n.attr) + n = n.value + if not isinstance(n, ast.Name): + fail("Unsupported default value " + repr(default) + " (looked like a Python constant)") + a.append(n.id) + py_default = ".".join(reversed(a)) + + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + + try: + value = eval(py_default) + except NameError: + value = unknown + else: + value = ast.literal_eval(expr) + py_default = repr(value) + if isinstance(value, (bool, None.__class__)): + c_default = "Py_" + py_default + elif isinstance(value, str): + c_default = '"' + quoted_for_c_string(value) + '"' + else: + c_default = py_default + + except SyntaxError as e: + fail("Syntax error: " + repr(e.text)) + except (ValueError, AttributeError): + value = unknown c_default = kwargs.get("c_default") + py_default = default if not (isinstance(c_default, str) and c_default): fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") - a = [] - n = expr - while isinstance(n, ast.Attribute): - a.append(n.attr) - n = n.value - if not isinstance(n, ast.Name): - fail("Malformed default value (looked like a Python constant)") - a.append(n.id) - py_default = ".".join(reversed(a)) - kwargs["py_default"] = py_default - value = eval(py_default) - else: - value = ast.literal_eval(expr) - else: - value = unspecified + kwargs.setdefault('c_default', c_default) + kwargs.setdefault('py_default', py_default) dict = legacy_converters if legacy else converters legacy_str = "legacy " if legacy else "" @@ -2777,7 +2848,7 @@ class DSLParser: if p.converter.is_optional(): a.append('=') value = p.converter.default - a.append(p.converter.doc_default) + a.append(p.converter.py_default) s = fix_right_bracket_count(p.right_bracket_count) s += "".join(a) if add_comma: @@ -2788,9 +2859,18 @@ class DSLParser: add(fix_right_bracket_count(0)) add(')') - # if f.return_converter.doc_default: + # PEP 8 says: + # + # The Python standard library will not use function annotations + # as that would result in a premature commitment to a particular + # annotation style. Instead, the annotations are left for users + # to discover and experiment with useful annotation styles. + # + # therefore this is commented out: + # + # if f.return_converter.py_default: # add(' -> ') - # add(f.return_converter.doc_default) + # add(f.return_converter.py_default) docstring_first_line = output() @@ -2998,8 +3078,8 @@ def main(argv): # print(" ", short_name + "".join(parameters)) print() - print("All converters also accept (doc_default=None, required=False, annotation=None).") - print("All return converters also accept (doc_default=None).") + print("All converters also accept (c_default=None, py_default=None, annotation=None).") + print("All return converters also accept (py_default=None).") sys.exit(0) if ns.make: diff --git a/Tools/clinic/clinic_test.py b/Tools/clinic/clinic_test.py index 27ff0cb..7de5429 100644 --- a/Tools/clinic/clinic_test.py +++ b/Tools/clinic/clinic_test.py @@ -231,20 +231,20 @@ xyz self._test_clinic(""" verbatim text here lah dee dah -/*[copy] +/*[copy input] def -[copy]*/ +[copy start generated code]*/ abc -/*[copy checksum: 03cfd743661f07975fa2f1220c5194cbaff48451]*/ +/*[copy end generated code: checksum=03cfd743661f07975fa2f1220c5194cbaff48451]*/ xyz """, """ verbatim text here lah dee dah -/*[copy] +/*[copy input] def -[copy]*/ +[copy start generated code]*/ def -/*[copy checksum: 7b18d017f89f61cf17d47f92749ea6930a3f1deb]*/ +/*[copy end generated code: checksum=7b18d017f89f61cf17d47f92749ea6930a3f1deb]*/ xyz """) @@ -292,17 +292,6 @@ os.access p = function.parameters['path'] self.assertEqual(1, p.converter.args['allow_fd']) - def test_param_docstring(self): - function = self.parse_function(""" -module os -os.stat as os_stat_fn -> object(doc_default='stat_result') - - path: str - Path to be examined""") - p = function.parameters['path'] - self.assertEqual("Path to be examined", p.docstring) - self.assertEqual(function.return_converter.doc_default, 'stat_result') - def test_function_docstring(self): function = self.parse_function(""" module os |