Issue #20294: Argument Clinic now supports argument parsing for __new__ and

__init__ functions.

6 files changed, 365 insertions, 239 deletions

diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
index 1f96e82..5ee6c3d 100644
--- a/Doc/howto/clinic.rst
+++ b/Doc/howto/clinic.rst
@@ -784,8 +784,8 @@ Argument Clinic converters.  On the left is the legacy converter,
 on the right is the text you'd replace it with.
 
 =========   =================================================================================
-``'B'``     ``byte(bitwise=True)``
-``'b'``     ``byte``
+``'B'``     ``unsigned_char(bitwise=True)``
+``'b'``     ``unsigned_char``
 ``'c'``     ``char``
 ``'C'``     ``int(types='str')``
 ``'d'``     ``double``
@@ -1275,6 +1275,25 @@ any arguments.
 You can still use a self converter, a return converter, and specify
 a ``type`` argument to the object converter for ``METH_O``.
 
+tp_new and tp_init functions
+----------------------------------------------
+
+You can convert ``tp_new`` and ``tp_init``
+functions.  Just name them ``__new__`` or
+``__init__`` as appropriate.  Notes:
+
+* The function name generated for ``__new__`` doesn't end in ``__new__``
+  like it would by default.  It's just the name of the class, converted
+  into a valid C identifier.
+
+* No ``PyMethodDef`` ``#define`` is generated for these functions.
+
+* ``__init__`` functions return ``int``, not ``PyObject *``.
+
+* Argument Clinic supports any signature for these functions, even though
+  the parsing function is required to always take ``args`` and ``kwargs``
+  objects.
+
 The #ifdef trick
 ----------------------------------------------
 
diff --git a/Include/modsupport.h b/Include/modsupport.h
index ab725f6..5de0458 100644
--- a/Include/modsupport.h
+++ b/Include/modsupport.h
@@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...);
 #endif
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kw);
+PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args);
 
 PyAPI_FUNC(int) PyArg_VaParse(PyObject *, const char *, va_list);
 PyAPI_FUNC(int) PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *,
diff --git a/Misc/NEWS b/Misc/NEWS
index b66ae0b..faac326 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -101,6 +101,9 @@ Tests
 Tools/Demos
 -----------
 
+- Issue #20294: Argument Clinic now supports argument parsing for __new__ and
+  __init__ functions.
+
 - Issue #20299: Argument Clinic custom converters may now change the default
   value of c_default and py_default with a class member.
 
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 13c3ae9..36abfd1 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -4064,13 +4064,13 @@ PyDoc_STRVAR(_pickle_Pickler___init____doc__,
 "to map the new Python 3 names to the old module names used in Python\n"
 "2, so that the pickle data stream is readable with Python 2.");
 
-static PyObject *
+static int
 _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *protocol, int fix_imports);
 
-static PyObject *
+static int
 _pickle_Pickler___init__(PyObject *self, PyObject *args, PyObject *kwargs)
 {
-    PyObject *return_value = NULL;
+    int return_value = -1;
     static char *_keywords[] = {"file", "protocol", "fix_imports", NULL};
     PyObject *file;
     PyObject *protocol = NULL;
@@ -4086,9 +4086,9 @@ exit:
     return return_value;
 }
 
-static PyObject *
+static int
 _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *protocol, int fix_imports)
-/*[clinic end generated code: checksum=defa3d9e9f8b51fb257d4fdfca99db503db0e6df]*/
+/*[clinic end generated code: checksum=10c8ea05194d08108471163d8202cf5e12975544]*/
 {
     _Py_IDENTIFIER(persistent_id);
     _Py_IDENTIFIER(dispatch_table);
@@ -4098,16 +4098,16 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *pro
         (void)Pickler_clear(self);
 
     if (_Pickler_SetProtocol(self, protocol, fix_imports) < 0)
-        return NULL;
+        return -1;
 
     if (_Pickler_SetOutputStream(self, file) < 0)
-        return NULL;
+        return -1;
 
     /* memo and output_buffer may have already been created in _Pickler_New */
     if (self->memo == NULL) {
         self->memo = PyMemoTable_New();
         if (self->memo == NULL)
-            return NULL;
+            return -1;
     }
     self->output_len = 0;
     if (self->output_buffer == NULL) {
@@ -4115,7 +4115,7 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *pro
         self->output_buffer = PyBytes_FromStringAndSize(NULL,
                                                         self->max_output_len);
         if (self->output_buffer == NULL)
-            return NULL;
+            return -1;
     }
 
     self->fast = 0;
@@ -4126,31 +4126,20 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *pro
         self->pers_func = _PyObject_GetAttrId((PyObject *)self,
                                               &PyId_persistent_id);
         if (self->pers_func == NULL)
-            return NULL;
+            return -1;
     }
     self->dispatch_table = NULL;
     if (_PyObject_HasAttrId((PyObject *)self, &PyId_dispatch_table)) {
         self->dispatch_table = _PyObject_GetAttrId((PyObject *)self,
                                                    &PyId_dispatch_table);
         if (self->dispatch_table == NULL)
-            return NULL;
+            return -1;
     }
 
-    Py_RETURN_NONE;
-}
-
-/* Wrap the Clinic generated signature to slot it in tp_init. */
-static int
-Pickler_init(PyObject *self, PyObject *args, PyObject *kwargs)
-{
-    PyObject *result = _pickle_Pickler___init__(self, args, kwargs);
-    if (result == NULL) {
-        return -1;
-    }
-    Py_DECREF(result);
     return 0;
 }
 
+
 /* Define a proxy object for the Pickler's internal memo object. This is to
  * avoid breaking code like:
  *  pickler.memo.clear()
@@ -4543,7 +4532,7 @@ static PyTypeObject Pickler_Type = {
     0,                                  /*tp_descr_get*/
     0,                                  /*tp_descr_set*/
     0,                                  /*tp_dictoffset*/
-    Pickler_init,                       /*tp_init*/
+    _pickle_Pickler___init__,           /*tp_init*/
     PyType_GenericAlloc,                /*tp_alloc*/
     PyType_GenericNew,                  /*tp_new*/
     PyObject_GC_Del,                    /*tp_free*/
@@ -6614,13 +6603,13 @@ PyDoc_STRVAR(_pickle_Unpickler___init____doc__,
 "respectively.  The *encoding* can be \'bytes\' to read these 8-bit\n"
 "string instances as bytes objects.");
 
-static PyObject *
+static int
 _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, int fix_imports, const char *encoding, const char *errors);
 
-static PyObject *
+static int
 _pickle_Unpickler___init__(PyObject *self, PyObject *args, PyObject *kwargs)
 {
-    PyObject *return_value = NULL;
+    int return_value = -1;
     static char *_keywords[] = {"file", "fix_imports", "encoding", "errors", NULL};
     PyObject *file;
     int fix_imports = 1;
@@ -6637,9 +6626,9 @@ exit:
     return return_value;
 }
 
-static PyObject *
+static int
 _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, int fix_imports, const char *encoding, const char *errors)
-/*[clinic end generated code: checksum=26c1d4a06841a8e51d29a0c244ba7f4607ff358a]*/
+/*[clinic end generated code: checksum=6936e9188104e45b1b15e1c11fe77b3965409471]*/
 {
     _Py_IDENTIFIER(persistent_load);
 
@@ -6648,20 +6637,20 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, int fix_i
         (void)Unpickler_clear(self);
 
     if (_Unpickler_SetInputStream(self, file) < 0)
-        return NULL;
+        return -1;
 
     if (_Unpickler_SetInputEncoding(self, encoding, errors) < 0)
-        return NULL;
+        return -1;
 
     self->fix_imports = fix_imports;
     if (self->fix_imports == -1)
-        return NULL;
+        return -1;
 
     if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_load)) {
         self->pers_func = _PyObject_GetAttrId((PyObject *)self,
                                               &PyId_persistent_load);
         if (self->pers_func == NULL)
-            return NULL;
+            return 1;
     }
     else {
         self->pers_func = NULL;
@@ -6669,30 +6658,19 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, int fix_i
 
     self->stack = (Pdata *)Pdata_New();
     if (self->stack == NULL)
-        return NULL;
+        return 1;
 
     self->memo_size = 32;
     self->memo = _Unpickler_NewMemo(self->memo_size);
     if (self->memo == NULL)
-        return NULL;
+        return -1;
 
     self->proto = 0;
 
-    Py_RETURN_NONE;
-}
-
-/* Wrap the Clinic generated signature to slot it in tp_init. */
-static int
-Unpickler_init(PyObject *self, PyObject *args, PyObject *kwargs)
-{
-    PyObject *result = _pickle_Unpickler___init__(self, args, kwargs);
-    if (result == NULL) {
-        return -1;
-    }
-    Py_DECREF(result);
     return 0;
 }
 
+
 /* Define a proxy object for the Unpickler's internal memo object. This is to
  * avoid breaking code like:
  *  unpickler.memo.clear()
@@ -7096,7 +7074,7 @@ static PyTypeObject Unpickler_Type = {
     0,                                  /*tp_descr_get*/
     0,                                  /*tp_descr_set*/
     0,                                  /*tp_dictoffset*/
-    Unpickler_init,                     /*tp_init*/
+    _pickle_Unpickler___init__,         /*tp_init*/
     PyType_GenericAlloc,                /*tp_alloc*/
     PyType_GenericNew,                  /*tp_new*/
     PyObject_GC_Del,                    /*tp_free*/
diff --git a/Python/getargs.c b/Python/getargs.c
index 6084568..bfea111 100644
--- a/Python/getargs.c
+++ b/Python/getargs.c
@@ -1805,7 +1805,7 @@ PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t m
 
 /* For type constructors that don't take keyword args
  *
- * Sets a TypeError and returns 0 if the kwds dict is
+ * Sets a TypeError and returns 0 if the args/kwargs is
  * not empty, returns 1 otherwise
  */
 int
@@ -1824,6 +1824,25 @@ _PyArg_NoKeywords(const char *funcname, PyObject *kw)
                     funcname);
     return 0;
 }
+
+
+int
+_PyArg_NoPositional(const char *funcname, PyObject *args)
+{
+    if (args == NULL)
+        return 1;
+    if (!PyTuple_CheckExact(args)) {
+        PyErr_BadInternalCall();
+        return 0;
+    }
+    if (PyTuple_GET_SIZE(args) == 0)
+        return 1;
+
+    PyErr_Format(PyExc_TypeError, "%s does not take positional arguments",
+                    funcname);
+    return 0;
+}
+
 #ifdef __cplusplus
 };
 #endif
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index ba0bc17..4d58056 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -169,6 +169,8 @@ def linear_format(s, **kwargs):
         themselves.  (This line is the "source line".)
       * If the substitution text is empty, the source line
         is removed in the output.
+      * If the field is not recognized, the original line
+        is passed unmodified through to the output.
       * If the substitution text is not empty:
           * Each line of the substituted text is indented
             by the indent of the source line.
@@ -454,6 +456,182 @@ class CLanguage(Language):
         add('"')
         return ''.join(text)
 
+    _templates = {}
+    # the templates will be run through str.format(),
+    # so actual curly-braces need to be doubled up.
+    templates_source = """
+__________________________________________________
+
+docstring_prototype
+
+PyDoc_VAR({c_basename}__doc__);
+__________________________________________________
+
+docstring_definition
+
+PyDoc_STRVAR({c_basename}__doc__,
+{docstring});
+__________________________________________________
+
+impl_definition
+
+static {impl_return_type}
+{c_basename}_impl({impl_parameters})
+__________________________________________________
+
+parser_prototype_noargs
+
+static PyObject *
+{c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored))
+__________________________________________________
+
+parser_prototype_meth_o
+
+# SLIGHT HACK
+# METH_O uses {impl_parameters} for the parser!
+
+static PyObject *
+{c_basename}({impl_parameters})
+__________________________________________________
+
+parser_prototype_varargs
+
+static PyObject *
+{c_basename}({self_type}{self_name}, PyObject *args)
+__________________________________________________
+
+parser_prototype_keyword
+
+static PyObject *
+{c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs)
+__________________________________________________
+
+parser_prototype_init
+
+static int
+{c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs)
+__________________________________________________
+
+parser_definition_simple_no_parsing
+
+{{
+    return {c_basename}_impl({impl_arguments});
+}}
+__________________________________________________
+
+parser_definition_start
+
+{{
+    {return_value_declaration}
+    {declarations}
+    {initializers}
+{empty line}
+__________________________________________________
+
+parser_definition_end
+
+    {return_conversion}
+
+{exit_label}
+    {cleanup}
+    return return_value;
+}}
+__________________________________________________
+
+parser_definition_impl_call
+
+    {return_value} = {c_basename}_impl({impl_arguments});
+__________________________________________________
+
+parser_definition_unpack_tuple
+
+    if (!PyArg_UnpackTuple(args, "{name}",
+        {unpack_min}, {unpack_max},
+        {parse_arguments}))
+        goto exit;
+__________________________________________________
+
+parser_definition_parse_tuple
+
+    if (!PyArg_ParseTuple(args,
+        "{format_units}:{name}",
+        {parse_arguments}))
+        goto exit;
+__________________________________________________
+
+parser_definition_option_groups
+    {option_group_parsing}
+
+__________________________________________________
+
+parser_definition_parse_tuple_and_keywords
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs,
+        "{format_units}:{name}", _keywords,
+        {parse_arguments}))
+        goto exit;
+__________________________________________________
+
+parser_definition_no_positional
+
+    if (!_PyArg_NoPositional("{name}", args))
+        goto exit;
+
+__________________________________________________
+
+parser_definition_no_keywords
+
+    if (!_PyArg_NoKeywords("{name}", kwargs))
+        goto exit;
+
+__________________________________________________
+
+methoddef_define
+
+#define {methoddef_name}    \\
+    {{"{name}", (PyCFunction){c_basename}, {methoddef_flags}, {c_basename}__doc__}},
+__________________________________________________
+""".rstrip()
+
+    title = ''
+    buffer = []
+    line = None
+    for line in templates_source.split('\n'):
+        line = line.rstrip()
+        if line.startswith('# '):
+            # comment
+            continue
+        if line.startswith("_____"):
+            if not buffer:
+                continue
+            assert title not in _templates, "defined template twice: " + repr(title)
+            buffer = '\n'.join(buffer).rstrip()
+            buffer = buffer.replace('{empty line}', '')
+            _templates[title] = buffer
+            buffer = []
+            title = ''
+            continue
+        if not title:
+            if not line:
+                continue
+            title = line
+            continue
+        if not (line or buffer):
+            # throw away leading blank lines
+            continue
+        buffer.append(line)
+
+    assert not title, 'ensure templates_source ends with ______ (still adding to ' + repr(title) + ")"
+
+    del templates_source
+    del title
+    del buffer
+    del line
+
+    # for name, value in _templates.items():
+    #     print(name + ":")
+    #     pprint.pprint(value)
+    #     print()
 
     def output_templates(self, f):
         parameters = list(f.parameters.values())
@@ -477,12 +655,14 @@ class CLanguage(Language):
         else:
             all_boring_objects = True
 
+        new_or_init = f.kind in (METHOD_NEW, METHOD_INIT)
+
         meth_o = (len(parameters) == 1 and
               parameters[0].kind == inspect.Parameter.POSITIONAL_ONLY and
               not converters[0].is_optional() and
               isinstance(converters[0], object_converter) and
-              converters[0].format_unit == 'O')
-
+              converters[0].format_unit == 'O' and
+              not new_or_init)
 
         # we have to set seven things before we're done:
         #
@@ -493,246 +673,144 @@ class CLanguage(Language):
         # parser_prototype
         # parser_definition
         # impl_definition
-        #
-        # since impl_prototype is always just impl_definition + ';'
-        # we just define impl_definition at the top
 
-        docstring_prototype = "PyDoc_VAR({c_basename}__doc__);"
+        templates = self._templates
 
-        docstring_definition = """
-PyDoc_STRVAR({c_basename}__doc__,
-{docstring});
-""".strip()
-
-        impl_definition = """
-static {impl_return_type}
-{c_basename}_impl({impl_parameters})""".strip()
+        return_value_declaration = "PyObject *return_value = NULL;"
 
+        methoddef_define = templates['methoddef_define']
+        docstring_prototype = templates['docstring_prototype']
+        docstring_definition = templates['docstring_definition']
+        impl_definition = templates['impl_definition']
         impl_prototype = parser_prototype = parser_definition = None
 
-        def meth_varargs():
-            nonlocal flags
-            nonlocal parser_prototype
-
-            flags = "METH_VARARGS"
+        parser_body_fields = None
+        def parser_body(prototype, *fields):
+            nonlocal parser_body_fields
+            add, output = text_accumulator()
+            add(prototype)
+            parser_body_fields = fields
+            fields = list(fields)
+            fields.insert(0, 'parser_definition_start')
+            fields.append('parser_definition_impl_call')
+            fields.append('parser_definition_end')
+            for field in fields:
+                add('\n')
+                add(templates[field])
+            return output()
 
-            parser_prototype = """
-static PyObject *
-{c_basename}({self_type}{self_name}, PyObject *args)
-""".strip()
+        def insert_keywords(s):
+            return linear_format(s, declarations="static char *_keywords[] = {{{keywords}, NULL}};\n{declarations}")
 
         if not parameters:
             # no parameters, METH_NOARGS
 
             flags = "METH_NOARGS"
 
-            parser_prototype = """
-static PyObject *
-{c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored))
-""".strip()
+            parser_prototype = templates['parser_prototype_noargs']
+            parser_definition = parser_prototype
 
             if default_return_converter:
-                parser_definition = parser_prototype + """
-{{
-    return {c_basename}_impl({impl_arguments});
-}}
-""".rstrip()
+                parser_definition = parser_prototype + '\n' + templates['parser_definition_simple_no_parsing']
             else:
-                parser_definition = parser_prototype + """
-{{
-    PyObject *return_value = NULL;
-    {declarations}
-    {initializers}
-
-    {return_value} = {c_basename}_impl({impl_arguments});
-    {return_conversion}
-
-{exit_label}
-    {cleanup}
-    return return_value;
-}}
-""".rstrip()
+                parser_definition = parser_body(parser_prototype)
 
         elif meth_o:
-            if default_return_converter:
-                # maps perfectly to METH_O, doesn't need a return converter,
-                # so we skip the parse function and call
-                # directly into the impl function
-
-                # SLIGHT HACK
-                # METH_O uses {impl_parameters} for the parser.
-
-                flags = "METH_O"
-
-                impl_definition = """
-static PyObject *
-{c_basename}({impl_parameters})
-""".strip()
+            flags = "METH_O"
+            # impl_definition = templates['parser_prototype_meth_o']
 
+            if default_return_converter:
+                # maps perfectly to METH_O, doesn't need a return converter.
+                # so we skip making a parse function
+                # and call directly into the impl function.
                 impl_prototype = parser_prototype = parser_definition = ''
-
+                impl_definition = templates['parser_prototype_meth_o']
             else:
-                # SLIGHT HACK
-                # METH_O uses {impl_parameters} for the parser.
-
-                flags = "METH_O"
-
-                parser_prototype = """
-static PyObject *
-{c_basename}({impl_parameters})
-""".strip()
-
-                parser_definition = parser_prototype + """
-{{
-    PyObject *return_value = NULL;
-    {declarations}
-    {initializers}
-
-    _return_value = {c_basename}_impl({impl_arguments});
-    {return_conversion}
-
-{exit_label}
-    {cleanup}
-    return return_value;
-}}
-""".rstrip()
+                parser_prototype = templates['parser_prototype_meth_o']
+                parser_definition = parser_body(parser_prototype)
 
         elif has_option_groups:
             # positional parameters with option groups
             # (we have to generate lots of PyArg_ParseTuple calls
             #  in a big switch statement)
 
-            meth_varargs()
-
-            parser_definition = parser_prototype + """
-{{
-    PyObject *return_value = NULL;
-    {declarations}
-    {initializers}
-
-    {option_group_parsing}
-    {return_value} = {c_basename}_impl({impl_arguments});
-    {return_conversion}
+            flags = "METH_VARARGS"
+            parser_prototype = templates['parser_prototype_varargs']
 
-{exit_label}
-    {cleanup}
-    return return_value;
-}}
-""".rstrip()
+            parser_definition = parser_body(parser_prototype, 'parser_definition_option_groups')
 
         elif positional and all_boring_objects:
             # positional-only, but no option groups,
             # and nothing but normal objects:
             # PyArg_UnpackTuple!
 
-            meth_varargs()
-
-            # substitute in the min and max by hand right here
-            assert parameters
-            min_o = first_optional
-            max_o = len(parameters)
-            if isinstance(parameters[0].converter, self_converter):
-                min_o -= 1
-                max_o -= 1
-            min_o = str(min_o)
-            max_o = str(max_o)
-
-            parser_definition = parser_prototype + """
-{{
-    PyObject *return_value = NULL;
-    {declarations}
-    {initializers}
-
-    if (!PyArg_UnpackTuple(args, "{name}",
-        {min}, {max},
-        {parse_arguments}))
-        goto exit;
-    {return_value} = {c_basename}_impl({impl_arguments});
-    {return_conversion}
+            flags = "METH_VARARGS"
+            parser_prototype = templates['parser_prototype_varargs']
 
-exit:
-    {cleanup}
-    return return_value;
-}}
-""".rstrip().replace('{min}', min_o).replace('{max}', max_o)
+            parser_definition = parser_body(parser_prototype, 'parser_definition_unpack_tuple')
 
         elif positional:
             # positional-only, but no option groups
             # we only need one call to PyArg_ParseTuple
 
-            meth_varargs()
-
-            parser_definition = parser_prototype + """
-{{
-    PyObject *return_value = NULL;
-    {declarations}
-    {initializers}
-
-    if (!PyArg_ParseTuple(args,
-        "{format_units}:{name}",
-        {parse_arguments}))
-        goto exit;
-    {return_value} = {c_basename}_impl({impl_arguments});
-    {return_conversion}
+            flags = "METH_VARARGS"
+            parser_prototype = templates['parser_prototype_varargs']
 
-exit:
-    {cleanup}
-    return return_value;
-}}
-""".rstrip()
+            parser_definition = parser_body(parser_prototype, 'parser_definition_parse_tuple')
 
         else:
             # positional-or-keyword arguments
             flags = "METH_VARARGS|METH_KEYWORDS"
 
-            parser_prototype = """
-static PyObject *
-{c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs)
-""".strip()
+            parser_prototype = templates['parser_prototype_keyword']
 
-            parser_definition = parser_prototype + """
-{{
-    PyObject *return_value = NULL;
-    static char *_keywords[] = {{{keywords}, NULL}};
-    {declarations}
-    {initializers}
+            parser_definition = parser_body(parser_prototype, 'parser_definition_parse_tuple_and_keywords')
+            parser_definition = insert_keywords(parser_definition)
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwargs,
-        "{format_units}:{name}", _keywords,
-        {parse_arguments}))
-        goto exit;
-    {return_value} = {c_basename}_impl({impl_arguments});
-    {return_conversion}
 
-exit:
-    {cleanup}
-    return return_value;
-}}
-""".rstrip()
+        if new_or_init:
+            methoddef_define = ''
+
+            if f.kind == METHOD_NEW:
+                parser_prototype = templates['parser_prototype_keyword']
+            else:
+                return_value_declaration = "int return_value = -1;"
+                parser_prototype = templates['parser_prototype_init']
+
+            fields = list(parser_body_fields)
+            parses_positional = 'METH_NOARGS' not in flags
+            parses_keywords = 'METH_KEYWORDS' in flags
+            if parses_keywords:
+                assert parses_positional
+
+            if not parses_keywords:
+                fields.insert(0, 'parser_definition_no_keywords')
+                if not parses_positional:
+                    fields.insert(0, 'parser_definition_no_positional')
+
+            parser_definition = parser_body(parser_prototype, *fields)
+            if parses_keywords:
+                parser_definition = insert_keywords(parser_definition)
+
 
         if f.methoddef_flags:
-            assert flags
             flags += '|' + f.methoddef_flags
 
-        methoddef_define = """
-#define {methoddef_name}    \\
-    {{"{name}", (PyCFunction){c_basename}, {methoddef_flags}, {c_basename}__doc__}},
-""".strip().replace('{methoddef_flags}', flags)
+        methoddef_define = methoddef_define.replace('{methoddef_flags}', flags)
 
-        # parser_prototype mustn't be None, but it could be an empty string.
+        # add ';' to the end of parser_prototype and impl_prototype
+        # (they mustn't be None, but they could be an empty string.)
         assert parser_prototype is not None
-        assert not parser_prototype.endswith(';')
-
         if parser_prototype:
+            assert not parser_prototype.endswith(';')
             parser_prototype += ';'
 
-        assert impl_definition
         if impl_prototype is None:
-            impl_prototype = impl_definition + ";"
+            impl_prototype = impl_definition
+        if impl_prototype:
+            impl_prototype += ";"
 
-        # __new__ and __init__ don't need methoddefs
-        if f.kind in (METHOD_NEW, METHOD_INIT):
-            methoddef_define = ''
+        parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration)
 
         d = {
             "docstring_prototype" : docstring_prototype,
@@ -744,8 +822,11 @@ exit:
             "impl_definition" : impl_definition,
         }
 
+        # make sure we didn't forget to assign something,
+        # and wrap each non-empty value in \n's
         d2 = {}
         for name, value in d.items():
+            assert value is not None, "got a None value for template " + repr(name)
             if value:
                 value = '\n' + value + '\n'
             d2[name] = value
@@ -881,12 +962,17 @@ exit:
 
         positional = has_option_groups =  False
 
+        first_optional = len(parameters)
+
         if parameters:
             last_group = 0
 
-            for p in parameters:
+            for i, p in enumerate(parameters):
                 c = p.converter
 
+                if p.default is not unspecified:
+                    first_optional = min(first_optional, i)
+
                 # insert group variable
                 group = p.group
                 if last_group != group:
@@ -950,6 +1036,10 @@ exit:
         template_dict['cleanup'] = "".join(data.cleanup)
         template_dict['return_value'] = data.return_value
 
+        # used by unpack tuple
+        template_dict['unpack_min'] = str(first_optional)
+        template_dict['unpack_max'] = str(len(parameters))
+
         if has_option_groups:
             self.render_option_group_parsing(f, template_dict)
 
@@ -2063,7 +2153,7 @@ class char_converter(CConverter):
 
 
 @add_legacy_c_converter('B', bitwise=True)
-class byte_converter(CConverter):
+class unsigned_char_converter(CConverter):
     type = 'unsigned char'
     default_type = int
     format_unit = 'b'
@@ -2073,6 +2163,8 @@ class byte_converter(CConverter):
         if bitwise:
             self.format_unit = 'B'
 
+class byte_converter(unsigned_char_converter): pass
+
 class short_converter(CConverter):
     type = 'short'
     default_type = int
@@ -2455,6 +2547,16 @@ class int_return_converter(long_return_converter):
     type = 'int'
     cast = '(long)'
 
+class init_return_converter(long_return_converter):
+    """
+    Special return converter for __init__ functions.
+    """
+    type = 'int'
+    cast = '(long)'
+
+    def render(self, function, data):
+        pass
+
 class unsigned_long_return_converter(long_return_converter):
     type = 'unsigned long'
     conversion_fn = 'PyLong_FromUnsignedLong'
@@ -2858,9 +2960,8 @@ class DSLParser:
         if c_basename and not is_legal_c_identifier(c_basename):
             fail("Illegal C basename: {}".format(c_basename))
 
-        if not returns:
-            return_converter = CReturnConverter()
-        else:
+        return_converter = None
+        if returns:
             ast_input = "def x() -> {}: pass".format(returns)
             module = None
             try:
@@ -2893,9 +2994,14 @@ class DSLParser:
             if (self.kind != CALLABLE) or (not cls):
                 fail("__init__ must be a normal method, not a class or static method!")
             self.kind = METHOD_INIT
+            if not return_converter:
+                return_converter = init_return_converter()
         elif fields[-1] in unsupported_special_methods:
             fail(fields[-1] + " should not be converted to Argument Clinic!  (Yet.)")
 
+        if not return_converter:
+            return_converter = CReturnConverter()
+
         if not module:
             fail("Undefined module used in declaration of " + repr(full_name.strip()) + ".")
         self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename,