Issue #20530: Argument Clinic's signature format has been revised again.

The new syntax is highly human readable while still preventing false
positives.  The syntax also extends Python syntax to denote "self" and
positional-only parameters, allowing inspect.Signature objects to be
totally accurate for all supported builtins in Python 3.4.

3 files changed, 170 insertions, 35 deletions

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 7a2739f..017a7e8 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -39,6 +39,7 @@ import os
 import re
 import sys
 import tokenize
+import token
 import types
 import warnings
 import functools
@@ -1648,25 +1649,88 @@ def _signature_get_bound_param(spec):
     return spec[2:pos]
 
 
+def _signature_strip_non_python_syntax(signature):
+    """
+    Takes a signature in Argument Clinic's extended signature format.
+    Returns a tuple of three things:
+      * that signature re-rendered in standard Python syntax,
+      * the index of the "self" parameter (generally 0), or None if
+        the function does not have a "self" parameter, and
+      * the index of the last "positional only" parameter,
+        or None if the signature has no positional-only parameters.
+    """
+
+    if not signature:
+        return signature, None, None
+
+    self_parameter = None
+    last_positional_only = None
+
+    lines = [l.encode('ascii') for l in signature.split('\n')]
+    generator = iter(lines).__next__
+    token_stream = tokenize.tokenize(generator)
+
+    delayed_comma = False
+    skip_next_comma = False
+    text = []
+    add = text.append
+
+    current_parameter = 0
+    OP = token.OP
+    ERRORTOKEN = token.ERRORTOKEN
+
+    # token stream always starts with ENCODING token, skip it
+    t = next(token_stream)
+    assert t.type == tokenize.ENCODING
+
+    for t in token_stream:
+        type, string = t.type, t.string
+
+        if type == OP:
+            if string == ',':
+                if skip_next_comma:
+                    skip_next_comma = False
+                else:
+                    assert not delayed_comma
+                    delayed_comma = True
+                    current_parameter += 1
+                continue
+
+            if string == '/':
+                assert not skip_next_comma
+                assert last_positional_only is None
+                skip_next_comma = True
+                last_positional_only = current_parameter - 1
+                continue
+
+        if (type == ERRORTOKEN) and (string == '$'):
+            assert self_parameter is None
+            self_parameter = current_parameter
+            continue
+
+        if delayed_comma:
+            delayed_comma = False
+            if not ((type == OP) and (string == ')')):
+                add(', ')
+        add(string)
+        if (string == ','):
+            add(' ')
+    clean_signature = ''.join(text)
+    return clean_signature, self_parameter, last_positional_only
+
+
 def _signature_fromstr(cls, obj, s):
     # Internal helper to parse content of '__text_signature__'
     # and return a Signature based on it
     Parameter = cls._parameter_cls
 
-    if s.endswith("/)"):
-        kind = Parameter.POSITIONAL_ONLY
-        s = s[:-2] + ')'
-    else:
-        kind = Parameter.POSITIONAL_OR_KEYWORD
-
-    first_parameter_is_self = s.startswith("($")
-    if first_parameter_is_self:
-        s = '(' + s[2:]
+    clean_signature, self_parameter, last_positional_only = \
+        _signature_strip_non_python_syntax(s)
 
-    s = "def foo" + s + ": pass"
+    program = "def foo" + clean_signature + ": pass"
 
     try:
-        module = ast.parse(s)
+        module = ast.parse(program)
     except SyntaxError:
         module = None
 
@@ -1750,8 +1814,14 @@ def _signature_fromstr(cls, obj, s):
     args = reversed(f.args.args)
     defaults = reversed(f.args.defaults)
     iter = itertools.zip_longest(args, defaults, fillvalue=None)
-    for name, default in reversed(list(iter)):
+    if last_positional_only is not None:
+        kind = Parameter.POSITIONAL_ONLY
+    else:
+        kind = Parameter.POSITIONAL_OR_KEYWORD
+    for i, (name, default) in enumerate(reversed(list(iter))):
         p(name, default)
+        if i == last_positional_only:
+            kind = Parameter.POSITIONAL_OR_KEYWORD
 
     # *args
     if f.args.vararg:
@@ -1768,7 +1838,7 @@ def _signature_fromstr(cls, obj, s):
         kind = Parameter.VAR_KEYWORD
         p(f.args.kwarg, empty)
 
-    if first_parameter_is_self:
+    if self_parameter is not None:
         assert parameters
         if getattr(obj, '__self__', None):
             # strip off self, it's already been bound
@@ -1861,12 +1931,13 @@ def signature(obj):
             # At this point we know, that `obj` is a class, with no user-
             # defined '__init__', '__new__', or class-level '__call__'
 
-            for base in obj.__mro__:
+            for base in obj.__mro__[:-1]:
                 # Since '__text_signature__' is implemented as a
                 # descriptor that extracts text signature from the
                 # class docstring, if 'obj' is derived from a builtin
                 # class, its own '__text_signature__' may be 'None'.
-                # Therefore, we go through the MRO to find the first
+                # Therefore, we go through the MRO (except the last
+                # class in there, which is 'object') to find the first
                 # class with non-empty text signature.
                 try:
                     text_sig = base.__text_signature__
@@ -1881,13 +1952,7 @@ def signature(obj):
             # No '__text_signature__' was found for the 'obj' class.
             # Last option is to check if its '__init__' is
             # object.__init__ or type.__init__.
-            if type in obj.__mro__:
-                # 'obj' is a metaclass without user-defined __init__
-                # or __new__.
-                if obj.__init__ is type.__init__:
-                    # Return a signature of 'type' builtin.
-                    return signature(type)
-            else:
+            if type not in obj.__mro__:
                 # We have a class (not metaclass), but no user-defined
                 # __init__ or __new__ for it
                 if obj.__init__ is object.__init__:
@@ -1901,7 +1966,11 @@ def signature(obj):
         # infinite recursion (and even potential segfault)
         call = _signature_get_user_defined_method(type(obj), '__call__')
         if call is not None:
-            sig = signature(call)
+            try:
+                sig = signature(call)
+            except ValueError as ex:
+                msg = 'no signature found for {!r}'.format(obj)
+                raise ValueError(msg) from ex
 
     if sig is not None:
         # For classes and objects we skip the first parameter of their
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 10e8c4e..ba7c38d 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -126,20 +126,29 @@ class CAPITest(unittest.TestCase):
         self.assertEqual(_testcapi.docstring_no_signature.__text_signature__, None)
 
         self.assertEqual(_testcapi.docstring_with_invalid_signature.__doc__,
-            "sig= (module, boo)\n"
+            "docstring_with_invalid_signature($module, /, boo)\n"
             "\n"
             "This docstring has an invalid signature."
             )
         self.assertEqual(_testcapi.docstring_with_invalid_signature.__text_signature__, None)
 
+        self.assertEqual(_testcapi.docstring_with_invalid_signature2.__doc__,
+            "docstring_with_invalid_signature2($module, /, boo)\n"
+            "\n"
+            "--\n"
+            "\n"
+            "This docstring also has an invalid signature."
+            )
+        self.assertEqual(_testcapi.docstring_with_invalid_signature2.__text_signature__, None)
+
         self.assertEqual(_testcapi.docstring_with_signature.__doc__,
             "This docstring has a valid signature.")
-        self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "(module, sig)")
+        self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "($module, /, sig)")
 
         self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__doc__,
-            "This docstring has a valid signature and some extra newlines.")
+            "\nThis docstring has a valid signature and some extra newlines.")
         self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__text_signature__,
-            "(module, parameter)")
+            "($module, /, parameter)")
 
 
 @unittest.skipUnless(threading, 'Threading required for this test.')
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 862ef82..3f20419 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -1684,7 +1684,6 @@ class TestSignatureObject(unittest.TestCase):
         self.assertEqual(p('sys'), sys.maxsize)
         self.assertEqual(p('exp'), sys.maxsize - 1)
 
-        test_callable(type)
         test_callable(object)
 
         # normal method
@@ -1710,9 +1709,12 @@ class TestSignatureObject(unittest.TestCase):
         # support for 'method-wrapper'
         test_callable(min.__call__)
 
-        class ThisWorksNow:
-            __call__ = type
-        test_callable(ThisWorksNow())
+        # This doesn't work now.
+        # (We don't have a valid signature for "type" in 3.4)
+        with self.assertRaisesRegex(ValueError, "no signature found"):
+            class ThisWorksNow:
+                __call__ = type
+            test_callable(ThisWorksNow())
 
     @cpython_only
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
@@ -2213,11 +2215,11 @@ class TestSignatureObject(unittest.TestCase):
 
         # Test meta-classes without user-defined __init__ or __new__
         class C(type): pass
-        self.assertEqual(str(inspect.signature(C)),
-                         '(object_or_name, bases, dict)')
         class D(C): pass
-        self.assertEqual(str(inspect.signature(D)),
-                         '(object_or_name, bases, dict)')
+        with self.assertRaisesRegex(ValueError, "callable.*is not supported"):
+            self.assertEqual(inspect.signature(C), None)
+        with self.assertRaisesRegex(ValueError, "callable.*is not supported"):
+            self.assertEqual(inspect.signature(D), None)
 
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings")
@@ -2768,6 +2770,61 @@ class TestSignaturePrivateHelpers(unittest.TestCase):
         self.assertEqual(getter('($self, obj)'), 'self')
         self.assertEqual(getter('($cls, /, obj)'), 'cls')
 
+    def _strip_non_python_syntax(self, input,
+        clean_signature, self_parameter, last_positional_only):
+        computed_clean_signature, \
+            computed_self_parameter, \
+            computed_last_positional_only = \
+            inspect._signature_strip_non_python_syntax(input)
+        self.assertEqual(computed_clean_signature, clean_signature)
+        self.assertEqual(computed_self_parameter, self_parameter)
+        self.assertEqual(computed_last_positional_only, last_positional_only)
+
+    def test_signature_strip_non_python_syntax(self):
+        self._strip_non_python_syntax(
+            "($module, /, path, mode, *, dir_fd=None, " +
+                "effective_ids=False,\n       follow_symlinks=True)",
+            "(module, path, mode, *, dir_fd=None, " +
+                "effective_ids=False, follow_symlinks=True)",
+            0,
+            0)
+
+        self._strip_non_python_syntax(
+            "($module, word, salt, /)",
+            "(module, word, salt)",
+            0,
+            2)
+
+        self._strip_non_python_syntax(
+            "(x, y=None, z=None, /)",
+            "(x, y=None, z=None)",
+            None,
+            2)
+
+        self._strip_non_python_syntax(
+            "(x, y=None, z=None)",
+            "(x, y=None, z=None)",
+            None,
+            None)
+
+        self._strip_non_python_syntax(
+            "(x,\n    y=None,\n      z = None  )",
+            "(x, y=None, z=None)",
+            None,
+            None)
+
+        self._strip_non_python_syntax(
+            "",
+            "",
+            None,
+            None)
+
+        self._strip_non_python_syntax(
+            None,
+            None,
+            None,
+            None)
+
 
 class TestUnwrap(unittest.TestCase):