bpo-33738: Fix macros which contradict PEP 384 (GH-7477)

During development of the limited API support for PySide,
we saw an error in a macro that accessed a type field.

This patch fixes the 7 errors in the Python headers.
Macros which were not written as capitals were implemented
as function.

To do the necessary analysis again, a script was included that
parses all headers and looks for "->tp_" in serctions which can
be reached with active limited API.

It is easily possible to call this script as a test.

Error listing:

../../Include/objimpl.h:243
#define PyObject_IS_GC(o) (PyType_IS_GC(Py_TYPE(o)) && \
    (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o)))
Action: commented only

../../Include/objimpl.h:362
#define PyType_SUPPORTS_WEAKREFS(t) ((t)->tp_weaklistoffset > 0)
Action: commented only

../../Include/objimpl.h:364
#define PyObject_GET_WEAKREFS_LISTPTR(o) \
    ((PyObject **) (((char *) (o)) + Py_TYPE(o)->tp_weaklistoffset))
Action: commented only

../../Include/pyerrors.h:143
#define PyExceptionClass_Name(x) \
     ((char *)(((PyTypeObject*)(x))->tp_name))
Action: implemented function

../../Include/abstract.h:593
#define PyIter_Check(obj) \
    ((obj)->ob_type->tp_iternext != NULL && \
     (obj)->ob_type->tp_iternext != &_PyObject_NextNotImplemented)
Action: implemented function

../../Include/abstract.h:713
#define PyIndex_Check(obj)                              \
    ((obj)->ob_type->tp_as_number != NULL &&            \
     (obj)->ob_type->tp_as_number->nb_index != NULL)
Action: implemented function

../../Include/abstract.h:924
#define PySequence_ITEM(o, i)\
    ( Py_TYPE(o)->tp_as_sequence->sq_item(o, i) )
Action: commented only

1 files changed, 148 insertions, 0 deletions

diff --git a/Tools/scripts/pep384_macrocheck.py b/Tools/scripts/pep384_macrocheck.py
new file mode 100644
index 0000000..142d248
--- /dev/null
+++ b/Tools/scripts/pep384_macrocheck.py
@@ -0,0 +1,148 @@
+"""
+pep384_macrocheck.py
+
+This programm tries to locate errors in the relevant Python header
+files where macros access type fields when they are reachable from
+the limided API.
+
+The idea is to search macros with the string "->tp_" in it.
+When the macro name does not begin with an underscore,
+then we have found a dormant error.
+
+Christian Tismer
+2018-06-02
+"""
+
+import sys
+import os
+import re
+
+
+DEBUG = False
+
+def dprint(*args, **kw):
+    if DEBUG:
+        print(*args, **kw)
+
+def parse_headerfiles(startpath):
+    """
+    Scan all header files which are reachable fronm Python.h
+    """
+    search = "Python.h"
+    name = os.path.join(startpath, search)
+    if not os.path.exists(name):
+        raise ValueError("file {} was not found in {}\n"
+            "Please give the path to Python's include directory."
+            .format(search, startpath))
+    errors = 0
+    with open(name) as python_h:
+        while True:
+            line = python_h.readline()
+            if not line:
+                break
+            found = re.match(r'^\s*#\s*include\s*"(\w+\.h)"', line)
+            if not found:
+                continue
+            include = found.group(1)
+            dprint("Scanning", include)
+            name = os.path.join(startpath, include)
+            if not os.path.exists(name):
+                name = os.path.join(startpath, "../PC", include)
+            errors += parse_file(name)
+    return errors
+
+def ifdef_level_gen():
+    """
+    Scan lines for #ifdef and track the level.
+    """
+    level = 0
+    ifdef_pattern = r"^\s*#\s*if"  # covers ifdef and ifndef as well
+    endif_pattern = r"^\s*#\s*endif"
+    while True:
+        line = yield level
+        if re.match(ifdef_pattern, line):
+            level += 1
+        elif re.match(endif_pattern, line):
+            level -= 1
+
+def limited_gen():
+    """
+    Scan lines for Py_LIMITED_API yes(1) no(-1) or nothing (0)
+    """
+    limited = [0]   # nothing
+    unlimited_pattern = r"^\s*#\s*ifndef\s+Py_LIMITED_API"
+    limited_pattern = "|".join([
+        r"^\s*#\s*ifdef\s+Py_LIMITED_API",
+        r"^\s*#\s*(el)?if\s+!\s*defined\s*\(\s*Py_LIMITED_API\s*\)\s*\|\|",
+        r"^\s*#\s*(el)?if\s+defined\s*\(\s*Py_LIMITED_API"
+        ])
+    else_pattern =      r"^\s*#\s*else"
+    ifdef_level = ifdef_level_gen()
+    status = next(ifdef_level)
+    wait_for = -1
+    while True:
+        line = yield limited[-1]
+        new_status = ifdef_level.send(line)
+        dir = new_status - status
+        status = new_status
+        if dir == 1:
+            if re.match(unlimited_pattern, line):
+                limited.append(-1)
+                wait_for = status - 1
+            elif re.match(limited_pattern, line):
+                limited.append(1)
+                wait_for = status - 1
+        elif dir == -1:
+            # this must have been an endif
+            if status == wait_for:
+                limited.pop()
+                wait_for = -1
+        else:
+            # it could be that we have an elif
+            if re.match(limited_pattern, line):
+                limited.append(1)
+                wait_for = status - 1
+            elif re.match(else_pattern, line):
+                limited.append(-limited.pop())  # negate top
+
+def parse_file(fname):
+    errors = 0
+    with open(fname) as f:
+        lines = f.readlines()
+    type_pattern = r"^.*?->\s*tp_"
+    define_pattern = r"^\s*#\s*define\s+(\w+)"
+    limited = limited_gen()
+    status = next(limited)
+    for nr, line in enumerate(lines):
+        status = limited.send(line)
+        line = line.rstrip()
+        dprint(fname, nr, status, line)
+        if status != -1:
+            if re.match(define_pattern, line):
+                name = re.match(define_pattern, line).group(1)
+                if not name.startswith("_"):
+                    # found a candidate, check it!
+                    macro = line + "\n"
+                    idx = nr
+                    while line.endswith("\\"):
+                        idx += 1
+                        line = lines[idx].rstrip()
+                        macro += line + "\n"
+                    if re.match(type_pattern, macro, re.DOTALL):
+                        # this type field can reach the limited API
+                        report(fname, nr + 1, macro)
+                        errors += 1
+    return errors
+
+def report(fname, nr, macro):
+    f = sys.stderr
+    print(fname + ":" + str(nr), file=f)
+    print(macro, file=f)
+
+if __name__ == "__main__":
+    p = sys.argv[1] if sys.argv[1:] else "../../Include"
+    errors = parse_headerfiles(p)
+    if errors:
+        # somehow it makes sense to raise a TypeError :-)
+        raise TypeError("These {} locations contradict the limited API."
+                        .format(errors))