gh-91324: List feature macros in the stable ABI manifest, improve tests (GH-32415)

1 files changed, 88 insertions, 23 deletions

diff --git a/Tools/scripts/stable_abi.py b/Tools/scripts/stable_abi.py
index 7376a46..5407524 100755
--- a/Tools/scripts/stable_abi.py
+++ b/Tools/scripts/stable_abi.py
@@ -45,21 +45,6 @@ EXCLUDED_HEADERS = {
 MACOS = (sys.platform == "darwin")
 UNIXY = MACOS or (sys.platform == "linux")  # XXX should this be "not Windows"?
 
-IFDEF_DOC_NOTES = {
-    'MS_WINDOWS': 'on Windows',
-    'HAVE_FORK': 'on platforms with fork()',
-    'USE_STACKCHECK': 'on platforms with USE_STACKCHECK',
-    'PY_HAVE_THREAD_NATIVE_ID': 'on platforms with native thread IDs',
-}
-
-# To generate the DLL definition, we need to know which feature macros are
-# defined on Windows. On all platforms.
-# Best way to do that is to hardcode the list (and later test in on Windows).
-WINDOWS_IFDEFS = frozenset({
-    'MS_WINDOWS',
-    'PY_HAVE_THREAD_NATIVE_ID',
-    'USE_STACKCHECK',
-})
 
 # The stable ABI manifest (Misc/stable_abi.txt) exists only to fill the
 # following dataclasses.
@@ -130,9 +115,11 @@ class ABIItem:
     ifdef: str = None
     struct_abi_kind: str = None
     members: list = None
+    doc: str = None
+    windows: bool = False
 
     KINDS = frozenset({
-        'struct', 'function', 'macro', 'data', 'const', 'typedef',
+        'struct', 'function', 'macro', 'data', 'const', 'typedef', 'ifdef',
     })
 
     def dump(self, indent=0):
@@ -171,8 +158,8 @@ def parse_manifest(file):
             levels.pop()
         parent = levels[-1][0]
         entry = None
-        if kind in ABIItem.KINDS:
-            if parent.kind not in {'manifest'}:
+        if parent.kind == 'manifest':
+            if kind not in kind in ABIItem.KINDS:
                 raise_error(f'{kind} cannot go in {parent.kind}')
             entry = ABIItem(kind, content)
             parent.add(entry)
@@ -193,10 +180,29 @@ def parse_manifest(file):
             parent.struct_abi_kind = kind
             if kind == 'members':
                 parent.members = content.split()
+        elif kind in {'doc'}:
+            if parent.kind not in {'ifdef'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            parent.doc = content
+        elif kind in {'windows'}:
+            if parent.kind not in {'ifdef'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            if not content:
+                parent.windows = True
+            elif content == 'maybe':
+                parent.windows = content
+            else:
+                raise_error(f'Unexpected: {content}')
         else:
             raise_error(f"unknown kind {kind!r}")
             # When adding more, update the comment in stable_abi.txt.
         levels.append((entry, level))
+
+    ifdef_names = {i.name for i in manifest.select({'ifdef'})}
+    for item in manifest.contents.values():
+        if item.ifdef and item.ifdef not in ifdef_names:
+            raise ValueError(f'{item.name} uses undeclared ifdef {item.ifdef}')
+
     return manifest
 
 # The tool can run individual "actions".
@@ -240,9 +246,12 @@ def gen_python3dll(manifest, args, outfile):
     def sort_key(item):
         return item.name.lower()
 
+    windows_ifdefs = {
+        item.name for item in manifest.select({'ifdef'}) if item.windows
+    }
     for item in sorted(
             manifest.select(
-                {'function'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS),
+                {'function'}, include_abi_only=True, ifdef=windows_ifdefs),
             key=sort_key):
         write(f'EXPORT_FUNC({item.name})')
 
@@ -250,7 +259,7 @@ def gen_python3dll(manifest, args, outfile):
 
     for item in sorted(
             manifest.select(
-                {'data'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS),
+                {'data'}, include_abi_only=True, ifdef=windows_ifdefs),
             key=sort_key):
         write(f'EXPORT_DATA({item.name})')
 
@@ -273,7 +282,7 @@ def gen_doc_annotations(manifest, args, outfile):
     writer.writeheader()
     for item in manifest.select(REST_ROLES.keys(), include_abi_only=False):
         if item.ifdef:
-            ifdef_note = IFDEF_DOC_NOTES[item.ifdef]
+            ifdef_note = manifest.contents[item.ifdef].doc
         else:
             ifdef_note = None
         writer.writerow({
@@ -298,23 +307,42 @@ def gen_ctypes_test(manifest, args, outfile):
         """Test that all symbols of the Stable ABI are accessible using ctypes
         """
 
+        import sys
         import unittest
         from test.support.import_helper import import_module
+        from _testcapi import get_feature_macros
 
+        feature_macros = get_feature_macros()
         ctypes_test = import_module('ctypes')
 
         class TestStableABIAvailability(unittest.TestCase):
             def test_available_symbols(self):
+
                 for symbol_name in SYMBOL_NAMES:
                     with self.subTest(symbol_name):
                         ctypes_test.pythonapi[symbol_name]
 
+            def test_feature_macros(self):
+                self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
+
+            # The feature macros for Windows are used in creating the DLL
+            # definition, so they must be known on all platforms.
+            # If we are on Windows, we check that the hardcoded data matches
+            # the reality.
+            @unittest.skipIf(sys.platform != "win32", "Windows specific test")
+            def test_windows_feature_macros(self):
+                for name, value in WINDOWS_IFDEFS.items():
+                    if value != 'maybe':
+                        with self.subTest(name):
+                            self.assertEqual(feature_macros[name], value)
+
         SYMBOL_NAMES = (
     '''))
     items = manifest.select(
         {'function', 'data'},
         include_abi_only=True,
-        ifdef=set())
+    )
+    ifdef_items = {}
     for item in items:
         if item.name in (
                 # Some symbols aren't exported on all platforms.
@@ -322,8 +350,45 @@ def gen_ctypes_test(manifest, args, outfile):
                 'PyModule_Create2', 'PyModule_FromDefAndSpec2',
             ):
             continue
-        write(f'    "{item.name}",')
+        if item.ifdef:
+            ifdef_items.setdefault(item.ifdef, []).append(item.name)
+        else:
+            write(f'    "{item.name}",')
     write(")")
+    for ifdef, names in ifdef_items.items():
+        write(f"if feature_macros[{ifdef!r}]:")
+        write(f"    SYMBOL_NAMES += (")
+        for name in names:
+            write(f"        {name!r},")
+        write("    )")
+    write("")
+    write(f"EXPECTED_IFDEFS = set({sorted(ifdef_items)})")
+
+    windows_ifdef_values = {
+        name: manifest.contents[name].windows for name in ifdef_items
+    }
+    write(f"WINDOWS_IFDEFS = {windows_ifdef_values}")
+
+
+@generator("testcapi_feature_macros", 'Modules/_testcapi_feature_macros.inc')
+def gen_testcapi_feature_macros(manifest, args, outfile):
+    """Generate/check the stable ABI list for documentation annotations"""
+    write = partial(print, file=outfile)
+    write('// Generated by Tools/scripts/stable_abi.py')
+    write()
+    write('// Add an entry in dict `result` for each Stable ABI feature macro.')
+    write()
+    for macro in manifest.select({'ifdef'}):
+        name = macro.name
+        write(f'#ifdef {name}')
+        write(f'    res = PyDict_SetItemString(result, "{name}", Py_True);')
+        write('#else')
+        write(f'    res = PyDict_SetItemString(result, "{name}", Py_False);')
+        write('#endif')
+        write('if (res) {')
+        write('    Py_DECREF(result); return NULL;')
+        write('}')
+        write()
 
 
 def generate_or_check(manifest, args, path, func):