gh-107211: No longer export internal functions (7) (#108425)

No longer export _PyUnicode_FromId() internal C API function.

Change comment style to "// comment" and add comment explaining why
other functions have to be exported.

Update Tools/build/generate_token.py to update Include/internal/pycore_token.h
comments.

10 files changed, 69 insertions, 46 deletions

diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index 56abd57..ec003a1 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -98,16 +98,20 @@ extern int _Py_FdIsInteractive(FILE *fp, PyObject *filename);
 extern const char* _Py_gitidentifier(void);
 extern const char* _Py_gitversion(void);
 
+// Export for '_asyncio' shared extension
 PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp);
 
 /* Random */
 extern int _PyOS_URandom(void *buffer, Py_ssize_t size);
+
 // Export for '_random' shared extension
 PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size);
 
 /* Legacy locale support */
 extern int _Py_CoerceLegacyLocale(int warn);
 extern int _Py_LegacyLocaleDetected(int warn);
+
+// Export for 'readline' shared extension
 PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category);
 
 #ifdef __cplusplus
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index f6ca910..2e601cc 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -66,6 +66,9 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp)
 #if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE)
 extern _Py_thread_local PyThreadState *_Py_tss_tstate;
 #endif
+
+// Export for most shared extensions, used via _PyThreadState_GET() static
+// inline function.
 PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void);
 
 /* Get the current Python thread state.
diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h
index 4a46080..2ce46f3 100644
--- a/Include/internal/pycore_runtime.h
+++ b/Include/internal/pycore_runtime.h
@@ -272,6 +272,9 @@ typedef struct pyruntimestate {
 
 /* other API */
 
+// Export _PyRuntime for shared extensions which use it in static inline
+// functions for best performance, like _Py_IsMainThread() or _Py_ID().
+// It's also made accessible for debuggers and profilers.
 PyAPI_DATA(_PyRuntimeState) _PyRuntime;
 
 extern PyStatus _PyRuntimeState_Init(_PyRuntimeState *runtime);
diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h
index 1b63479..34a00e6 100644
--- a/Include/internal/pycore_setobject.h
+++ b/Include/internal/pycore_setobject.h
@@ -8,17 +8,17 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-// Export for 'pickle' shared extension
+// Export for '_pickle' shared extension
 PyAPI_FUNC(int) _PySet_NextEntry(
     PyObject *set,
     Py_ssize_t *pos,
     PyObject **key,
     Py_hash_t *hash);
 
-// Export for 'pickle' shared extension
+// Export for '_pickle' shared extension
 PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable);
 
-// Export _PySet_Dummy for the gdb plugin's benefit
+// Export for the gdb plugin's (python-gdb.py) benefit
 PyAPI_DATA(PyObject *) _PySet_Dummy;
 
 #ifdef __cplusplus
diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h
index 89a2f76..aec9c20 100644
--- a/Include/internal/pycore_sysmodule.h
+++ b/Include/internal/pycore_sysmodule.h
@@ -14,8 +14,8 @@ extern int _PySys_Audit(
     const char *argFormat,
     ...);
 
-/* We want minimal exposure of this function, so use extern rather than
-   PyAPI_FUNC() to not export the symbol. */
+// _PySys_ClearAuditHooks() must not be exported: use extern rather than
+// PyAPI_FUNC(). We want minimal exposure of this function.
 extern void _PySys_ClearAuditHooks(PyThreadState *tstate);
 
 extern int _PySys_SetAttr(PyObject *, PyObject *);
diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h
index 9c65cd8..571cd62 100644
--- a/Include/internal/pycore_token.h
+++ b/Include/internal/pycore_token.h
@@ -1,4 +1,4 @@
-/* Auto-generated by Tools/build/generate_token.py */
+// Auto-generated by Tools/build/generate_token.py
 
 /* Token types */
 #ifndef Py_INTERNAL_TOKEN_H
@@ -94,7 +94,7 @@ extern "C" {
                                  (x) == FSTRING_MIDDLE)
 
 
-// Symbols exported for test_peg_generator
+// Export these 4 symbols for 'test_peg_generator'
 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
 PyAPI_FUNC(int) _PyToken_OneChar(int);
 PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h
index 4d6a018..27c6c87 100644
--- a/Include/internal/pycore_typeobject.h
+++ b/Include/internal/pycore_typeobject.h
@@ -114,8 +114,10 @@ extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTyp
 extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type);
 extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *);
 
-// Export for 'math' shared extension via _PyType_IsReady() function
+// Export for 'math' shared extension, used via _PyType_IsReady() static inline
+// function
 PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *);
+
 extern PyObject * _PyType_GetBases(PyTypeObject *type);
 extern PyObject * _PyType_GetMRO(PyTypeObject *type);
 extern PyObject* _PyType_GetSubclasses(PyTypeObject *);
diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h
index 8ec80dd..08c3dfe 100644
--- a/Include/internal/pycore_unicodeobject.h
+++ b/Include/internal/pycore_unicodeobject.h
@@ -24,6 +24,7 @@ extern int _PyUnicode_IsCased(Py_UCS4 ch);
 
 /* --- Unicode API -------------------------------------------------------- */
 
+// Export for '_json' shared extension
 PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
     PyObject *op,
     int check_content);
@@ -31,10 +32,10 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
 extern void _PyUnicode_ExactDealloc(PyObject *op);
 extern Py_ssize_t _PyUnicode_InternedSize(void);
 
-/* Get a copy of a Unicode string. */
+// Get a copy of a Unicode string.
+// Export for '_datetime' shared extension.
 PyAPI_FUNC(PyObject*) _PyUnicode_Copy(
-    PyObject *unicode
-    );
+    PyObject *unicode);
 
 /* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash
    if parameters are invalid (e.g. if length is longer than the string). */
@@ -93,11 +94,13 @@ typedef struct {
     unsigned char readonly;
 } _PyUnicodeWriter ;
 
-/* Initialize a Unicode writer.
- *
- * By default, the minimum buffer size is 0 character and overallocation is
- * disabled. Set min_length, min_char and overallocate attributes to control
- * the allocation of the buffer. */
+// Initialize a Unicode writer.
+//
+// By default, the minimum buffer size is 0 character and overallocation is
+// disabled. Set min_length, min_char and overallocate attributes to control
+// the allocation of the buffer.
+//
+// Export the _PyUnicodeWriter API for '_multibytecodec' shared extension.
 PyAPI_FUNC(void)
 _PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
 
@@ -204,12 +207,14 @@ extern PyObject* _PyUnicode_EncodeUTF7(
 
 /* --- UTF-8 Codecs ------------------------------------------------------- */
 
+// Export for '_tkinter' shared extension.
 PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String(
     PyObject *unicode,
     const char *errors);
 
 /* --- UTF-32 Codecs ------------------------------------------------------ */
 
+// Export for '_tkinter' shared extension
 PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(
     PyObject *object,           /* Unicode object */
     const char *errors,         /* error handling */
@@ -217,20 +222,21 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(
 
 /* --- UTF-16 Codecs ------------------------------------------------------ */
 
-/* Returns a Python string object holding the UTF-16 encoded value of
-   the Unicode data.
-
-   If byteorder is not 0, output is written according to the following
-   byte order:
-
-   byteorder == -1: little endian
-   byteorder == 0:  native byte order (writes a BOM mark)
-   byteorder == 1:  big endian
-
-   If byteorder is 0, the output string will always start with the
-   Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
-   prepended.
-*/
+// Returns a Python string object holding the UTF-16 encoded value of
+// the Unicode data.
+//
+// If byteorder is not 0, output is written according to the following
+// byte order:
+//
+// byteorder == -1: little endian
+// byteorder == 0:  native byte order (writes a BOM mark)
+// byteorder == 1:  big endian
+//
+// If byteorder is 0, the output string will always start with the
+// Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
+// prepended.
+//
+// Export for '_tkinter' shared extension
 PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16(
     PyObject* unicode,          /* Unicode object */
     const char *errors,         /* error handling */
@@ -297,13 +303,14 @@ extern PyObject* _PyUnicode_EncodeCharmap(
 
 /* --- Decimal Encoder ---------------------------------------------------- */
 
-/* Coverts a Unicode object holding a decimal value to an ASCII string
-   for using in int, float and complex parsers.
-   Transforms code points that have decimal digit property to the
-   corresponding ASCII digit code points.  Transforms spaces to ASCII.
-   Transforms code points starting from the first non-ASCII code point that
-   is neither a decimal digit nor a space to the end into '?'. */
-
+// Coverts a Unicode object holding a decimal value to an ASCII string
+// for using in int, float and complex parsers.
+// Transforms code points that have decimal digit property to the
+// corresponding ASCII digit code points.  Transforms spaces to ASCII.
+// Transforms code points starting from the first non-ASCII code point that
+// is neither a decimal digit nor a space to the end into '?'.
+//
+// Export for '_testinternalcapi' shared extension.
 PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(
     PyObject *unicode);         /* Unicode object */
 
@@ -323,9 +330,10 @@ extern int _PyUnicode_EqualToASCIIId(
     _Py_Identifier *right       /* Right identifier */
     );
 
-/* Test whether a unicode is equal to ASCII string.  Return 1 if true,
-   0 otherwise.  The right argument must be ASCII-encoded string.
-   Any error occurs inside will be cleared before return. */
+// Test whether a unicode is equal to ASCII string.  Return 1 if true,
+// 0 otherwise.  The right argument must be ASCII-encoded string.
+// Any error occurs inside will be cleared before return.
+// Export for '_ctypes' shared extension
 PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString(
     PyObject *left,
     const char *right           /* ASCII-encoded string */
@@ -357,14 +365,17 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping(
 
 extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int);
 
-/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/
+// Return an interned Unicode object for an Identifier; may fail if there is no
+// memory.
+// Export for '_testembed' program.
 PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
 
 /* Fast equality check when the inputs are known to be exact unicode types
    and where the hash values are equal (i.e. a very probable match) */
 extern int _PyUnicode_EQ(PyObject *, PyObject *);
 
-/* Equality check. */
+// Equality check.
+// Export for '_pickle' shared extension.
 PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *);
 
 extern int _PyUnicode_WideCharString_Converter(PyObject *, void *);
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 5571ae4..7b98885 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -14,9 +14,9 @@
 #include "pycore_bitutils.h"      // _Py_bswap32()
 #include "pycore_bytesobject.h"   // _PyBytes_Find()
 #include "pycore_compile.h"       // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc
-#include "pycore_ceval.h"         // _PyEval_AddPendingCall
+#include "pycore_ceval.h"         // _PyEval_AddPendingCall()
 #include "pycore_dict.h"          // _PyDictOrValues_GetValues()
-#include "pycore_fileutils.h"     // _Py_normpath
+#include "pycore_fileutils.h"     // _Py_normpath()
 #include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_gc.h"            // PyGC_Head
 #include "pycore_hashtable.h"     // _Py_hashtable_new()
diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py
index 3bd307c..16c3884 100755
--- a/Tools/build/generate_token.py
+++ b/Tools/build/generate_token.py
@@ -50,7 +50,7 @@ def update_file(file, content):
 
 
 token_h_template = f"""\
-/* {AUTO_GENERATED_BY_SCRIPT} */
+// {AUTO_GENERATED_BY_SCRIPT}
 """
 token_h_template += """\
 
@@ -84,7 +84,7 @@ extern "C" {
                                  (x) == FSTRING_MIDDLE)
 
 
-// Symbols exported for test_peg_generator
+// Export these 4 symbols for 'test_peg_generator'
 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
 PyAPI_FUNC(int) _PyToken_OneChar(int);
 PyAPI_FUNC(int) _PyToken_TwoChars(int, int);