bpo-46541: Replace core use of _Py_IDENTIFIER() with statically initialized global objects. (gh-30928)

We're no longer using _Py_IDENTIFIER() (or _Py_static_string()) in any core CPython code.  It is still used in a number of non-builtin stdlib modules.

The replacement is: PyUnicodeObject (not pointer) fields under _PyRuntimeState, statically initialized as part of _PyRuntime.  A new _Py_GET_GLOBAL_IDENTIFIER() macro facilitates lookup of the fields (along with _Py_GET_GLOBAL_STRING() for non-identifier strings).

https://bugs.python.org/issue46541#msg411799 explains the rationale for this change.

The core of the change is in:

* (new) Include/internal/pycore_global_strings.h - the declarations for the global strings, along with the macros
* Include/internal/pycore_runtime_init.h - added the static initializers for the global strings
* Include/internal/pycore_global_objects.h - where the struct in pycore_global_strings.h is hooked into _PyRuntimeState
* Tools/scripts/generate_global_objects.py - added generation of the global string declarations and static initializers

I've also added a --check flag to generate_global_objects.py (along with make check-global-objects) to check for unused global strings.  That check is added to the PR CI config.

The remainder of this change updates the core code to use _Py_GET_GLOBAL_IDENTIFIER() instead of _Py_IDENTIFIER() and the related _Py*Id functions (likewise for _Py_GET_GLOBAL_STRING() instead of _Py_static_string()).  This includes adding a few functions where there wasn't already an alternative to _Py*Id(), replacing the _Py_Identifier * parameter with PyObject *.

The following are not changed (yet):

* stop using _Py_IDENTIFIER() in the stdlib modules
* (maybe) get rid of _Py_IDENTIFIER(), etc. entirely -- this may not be doable as at least one package on PyPI using this (private) API
* (maybe) intern the strings during runtime init

https://bugs.python.org/issue46541

1 files changed, 24 insertions, 37 deletions

diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c
index 1d649a7..bd5e561 100644
--- a/Objects/moduleobject.c
+++ b/Objects/moduleobject.c
@@ -10,13 +10,6 @@
 
 static Py_ssize_t max_module_number;
 
-_Py_IDENTIFIER(__doc__);
-_Py_IDENTIFIER(__name__);
-_Py_IDENTIFIER(__spec__);
-_Py_IDENTIFIER(__dict__);
-_Py_IDENTIFIER(__dir__);
-_Py_IDENTIFIER(__annotations__);
-
 static PyMemberDef module_members[] = {
     {"__dict__", T_OBJECT, offsetof(PyModuleObject, md_dict), READONLY},
     {0}
@@ -61,22 +54,19 @@ static int
 module_init_dict(PyModuleObject *mod, PyObject *md_dict,
                  PyObject *name, PyObject *doc)
 {
-    _Py_IDENTIFIER(__package__);
-    _Py_IDENTIFIER(__loader__);
-
     assert(md_dict != NULL);
     if (doc == NULL)
         doc = Py_None;
 
-    if (_PyDict_SetItemId(md_dict, &PyId___name__, name) != 0)
+    if (PyDict_SetItem(md_dict, &_Py_ID(__name__), name) != 0)
         return -1;
-    if (_PyDict_SetItemId(md_dict, &PyId___doc__, doc) != 0)
+    if (PyDict_SetItem(md_dict, &_Py_ID(__doc__), doc) != 0)
         return -1;
-    if (_PyDict_SetItemId(md_dict, &PyId___package__, Py_None) != 0)
+    if (PyDict_SetItem(md_dict, &_Py_ID(__package__), Py_None) != 0)
         return -1;
-    if (_PyDict_SetItemId(md_dict, &PyId___loader__, Py_None) != 0)
+    if (PyDict_SetItem(md_dict, &_Py_ID(__loader__), Py_None) != 0)
         return -1;
-    if (_PyDict_SetItemId(md_dict, &PyId___spec__, Py_None) != 0)
+    if (PyDict_SetItem(md_dict, &_Py_ID(__spec__), Py_None) != 0)
         return -1;
     if (PyUnicode_CheckExact(name)) {
         Py_INCREF(name);
@@ -474,7 +464,7 @@ PyModule_SetDocString(PyObject *m, const char *doc)
     PyObject *v;
 
     v = PyUnicode_FromString(doc);
-    if (v == NULL || _PyObject_SetAttrId(m, &PyId___doc__, v) != 0) {
+    if (v == NULL || PyObject_SetAttr(m, &_Py_ID(__doc__), v) != 0) {
         Py_XDECREF(v);
         return -1;
     }
@@ -503,7 +493,7 @@ PyModule_GetNameObject(PyObject *m)
     }
     d = ((PyModuleObject *)m)->md_dict;
     if (d == NULL || !PyDict_Check(d) ||
-        (name = _PyDict_GetItemIdWithError(d, &PyId___name__)) == NULL ||
+        (name = PyDict_GetItemWithError(d, &_Py_ID(__name__))) == NULL ||
         !PyUnicode_Check(name))
     {
         if (!PyErr_Occurred()) {
@@ -528,7 +518,6 @@ PyModule_GetName(PyObject *m)
 PyObject*
 PyModule_GetFilenameObject(PyObject *m)
 {
-    _Py_IDENTIFIER(__file__);
     PyObject *d;
     PyObject *fileobj;
     if (!PyModule_Check(m)) {
@@ -537,7 +526,7 @@ PyModule_GetFilenameObject(PyObject *m)
     }
     d = ((PyModuleObject *)m)->md_dict;
     if (d == NULL ||
-        (fileobj = _PyDict_GetItemIdWithError(d, &PyId___file__)) == NULL ||
+        (fileobj = PyDict_GetItemWithError(d, &_Py_ID(__file__))) == NULL ||
         !PyUnicode_Check(fileobj))
     {
         if (!PyErr_Occurred()) {
@@ -726,8 +715,7 @@ int
 _PyModuleSpec_IsInitializing(PyObject *spec)
 {
     if (spec != NULL) {
-        _Py_IDENTIFIER(_initializing);
-        PyObject *value = _PyObject_GetAttrId(spec, &PyId__initializing);
+        PyObject *value = PyObject_GetAttr(spec, &_Py_ID(_initializing));
         if (value != NULL) {
             int initializing = PyObject_IsTrue(value);
             Py_DECREF(value);
@@ -750,8 +738,7 @@ _PyModuleSpec_IsUninitializedSubmodule(PyObject *spec, PyObject *name)
          return 0;
     }
 
-    _Py_IDENTIFIER(_uninitialized_submodules);
-    PyObject *value = _PyObject_GetAttrId(spec, &PyId__uninitialized_submodules);
+    PyObject *value = PyObject_GetAttr(spec, &_Py_ID(_uninitialized_submodules));
     if (value == NULL) {
         return 0;
     }
@@ -774,18 +761,17 @@ module_getattro(PyModuleObject *m, PyObject *name)
     }
     PyErr_Clear();
     assert(m->md_dict != NULL);
-    _Py_IDENTIFIER(__getattr__);
-    getattr = _PyDict_GetItemIdWithError(m->md_dict, &PyId___getattr__);
+    getattr = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__getattr__));
     if (getattr) {
         return PyObject_CallOneArg(getattr, name);
     }
     if (PyErr_Occurred()) {
         return NULL;
     }
-    mod_name = _PyDict_GetItemIdWithError(m->md_dict, &PyId___name__);
+    mod_name = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__name__));
     if (mod_name && PyUnicode_Check(mod_name)) {
         Py_INCREF(mod_name);
-        PyObject *spec = _PyDict_GetItemIdWithError(m->md_dict, &PyId___spec__);
+        PyObject *spec = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__spec__));
         if (spec == NULL && PyErr_Occurred()) {
             Py_DECREF(mod_name);
             return NULL;
@@ -861,11 +847,11 @@ static PyObject *
 module_dir(PyObject *self, PyObject *args)
 {
     PyObject *result = NULL;
-    PyObject *dict = _PyObject_GetAttrId(self, &PyId___dict__);
+    PyObject *dict = PyObject_GetAttr(self, &_Py_ID(__dict__));
 
     if (dict != NULL) {
         if (PyDict_Check(dict)) {
-            PyObject *dirfunc = _PyDict_GetItemIdWithError(dict, &PyId___dir__);
+            PyObject *dirfunc = PyDict_GetItemWithError(dict, &_Py_ID(__dir__));
             if (dirfunc) {
                 result = _PyObject_CallNoArgs(dirfunc);
             }
@@ -891,7 +877,7 @@ static PyMethodDef module_methods[] = {
 static PyObject *
 module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored))
 {
-    PyObject *dict = _PyObject_GetAttrId((PyObject *)m, &PyId___dict__);
+    PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__));
 
     if ((dict == NULL) || !PyDict_Check(dict)) {
         PyErr_Format(PyExc_TypeError, "<module>.__dict__ is not a dictionary");
@@ -901,8 +887,8 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored))
 
     PyObject *annotations;
     /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */
-    if (_PyDict_ContainsId(dict, &PyId___annotations__)) {
-        annotations = _PyDict_GetItemIdWithError(dict, &PyId___annotations__);
+    if (PyDict_Contains(dict, &_Py_ID(__annotations__))) {
+        annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__));
         /*
         ** _PyDict_GetItemIdWithError could still fail,
         ** for instance with a well-timed Ctrl-C or a MemoryError.
@@ -914,7 +900,8 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored))
     } else {
         annotations = PyDict_New();
         if (annotations) {
-            int result = _PyDict_SetItemId(dict, &PyId___annotations__, annotations);
+            int result = PyDict_SetItem(
+                    dict, &_Py_ID(__annotations__), annotations);
             if (result) {
                 Py_CLEAR(annotations);
             }
@@ -928,7 +915,7 @@ static int
 module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored))
 {
     int ret = -1;
-    PyObject *dict = _PyObject_GetAttrId((PyObject *)m, &PyId___dict__);
+    PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__));
 
     if ((dict == NULL) || !PyDict_Check(dict)) {
         PyErr_Format(PyExc_TypeError, "<module>.__dict__ is not a dictionary");
@@ -937,17 +924,17 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor
 
     if (value != NULL) {
         /* set */
-        ret = _PyDict_SetItemId(dict, &PyId___annotations__, value);
+        ret = PyDict_SetItem(dict, &_Py_ID(__annotations__), value);
         goto exit;
     }
 
     /* delete */
-    if (!_PyDict_ContainsId(dict, &PyId___annotations__)) {
+    if (!PyDict_Contains(dict, &_Py_ID(__annotations__))) {
         PyErr_Format(PyExc_AttributeError, "__annotations__");
         goto exit;
     }
 
-    ret = _PyDict_DelItemId(dict, &PyId___annotations__);
+    ret = PyDict_DelItem(dict, &_Py_ID(__annotations__));
 
 exit:
     Py_XDECREF(dict);