bpo-42955: Add sys.modules_names (GH-24238)

Add sys.module_names, containing the list of the standard library
module names.

3 files changed, 226 insertions, 190 deletions

diff --git a/Python/module_names.h b/Python/module_names.h
index 533a732..0dc2633 100644
--- a/Python/module_names.h
+++ b/Python/module_names.h
@@ -1,63 +1,123 @@
 // Auto-generated by Tools/scripts/generate_module_names.py.
+// List used to create sys.module_names.
 
 static const char* _Py_module_names[] = {
-
-// Built-in modules
+"__future__",
 "_abc",
+"_aix_support",
 "_ast",
+"_asyncio",
+"_bisect",
+"_blake2",
+"_bootsubprocess",
+"_bz2",
 "_codecs",
+"_codecs_cn",
+"_codecs_hk",
+"_codecs_iso2022",
+"_codecs_jp",
+"_codecs_kr",
+"_codecs_tw",
 "_collections",
+"_collections_abc",
+"_compat_pickle",
+"_compression",
+"_contextvars",
+"_crypt",
+"_csv",
+"_ctypes",
+"_curses",
+"_curses_panel",
+"_datetime",
+"_dbm",
+"_decimal",
+"_elementtree",
 "_functools",
+"_gdbm",
+"_hashlib",
+"_heapq",
 "_imp",
 "_io",
+"_json",
 "_locale",
+"_lsprof",
+"_lzma",
+"_markupbase",
+"_md5",
+"_msi",
+"_multibytecodec",
+"_multiprocessing",
+"_opcode",
 "_operator",
+"_osx_support",
+"_pickle",
+"_posixshmem",
+"_posixsubprocess",
+"_py_abc",
+"_pydecimal",
+"_pyio",
+"_queue",
+"_random",
+"_sha1",
+"_sha256",
+"_sha3",
+"_sha512",
 "_signal",
+"_sitebuiltins",
+"_socket",
+"_sqlite3",
 "_sre",
+"_ssl",
 "_stat",
+"_statistics",
 "_string",
+"_strptime",
+"_struct",
 "_symtable",
 "_thread",
+"_threading_local",
+"_tkinter",
 "_tracemalloc",
+"_uuid",
 "_warnings",
 "_weakref",
-"atexit",
-"builtins",
-"errno",
-"faulthandler",
-"gc",
-"itertools",
-"marshal",
-"posix",
-"pwd",
-"sys",
-"time",
-
-// Pure Python modules (Lib/*.py)
-"__future__",
+"_weakrefset",
+"_winapi",
+"_xxsubinterpreters",
+"_zoneinfo",
 "abc",
 "aifc",
 "antigravity",
 "argparse",
+"array",
 "ast",
 "asynchat",
+"asyncio",
 "asyncore",
+"atexit",
+"audioop",
 "base64",
 "bdb",
+"binascii",
 "binhex",
 "bisect",
+"builtins",
 "bz2",
 "cProfile",
 "calendar",
 "cgi",
 "cgitb",
 "chunk",
+"cmath",
 "cmd",
 "code",
 "codecs",
 "codeop",
+"collections",
 "colorsys",
 "compileall",
+"concurrent",
+"concurrent.futures",
 "configparser",
 "contextlib",
 "contextvars",
@@ -65,45 +125,80 @@ static const char* _Py_module_names[] = {
 "copyreg",
 "crypt",
 "csv",
+"ctypes",
+"ctypes.macholib",
+"curses",
 "dataclasses",
 "datetime",
+"dbm",
 "decimal",
 "difflib",
 "dis",
+"distutils",
+"distutils.command",
 "doctest",
+"email",
+"email.mime",
+"encodings",
+"ensurepip",
+"ensurepip._bundled",
 "enum",
+"errno",
+"faulthandler",
+"fcntl",
 "filecmp",
 "fileinput",
 "fnmatch",
 "fractions",
 "ftplib",
 "functools",
+"gc",
 "genericpath",
 "getopt",
 "getpass",
 "gettext",
 "glob",
 "graphlib",
+"grp",
 "gzip",
 "hashlib",
 "heapq",
 "hmac",
+"html",
+"http",
+"idlelib",
 "imaplib",
 "imghdr",
 "imp",
+"importlib",
 "inspect",
 "io",
 "ipaddress",
+"itertools",
+"json",
 "keyword",
+"lib2to3",
+"lib2to3.fixes",
+"lib2to3.pgen2",
 "linecache",
 "locale",
+"logging",
 "lzma",
 "mailbox",
 "mailcap",
+"marshal",
+"math",
 "mimetypes",
+"mmap",
 "modulefinder",
+"msilib",
+"msvcrt",
+"multiprocessing",
+"multiprocessing.dummy",
 "netrc",
+"nis",
 "nntplib",
+"nt",
 "ntpath",
 "nturl2path",
 "numbers",
@@ -111,6 +206,7 @@ static const char* _Py_module_names[] = {
 "operator",
 "optparse",
 "os",
+"ossaudiodev",
 "pathlib",
 "pdb",
 "pickle",
@@ -120,23 +216,30 @@ static const char* _Py_module_names[] = {
 "platform",
 "plistlib",
 "poplib",
+"posix",
 "posixpath",
 "pprint",
 "profile",
 "pstats",
 "pty",
+"pwd",
 "py_compile",
 "pyclbr",
 "pydoc",
+"pydoc_data",
+"pyexpat",
 "queue",
 "quopri",
 "random",
 "re",
+"readline",
 "reprlib",
+"resource",
 "rlcompleter",
 "runpy",
 "sched",
 "secrets",
+"select",
 "selectors",
 "shelve",
 "shlex",
@@ -148,6 +251,8 @@ static const char* _Py_module_names[] = {
 "sndhdr",
 "socket",
 "socketserver",
+"spwd",
+"sqlite3",
 "sre_compile",
 "sre_constants",
 "sre_parse",
@@ -160,15 +265,20 @@ static const char* _Py_module_names[] = {
 "subprocess",
 "sunau",
 "symtable",
+"sys",
 "sysconfig",
+"syslog",
 "tabnanny",
 "tarfile",
 "telnetlib",
 "tempfile",
+"termios",
 "textwrap",
 "this",
 "threading",
+"time",
 "timeit",
+"tkinter",
 "token",
 "tokenize",
 "trace",
@@ -176,161 +286,32 @@ static const char* _Py_module_names[] = {
 "tracemalloc",
 "tty",
 "turtle",
+"turtledemo",
 "types",
 "typing",
+"unicodedata",
+"unittest",
+"urllib",
 "uu",
 "uuid",
+"venv",
 "warnings",
 "wave",
 "weakref",
 "webbrowser",
-"xdrlib",
-"zipapp",
-"zipfile",
-"zipimport",
-
-// Packages and sub-packages
-"asyncio",
-"collections",
-"concurrent",
-"concurrent.futures",
-"ctypes",
-"ctypes.macholib",
-"curses",
-"dbm",
-"distutils",
-"distutils.command",
-"email",
-"email.mime",
-"encodings",
-"ensurepip",
-"ensurepip._bundled",
-"html",
-"http",
-"idlelib",
-"importlib",
-"json",
-"lib2to3",
-"lib2to3.fixes",
-"lib2to3.pgen2",
-"logging",
-"msilib",
-"multiprocessing",
-"multiprocessing.dummy",
-"pydoc_data",
-"sqlite3",
-"tkinter",
-"turtledemo",
-"unittest",
-"urllib",
-"venv",
+"winreg",
+"winsound",
 "wsgiref",
+"xdrlib",
 "xml",
 "xml.dom",
 "xml.etree",
 "xml.parsers",
 "xml.sax",
 "xmlrpc",
-"zoneinfo",
-
-// Extension modules built by setup.py
-"_asyncio",
-"_bisect",
-"_blake2",
-"_bz2",
-"_codecs_cn",
-"_codecs_hk",
-"_codecs_iso2022",
-"_codecs_jp",
-"_codecs_kr",
-"_codecs_tw",
-"_contextvars",
-"_crypt",
-"_csv",
-"_ctypes",
-"_curses",
-"_curses_panel",
-"_datetime",
-"_dbm",
-"_decimal",
-"_elementtree",
-"_gdbm",
-"_hashlib",
-"_heapq",
-"_json",
-"_lsprof",
-"_lzma",
-"_md5",
-"_multibytecodec",
-"_multiprocessing",
-"_opcode",
-"_pickle",
-"_posixshmem",
-"_posixsubprocess",
-"_queue",
-"_random",
-"_sha1",
-"_sha256",
-"_sha3",
-"_sha512",
-"_socket",
-"_sqlite3",
-"_ssl",
-"_statistics",
-"_struct",
-"_tkinter",
-"_uuid",
-"_xxsubinterpreters",
-"_zoneinfo",
-"array",
-"audioop",
-"binascii",
-"cmath",
-"fcntl",
-"grp",
-"math",
-"mmap",
-"nis",
-"ossaudiodev",
-"pyexpat",
-"readline",
-"resource",
-"select",
-"spwd",
-"syslog",
-"termios",
-"unicodedata",
+"zipapp",
+"zipfile",
+"zipimport",
 "zlib",
-
-// Built-in and extension modules built by Modules/Setup
-"_abc",
-"_codecs",
-"_collections",
-"_functools",
-"_io",
-"_locale",
-"_operator",
-"_signal",
-"_sre",
-"_stat",
-"_symtable",
-"_thread",
-"_tracemalloc",
-"_weakref",
-"atexit",
-"errno",
-"faulthandler",
-"itertools",
-"posix",
-"pwd",
-"time",
-
-// Windows extension modules
-"_msi",
-"_winapi",
-"msvcrt",
-"nt",
-"winreg",
-"winsound",
-
+"zoneinfo",
 };
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index e9df8fb..a97f45d 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -18,8 +18,6 @@
 #include "pycore_sysmodule.h"     // _PySys_ClearAuditHooks()
 #include "pycore_traceback.h"     // _Py_DumpTracebackThreads()
 
-#include "module_names.h"         // _Py_module_names
-
 #include <locale.h>               // setlocale()
 
 #ifdef HAVE_SIGNAL_H
@@ -2499,7 +2497,7 @@ fatal_error_exit(int status)
 
 
 // Dump the list of extension modules of sys.modules, excluding stdlib modules
-// (_Py_module_names), into fd file descriptor.
+// (sys.module_names), into fd file descriptor.
 //
 // This function is called by a signal handler in faulthandler: avoid memory
 // allocations and keep the implementation simple. For example, the list is not
@@ -2515,10 +2513,31 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp)
         return;
     }
 
+    Py_ssize_t pos;
+    PyObject *key, *value;
+
+    // Avoid PyDict_GetItemString() which calls PyUnicode_FromString(),
+    // memory cannot be allocated on the heap in a signal handler.
+    // Iterate on the dict instead.
+    PyObject *module_names = NULL;
+    pos = 0;
+    while (PyDict_Next(interp->sysdict, &pos, &key, &value)) {
+        if (PyUnicode_Check(key)
+           && PyUnicode_CompareWithASCIIString(key, "module_names") == 0) {
+            module_names = value;
+            break;
+        }
+    }
+    // If we failed to get sys.module_names or it's not a frozenset,
+    // don't exclude stdlib modules.
+    if (module_names != NULL && !PyFrozenSet_Check(module_names)) {
+        module_names = NULL;
+    }
+
+    // List extensions
     int header = 1;
     Py_ssize_t count = 0;
-    Py_ssize_t pos = 0;
-    PyObject *key, *value;
+    pos = 0;
     while (PyDict_Next(modules, &pos, &key, &value)) {
         if (!PyUnicode_Check(key)) {
             continue;
@@ -2526,22 +2545,26 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp)
         if (!_PyModule_IsExtension(value)) {
             continue;
         }
-
-        // Check if it is a stdlib extension module.
         // Use the module name from the sys.modules key,
         // don't attempt to get the module object name.
-        const Py_ssize_t names_len = Py_ARRAY_LENGTH(_Py_module_names);
-        int is_stdlib_mod = 0;
-        for (Py_ssize_t i=0; i < names_len; i++) {
-            const char *name = _Py_module_names[i];
-            if (PyUnicode_CompareWithASCIIString(key, name) == 0) {
-                is_stdlib_mod = 1;
-                break;
+        if (module_names != NULL) {
+            int is_stdlib_ext = 0;
+
+            Py_ssize_t i;
+            PyObject *item;
+            Py_hash_t hash;
+            for (i=0; _PySet_NextEntry(module_names, &i, &item, &hash); ) {
+                if (PyUnicode_Check(item)
+                    && PyUnicode_Compare(key, item) == 0)
+                {
+                    is_stdlib_ext = 1;
+                    break;
+                }
+            }
+            if (is_stdlib_ext) {
+                // Ignore stdlib extension
+                continue;
             }
-        }
-        if (is_stdlib_mod) {
-            // Ignore stdlib extension module.
-            continue;
         }
 
         if (header) {
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 720532e..e2f7e39 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -29,6 +29,7 @@ Data members:
 #include "frameobject.h"          // PyFrame_GetBack()
 #include "pydtrace.h"
 #include "osdefs.h"               // DELIM
+#include "module_names.h"         // _Py_module_names
 #include <locale.h>
 
 #ifdef MS_WINDOWS
@@ -2020,33 +2021,63 @@ static PyMethodDef sys_methods[] = {
     {NULL,              NULL}           /* sentinel */
 };
 
+
 static PyObject *
 list_builtin_module_names(void)
 {
     PyObject *list = PyList_New(0);
-    int i;
-    if (list == NULL)
+    if (list == NULL) {
         return NULL;
-    for (i = 0; PyImport_Inittab[i].name != NULL; i++) {
-        PyObject *name = PyUnicode_FromString(
-            PyImport_Inittab[i].name);
-        if (name == NULL)
-            break;
-        PyList_Append(list, name);
+    }
+    for (Py_ssize_t i = 0; PyImport_Inittab[i].name != NULL; i++) {
+        PyObject *name = PyUnicode_FromString(PyImport_Inittab[i].name);
+        if (name == NULL) {
+            goto error;
+        }
+        if (PyList_Append(list, name) < 0) {
+            Py_DECREF(name);
+            goto error;
+        }
         Py_DECREF(name);
     }
     if (PyList_Sort(list) != 0) {
-        Py_DECREF(list);
-        list = NULL;
+        goto error;
+    }
+    PyObject *tuple = PyList_AsTuple(list);
+    Py_DECREF(list);
+    return tuple;
+
+error:
+    Py_DECREF(list);
+    return NULL;
+}
+
+
+static PyObject *
+list_module_names(void)
+{
+    Py_ssize_t len = Py_ARRAY_LENGTH(_Py_module_names);
+    PyObject *names = PyTuple_New(len);
+    if (names == NULL) {
+        return NULL;
     }
-    if (list) {
-        PyObject *v = PyList_AsTuple(list);
-        Py_DECREF(list);
-        list = v;
+
+    for (Py_ssize_t i = 0; i < len; i++) {
+        PyObject *name = PyUnicode_FromString(_Py_module_names[i]);
+        if (name == NULL) {
+            Py_DECREF(names);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(names, i, name);
     }
-    return list;
+
+    PyObject *set = PyObject_CallFunction((PyObject *)&PyFrozenSet_Type,
+                                          "(O)", names);
+    Py_DECREF(names);
+    return set;
 }
 
+
 /* Pre-initialization support for sys.warnoptions and sys._xoptions
  *
  * Modern internal code paths:
@@ -2753,6 +2784,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict)
     SET_SYS("hash_info", get_hash_info(tstate));
     SET_SYS("maxunicode", PyLong_FromLong(0x10FFFF));
     SET_SYS("builtin_module_names", list_builtin_module_names());
+    SET_SYS("module_names", list_module_names());
 #if PY_BIG_ENDIAN
     SET_SYS_FROM_STRING("byteorder", "big");
 #else