gh-116738: Make `_codecs` module thread-safe (#117530)

The module itself is a thin wrapper around calls to functions in
`Python/codecs.c`, so that's where the meaningful changes happened:

- Move codecs-related state that lives on `PyInterpreterState` to a
  struct declared in `pycore_codecs.h`.

- In free-threaded builds, add a mutex to `codecs_state` to synchronize
  operations on `search_path`. Because `search_path_mutex` is used as a
  normal mutex and not a critical section, we must be extremely careful
  with operations called while holding it.

- The codec registry is explicitly initialized as part of
  `_PyUnicode_InitEncodings` to simplify thread-safety.

2 files changed, 33 insertions, 4 deletions

diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h
index a2a7151..5e2d5c5 100644
--- a/Include/internal/pycore_codecs.h
+++ b/Include/internal/pycore_codecs.h
@@ -8,6 +8,17 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_lock.h"          // PyMutex
+
+/* Initialize codecs-related state for the given interpreter, including
+   registering the first codec search function. Must be called before any other
+   PyCodec-related functions, and while only one thread is active. */
+extern PyStatus _PyCodec_InitRegistry(PyInterpreterState *interp);
+
+/* Finalize codecs-related state for the given interpreter. No PyCodec-related
+   functions other than PyCodec_Unregister() may be called after this. */
+extern void _PyCodec_Fini(PyInterpreterState *interp);
+
 extern PyObject* _PyCodec_Lookup(const char *encoding);
 
 /* Text codec specific encoding and decoding API.
@@ -48,6 +59,26 @@ extern PyObject* _PyCodecInfo_GetIncrementalEncoder(
    PyObject *codec_info,
    const char *errors);
 
+// Per-interpreter state used by codecs.c.
+struct codecs_state {
+    // A list of callable objects used to search for codecs.
+    PyObject *search_path;
+
+    // A dict mapping codec names to codecs returned from a callable in
+    // search_path.
+    PyObject *search_cache;
+
+    // A dict mapping error handling strategies to functions to implement them.
+    PyObject *error_registry;
+
+#ifdef Py_GIL_DISABLED
+    // Used to safely delete a specific item from search_path.
+    PyMutex search_path_mutex;
+#endif
+
+    // Whether or not the rest of the state is initialized.
+    int initialized;
+};
 
 #ifdef __cplusplus
 }
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index d38959e..a2aec6d 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -14,6 +14,7 @@ extern "C" {
 #include "pycore_atexit.h"        // struct atexit_state
 #include "pycore_ceval_state.h"   // struct _ceval_state
 #include "pycore_code.h"          // struct callable_cache
+#include "pycore_codecs.h"        // struct codecs_state
 #include "pycore_context.h"       // struct _Py_context_state
 #include "pycore_crossinterp.h"   // struct _xidregistry
 #include "pycore_dict_state.h"    // struct _Py_dict_state
@@ -182,10 +183,7 @@ struct _is {
      possible to facilitate out-of-process observability
      tools. */
 
-    PyObject *codec_search_path;
-    PyObject *codec_search_cache;
-    PyObject *codec_error_registry;
-    int codecs_initialized;
+    struct codecs_state codecs;
 
     PyConfig config;
     unsigned long feature_flags;