From 2731913dd5234ff5ab630a3b7f1c98ad79d4d9df Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Mon, 11 Mar 2024 11:02:58 -0400 Subject: gh-116167: Allow disabling the GIL with `PYTHON_GIL=0` or `-X gil=0` (#116338) In free-threaded builds, running with `PYTHON_GIL=0` will now disable the GIL. Follow-up issues track work to re-enable the GIL when loading an incompatible extension, and to disable the GIL by default. In order to support re-enabling the GIL at runtime, all GIL-related data structures are initialized as usual, and disabling the GIL simply sets a flag that causes `take_gil()` and `drop_gil()` to return early. --- Doc/using/cmdline.rst | 18 +++++++++++++++ Include/cpython/initconfig.h | 3 +++ Include/internal/pycore_gil.h | 5 ++++ Include/internal/pycore_initconfig.h | 12 ++++++++++ Lib/subprocess.py | 2 +- Lib/test/_test_embed_set_config.py | 14 +++++++++++ Lib/test/test_cmd_line.py | 33 ++++++++++++++++++++++++++ Lib/test/test_embed.py | 2 ++ Misc/python.man | 4 ++++ Python/ceval_gil.c | 15 ++++++++++++ Python/initconfig.c | 45 ++++++++++++++++++++++++++++++++++++ Python/sysmodule.c | 11 +++++++++ 12 files changed, 163 insertions(+), 1 deletion(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 0a7f636..36cddff 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -559,6 +559,9 @@ Miscellaneous options :mod:`__main__`. This can be used to execute code early during Python initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. + * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, + respectively. Only available in builds configured with + :option:`--disable-gil`. See also :envvar:`PYTHON_GIL`. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -601,6 +604,9 @@ Miscellaneous options .. versionchanged:: 3.13 Added the ``-X cpu_count`` and ``-X presite`` options. + .. versionchanged:: 3.13 + Added the ``-X gil`` option. + .. _using-on-controlling-color: Controlling color @@ -1138,6 +1144,18 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_GIL + + If this variable is set to ``1``, the global interpreter lock (GIL) will be + forced on. Setting it to ``0`` forces the GIL off. + + See also the :option:`-X gil <-X>` command-line option, which takes + precedence over this variable. + + Needs Python configured with the :option:`--disable-gil` build option. + + .. versionadded:: 3.13 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 87c059c..5da5ef9 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -181,6 +181,9 @@ typedef struct PyConfig { int int_max_str_digits; int cpu_count; +#ifdef Py_GIL_DISABLED + int enable_gil; +#endif /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 19b0d23..d36b4c0 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -20,6 +20,11 @@ extern "C" { #define FORCE_SWITCHING struct _gil_runtime_state { +#ifdef Py_GIL_DISABLED + /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime + if, for example, a module that requires the GIL is loaded. */ + int enabled; +#endif /* microseconds (the Python API uses seconds, though) */ unsigned long interval; /* Last PyThreadState holding / having held the GIL. This helps us diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index c869882..1c68161 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,6 +153,18 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; +typedef enum { + /* For now, this means the GIL is enabled. + + gh-116329: This will eventually change to "the GIL is disabled but can + be reenabled by loading an incompatible extension module." */ + _PyConfig_GIL_DEFAULT = -1, + + /* The GIL has been forced off or on, and will not be affected by module loading. */ + _PyConfig_GIL_DISABLE = 0, + _PyConfig_GIL_ENABLE = 1, +} _PyConfigGILEnum; + // Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 20db774..1437bf8 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -350,7 +350,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index 75b6b7d..5ff5218 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -9,6 +9,7 @@ import _testinternalcapi import os import sys import unittest +from test import support from test.support import MS_WINDOWS @@ -211,6 +212,19 @@ class SetConfigTests(unittest.TestCase): self.set_config(use_hash_seed=1, hash_seed=123) self.assertEqual(sys.flags.hash_randomization, 1) + if support.Py_GIL_DISABLED: + self.set_config(enable_gil=-1) + self.assertEqual(sys.flags.gil, None) + self.set_config(enable_gil=0) + self.assertEqual(sys.flags.gil, 0) + self.set_config(enable_gil=1) + self.assertEqual(sys.flags.gil, 1) + else: + # Builds without Py_GIL_DISABLED don't have + # PyConfig.enable_gil. sys.flags.gil is always defined to 1, for + # consistency. + self.assertEqual(sys.flags.gil, 1) + def test_options(self): self.check(warnoptions=[]) self.check(warnoptions=["default", "ignore"]) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 6796dc6..c633f64 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -869,6 +869,39 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_GIL and -X gil only supported in Py_GIL_DISABLED builds") + def test_python_gil(self): + cases = [ + # (env, opt, expected, msg) + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', None, '1', "PYTHON_GIL=1"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + (None, '1', '1', "-X gil=1"), + ] + + code = "import sys; print(sys.flags.gil)" + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') + @unittest.skipUnless(sys.platform == 'win32', 'bpo-32457 only applies on Windows') def test_argv0_normalization(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 55d3acf..ab1d579 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -523,6 +523,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT['_pystats'] = 0 if support.Py_DEBUG: CONFIG_COMPAT['run_presite'] = None + if support.Py_GIL_DISABLED: + CONFIG_COMPAT['enable_gil'] = -1 if MS_WINDOWS: CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, diff --git a/Misc/python.man b/Misc/python.man index 0f5dfa2..4c90c0e 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -607,6 +607,10 @@ output. Setting it to 0 deactivates this behavior. .IP PYTHON_HISTORY This environment variable can be used to set the location of a history file (on Unix, it is \fI~/.python_history\fP by default). +.IP PYTHON_GIL +If this variable is set to 1, the global interpreter lock (GIL) will be forced +on. Setting it to 0 forces the GIL off. Only available in builds configured +with \fB--disable-gil\fP. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index edfc466..d2cd35d 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -219,6 +219,11 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) // XXX assert(tstate == NULL || !tstate->_status.cleared); struct _gil_runtime_state *gil = ceval->gil; +#ifdef Py_GIL_DISABLED + if (!gil->enabled) { + return; + } +#endif if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -294,6 +299,11 @@ take_gil(PyThreadState *tstate) assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; struct _gil_runtime_state *gil = interp->ceval.gil; +#ifdef Py_GIL_DISABLED + if (!gil->enabled) { + return; + } +#endif /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -440,6 +450,11 @@ static void init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { assert(!gil_created(gil)); +#ifdef Py_GIL_DISABLED + // gh-116329: Once it is safe to do so, change this condition to + // (enable_gil == _PyConfig_GIL_ENABLE), so the GIL is disabled by default. + gil->enabled = _PyInterpreterState_GetConfig(interp)->enable_gil != _PyConfig_GIL_DISABLE; +#endif create_gil(gil); assert(gil_created(gil)); interp->ceval.gil = gil; diff --git a/Python/initconfig.c b/Python/initconfig.c index 17c9517..e3a62e5 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -95,6 +95,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(safe_path, BOOL), SPEC(int_max_str_digits, INT), SPEC(cpu_count, INT), +#ifdef Py_GIL_DISABLED + SPEC(enable_gil, INT), +#endif SPEC(pathconfig_warnings, BOOL), SPEC(program_name, WSTR), SPEC(pythonpath_env, WSTR_OPT), @@ -278,6 +281,9 @@ static const char usage_envvars[] = "PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" " os.cpu_count(), and multiprocessing.cpu_count() if set to\n" " a positive integer.\n" +#ifdef Py_GIL_DISABLED +"PYTHON_GIL : When set to 0, disables the GIL.\n" +#endif "PYTHONDEVMODE : enable the development mode.\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" @@ -862,6 +868,9 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef Py_GIL_DISABLED + config->enable_gil = _PyConfig_GIL_DEFAULT; +#endif } @@ -1574,6 +1583,24 @@ config_wstr_to_int(const wchar_t *wstr, int *result) return 0; } +static PyStatus +config_read_gil(PyConfig *config, size_t len, wchar_t first_char) +{ +#ifdef Py_GIL_DISABLED + if (len == 1 && first_char == L'0') { + config->enable_gil = _PyConfig_GIL_DISABLE; + } + else if (len == 1 && first_char == L'1') { + config->enable_gil = _PyConfig_GIL_ENABLE; + } + else { + return _PyStatus_ERR("PYTHON_GIL / -X gil must be \"0\" or \"1\""); + } + return _PyStatus_OK(); +#else + return _PyStatus_ERR("PYTHON_GIL / -X gil are not supported by this build"); +#endif +} static PyStatus config_read_env_vars(PyConfig *config) @@ -1652,6 +1679,15 @@ config_read_env_vars(PyConfig *config) config->safe_path = 1; } + const char *gil = config_get_env(config, "PYTHON_GIL"); + if (gil != NULL) { + size_t len = strlen(gil); + status = config_read_gil(config, len, gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + return _PyStatus_OK(); } @@ -2207,6 +2243,15 @@ config_read(PyConfig *config, int compute_path_config) config->show_ref_count = 1; } + const wchar_t *x_gil = config_get_xoption_value(config, L"gil"); + if (x_gil != NULL) { + size_t len = wcslen(x_gil); + status = config_read_gil(config, len, x_gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + #ifdef Py_STATS if (config_get_xoption(config, L"pystats")) { config->_pystats = 1; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a4161da..cd193c1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3048,6 +3048,7 @@ static PyStructSequence_Field flags_fields[] = { {"warn_default_encoding", "-X warn_default_encoding"}, {"safe_path", "-P"}, {"int_max_str_digits", "-X int_max_str_digits"}, + {"gil", "-X gil"}, {0} }; @@ -3097,6 +3098,16 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) SetFlag(config->warn_default_encoding); SetFlagObj(PyBool_FromLong(config->safe_path)); SetFlag(config->int_max_str_digits); +#ifdef Py_GIL_DISABLED + if (config->enable_gil == _PyConfig_GIL_DEFAULT) { + SetFlagObj(Py_NewRef(Py_None)); + } + else { + SetFlag(config->enable_gil); + } +#else + SetFlagObj(PyLong_FromLong(1)); +#endif #undef SetFlagObj #undef SetFlag return 0; -- cgit v0.12