summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Doc/library/compileall.rst2
-rw-r--r--Doc/library/concurrent.futures.rst10
-rw-r--r--Doc/library/multiprocessing.rst12
-rw-r--r--Doc/whatsnew/3.13.rst7
-rw-r--r--Lib/concurrent/futures/process.py2
-rw-r--r--Lib/concurrent/futures/thread.py4
-rw-r--r--Lib/multiprocessing/pool.py2
-rw-r--r--Lib/test/libregrtest/main.py2
-rw-r--r--Lib/test/libregrtest/utils.py3
-rw-r--r--Lib/test/pythoninfo.py1
-rw-r--r--Lib/test/test_concurrent_futures/test_thread_pool.py2
-rw-r--r--Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst4
-rw-r--r--Modules/_decimal/tests/deccheck.py2
-rw-r--r--Tools/freeze/test/freeze.py2
-rwxr-xr-xTools/ssl/multissltests.py5
-rwxr-xr-xTools/wasm/wasm_build.py6
16 files changed, 50 insertions, 16 deletions
diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index a7455ae..b4723b9 100644
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -90,7 +90,7 @@ compile Python sources.
.. cmdoption:: -j N
Use *N* workers to compile the files within the given directory.
- If ``0`` is used, then the result of :func:`os.cpu_count()`
+ If ``0`` is used, then the result of :func:`os.process_cpu_count()`
will be used.
.. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash]
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index 6503d1f..dca5145 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -188,6 +188,10 @@ And::
ThreadPoolExecutor now reuses idle worker threads before starting
*max_workers* worker threads too.
+ .. versionchanged:: 3.13
+ Default value of *max_workers* is changed to
+ ``min(32, (os.process_cpu_count() or 1) + 4)``.
+
.. _threadpoolexecutor-example:
@@ -243,7 +247,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
An :class:`Executor` subclass that executes calls asynchronously using a pool
of at most *max_workers* processes. If *max_workers* is ``None`` or not
- given, it will default to the number of processors on the machine.
+ given, it will default to :func:`os.process_cpu_count`.
If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError`
will be raised.
On Windows, *max_workers* must be less than or equal to ``61``. If it is not
@@ -301,6 +305,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
different start method. See the :func:`os.fork` documentation for
further explanation.
+ .. versionchanged:: 3.13
+ *max_workers* uses :func:`os.process_cpu_count` by default, instead of
+ :func:`os.cpu_count`.
+
.. _processpoolexecutor-example:
ProcessPoolExecutor Example
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index 2f0f1f8..d19f911 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -996,13 +996,13 @@ Miscellaneous
This number is not equivalent to the number of CPUs the current process can
use. The number of usable CPUs can be obtained with
- ``len(os.sched_getaffinity(0))``
+ :func:`os.process_cpu_count`.
When the number of CPUs cannot be determined a :exc:`NotImplementedError`
is raised.
.. seealso::
- :func:`os.cpu_count`
+ :func:`os.cpu_count` and :func:`os.process_cpu_count`
.. function:: current_process()
@@ -2214,7 +2214,7 @@ with the :class:`Pool` class.
callbacks and has a parallel map implementation.
*processes* is the number of worker processes to use. If *processes* is
- ``None`` then the number returned by :func:`os.cpu_count` is used.
+ ``None`` then the number returned by :func:`os.process_cpu_count` is used.
If *initializer* is not ``None`` then each worker process will call
``initializer(*initargs)`` when it starts.
@@ -2249,6 +2249,10 @@ with the :class:`Pool` class.
.. versionadded:: 3.4
*context*
+ .. versionchanged:: 3.13
+ *processes* uses :func:`os.process_cpu_count` by default, instead of
+ :func:`os.cpu_count`.
+
.. note::
Worker processes within a :class:`Pool` typically live for the complete
@@ -2775,7 +2779,7 @@ worker threads rather than worker processes.
:meth:`~multiprocessing.pool.Pool.terminate` manually.
*processes* is the number of worker threads to use. If *processes* is
- ``None`` then the number returned by :func:`os.cpu_count` is used.
+ ``None`` then the number returned by :func:`os.process_cpu_count` is used.
If *initializer* is not ``None`` then each worker process will call
``initializer(*initargs)`` when it starts.
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index 484443a..a789084 100644
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -91,6 +91,13 @@ Other Language Changes
of the ``optimize`` argument.
(Contributed by Irit Katriel in :gh:`108113`).
+* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
+ Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
+ default number of worker threads and processes. Get the CPU affinity
+ if supported.
+ (Contributed by Victor Stinner in :gh:`109649`.)
+
+
New Modules
===========
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index 3990e6b..ffaffdb 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -666,7 +666,7 @@ class ProcessPoolExecutor(_base.Executor):
_check_system_limits()
if max_workers is None:
- self._max_workers = os.cpu_count() or 1
+ self._max_workers = os.process_cpu_count() or 1
if sys.platform == 'win32':
self._max_workers = min(_MAX_WINDOWS_WORKERS,
self._max_workers)
diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py
index 3b3a36a..a024033 100644
--- a/Lib/concurrent/futures/thread.py
+++ b/Lib/concurrent/futures/thread.py
@@ -139,10 +139,10 @@ class ThreadPoolExecutor(_base.Executor):
# * CPU bound task which releases GIL
# * I/O bound task (which releases GIL, of course)
#
- # We use cpu_count + 4 for both types of tasks.
+ # We use process_cpu_count + 4 for both types of tasks.
# But we limit it to 32 to avoid consuming surprisingly large resource
# on many core machine.
- max_workers = min(32, (os.cpu_count() or 1) + 4)
+ max_workers = min(32, (os.process_cpu_count() or 1) + 4)
if max_workers <= 0:
raise ValueError("max_workers must be greater than 0")
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py
index 4f5d88c..f979890 100644
--- a/Lib/multiprocessing/pool.py
+++ b/Lib/multiprocessing/pool.py
@@ -200,7 +200,7 @@ class Pool(object):
self._initargs = initargs
if processes is None:
- processes = os.cpu_count() or 1
+ processes = os.process_cpu_count() or 1
if processes < 1:
raise ValueError("Number of processes must be at least 1")
if maxtasksperchild is not None:
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 19bf235..5f2baac 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -426,7 +426,7 @@ class Regrtest:
if self.num_workers < 0:
# Use all CPUs + 2 extra worker processes for tests
# that like to sleep
- self.num_workers = (os.cpu_count() or 1) + 2
+ self.num_workers = (os.process_cpu_count() or 1) + 2
# For a partial run, we do not need to clutter the output.
if (self.want_header
diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py
index d2c274d..86fb820 100644
--- a/Lib/test/libregrtest/utils.py
+++ b/Lib/test/libregrtest/utils.py
@@ -546,6 +546,9 @@ def display_header(use_resources: tuple[str, ...],
cpu_count = os.cpu_count()
if cpu_count:
+ process_cpu_count = os.process_cpu_count()
+ if process_cpu_count and process_cpu_count != cpu_count:
+ cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)"
print("== CPU count:", cpu_count)
print("== encodings: locale=%s, FS=%s"
% (locale.getencoding(), sys.getfilesystemencoding()))
diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py
index 0e7528e..58d906f 100644
--- a/Lib/test/pythoninfo.py
+++ b/Lib/test/pythoninfo.py
@@ -239,6 +239,7 @@ def collect_os(info_add):
'getresgid',
'getresuid',
'getuid',
+ 'process_cpu_count',
'uname',
):
call_func(info_add, 'os.%s' % func, os, func)
diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py
index 812f989..5926a63 100644
--- a/Lib/test/test_concurrent_futures/test_thread_pool.py
+++ b/Lib/test/test_concurrent_futures/test_thread_pool.py
@@ -25,7 +25,7 @@ class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, BaseTestCase):
def test_default_workers(self):
executor = self.executor_type()
- expected = min(32, (os.cpu_count() or 1) + 4)
+ expected = min(32, (os.process_cpu_count() or 1) + 4)
self.assertEqual(executor._max_workers, expected)
def test_saturation(self):
diff --git a/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst
new file mode 100644
index 0000000..888fd79
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst
@@ -0,0 +1,4 @@
+:mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
+Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
+default number of worker threads and processes. Get the CPU affinity if
+supported. Patch by Victor Stinner.
diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py
index edf753f..bf277dd 100644
--- a/Modules/_decimal/tests/deccheck.py
+++ b/Modules/_decimal/tests/deccheck.py
@@ -1301,7 +1301,7 @@ if __name__ == '__main__':
out, _ = p.communicate()
write_output(out, p.returncode)
- N = os.cpu_count()
+ N = os.process_cpu_count()
t = N * [None]
for i in range(N):
diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py
index cdf77c5..9030ad4 100644
--- a/Tools/freeze/test/freeze.py
+++ b/Tools/freeze/test/freeze.py
@@ -130,7 +130,7 @@ def prepare(script=None, outdir=None):
if not MAKE:
raise UnsupportedError('make')
- cores = os.cpu_count()
+ cores = os.process_cpu_count()
if cores and cores >= 3:
# this test is most often run as part of the whole suite with a lot
# of other tests running in parallel, from 1-2 vCPU systems up to
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index f066fb5..120e388 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -151,7 +151,10 @@ class AbstractBuilder(object):
build_template = None
depend_target = None
install_target = 'install'
- jobs = os.cpu_count()
+ if hasattr(os, 'process_cpu_count'):
+ jobs = os.process_cpu_count()
+ else:
+ jobs = os.cpu_count()
module_files = (
os.path.join(PYTHONROOT, "Modules/_ssl.c"),
diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
index 3558ecd..c0b9999 100755
--- a/Tools/wasm/wasm_build.py
+++ b/Tools/wasm/wasm_build.py
@@ -516,7 +516,11 @@ class BuildProfile:
def getenv(self) -> Dict[str, Any]:
"""Generate environ dict for platform"""
env = os.environ.copy()
- env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
+ if hasattr(os, 'process_cpu_count'):
+ cpu_count = os.process_cpu_count()
+ else:
+ cpu_count = os.cpu_count()
+ env.setdefault("MAKEFLAGS", f"-j{cpu_count}")
platenv = self.host.platform.getenv(self)
for key, value in platenv.items():
if value is None:
generated by cgit v0.12 at 2025-01-10 20:17:25 (GMT)