From 6b15e90e8a2238415a6033690fc26a88f3e527b4 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Sun, 1 Oct 2023 01:30:42 +0200
Subject: [PATCH 1/4] gh-109649: Use os.process_cpu_count()

Replace os.cpu_count() with os.process_cpu_count() in modules:

* compileall
* concurrent.futures
* multiprocessing

Replace os.cpu_count() with os.process_cpu_count() in programs:

* _decimal deccheck.py test
* freeze.py
* multissltests.py
* python -m test (regrtest)
* wasm_build.py

Other changes:

* test.pythoninfo logs os.process_cpu_count().
* regrtest gets os.process_cpu_count() / os.cpu_count() in headers.
---
 Doc/library/compileall.rst                                | 2 +-
 Doc/library/concurrent.futures.rst                        | 6 +++++-
 Doc/library/multiprocessing.rst                           | 8 ++++----
 Doc/whatsnew/3.13.rst                                     | 6 ++++++
 Lib/concurrent/futures/process.py                         | 2 +-
 Lib/concurrent/futures/thread.py                          | 4 ++--
 Lib/multiprocessing/pool.py                               | 2 +-
 Lib/test/libregrtest/main.py                              | 2 +-
 Lib/test/libregrtest/utils.py                             | 3 +++
 Lib/test/pythoninfo.py                                    | 1 +
 Lib/test/test_concurrent_futures/test_thread_pool.py      | 2 +-
 .../2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst        | 3 +++
 Modules/_decimal/tests/deccheck.py                        | 2 +-
 Tools/freeze/test/freeze.py                               | 2 +-
 Tools/ssl/multissltests.py                                | 2 +-
 Tools/wasm/wasm_build.py                                  | 2 +-
 16 files changed, 33 insertions(+), 16 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst

diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index a7455aeb0ec1cd..b4723b98f67bb2 100644
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -90,7 +90,7 @@ compile Python sources.
 .. cmdoption:: -j N
 
    Use *N* workers to compile the files within the given directory.
-   If ``0`` is used, then the result of :func:`os.cpu_count()`
+   If ``0`` is used, then the result of :func:`os.process_cpu_count()`
    will be used.
 
 .. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash]
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index 6503d1fcf70a32..f6ece7acb11209 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -188,6 +188,10 @@ And::
       ThreadPoolExecutor now reuses idle worker threads before starting
       *max_workers* worker threads too.
 
+   .. versionchanged:: 3.8
+      Default value of *max_workers* is changed to
+      ``min(32, (os.process_cpu_count() or 1) + 4)``.
+
 
 .. _threadpoolexecutor-example:
 
@@ -243,7 +247,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
 
    An :class:`Executor` subclass that executes calls asynchronously using a pool
    of at most *max_workers* processes.  If *max_workers* is ``None`` or not
-   given, it will default to the number of processors on the machine.
+   given, it will default to :func:`os.process_cpu_count`.
    If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError`
    will be raised.
    On Windows, *max_workers* must be less than or equal to ``61``. If it is not
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index 2f0f1f800fdc94..cbe9efd4a18fde 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -996,13 +996,13 @@ Miscellaneous
 
    This number is not equivalent to the number of CPUs the current process can
    use.  The number of usable CPUs can be obtained with
-   ``len(os.sched_getaffinity(0))``
+   :func:`os.process_cpu_count`.
 
    When the number of CPUs cannot be determined a :exc:`NotImplementedError`
    is raised.
 
    .. seealso::
-      :func:`os.cpu_count`
+      :func:`os.cpu_count` and :func:`os.process_cpu_count`
 
 .. function:: current_process()
 
@@ -2214,7 +2214,7 @@ with the :class:`Pool` class.
    callbacks and has a parallel map implementation.
 
    *processes* is the number of worker processes to use.  If *processes* is
-   ``None`` then the number returned by :func:`os.cpu_count` is used.
+   ``None`` then the number returned by :func:`os.process_cpu_count` is used.
 
    If *initializer* is not ``None`` then each worker process will call
    ``initializer(*initargs)`` when it starts.
@@ -2775,7 +2775,7 @@ worker threads rather than worker processes.
    :meth:`~multiprocessing.pool.Pool.terminate` manually.
 
    *processes* is the number of worker threads to use.  If *processes* is
-   ``None`` then the number returned by :func:`os.cpu_count` is used.
+   ``None`` then the number returned by :func:`os.process_cpu_count` is used.
 
    If *initializer* is not ``None`` then each worker process will call
    ``initializer(*initargs)`` when it starts.
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index 484443a086fdd6..adf0784e4bbf4f 100644
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -91,6 +91,12 @@ Other Language Changes
   of the ``optimize`` argument.
   (Contributed by Irit Katriel in :gh:`108113`).
 
+* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
+  Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
+  default number of worker threads and processes.
+  (Contributed by Victor Stinner in :gh:`109649`.)
+
+
 New Modules
 ===========
 
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index 3990e6b1833d78..ffaffdb8b3d0aa 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -666,7 +666,7 @@ def __init__(self, max_workers=None, mp_context=None,
         _check_system_limits()
 
         if max_workers is None:
-            self._max_workers = os.cpu_count() or 1
+            self._max_workers = os.process_cpu_count() or 1
             if sys.platform == 'win32':
                 self._max_workers = min(_MAX_WINDOWS_WORKERS,
                                         self._max_workers)
diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py
index 3b3a36a5093336..a024033f35fb54 100644
--- a/Lib/concurrent/futures/thread.py
+++ b/Lib/concurrent/futures/thread.py
@@ -139,10 +139,10 @@ def __init__(self, max_workers=None, thread_name_prefix='',
             # * CPU bound task which releases GIL
             # * I/O bound task (which releases GIL, of course)
             #
-            # We use cpu_count + 4 for both types of tasks.
+            # We use process_cpu_count + 4 for both types of tasks.
             # But we limit it to 32 to avoid consuming surprisingly large resource
             # on many core machine.
-            max_workers = min(32, (os.cpu_count() or 1) + 4)
+            max_workers = min(32, (os.process_cpu_count() or 1) + 4)
         if max_workers <= 0:
             raise ValueError("max_workers must be greater than 0")
 
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py
index 4f5d88cb975cb7..f979890170b1a1 100644
--- a/Lib/multiprocessing/pool.py
+++ b/Lib/multiprocessing/pool.py
@@ -200,7 +200,7 @@ def __init__(self, processes=None, initializer=None, initargs=(),
         self._initargs = initargs
 
         if processes is None:
-            processes = os.cpu_count() or 1
+            processes = os.process_cpu_count() or 1
         if processes < 1:
             raise ValueError("Number of processes must be at least 1")
         if maxtasksperchild is not None:
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 19bf2358456036..5f2baac9cba9e0 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -426,7 +426,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
         if self.num_workers < 0:
             # Use all CPUs + 2 extra worker processes for tests
             # that like to sleep
-            self.num_workers = (os.cpu_count() or 1) + 2
+            self.num_workers = (os.process_cpu_count() or 1) + 2
 
         # For a partial run, we do not need to clutter the output.
         if (self.want_header
diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py
index d2c274d9970738..86fb820a23f535 100644
--- a/Lib/test/libregrtest/utils.py
+++ b/Lib/test/libregrtest/utils.py
@@ -546,6 +546,9 @@ def display_header(use_resources: tuple[str, ...],
 
     cpu_count = os.cpu_count()
     if cpu_count:
+        process_cpu_count = os.process_cpu_count()
+        if process_cpu_count and process_cpu_count != cpu_count:
+            cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)"
         print("== CPU count:", cpu_count)
     print("== encodings: locale=%s, FS=%s"
           % (locale.getencoding(), sys.getfilesystemencoding()))
diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py
index 0e7528ef97c5f6..58d906ffc62a53 100644
--- a/Lib/test/pythoninfo.py
+++ b/Lib/test/pythoninfo.py
@@ -239,6 +239,7 @@ def format_attr(attr, value):
         'getresgid',
         'getresuid',
         'getuid',
+        'process_cpu_count',
         'uname',
     ):
         call_func(info_add, 'os.%s' % func, os, func)
diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py
index 812f989d8f3ad2..5926a632aa4bec 100644
--- a/Lib/test/test_concurrent_futures/test_thread_pool.py
+++ b/Lib/test/test_concurrent_futures/test_thread_pool.py
@@ -25,7 +25,7 @@ def record_finished(n):
 
     def test_default_workers(self):
         executor = self.executor_type()
-        expected = min(32, (os.cpu_count() or 1) + 4)
+        expected = min(32, (os.process_cpu_count() or 1) + 4)
         self.assertEqual(executor._max_workers, expected)
 
     def test_saturation(self):
diff --git a/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst
new file mode 100644
index 00000000000000..821e5217b9fae7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst
@@ -0,0 +1,3 @@
+:mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
+Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
+default number of worker threads and processes. Patch by Victor Stinner.
diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py
index edf753f3704a18..bf277dd6879ffe 100644
--- a/Modules/_decimal/tests/deccheck.py
+++ b/Modules/_decimal/tests/deccheck.py
@@ -1301,7 +1301,7 @@ def tfunc():
                 out, _ = p.communicate()
                 write_output(out, p.returncode)
 
-        N = os.cpu_count()
+        N = os.process_cpu_count()
         t = N * [None]
 
         for i in range(N):
diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py
index cdf77c57bbb6ae..9030ad4d4e5f93 100644
--- a/Tools/freeze/test/freeze.py
+++ b/Tools/freeze/test/freeze.py
@@ -130,7 +130,7 @@ def prepare(script=None, outdir=None):
     if not MAKE:
         raise UnsupportedError('make')
 
-    cores = os.cpu_count()
+    cores = os.process_cpu_count()
     if cores and cores >= 3:
         # this test is most often run as part of the whole suite with a lot
         # of other tests running in parallel, from 1-2 vCPU systems up to
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index f066fb52cfd496..bd8c8b28430ab6 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -151,7 +151,7 @@ class AbstractBuilder(object):
     build_template = None
     depend_target = None
     install_target = 'install'
-    jobs = os.cpu_count()
+    jobs = os.process_cpu_count()
 
     module_files = (
         os.path.join(PYTHONROOT, "Modules/_ssl.c"),
diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
index 3558ecd869dfc5..ab85295b59c2e1 100755
--- a/Tools/wasm/wasm_build.py
+++ b/Tools/wasm/wasm_build.py
@@ -516,7 +516,7 @@ def make_cmd(self) -> List[str]:
     def getenv(self) -> Dict[str, Any]:
         """Generate environ dict for platform"""
         env = os.environ.copy()
-        env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
+        env.setdefault("MAKEFLAGS", f"-j{os.process_cpu_count()}")
         platenv = self.host.platform.getenv(self)
         for key, value in platenv.items():
             if value is None:

From 6c336bc0a43ff0453bfe9258bcbb68c94c74b112 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Sun, 1 Oct 2023 01:51:13 +0200
Subject: [PATCH 2/4] wasm_build.py: keep support with old Python

---
 Tools/wasm/wasm_build.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
index ab85295b59c2e1..c0b9999a5dad03 100755
--- a/Tools/wasm/wasm_build.py
+++ b/Tools/wasm/wasm_build.py
@@ -516,7 +516,11 @@ def make_cmd(self) -> List[str]:
     def getenv(self) -> Dict[str, Any]:
         """Generate environ dict for platform"""
         env = os.environ.copy()
-        env.setdefault("MAKEFLAGS", f"-j{os.process_cpu_count()}")
+        if hasattr(os, 'process_cpu_count'):
+            cpu_count = os.process_cpu_count()
+        else:
+            cpu_count = os.cpu_count()
+        env.setdefault("MAKEFLAGS", f"-j{cpu_count}")
         platenv = self.host.platform.getenv(self)
         for key, value in platenv.items():
             if value is None:

From 0cc90be360e9b2bf775982a54a764d014f1f1b3a Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Sun, 1 Oct 2023 01:53:24 +0200
Subject: [PATCH 3/4] multissltests.py: keep support with old Python

---
 Tools/ssl/multissltests.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index bd8c8b28430ab6..120e3883adc795 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -151,7 +151,10 @@ class AbstractBuilder(object):
     build_template = None
     depend_target = None
     install_target = 'install'
-    jobs = os.process_cpu_count()
+    if hasattr(os, 'process_cpu_count'):
+        jobs = os.process_cpu_count()
+    else:
+        jobs = os.cpu_count()
 
     module_files = (
         os.path.join(PYTHONROOT, "Modules/_ssl.c"),

From 6059deec8634e1ecb9675fb12e5ba95b348155a2 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Sun, 1 Oct 2023 02:12:15 +0200
Subject: [PATCH 4/4] Complete the doc

---
 Doc/library/concurrent.futures.rst                          | 6 +++++-
 Doc/library/multiprocessing.rst                             | 4 ++++
 Doc/whatsnew/3.13.rst                                       | 3 ++-
 .../Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst  | 3 ++-
 4 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index f6ece7acb11209..dca51459a2df98 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -188,7 +188,7 @@ And::
       ThreadPoolExecutor now reuses idle worker threads before starting
       *max_workers* worker threads too.
 
-   .. versionchanged:: 3.8
+   .. versionchanged:: 3.13
       Default value of *max_workers* is changed to
       ``min(32, (os.process_cpu_count() or 1) + 4)``.
 
@@ -305,6 +305,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
       different start method. See the :func:`os.fork` documentation for
       further explanation.
 
+   .. versionchanged:: 3.13
+      *max_workers* uses :func:`os.process_cpu_count` by default, instead of
+      :func:`os.cpu_count`.
+
 .. _processpoolexecutor-example:
 
 ProcessPoolExecutor Example
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index cbe9efd4a18fde..d19f911dd7016c 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -2249,6 +2249,10 @@ with the :class:`Pool` class.
    .. versionadded:: 3.4
       *context*
 
+   .. versionchanged:: 3.13
+      *processes* uses :func:`os.process_cpu_count` by default, instead of
+      :func:`os.cpu_count`.
+
    .. note::
 
       Worker processes within a :class:`Pool` typically live for the complete
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index adf0784e4bbf4f..a789084a79c397 100644
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -93,7 +93,8 @@ Other Language Changes
 
 * :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
   Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
-  default number of worker threads and processes.
+  default number of worker threads and processes. Get the CPU affinity
+  if supported.
   (Contributed by Victor Stinner in :gh:`109649`.)
 
 
diff --git a/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst
index 821e5217b9fae7..888fd79962b412 100644
--- a/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst
+++ b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst
@@ -1,3 +1,4 @@
 :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
 Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
-default number of worker threads and processes. Patch by Victor Stinner.
+default number of worker threads and processes. Get the CPU affinity if
+supported. Patch by Victor Stinner.