From 41be17016ef71d39725a088f4561232c323e0e7b Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 01:44:58 +0900 Subject: [PATCH 01/68] gh-109595: Add -Xcpu_count= cmdline for container users --- Doc/library/os.rst | 4 +++ Doc/using/cmdline.rst | 7 ++++ Include/cpython/initconfig.h | 1 + Lib/test/test_cmd_line.py | 8 +++++ ...-09-22-01-44-53.gh-issue-109595.fVINgD.rst | 2 ++ Modules/posixmodule.c | 5 +++ Python/initconfig.c | 32 +++++++++++++++++++ 7 files changed, 59 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index c67b966f777db8..14c6a6491e5d7b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5164,6 +5164,10 @@ Miscellaneous System Information .. versionadded:: 3.4 + .. versionchanged:: 3.13 + if ``-X cpu_count=`` is given, :func:`cpu_count` will return the overrided + value. + .. function:: getloadavg() diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 921b6a6961c7b2..cc1658c9a200fd 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -546,6 +546,10 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. + * ``-X cpu_count=`` will override the number of cpu count from system. + If this option is provided, :func:`os.cpu_count` will return the overrided value. + This option will be useful for users who need to limit cpu resource of the container system. + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -593,6 +597,9 @@ Miscellaneous options .. versionadded:: 3.12 The ``-X perf`` option. + .. versionadded:: 3.13 + The ``-X cpu_count`` option + Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index ee130467824daa..2d5c7ee8e85713 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -179,6 +179,7 @@ typedef struct PyConfig { int use_frozen_modules; int safe_path; int int_max_str_digits; + int cpu_count; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index e88b7c8572d9e8..7467448a85b426 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -901,6 +901,14 @@ def res2int(res): ) self.assertEqual(res2int(res), (6000, 6000)) + def test_cpu_count(self): + def res2int(res): + out = res.out.strip().decode("utf-8") + return tuple(int(i) for i in out.split()) + code = "import os; print(os.cpu_count())" + res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) + self.assertEqual(res2int(res), (4321,)) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst new file mode 100644 index 00000000000000..7646fc4b0b7948 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst @@ -0,0 +1,2 @@ +Add `-Xcpu_count=` cmdline option for limiting system cpu resources from +Python program. Patch by Donghee Na. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 096aa043514c85..a01051f91feb7b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14349,6 +14349,11 @@ static PyObject * os_cpu_count_impl(PyObject *module) /*[clinic end generated code: output=5fc29463c3936a9c input=e7c8f4ba6dbbadd3]*/ { + PyThreadState *tstate = _PyThreadState_GET(); + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->cpu_count > 0) { + return PyLong_FromLong(config->cpu_count); + } int ncpu = 0; #ifdef MS_WINDOWS #ifdef MS_WINDOWS_DESKTOP diff --git a/Python/initconfig.c b/Python/initconfig.c index a0467f51d4834e..9608a9a93d479d 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -732,6 +732,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->int_max_str_digits = -1; config->_is_python_build = 0; config->code_debug_ranges = 1; + config->cpu_count = -1; } @@ -789,6 +790,7 @@ PyConfig_InitIsolatedConfig(PyConfig *config) config->int_max_str_digits = _PY_LONG_DEFAULT_MAX_STR_DIGITS; config->safe_path = 1; config->pathconfig_warnings = 0; + config->cpu_count = -1; #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif @@ -959,6 +961,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTRLIST(orig_argv); COPY_ATTR(_is_python_build); COPY_ATTR(int_max_str_digits); + COPY_ATTR(cpu_count); #ifdef Py_STATS COPY_ATTR(_pystats); #endif @@ -1678,6 +1681,28 @@ config_read_env_vars(PyConfig *config) return _PyStatus_OK(); } +static PyStatus +config_init_cpu_count(PyConfig *config) +{ + int cpu_count = -1; + const wchar_t *xoption = config_get_xoption(config, L"cpu_count"); + if (xoption) { + const wchar_t *sep = wcschr(xoption, L'='); + if (sep) { + if (config_wstr_to_int(sep + 1, &cpu_count) < 0 || cpu_count < 1) { + goto error; + } + } + else { + goto error; + } + config->cpu_count = cpu_count; + } + return _PyStatus_OK(); +error: + return _PyStatus_ERR("-X cpu_count= invalid cpu_count format"); +} + static PyStatus config_init_perf_profiling(PyConfig *config) { @@ -1860,6 +1885,13 @@ config_read_complex_options(PyConfig *config) } } + if (config->cpu_count < 0) { + status = config_init_cpu_count(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + if (config->pycache_prefix == NULL) { status = config_init_pycache_prefix(config); if (_PyStatus_EXCEPTION(status)) { From f7a7428efd387c9fae0cdd08c6538250afb276e4 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 01:47:51 +0900 Subject: [PATCH 02/68] Check style --- .../2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst index 7646fc4b0b7948..6ca3e87a375be4 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst @@ -1,2 +1,2 @@ -Add `-Xcpu_count=` cmdline option for limiting system cpu resources from +Add ``-Xcpu_count=`` cmdline option for limiting system cpu resources from Python program. Patch by Donghee Na. From 7009bbe8d16f7c3be79d05e884a7a5e4699b1132 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 01:54:46 +0900 Subject: [PATCH 03/68] Add help --- Python/initconfig.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 9608a9a93d479d..d08c725a7a2666 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -130,7 +130,10 @@ The following implementation-specific options are available:\n\ \n\ -X int_max_str_digits=number: limit the size of int<->str conversions.\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ - The default is sys.int_info.default_max_str_digits. 0 disables." + The default is sys.int_info.default_max_str_digits. 0 disables.\n\ +\n\ +-X cpu_count=number: limit cpu count of os.cpu_count.\n\ + This helps for users who need to limit cpu resource of the container system." #ifdef Py_STATS "\n\ From d1f91d8a336c679660a8d20f601d0c7338149093 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:23:40 +0900 Subject: [PATCH 04/68] Apply suggestions from code review Co-authored-by: Victor Stinner --- Doc/library/os.rst | 4 ++-- Doc/using/cmdline.rst | 4 ++-- Include/cpython/initconfig.h | 2 ++ Modules/posixmodule.c | 3 +-- Python/initconfig.c | 5 +++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 14c6a6491e5d7b..327ccdc44e3cb3 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5165,8 +5165,8 @@ Miscellaneous System Information .. versionadded:: 3.4 .. versionchanged:: 3.13 - if ``-X cpu_count=`` is given, :func:`cpu_count` will return the overrided - value. + if :samp:`-X cpu_count={n}` is given, :func:`cpu_count` will return the overrided + value *n*. .. function:: getloadavg() diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index cc1658c9a200fd..8dc6fbaff72c20 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -546,7 +546,7 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. - * ``-X cpu_count=`` will override the number of cpu count from system. + * :samp:`-X cpu_count={n}` will override the number of CPU count from system. If this option is provided, :func:`os.cpu_count` will return the overrided value. This option will be useful for users who need to limit cpu resource of the container system. @@ -598,7 +598,7 @@ Miscellaneous options The ``-X perf`` option. .. versionadded:: 3.13 - The ``-X cpu_count`` option + The ``-X cpu_count`` option. Options you shouldn't use diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 2d5c7ee8e85713..11abb23c648b3e 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -179,6 +179,8 @@ typedef struct PyConfig { int use_frozen_modules; int safe_path; int int_max_str_digits; + + // Python 3.13 int cpu_count; /* --- Path configuration inputs ------------ */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a01051f91feb7b..78c261e134e01d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14349,8 +14349,7 @@ static PyObject * os_cpu_count_impl(PyObject *module) /*[clinic end generated code: output=5fc29463c3936a9c input=e7c8f4ba6dbbadd3]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + const PyConfig *config = _Py_GetConfig(); if (config->cpu_count > 0) { return PyLong_FromLong(config->cpu_count); } diff --git a/Python/initconfig.c b/Python/initconfig.c index d08c725a7a2666..1aca99b83a7594 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -133,7 +133,7 @@ The following implementation-specific options are available:\n\ The default is sys.int_info.default_max_str_digits. 0 disables.\n\ \n\ -X cpu_count=number: limit cpu count of os.cpu_count.\n\ - This helps for users who need to limit cpu resource of the container system." + This helps for users who need to limit CPU resources of a container system." #ifdef Py_STATS "\n\ @@ -1702,8 +1702,9 @@ config_init_cpu_count(PyConfig *config) config->cpu_count = cpu_count; } return _PyStatus_OK(); + error: - return _PyStatus_ERR("-X cpu_count= invalid cpu_count format"); + return _PyStatus_ERR("-X cpu_count=n option: n is missing or an invalid number"); } static PyStatus From 8c92ed6a40908ca69691b7e2fbbefa89d1796e77 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:24:23 +0900 Subject: [PATCH 05/68] Apply suggestions from code review Co-authored-by: Victor Stinner --- Doc/using/cmdline.rst | 2 +- .../2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst | 2 +- Python/initconfig.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 8dc6fbaff72c20..94285b44dfd4d8 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -548,7 +548,7 @@ Miscellaneous options is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. * :samp:`-X cpu_count={n}` will override the number of CPU count from system. If this option is provided, :func:`os.cpu_count` will return the overrided value. - This option will be useful for users who need to limit cpu resource of the container system. + This option will be useful for users who need to limit CPU resources of a container system. It also allows passing arbitrary values and retrieving them through the diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst index 6ca3e87a375be4..3b4b091c39aadc 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst @@ -1,2 +1,2 @@ -Add ``-Xcpu_count=`` cmdline option for limiting system cpu resources from +Add :samp:`-Xcpu_count={n}` command line option for limiting system CPU resources from Python program. Patch by Donghee Na. diff --git a/Python/initconfig.c b/Python/initconfig.c index 1aca99b83a7594..9f9a346efefa5b 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -132,7 +132,7 @@ The following implementation-specific options are available:\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ The default is sys.int_info.default_max_str_digits. 0 disables.\n\ \n\ --X cpu_count=number: limit cpu count of os.cpu_count.\n\ +-X cpu_count=n: limit CPU count of os.cpu_count().\n\ This helps for users who need to limit CPU resources of a container system." #ifdef Py_STATS From c27bdfca61f5d8f495adf9fba06bc54792c8f82c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:26:55 +0900 Subject: [PATCH 06/68] Check style --- Python/initconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 9f9a346efefa5b..57c52b0f2a1687 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1702,7 +1702,7 @@ config_init_cpu_count(PyConfig *config) config->cpu_count = cpu_count; } return _PyStatus_OK(); - + error: return _PyStatus_ERR("-X cpu_count=n option: n is missing or an invalid number"); } From b13e5ee3b8547edefe28c0766a5323915d7ea46e Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:31:23 +0900 Subject: [PATCH 07/68] Address code review --- Doc/using/cmdline.rst | 1 + Lib/test/test_cmd_line.py | 5 +---- Python/initconfig.c | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 94285b44dfd4d8..4131bc40247697 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -548,6 +548,7 @@ Miscellaneous options is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. * :samp:`-X cpu_count={n}` will override the number of CPU count from system. If this option is provided, :func:`os.cpu_count` will return the overrided value. + And *n* must be greater equal then 1. This option will be useful for users who need to limit CPU resources of a container system. diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 7467448a85b426..0ad431c40c446b 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -902,12 +902,9 @@ def res2int(res): self.assertEqual(res2int(res), (6000, 6000)) def test_cpu_count(self): - def res2int(res): - out = res.out.strip().decode("utf-8") - return tuple(int(i) for i in out.split()) code = "import os; print(os.cpu_count())" res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) - self.assertEqual(res2int(res), (4321,)) + self.assertEqual(res.out.strip().decode("utf-8"), '4321') @unittest.skipIf(interpreter_requires_environment(), diff --git a/Python/initconfig.c b/Python/initconfig.c index 57c52b0f2a1687..1fd5d45a620864 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -793,7 +793,6 @@ PyConfig_InitIsolatedConfig(PyConfig *config) config->int_max_str_digits = _PY_LONG_DEFAULT_MAX_STR_DIGITS; config->safe_path = 1; config->pathconfig_warnings = 0; - config->cpu_count = -1; #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif From 45fce16d31118a8f5706b6d416911b2b0e611e50 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:32:16 +0900 Subject: [PATCH 08/68] Address code review --- Python/initconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 1fd5d45a620864..df9574f9435146 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -132,7 +132,7 @@ The following implementation-specific options are available:\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ The default is sys.int_info.default_max_str_digits. 0 disables.\n\ \n\ --X cpu_count=n: limit CPU count of os.cpu_count().\n\ +-X cpu_count=n: override CPU count of os.cpu_count().\n\ This helps for users who need to limit CPU resources of a container system." #ifdef Py_STATS From 49a48e4645f7a89ca8abde26fef71c188b183e09 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:36:06 +0900 Subject: [PATCH 09/68] Address code review --- Python/initconfig.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/initconfig.c b/Python/initconfig.c index df9574f9435146..9f88d69d1257ab 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -636,6 +636,8 @@ config_check_consistency(const PyConfig *config) assert(config->_is_python_build >= 0); assert(config->safe_path >= 0); assert(config->int_max_str_digits >= 0); + // cpu_count can be -1 if the user doesn't override it. + assert(config->cpu_count != 0); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). #ifdef Py_STATS @@ -1074,6 +1076,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_INT(safe_path); SET_ITEM_INT(_is_python_build); SET_ITEM_INT(int_max_str_digits); + SET_ITEM_INT(cpu_count); #ifdef Py_STATS SET_ITEM_INT(_pystats); #endif @@ -1384,6 +1387,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(safe_path); GET_UINT(_is_python_build); GET_INT(int_max_str_digits); + GET_INT(cpu_count); #ifdef Py_STATS GET_UINT(_pystats); #endif From 829a8e87fbc2d27735315a1b7f7faddb8057ba32 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:39:47 +0900 Subject: [PATCH 10/68] Address code review --- Programs/_testembed.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index bc991020d0fa77..dde4b33b13f300 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -715,6 +715,7 @@ static int test_init_from_config(void) putenv("PYTHONINTMAXSTRDIGITS=6666"); config.int_max_str_digits = 31337; + config.cpu_count = 4; init_from_config_clear(&config); From cfb33a44773f02a4b9474982aef7c2a9ae32274b Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:48:27 +0900 Subject: [PATCH 11/68] Fix test --- Lib/test/test_embed.py | 3 +++ Programs/_testembed.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7f1a4e665f3b5d..5492cf41aae912 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -448,6 +448,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'use_hash_seed': 0, 'hash_seed': 0, 'int_max_str_digits': sys.int_info.default_max_str_digits, + 'cpu_count': -1, 'faulthandler': 0, 'tracemalloc': 0, 'perf_profiling': 0, @@ -894,6 +895,7 @@ def test_init_from_config(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': 1, 'int_max_str_digits': 31337, + 'cpu_count': -1, 'check_hash_pycs_mode': 'always', 'pathconfig_warnings': 0, @@ -966,6 +968,7 @@ def test_init_python_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': 1, 'int_max_str_digits': 4567, + 'cpu_count': -1, } if Py_STATS: config['_pystats'] = 1 diff --git a/Programs/_testembed.c b/Programs/_testembed.c index dde4b33b13f300..bede32205ca057 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -715,7 +715,7 @@ static int test_init_from_config(void) putenv("PYTHONINTMAXSTRDIGITS=6666"); config.int_max_str_digits = 31337; - config.cpu_count = 4; + config.cpu_count = -1; init_from_config_clear(&config); From 95a2173e9a3d8acfc1d1156267ad0dc8c65c60fb Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:49:56 +0900 Subject: [PATCH 12/68] Update NEWS.d --- .../2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst index 3b4b091c39aadc..9e3d768f820e7e 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst @@ -1,2 +1,2 @@ -Add :samp:`-Xcpu_count={n}` command line option for limiting system CPU resources from +Add :samp:`-Xcpu_count={n}` command line option for overriding system CPU resources from Python program. Patch by Donghee Na. From 0394d1d75b9f1437eae1c82dcc31b06e40c850f0 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:56:57 +0900 Subject: [PATCH 13/68] Address code review --- Lib/test/test_embed.py | 2 +- Programs/_testembed.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 5492cf41aae912..58903c05ec5465 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -895,7 +895,7 @@ def test_init_from_config(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': 1, 'int_max_str_digits': 31337, - 'cpu_count': -1, + 'cpu_count': 4, 'check_hash_pycs_mode': 'always', 'pathconfig_warnings': 0, diff --git a/Programs/_testembed.c b/Programs/_testembed.c index bede32205ca057..dde4b33b13f300 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -715,7 +715,7 @@ static int test_init_from_config(void) putenv("PYTHONINTMAXSTRDIGITS=6666"); config.int_max_str_digits = 31337; - config.cpu_count = -1; + config.cpu_count = 4; init_from_config_clear(&config); From f4a3f01393a5d8f4a4b7aa9043a53adf554fb7da Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 02:58:15 +0900 Subject: [PATCH 14/68] Update --- Lib/test/test_embed.py | 2 +- Programs/_testembed.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 58903c05ec5465..4821a2bd8e79ac 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -895,7 +895,7 @@ def test_init_from_config(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': 1, 'int_max_str_digits': 31337, - 'cpu_count': 4, + 'cpu_count': 4321, 'check_hash_pycs_mode': 'always', 'pathconfig_warnings': 0, diff --git a/Programs/_testembed.c b/Programs/_testembed.c index dde4b33b13f300..cd4a56d16b9fa1 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -715,7 +715,7 @@ static int test_init_from_config(void) putenv("PYTHONINTMAXSTRDIGITS=6666"); config.int_max_str_digits = 31337; - config.cpu_count = 4; + config.cpu_count = 4321; init_from_config_clear(&config); From 60a28fe70d336a9192dcf95982941d80b0b7c765 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:01:45 +0900 Subject: [PATCH 15/68] Update --- Doc/using/cmdline.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4131bc40247697..b15c5e9bb631c3 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -546,8 +546,7 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. - * :samp:`-X cpu_count={n}` will override the number of CPU count from system. - If this option is provided, :func:`os.cpu_count` will return the overrided value. + * :samp:`-X cpu_count={n}` will override func:`os.cpu_count`. And *n* must be greater equal then 1. This option will be useful for users who need to limit CPU resources of a container system. From 7e5595c202d4a9fb2e16471e32c2a9fa5d52d65e Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:02:14 +0900 Subject: [PATCH 16/68] nit --- Doc/using/cmdline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b15c5e9bb631c3..9d7230ee355ca5 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -546,7 +546,7 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. - * :samp:`-X cpu_count={n}` will override func:`os.cpu_count`. + * :samp:`-X cpu_count={n}` will override :func:`os.cpu_count`. And *n* must be greater equal then 1. This option will be useful for users who need to limit CPU resources of a container system. From 86780126a72ab6b8fa5cea2f7a1d30172fecf81c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:04:47 +0900 Subject: [PATCH 17/68] Update --- Lib/test/test_embed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4821a2bd8e79ac..261c7cf1ccf2d3 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -968,7 +968,6 @@ def test_init_python_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': 1, 'int_max_str_digits': 4567, - 'cpu_count': -1, } if Py_STATS: config['_pystats'] = 1 From cbc5484e3b402d6bd020120bd6c6cdd6db38d1a1 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:09:35 +0900 Subject: [PATCH 18/68] Add PYTHONCPUCOUNT --- Python/initconfig.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Python/initconfig.c b/Python/initconfig.c index 9f88d69d1257ab..86aa8590feb951 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1691,6 +1691,15 @@ static PyStatus config_init_cpu_count(PyConfig *config) { int cpu_count = -1; + const char *env = config_get_env(config, "PYTHONCPUCOUNT"); + if (env) { + if (_Py_str_to_int(env, &cpu_count) != 0) { + cpu_count = -1; + } + if (cpu_count >= 1) { + config->cpu_count = cpu_count; + } + } const wchar_t *xoption = config_get_xoption(config, L"cpu_count"); if (xoption) { const wchar_t *sep = wcschr(xoption, L'='); From 4622b60f1d1978b48d04dae13cae46e53e6346b1 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:13:32 +0900 Subject: [PATCH 19/68] Add PYTHONCPUCOUNT --- Doc/using/cmdline.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9d7230ee355ca5..1fb6bbd8747628 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -549,6 +549,7 @@ Miscellaneous options * :samp:`-X cpu_count={n}` will override :func:`os.cpu_count`. And *n* must be greater equal then 1. This option will be useful for users who need to limit CPU resources of a container system. + See also :envar:`PYTHONCPUCOUNT`. It also allows passing arbitrary values and retrieving them through the @@ -1070,6 +1071,15 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHONCPUCOUNT + + If this variable is set to a positive integer, it will override + override :func:`os.cpu_count` + + See also the :samp:`-X cpu_count={n}` command-line option. + + .. versionadded:: 3.13 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ From 50f0178849f1fecb4f313514c0bee979d5898e74 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:14:26 +0900 Subject: [PATCH 20/68] Apply suggestions from code review Co-authored-by: Victor Stinner --- Doc/using/cmdline.rst | 2 +- .../2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 1fb6bbd8747628..f0eb7b4fee7ba3 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -548,7 +548,7 @@ Miscellaneous options is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. * :samp:`-X cpu_count={n}` will override :func:`os.cpu_count`. And *n* must be greater equal then 1. - This option will be useful for users who need to limit CPU resources of a container system. + This option is useful for users who need to limit CPU resources of a container system. See also :envar:`PYTHONCPUCOUNT`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst index 9e3d768f820e7e..4b27901b4aa4a5 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst @@ -1,2 +1,3 @@ -Add :samp:`-Xcpu_count={n}` command line option for overriding system CPU resources from -Python program. Patch by Donghee Na. +Add :samp:`-Xcpu_count={n}` command line option to override CPU count of :func:`os.cpu_count`. +This option is useful for users who need to limit CPU resources of a container system. +Patch by Donghee Na. From 5c3ac68b288e249cabe5a182f18309ff3ce3037a Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:15:30 +0900 Subject: [PATCH 21/68] Update Python/initconfig.c Co-authored-by: Victor Stinner --- Python/initconfig.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 86aa8590feb951..cfa915308aa8e8 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1716,7 +1716,8 @@ config_init_cpu_count(PyConfig *config) return _PyStatus_OK(); error: - return _PyStatus_ERR("-X cpu_count=n option: n is missing or an invalid number"); + return _PyStatus_ERR("-X cpu_count=n option: n is missing or an invalid number, "); + "n must be greater than 0"); } static PyStatus From a276bfd96855538f03746b12bcffc3526508183d Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:19:48 +0900 Subject: [PATCH 22/68] Fix --- Python/initconfig.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index cfa915308aa8e8..d5966cef60b059 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1716,8 +1716,8 @@ config_init_cpu_count(PyConfig *config) return _PyStatus_OK(); error: - return _PyStatus_ERR("-X cpu_count=n option: n is missing or an invalid number, "); - "n must be greater than 0"); + return _PyStatus_ERR("-X cpu_count=n option: n is missing or an invalid number, " + "n must be greater than 0"); } static PyStatus From 9c582be6a1ccf74a1e5d1fc82415c3afcc407276 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:24:06 +0900 Subject: [PATCH 23/68] Update whatsnew --- Doc/whatsnew/3.13.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index b4411a587c8bc7..72c99e8d6eee96 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -163,6 +163,15 @@ opcode documented or exposed through ``dis``, and were not intended to be used externally. + +os +-- + +* :func:`cpu_count` can be overrided through the new + environment variable :envvar:`PYTHONCPUCOUNT`, the new command-line option + :option:`-X cpu_count={n}`. This option is useful for users who need to limit + CPU resources of a container system.(Contributed by Donghee Na in :gh:`109595`) + pathlib ------- From 35b952f993c783315ba7c7b2d2e728d4517846ba Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:25:48 +0900 Subject: [PATCH 24/68] nit --- Doc/whatsnew/3.13.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 72c99e8d6eee96..9cb0db05bb24ae 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -163,7 +163,6 @@ opcode documented or exposed through ``dis``, and were not intended to be used externally. - os -- From f04ea58a135cb7a49d9b87d93d1eedb9985805c9 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:29:40 +0900 Subject: [PATCH 25/68] Check style --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9cb0db05bb24ae..bdb8c785e68f3a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -168,7 +168,7 @@ os * :func:`cpu_count` can be overrided through the new environment variable :envvar:`PYTHONCPUCOUNT`, the new command-line option - :option:`-X cpu_count={n}`. This option is useful for users who need to limit + :option:`-X cpu_count={n}`. This option is useful for users who need to limit CPU resources of a container system.(Contributed by Donghee Na in :gh:`109595`) pathlib From 7ecf7056fdef3d62950ac6ff81bd8765f8ccdd00 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:33:18 +0900 Subject: [PATCH 26/68] Fix docs --- Doc/using/cmdline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index f0eb7b4fee7ba3..0e473d655b1713 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -549,7 +549,7 @@ Miscellaneous options * :samp:`-X cpu_count={n}` will override :func:`os.cpu_count`. And *n* must be greater equal then 1. This option is useful for users who need to limit CPU resources of a container system. - See also :envar:`PYTHONCPUCOUNT`. + See also :envvar:`PYTHONCPUCOUNT`. It also allows passing arbitrary values and retrieving them through the From 18dcd44db21ebb46ce4e061c7eb9a5e7cd622b7c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:38:53 +0900 Subject: [PATCH 27/68] Update doc --- Doc/library/os.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 327ccdc44e3cb3..a5465be82a72cb 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5165,8 +5165,8 @@ Miscellaneous System Information .. versionadded:: 3.4 .. versionchanged:: 3.13 - if :samp:`-X cpu_count={n}` is given, :func:`cpu_count` will return the overrided - value *n*. + if :samp:`-X cpu_count={n}` is given or :envvar:`PYTHONCPUCOUNT` is set, + :func:`cpu_count` will return the overrided value *n*. .. function:: getloadavg() From b344f4f8dd0082986120ab85520e7b9bc751ec91 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:39:22 +0900 Subject: [PATCH 28/68] nit --- Doc/library/os.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index a5465be82a72cb..1fd2cde9bd61c4 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5165,7 +5165,7 @@ Miscellaneous System Information .. versionadded:: 3.4 .. versionchanged:: 3.13 - if :samp:`-X cpu_count={n}` is given or :envvar:`PYTHONCPUCOUNT` is set, + If :samp:`-X cpu_count={n}` is given or :envvar:`PYTHONCPUCOUNT` is set, :func:`cpu_count` will return the overrided value *n*. From 8069d216b3ab32f017c7e391d02c6ec4b0eb2fb9 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:41:13 +0900 Subject: [PATCH 29/68] Update --- Doc/library/os.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 1fd2cde9bd61c4..5bc9b7683e2dc7 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5166,7 +5166,7 @@ Miscellaneous System Information .. versionchanged:: 3.13 If :samp:`-X cpu_count={n}` is given or :envvar:`PYTHONCPUCOUNT` is set, - :func:`cpu_count` will return the overrided value *n*. + :func:`cpu_count` returns the overrided value *n*. .. function:: getloadavg() From c70bc82241c00bdb147f29e14db5f7df655ac934 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:50:04 +0900 Subject: [PATCH 30/68] Update --- Doc/using/cmdline.rst | 8 ++++---- Doc/whatsnew/3.13.rst | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 0e473d655b1713..cbaa297798107a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -546,8 +546,8 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. - * :samp:`-X cpu_count={n}` will override :func:`os.cpu_count`. - And *n* must be greater equal then 1. + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`. + And *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHONCPUCOUNT`. @@ -1073,8 +1073,8 @@ conflict. .. envvar:: PYTHONCPUCOUNT - If this variable is set to a positive integer, it will override - override :func:`os.cpu_count` + If this variable is set to a positive integer, it overrides + :func:`os.cpu_count` See also the :samp:`-X cpu_count={n}` command-line option. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bdb8c785e68f3a..629858fa48de91 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -166,9 +166,9 @@ opcode os -- -* :func:`cpu_count` can be overrided through the new +* :func:`os.cpu_count` can be overrided through the new environment variable :envvar:`PYTHONCPUCOUNT`, the new command-line option - :option:`-X cpu_count={n}`. This option is useful for users who need to limit + :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system.(Contributed by Donghee Na in :gh:`109595`) pathlib From c8abc2983d62c844d27d5b98844ed5f9da78e8a2 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 03:52:20 +0900 Subject: [PATCH 31/68] Update --- Doc/using/cmdline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index cbaa297798107a..b939302286ed83 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -547,7 +547,7 @@ Miscellaneous options will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`. - And *n* must be greater than or equal to 1. + And *n* must be greater than 0. This option is useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHONCPUCOUNT`. From 2ac6901e4351a837180d928b96e460bd53aafe3c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 08:24:13 +0900 Subject: [PATCH 32/68] Address code review --- Doc/using/cmdline.rst | 2 +- Doc/whatsnew/3.13.rst | 2 +- Modules/posixmodule.c | 1 + Python/initconfig.c | 3 ++- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b939302286ed83..4e170ccb230b42 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -547,7 +547,7 @@ Miscellaneous options will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`. - And *n* must be greater than 0. + *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHONCPUCOUNT`. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 629858fa48de91..2550ecd2cc7b09 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -167,7 +167,7 @@ os -- * :func:`os.cpu_count` can be overrided through the new - environment variable :envvar:`PYTHONCPUCOUNT`, the new command-line option + environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system.(Contributed by Donghee Na in :gh:`109595`) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 78c261e134e01d..4b9b03510d7795 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14353,6 +14353,7 @@ os_cpu_count_impl(PyObject *module) if (config->cpu_count > 0) { return PyLong_FromLong(config->cpu_count); } + int ncpu = 0; #ifdef MS_WINDOWS #ifdef MS_WINDOWS_DESKTOP diff --git a/Python/initconfig.c b/Python/initconfig.c index d5966cef60b059..d38cadcaadc33f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1690,9 +1690,9 @@ config_read_env_vars(PyConfig *config) static PyStatus config_init_cpu_count(PyConfig *config) { - int cpu_count = -1; const char *env = config_get_env(config, "PYTHONCPUCOUNT"); if (env) { + int cpu_count = -1; if (_Py_str_to_int(env, &cpu_count) != 0) { cpu_count = -1; } @@ -1702,6 +1702,7 @@ config_init_cpu_count(PyConfig *config) } const wchar_t *xoption = config_get_xoption(config, L"cpu_count"); if (xoption) { + int cpu_count = -1; const wchar_t *sep = wcschr(xoption, L'='); if (sep) { if (config_wstr_to_int(sep + 1, &cpu_count) < 0 || cpu_count < 1) { From f0a3ebfe2c4e9f89c393260443b489533eb01067 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 22 Sep 2023 08:37:47 +0900 Subject: [PATCH 33/68] Address code review --- Python/initconfig.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/initconfig.c b/Python/initconfig.c index d38cadcaadc33f..29df4003112d74 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1700,6 +1700,7 @@ config_init_cpu_count(PyConfig *config) config->cpu_count = cpu_count; } } + const wchar_t *xoption = config_get_xoption(config, L"cpu_count"); if (xoption) { int cpu_count = -1; From 89d8bb203e8a0e8172db6cc3d82afdc7f1341c1a Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 24 Sep 2023 16:50:49 +0900 Subject: [PATCH 34/68] Address Victor's suggestion --- Doc/using/cmdline.rst | 2 ++ Lib/test/test_cmd_line.py | 5 +++++ Python/initconfig.c | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4e170ccb230b42..5c3ae98b54f6ca 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -550,6 +550,8 @@ Miscellaneous options *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHONCPUCOUNT`. + If *n* is "default", it follows default :func:`os.cpu_count` even if :envvar:`PYTHONCPUCOUNT` + is set. It also allows passing arbitrary values and retrieving them through the diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 0ad431c40c446b..cb178c13481f52 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -906,6 +906,11 @@ def test_cpu_count(self): res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) self.assertEqual(res.out.strip().decode("utf-8"), '4321') + def test_cpu_count_default(self): + code = "import os; print(os.cpu_count())" + res = assert_python_ok('-X', 'cpu_count=default', '-c', code) + self.assertEqual(int(res.out.strip().decode("utf-8")), os.cpu_count()) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') diff --git a/Python/initconfig.c b/Python/initconfig.c index 29df4003112d74..6e3cca142a4699 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1706,6 +1706,10 @@ config_init_cpu_count(PyConfig *config) int cpu_count = -1; const wchar_t *sep = wcschr(xoption, L'='); if (sep) { + if (wcscmp(sep + 1, L"default") == 0) { + config->cpu_count = -1; + return _PyStatus_OK(); + } if (config_wstr_to_int(sep + 1, &cpu_count) < 0 || cpu_count < 1) { goto error; } From 66c617f2591646dcf5b25b1121307cfc801d7ad5 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 24 Sep 2023 16:53:42 +0900 Subject: [PATCH 35/68] nit --- Python/initconfig.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 6e3cca142a4699..eae25c3b3f5f93 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1707,10 +1707,9 @@ config_init_cpu_count(PyConfig *config) const wchar_t *sep = wcschr(xoption, L'='); if (sep) { if (wcscmp(sep + 1, L"default") == 0) { - config->cpu_count = -1; - return _PyStatus_OK(); + cpu_count = -1; } - if (config_wstr_to_int(sep + 1, &cpu_count) < 0 || cpu_count < 1) { + else if (config_wstr_to_int(sep + 1, &cpu_count) < 0 || cpu_count < 1) { goto error; } } From 4a72ed4396c88963508c707bf21a555f919f281c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 26 Sep 2023 23:42:32 +0900 Subject: [PATCH 36/68] Address Erlend's review --- Doc/c-api/init_config.rst | 11 +++++++++++ Include/cpython/initconfig.h | 1 - 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 56b4ee03c1a8fd..647aca02b9bdad 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -878,6 +878,17 @@ PyConfig .. versionadded:: 3.12 + .. c:member:: int cpu_count + If the value of is not ``-1`` then it will override + the return value of :func:`cpu_count` into . + + Configured by the :samp:`-X cpu_count={n}` command line + flag or the :envvar:`PYTHONCPUCOUNT` environment variable. + + Default: ``-1``, it will follow the original behavior of :func:`cpu_count`. + + .. versionadded:: 3.13 + .. c:member:: int isolated If greater than ``0``, enable isolated mode: diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 11abb23c648b3e..ef73ff6b5fa151 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -180,7 +180,6 @@ typedef struct PyConfig { int safe_path; int int_max_str_digits; - // Python 3.13 int cpu_count; /* --- Path configuration inputs ------------ */ From 0426e3e84c793cd076d36c94a1aec3932b37f147 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 26 Sep 2023 23:46:50 +0900 Subject: [PATCH 37/68] fix --- Doc/c-api/init_config.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 647aca02b9bdad..681e8b5f79e46e 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -879,6 +879,7 @@ PyConfig .. versionadded:: 3.12 .. c:member:: int cpu_count + If the value of is not ``-1`` then it will override the return value of :func:`cpu_count` into . From e50e678957ab4ae62bfb74ea78ce680805eb9462 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 07:23:17 +0900 Subject: [PATCH 38/68] fix --- Python/initconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 5f0ca24a21443f..0f8454219933b2 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -92,7 +92,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(use_frozen_modules, UINT), SPEC(safe_path, UINT), SPEC(int_max_str_digits, INT), - SPEC(int_cpu_coumt, INT), + SPEC(cpu_count, INT), SPEC(pathconfig_warnings, UINT), SPEC(program_name, WSTR), SPEC(pythonpath_env, WSTR_OPT), From 24fe0e4ff8f146bc5b099842e55fb19762461eae Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 08:17:10 +0900 Subject: [PATCH 39/68] Add space --- Doc/whatsnew/3.13.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 64ba45134aadc0..967aad0fb86cae 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -170,6 +170,7 @@ os environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system.(Contributed by Donghee Na in :gh:`109595`) + * Add :func:`os.process_cpu_count` function to get the number of logical CPUs usable by the calling thread of the current process. (Contributed by Victor Stinner in :gh:`109649`.) From 32843edd696f715b53616f3dfc6ae444bac6d63c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 08:31:03 +0900 Subject: [PATCH 40/68] Update os.py --- Lib/os.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 35842cedf14fc7..bb16aafb145623 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1138,7 +1138,18 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity'): +def _is_cpu_overrided(): + is_overrided = False + env_opt = environ.get('PYTHONCPUCOUNT', None) + xopt = sys._xoptions.get('cpu_count', None) + if not any((env_opt, xopt)): + return False + if xopt == 'default': + return False + return True + + +if _exists('sched_getaffinity') and not _is_cpu_overrided(): def process_cpu_count(): """ Get the number of CPUs of the current process. From a954f1c6c3e7b1d48c4afb92d15ee7ba186ec463 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 08:32:28 +0900 Subject: [PATCH 41/68] nit --- Lib/os.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index bb16aafb145623..8484f07e2796b0 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1139,7 +1139,6 @@ def add_dll_directory(path): def _is_cpu_overrided(): - is_overrided = False env_opt = environ.get('PYTHONCPUCOUNT', None) xopt = sys._xoptions.get('cpu_count', None) if not any((env_opt, xopt)): From 3ab2bc4f457cc562cda87a410139339a9ecafdb6 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 08:35:57 +0900 Subject: [PATCH 42/68] Add test code --- Lib/test/test_cmd_line.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index c2e284ba4a76b0..ae13f53c9f8208 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -907,11 +907,21 @@ def test_cpu_count(self): res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) self.assertEqual(res.out.strip().decode("utf-8"), '4321') + def test_process_cpu_count(self): + code = "import os; print(os.process_cpu_count())" + res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) + self.assertEqual(res.out.strip().decode("utf-8"), '4321') + def test_cpu_count_default(self): code = "import os; print(os.cpu_count())" res = assert_python_ok('-X', 'cpu_count=default', '-c', code) self.assertEqual(int(res.out.strip().decode("utf-8")), os.cpu_count()) + def test_process_cpu_count_default(self): + code = "import os; print(os.process_cpu_count())" + res = assert_python_ok('-X', 'cpu_count=default', '-c', code) + self.assertEqual(int(res.out.strip().decode("utf-8")), os.process_cpu_count()) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') From 7231697892fa48997e91fc9702a456cabad19829 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 08:41:01 +0900 Subject: [PATCH 43/68] Update docs --- Doc/c-api/init_config.rst | 4 ++-- Doc/library/os.rst | 3 +++ Doc/using/cmdline.rst | 4 ++-- Doc/whatsnew/3.13.rst | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 681e8b5f79e46e..418f0ab5a28cc0 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -881,12 +881,12 @@ PyConfig .. c:member:: int cpu_count If the value of is not ``-1`` then it will override - the return value of :func:`cpu_count` into . + the return value of :func:`cpu_count` and :func:`process_cpu_count` into . Configured by the :samp:`-X cpu_count={n}` command line flag or the :envvar:`PYTHONCPUCOUNT` environment variable. - Default: ``-1``, it will follow the original behavior of :func:`cpu_count`. + Default: ``-1``, it will follow the original behavior of :func:`cpu_count` and :func:`process_cpu_count. .. versionadded:: 3.13 diff --git a/Doc/library/os.rst b/Doc/library/os.rst index e6a32363bea266..bec40eb48330bd 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5218,6 +5218,9 @@ Miscellaneous System Information The :func:`cpu_count` function can be used to get the number of logical CPUs in the **system**. + If :samp:`-X cpu_count={n}` is given or :envvar:`PYTHONCPUCOUNT` is set, + :func:`process_cpu_count` returns the overrided value *n*. + See also the :func:`sched_getaffinity` functions. .. versionadded:: 3.13 diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 5c3ae98b54f6ca..0c4adc24ea517a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -546,7 +546,7 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. - * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`. + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count` and :func:`os.process_cpu_count`. *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHONCPUCOUNT`. @@ -1076,7 +1076,7 @@ conflict. .. envvar:: PYTHONCPUCOUNT If this variable is set to a positive integer, it overrides - :func:`os.cpu_count` + :func:`os.cpu_count` and and :func:`os.process_cpu_count`. See also the :samp:`-X cpu_count={n}` command-line option. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 967aad0fb86cae..5035a022c7dc69 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -166,8 +166,8 @@ opcode os -- -* :func:`os.cpu_count` can be overrided through the new - environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option +* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overrided through + the new environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system.(Contributed by Donghee Na in :gh:`109595`) From b18da0dd42e0aaecee5506d345c2e3e5544432d9 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 09:04:03 +0900 Subject: [PATCH 44/68] fix --- Doc/c-api/init_config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 418f0ab5a28cc0..e086d6d386cdc8 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -886,7 +886,7 @@ PyConfig Configured by the :samp:`-X cpu_count={n}` command line flag or the :envvar:`PYTHONCPUCOUNT` environment variable. - Default: ``-1``, it will follow the original behavior of :func:`cpu_count` and :func:`process_cpu_count. + Default: ``-1``, it will follow the original behavior of :func:`cpu_count` and :func:`process_cpu_count`. .. versionadded:: 3.13 From 3579fc4e501054ba24bcf00fd9b9677436eaa45a Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 09:24:42 +0900 Subject: [PATCH 45/68] Address code reivew --- Doc/c-api/init_config.rst | 8 ++++---- Doc/using/cmdline.rst | 6 +++--- Doc/whatsnew/3.13.rst | 11 ++++++----- Lib/test/test_cmd_line.py | 38 ++++++++++++++------------------------ 4 files changed, 27 insertions(+), 36 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index e086d6d386cdc8..ccb943b7e1112b 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -880,13 +880,13 @@ PyConfig .. c:member:: int cpu_count - If the value of is not ``-1`` then it will override - the return value of :func:`cpu_count` and :func:`process_cpu_count` into . + If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will override + the return value of :func:`os.cpu_count` and :func:`os.process_cpu_count` into . - Configured by the :samp:`-X cpu_count={n}` command line + Configured by the :samp:`-X cpu_count={n|default}` command line flag or the :envvar:`PYTHONCPUCOUNT` environment variable. - Default: ``-1``, it will follow the original behavior of :func:`cpu_count` and :func:`process_cpu_count`. + Default: ``-1``. .. versionadded:: 3.13 diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 0c4adc24ea517a..b4e584457147be 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -550,7 +550,7 @@ Miscellaneous options *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHONCPUCOUNT`. - If *n* is "default", it follows default :func:`os.cpu_count` even if :envvar:`PYTHONCPUCOUNT` + If *n* is ``default``, it follows default :func:`os.cpu_count` even if :envvar:`PYTHONCPUCOUNT` is set. @@ -1076,9 +1076,9 @@ conflict. .. envvar:: PYTHONCPUCOUNT If this variable is set to a positive integer, it overrides - :func:`os.cpu_count` and and :func:`os.process_cpu_count`. + :func:`os.cpu_count` and and :func:`os.process_cpu_count` result. - See also the :samp:`-X cpu_count={n}` command-line option. + See also the :option:`-X cpu_count <-X>` command-line option. .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5035a022c7dc69..adc76c87c11b0f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -166,15 +166,16 @@ opcode os -- -* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overrided through - the new environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option - :option:`-X cpu_count <-X>`. This option is useful for users who need to limit - CPU resources of a container system.(Contributed by Donghee Na in :gh:`109595`) - * Add :func:`os.process_cpu_count` function to get the number of logical CPUs usable by the calling thread of the current process. (Contributed by Victor Stinner in :gh:`109649`.) +* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overrided through + the new environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option + :option:`-X cpu_count <-X>`. This option is useful for users who need to limit + CPU resources of a container system. + (Contributed by Donghee Na in :gh:`109595`) + pathlib ------- diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index ae13f53c9f8208..8fe50b49bf7bca 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -878,49 +878,39 @@ def test_int_max_str_digits(self): assert_python_failure('-c', code, PYTHONINTMAXSTRDIGITS='foo') assert_python_failure('-c', code, PYTHONINTMAXSTRDIGITS='100') - def res2int(res): - out = res.out.strip().decode("utf-8") - return tuple(int(i) for i in out.split()) - res = assert_python_ok('-c', code) current_max = sys.get_int_max_str_digits() - self.assertEqual(res2int(res), (current_max, current_max)) + self.assertEqual(self.res2int(res), (current_max, current_max)) res = assert_python_ok('-X', 'int_max_str_digits=0', '-c', code) - self.assertEqual(res2int(res), (0, 0)) + self.assertEqual(self.res2int(res), (0, 0)) res = assert_python_ok('-X', 'int_max_str_digits=4000', '-c', code) - self.assertEqual(res2int(res), (4000, 4000)) + self.assertEqual(self.res2int(res), (4000, 4000)) res = assert_python_ok('-X', 'int_max_str_digits=100000', '-c', code) - self.assertEqual(res2int(res), (100000, 100000)) + self.assertEqual(self.res2int(res), (100000, 100000)) res = assert_python_ok('-c', code, PYTHONINTMAXSTRDIGITS='0') - self.assertEqual(res2int(res), (0, 0)) + self.assertEqual(self.res2int(res), (0, 0)) res = assert_python_ok('-c', code, PYTHONINTMAXSTRDIGITS='4000') - self.assertEqual(res2int(res), (4000, 4000)) + self.assertEqual(self.res2int(res), (4000, 4000)) res = assert_python_ok( '-X', 'int_max_str_digits=6000', '-c', code, PYTHONINTMAXSTRDIGITS='4000' ) - self.assertEqual(res2int(res), (6000, 6000)) + self.assertEqual(self.res2int(res), (6000, 6000)) def test_cpu_count(self): - code = "import os; print(os.cpu_count())" - res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) - self.assertEqual(res.out.strip().decode("utf-8"), '4321') - - def test_process_cpu_count(self): - code = "import os; print(os.process_cpu_count())" + code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) - self.assertEqual(res.out.strip().decode("utf-8"), '4321') + self.assertEqual(self.res2int(res), (4321, 4321)) def test_cpu_count_default(self): - code = "import os; print(os.cpu_count())" + code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=default', '-c', code) - self.assertEqual(int(res.out.strip().decode("utf-8")), os.cpu_count()) + self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - def test_process_cpu_count_default(self): - code = "import os; print(os.process_cpu_count())" - res = assert_python_ok('-X', 'cpu_count=default', '-c', code) - self.assertEqual(int(res.out.strip().decode("utf-8")), os.process_cpu_count()) + def res2int(self, res): + out = res.out.strip().decode("utf-8") + return tuple(int(i) for i in out.split()) @unittest.skipIf(interpreter_requires_environment(), From 134ed9eea8fc89d888b9ae27a04ccad8ff5f8864 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 09:30:50 +0900 Subject: [PATCH 46/68] Address code review --- Lib/os.py | 13 +------------ Modules/clinic/posixmodule.c.h | 30 +++++++++++++++++++++++++++++- Modules/posixmodule.c | 14 ++++++++++++++ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 8484f07e2796b0..722f2462ac2a47 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1137,18 +1137,7 @@ def add_dll_directory(path): nt._remove_dll_directory ) - -def _is_cpu_overrided(): - env_opt = environ.get('PYTHONCPUCOUNT', None) - xopt = sys._xoptions.get('cpu_count', None) - if not any((env_opt, xopt)): - return False - if xopt == 'default': - return False - return True - - -if _exists('sched_getaffinity') and not _is_cpu_overrided(): +if _exists('sched_getaffinity') and os._get_cpu_config() < 0: def process_cpu_count(): """ Get the number of CPUs of the current process. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index fc39ab72bf2a51..67b9e6dd3989d8 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10421,6 +10421,34 @@ os_get_terminal_size(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* (defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL)) */ +PyDoc_STRVAR(os__get_cpu_count_config__doc__, +"_get_cpu_count_config($module, /)\n" +"--\n" +"\n" +"Private function for get PyConfig.cpu_count"); + +#define OS__GET_CPU_COUNT_CONFIG_METHODDEF \ + {"_get_cpu_count_config", (PyCFunction)os__get_cpu_count_config, METH_NOARGS, os__get_cpu_count_config__doc__}, + +static int +os__get_cpu_count_config_impl(PyObject *module); + +static PyObject * +os__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = os__get_cpu_count_config_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(os_cpu_count__doc__, "cpu_count($module, /)\n" "--\n" @@ -11986,4 +12014,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=8b60de6ddb925bc3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=726046e386a960c9 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 40e6adf2bbe53e..49b98b059cd610 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14335,6 +14335,19 @@ os_get_terminal_size_impl(PyObject *module, int fd) } #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ +/*[clinic input] +os._get_cpu_count_config -> int + +Private function for get PyConfig.cpu_count +[clinic start generated code]*/ + +static int +os__get_cpu_count_config_impl(PyObject *module) +/*[clinic end generated code: output=4796d9e355e66257 input=fd76f60fcec3e42f]*/ +{ + const PyConfig *config = _Py_GetConfig(); + return config->cpu_count; +} /*[clinic input] os.cpu_count @@ -15982,6 +15995,7 @@ static PyMethodDef posix_methods[] = { OS_LISTXATTR_METHODDEF OS_GET_TERMINAL_SIZE_METHODDEF + OS__GET_CPU_COUNT_CONFIG_METHODDEF OS_CPU_COUNT_METHODDEF OS_GET_INHERITABLE_METHODDEF OS_SET_INHERITABLE_METHODDEF From 2bec7f41cec494209f9a09f2becf7f532f15e8e6 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 09:34:28 +0900 Subject: [PATCH 47/68] Add test --- Lib/os.py | 1 + Lib/test/test_cmd_line.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/Lib/os.py b/Lib/os.py index 722f2462ac2a47..16a5b4c12ddcf6 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1137,6 +1137,7 @@ def add_dll_directory(path): nt._remove_dll_directory ) + if _exists('sched_getaffinity') and os._get_cpu_config() < 0: def process_cpu_count(): """ diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 8fe50b49bf7bca..5517d3b67b2554 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -902,11 +902,15 @@ def test_cpu_count(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) self.assertEqual(self.res2int(res), (4321, 4321)) + res = assert_python_ok('-c', code, PYTHONCPUCOUNT='1234') + self.assertEqual(self.res2int(res), (1234, 1234)) def test_cpu_count_default(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=default', '-c', code) self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) + res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHONCPUCOUNT='1234') + self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): out = res.out.strip().decode("utf-8") From 9f7cb5ed12b7f0f7caea24bbc36ed6ed28a31867 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 09:41:29 +0900 Subject: [PATCH 48/68] fix --- Lib/os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 16a5b4c12ddcf6..ff19f44a5aa49b 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1138,7 +1138,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and os._get_cpu_config() < 0: +if _exists('sched_getaffinity') and _get_cpu_config() < 0: def process_cpu_count(): """ Get the number of CPUs of the current process. From 1217ab518adc50989a299803f4925d743f831c2a Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 09:46:00 +0900 Subject: [PATCH 49/68] fix --- Lib/os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index ff19f44a5aa49b..03e7682fe8855c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1138,7 +1138,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and _get_cpu_config() < 0: +if _exists('sched_getaffinity') and _get_cpu_count_config() < 0: def process_cpu_count(): """ Get the number of CPUs of the current process. From 64da2f9557e9e3f6fff936ace4a6697b60ac3582 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 10:01:48 +0900 Subject: [PATCH 50/68] fix --- Lib/os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 03e7682fe8855c..e92257c920e57c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1138,7 +1138,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and _get_cpu_count_config() < 0: +if _exists('sched_getaffinity') and posix._get_cpu_count_config() < 0: def process_cpu_count(): """ Get the number of CPUs of the current process. From 64c732960da3527a3d51effc9116d5bf7df3cd26 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 10:06:02 +0900 Subject: [PATCH 51/68] fix --- Lib/os.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index e92257c920e57c..694e6987897a91 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -26,6 +26,7 @@ import sys import stat as st +from posix import _get_cpu_count_config from _collections_abc import _check_methods GenericAlias = type(list[int]) @@ -1138,7 +1139,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and posix._get_cpu_count_config() < 0: +if _exists('sched_getaffinity') and _get_cpu_count_config() < 0: def process_cpu_count(): """ Get the number of CPUs of the current process. From cc54afbdfe578fe218c25eb9ea4a78dbaaddae3a Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 10:08:34 +0900 Subject: [PATCH 52/68] Update --- Lib/os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 694e6987897a91..4eb3d20f0fa46c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -26,7 +26,6 @@ import sys import stat as st -from posix import _get_cpu_count_config from _collections_abc import _check_methods GenericAlias = type(list[int]) @@ -56,6 +55,7 @@ def _get_exports_list(module): from posix import * try: from posix import _exit + from posix import _get_cpu_count_config __all__.append('_exit') except ImportError: pass From ba421c77f14d97a92efebac48602e4e987aa6000 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 10:20:08 +0900 Subject: [PATCH 53/68] Address code review --- Doc/c-api/init_config.rst | 3 ++- Doc/using/cmdline.rst | 6 ++---- Doc/whatsnew/3.13.rst | 2 +- Lib/test/test_cmd_line.py | 15 ++++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index ccb943b7e1112b..79feb3e502af08 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -881,7 +881,8 @@ PyConfig .. c:member:: int cpu_count If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will override - the return value of :func:`os.cpu_count` and :func:`os.process_cpu_count` into . + the return value of :func:`os.cpu_count` and :func:`os.process_cpu_count` functions + into *cpu_count*. Configured by the :samp:`-X cpu_count={n|default}` command line flag or the :envvar:`PYTHONCPUCOUNT` environment variable. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b4e584457147be..8814338087ad9a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -550,9 +550,7 @@ Miscellaneous options *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHONCPUCOUNT`. - If *n* is ``default``, it follows default :func:`os.cpu_count` even if :envvar:`PYTHONCPUCOUNT` - is set. - + If *n* is ``default``, :func:`os.cpu_count` and :func:`os.process_cpu_count` are not overridden. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -1076,7 +1074,7 @@ conflict. .. envvar:: PYTHONCPUCOUNT If this variable is set to a positive integer, it overrides - :func:`os.cpu_count` and and :func:`os.process_cpu_count` result. + :func:`os.cpu_count` and and :func:`os.process_cpu_count` return result. See also the :option:`-X cpu_count <-X>` command-line option. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index adc76c87c11b0f..85837c1029a6aa 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -173,7 +173,7 @@ os * :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overrided through the new environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit - CPU resources of a container system. + CPU resources of a container system without having to modify the container (application code). (Contributed by Donghee Na in :gh:`109595`) pathlib diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 5517d3b67b2554..b416a3dbdbf581 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -879,24 +879,25 @@ def test_int_max_str_digits(self): assert_python_failure('-c', code, PYTHONINTMAXSTRDIGITS='100') res = assert_python_ok('-c', code) + res2int = self.res2int current_max = sys.get_int_max_str_digits() - self.assertEqual(self.res2int(res), (current_max, current_max)) + self.assertEqual(res2int(res), (current_max, current_max)) res = assert_python_ok('-X', 'int_max_str_digits=0', '-c', code) - self.assertEqual(self.res2int(res), (0, 0)) + self.assertEqual(res2int(res), (0, 0)) res = assert_python_ok('-X', 'int_max_str_digits=4000', '-c', code) - self.assertEqual(self.res2int(res), (4000, 4000)) + self.assertEqual(res2int(res), (4000, 4000)) res = assert_python_ok('-X', 'int_max_str_digits=100000', '-c', code) - self.assertEqual(self.res2int(res), (100000, 100000)) + self.assertEqual(res2int(res), (100000, 100000)) res = assert_python_ok('-c', code, PYTHONINTMAXSTRDIGITS='0') - self.assertEqual(self.res2int(res), (0, 0)) + self.assertEqual(res2int(res), (0, 0)) res = assert_python_ok('-c', code, PYTHONINTMAXSTRDIGITS='4000') - self.assertEqual(self.res2int(res), (4000, 4000)) + self.assertEqual(res2int(res), (4000, 4000)) res = assert_python_ok( '-X', 'int_max_str_digits=6000', '-c', code, PYTHONINTMAXSTRDIGITS='4000' ) - self.assertEqual(self.res2int(res), (6000, 6000)) + self.assertEqual(res2int(res), (6000, 6000)) def test_cpu_count(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" From 5f20bf6868c6088d835888eb26a7cca69cbd6864 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 10:26:23 +0900 Subject: [PATCH 54/68] Update NEWS.d --- .../2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst index 4b27901b4aa4a5..b63a3a89222901 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst @@ -1,3 +1,5 @@ -Add :samp:`-Xcpu_count={n}` command line option to override CPU count of :func:`os.cpu_count`. -This option is useful for users who need to limit CPU resources of a container system. +Add :samp:`-Xcpu_count={n}` command line option to override return results of +:func:`os.cpu_count` and :func:`os.process_cpu_count`. +This option is useful for users who need to limit CPU resources of a container system +without having to modify the container (application code). Patch by Donghee Na. From c11789b01a60aee3b523199d41994d55a132207b Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 10:26:56 +0900 Subject: [PATCH 55/68] Update --- .../2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst index b63a3a89222901..f182f965834fa1 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-22-01-44-53.gh-issue-109595.fVINgD.rst @@ -1,4 +1,4 @@ -Add :samp:`-Xcpu_count={n}` command line option to override return results of +Add :option:`-X cpu_count <-X>` command line option to override return results of :func:`os.cpu_count` and :func:`os.process_cpu_count`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). From 936c182f1fb6cc99079a6f00b3d33c901d51155c Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 10:28:29 +0900 Subject: [PATCH 56/68] Update --- Doc/library/os.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index bec40eb48330bd..b94daa0981c35d 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5196,7 +5196,7 @@ Miscellaneous System Information .. versionadded:: 3.4 .. versionchanged:: 3.13 - If :samp:`-X cpu_count={n}` is given or :envvar:`PYTHONCPUCOUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, :func:`cpu_count` returns the overrided value *n*. @@ -5218,7 +5218,7 @@ Miscellaneous System Information The :func:`cpu_count` function can be used to get the number of logical CPUs in the **system**. - If :samp:`-X cpu_count={n}` is given or :envvar:`PYTHONCPUCOUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, :func:`process_cpu_count` returns the overrided value *n*. See also the :func:`sched_getaffinity` functions. From 2f0dc1cefad8138df37620daece40397b547df70 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 1 Oct 2023 19:56:17 +0900 Subject: [PATCH 57/68] Address code review --- Lib/test/test_cmd_line.py | 2 ++ Python/initconfig.c | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index b416a3dbdbf581..5beb72096e2a3f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -912,6 +912,8 @@ def test_cpu_count_default(self): self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHONCPUCOUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) + es = assert_python_ok('-c', code, PYTHONCPUCOUNT='default') + self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): out = res.out.strip().decode("utf-8") diff --git a/Python/initconfig.c b/Python/initconfig.c index 0f8454219933b2..cb02c60a038e0e 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -231,7 +231,7 @@ The following implementation-specific options are available:\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ The default is sys.int_info.default_max_str_digits. 0 disables.\n\ \n\ --X cpu_count=n: override CPU count of os.cpu_count().\n\ +-X cpu_count=[n|default]: override CPU count of os.cpu_count() and os.process_cpu_count().\n\ This helps for users who need to limit CPU resources of a container system." #ifdef Py_STATS @@ -1628,9 +1628,12 @@ config_init_cpu_count(PyConfig *config) const char *env = config_get_env(config, "PYTHONCPUCOUNT"); if (env) { int cpu_count = -1; - if (_Py_str_to_int(env, &cpu_count) != 0) { + if (strcmp(env, "default") == 0) { cpu_count = -1; } + else if (_Py_str_to_int(env, &cpu_count) != 0) { + goto error; + } if (cpu_count >= 1) { config->cpu_count = cpu_count; } From a7b2c883678489fd9cfcae03ca318dc8ab7aa0db Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 2 Oct 2023 00:09:30 +0900 Subject: [PATCH 58/68] Rename to PYTHON_CPU_COUNT --- Doc/c-api/init_config.rst | 2 +- Doc/library/os.rst | 4 ++-- Doc/using/cmdline.rst | 4 ++-- Doc/whatsnew/3.13.rst | 2 +- Lib/test/test_cmd_line.py | 6 +++--- Python/initconfig.c | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 79feb3e502af08..c632e6a1e6a1b9 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -885,7 +885,7 @@ PyConfig into *cpu_count*. Configured by the :samp:`-X cpu_count={n|default}` command line - flag or the :envvar:`PYTHONCPUCOUNT` environment variable. + flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. Default: ``-1``. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b94daa0981c35d..44732a750eb826 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5196,7 +5196,7 @@ Miscellaneous System Information .. versionadded:: 3.4 .. versionchanged:: 3.13 - If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, :func:`cpu_count` returns the overrided value *n*. @@ -5218,7 +5218,7 @@ Miscellaneous System Information The :func:`cpu_count` function can be used to get the number of logical CPUs in the **system**. - If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, :func:`process_cpu_count` returns the overrided value *n*. See also the :func:`sched_getaffinity` functions. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 8814338087ad9a..aac0a5a85ae2ab 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -549,7 +549,7 @@ Miscellaneous options * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count` and :func:`os.process_cpu_count`. *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. - See also :envvar:`PYTHONCPUCOUNT`. + See also :envvar:`PYTHON_CPU_COUNT`. If *n* is ``default``, :func:`os.cpu_count` and :func:`os.process_cpu_count` are not overridden. It also allows passing arbitrary values and retrieving them through the @@ -1071,7 +1071,7 @@ conflict. .. versionadded:: 3.12 -.. envvar:: PYTHONCPUCOUNT +.. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides :func:`os.cpu_count` and and :func:`os.process_cpu_count` return result. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 85837c1029a6aa..64422fc11a3a3a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -171,7 +171,7 @@ os (Contributed by Victor Stinner in :gh:`109649`.) * :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overrided through - the new environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option + the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). (Contributed by Donghee Na in :gh:`109595`) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 5beb72096e2a3f..eaf19aa160e860 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -903,16 +903,16 @@ def test_cpu_count(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) self.assertEqual(self.res2int(res), (4321, 4321)) - res = assert_python_ok('-c', code, PYTHONCPUCOUNT='1234') + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (1234, 1234)) def test_cpu_count_default(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=default', '-c', code) self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHONCPUCOUNT='1234') + res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - es = assert_python_ok('-c', code, PYTHONCPUCOUNT='default') + es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): diff --git a/Python/initconfig.c b/Python/initconfig.c index cb02c60a038e0e..a59e06cb161b57 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1625,7 +1625,7 @@ config_read_env_vars(PyConfig *config) static PyStatus config_init_cpu_count(PyConfig *config) { - const char *env = config_get_env(config, "PYTHONCPUCOUNT"); + const char *env = config_get_env(config, "PYTHON_CPU_COUNT"); if (env) { int cpu_count = -1; if (strcmp(env, "default") == 0) { From 551c76d2db08c127da07d4e575af2f70df6c851e Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 2 Oct 2023 10:18:43 +0900 Subject: [PATCH 59/68] Hidden overrided cpu count --- Lib/os.py | 2 +- Modules/clinic/posixmodule.c.h | 16 +++------------- Modules/posixmodule.c | 11 +++++++---- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 4eb3d20f0fa46c..de43e97745695a 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1139,7 +1139,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and _get_cpu_count_config() < 0: +if _exists('sched_getaffinity') and _get_cpu_count_config() != 'overrided': def process_cpu_count(): """ Get the number of CPUs of the current process. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 67b9e6dd3989d8..11ca371514c809 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10430,23 +10430,13 @@ PyDoc_STRVAR(os__get_cpu_count_config__doc__, #define OS__GET_CPU_COUNT_CONFIG_METHODDEF \ {"_get_cpu_count_config", (PyCFunction)os__get_cpu_count_config, METH_NOARGS, os__get_cpu_count_config__doc__}, -static int +static PyObject * os__get_cpu_count_config_impl(PyObject *module); static PyObject * os__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) { - PyObject *return_value = NULL; - int _return_value; - - _return_value = os__get_cpu_count_config_impl(module); - if ((_return_value == -1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromLong((long)_return_value); - -exit: - return return_value; + return os__get_cpu_count_config_impl(module); } PyDoc_STRVAR(os_cpu_count__doc__, @@ -12014,4 +12004,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=726046e386a960c9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7c6a691f8d40bf1e input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 49b98b059cd610..94d7a37984caf0 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14336,17 +14336,20 @@ os_get_terminal_size_impl(PyObject *module, int fd) #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ /*[clinic input] -os._get_cpu_count_config -> int +os._get_cpu_count_config Private function for get PyConfig.cpu_count [clinic start generated code]*/ -static int +static PyObject * os__get_cpu_count_config_impl(PyObject *module) -/*[clinic end generated code: output=4796d9e355e66257 input=fd76f60fcec3e42f]*/ +/*[clinic end generated code: output=3b0ba7e445c9c6b2 input=c52e5bc863732c21]*/ { const PyConfig *config = _Py_GetConfig(); - return config->cpu_count; + if (config->cpu_count > 0) { + return PyUnicode_FromString("overrided"); + } + return PyUnicode_FromString("default"); } /*[clinic input] From 75021be4560c950ec30dcfe7e9799eb2b0ac3c04 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 2 Oct 2023 10:24:31 +0900 Subject: [PATCH 60/68] Use overridden --- Doc/library/os.rst | 4 ++-- Doc/whatsnew/3.13.rst | 2 +- Lib/os.py | 2 +- Modules/posixmodule.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 44732a750eb826..15e5f5c951db51 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5197,7 +5197,7 @@ Miscellaneous System Information .. versionchanged:: 3.13 If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`cpu_count` returns the overrided value *n*. + :func:`cpu_count` returns the overridden value *n*. .. function:: getloadavg() @@ -5219,7 +5219,7 @@ Miscellaneous System Information in the **system**. If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`process_cpu_count` returns the overrided value *n*. + :func:`process_cpu_count` returns the overridden value *n*. See also the :func:`sched_getaffinity` functions. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 64422fc11a3a3a..eb347e82ada4cf 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -170,7 +170,7 @@ os usable by the calling thread of the current process. (Contributed by Victor Stinner in :gh:`109649`.) -* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overrided through +* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overridden through the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). diff --git a/Lib/os.py b/Lib/os.py index de43e97745695a..6f7f97d3b18224 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1139,7 +1139,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and _get_cpu_count_config() != 'overrided': +if _exists('sched_getaffinity') and _get_cpu_count_config() != 'overridden': def process_cpu_count(): """ Get the number of CPUs of the current process. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 94d7a37984caf0..9ad04f27d5ca5a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14347,7 +14347,7 @@ os__get_cpu_count_config_impl(PyObject *module) { const PyConfig *config = _Py_GetConfig(); if (config->cpu_count > 0) { - return PyUnicode_FromString("overrided"); + return PyUnicode_FromString("overridden"); } return PyUnicode_FromString("default"); } From 57dd53bb131491fda8171d90423e04d009cde2aa Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 2 Oct 2023 10:32:03 +0900 Subject: [PATCH 61/68] Minor refactoring --- Python/initconfig.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index a59e06cb161b57..210a8294fd57c0 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1631,12 +1631,10 @@ config_init_cpu_count(PyConfig *config) if (strcmp(env, "default") == 0) { cpu_count = -1; } - else if (_Py_str_to_int(env, &cpu_count) != 0) { + else if (_Py_str_to_int(env, &cpu_count) < 0 || cpu_count < 1) { goto error; } - if (cpu_count >= 1) { - config->cpu_count = cpu_count; - } + config->cpu_count = cpu_count; } const wchar_t *xoption = config_get_xoption(config, L"cpu_count"); From 3f9da50d130de208107ae7a62dab4f8b31dc88b8 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 2 Oct 2023 20:02:27 +0900 Subject: [PATCH 62/68] Revert to PYTHONCPUCOUNT --- Doc/c-api/init_config.rst | 2 +- Doc/library/os.rst | 4 ++-- Doc/using/cmdline.rst | 4 ++-- Doc/whatsnew/3.13.rst | 2 +- Lib/test/test_cmd_line.py | 6 +++--- Modules/posixmodule.c | 4 ++-- Python/initconfig.c | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index c632e6a1e6a1b9..79feb3e502af08 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -885,7 +885,7 @@ PyConfig into *cpu_count*. Configured by the :samp:`-X cpu_count={n|default}` command line - flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. + flag or the :envvar:`PYTHONCPUCOUNT` environment variable. Default: ``-1``. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 15e5f5c951db51..7fd04cf08f21bc 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5196,7 +5196,7 @@ Miscellaneous System Information .. versionadded:: 3.4 .. versionchanged:: 3.13 - If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, :func:`cpu_count` returns the overridden value *n*. @@ -5218,7 +5218,7 @@ Miscellaneous System Information The :func:`cpu_count` function can be used to get the number of logical CPUs in the **system**. - If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, :func:`process_cpu_count` returns the overridden value *n*. See also the :func:`sched_getaffinity` functions. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index aac0a5a85ae2ab..8814338087ad9a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -549,7 +549,7 @@ Miscellaneous options * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count` and :func:`os.process_cpu_count`. *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. - See also :envvar:`PYTHON_CPU_COUNT`. + See also :envvar:`PYTHONCPUCOUNT`. If *n* is ``default``, :func:`os.cpu_count` and :func:`os.process_cpu_count` are not overridden. It also allows passing arbitrary values and retrieving them through the @@ -1071,7 +1071,7 @@ conflict. .. versionadded:: 3.12 -.. envvar:: PYTHON_CPU_COUNT +.. envvar:: PYTHONCPUCOUNT If this variable is set to a positive integer, it overrides :func:`os.cpu_count` and and :func:`os.process_cpu_count` return result. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index eb347e82ada4cf..d75cd0f8c4dcd3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -171,7 +171,7 @@ os (Contributed by Victor Stinner in :gh:`109649`.) * :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overridden through - the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option + the new environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). (Contributed by Donghee Na in :gh:`109595`) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index eaf19aa160e860..5beb72096e2a3f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -903,16 +903,16 @@ def test_cpu_count(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) self.assertEqual(self.res2int(res), (4321, 4321)) - res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='1234') + res = assert_python_ok('-c', code, PYTHONCPUCOUNT='1234') self.assertEqual(self.res2int(res), (1234, 1234)) def test_cpu_count_default(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=default', '-c', code) self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') + res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHONCPUCOUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') + es = assert_python_ok('-c', code, PYTHONCPUCOUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9ad04f27d5ca5a..740650447c6e38 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14336,9 +14336,9 @@ os_get_terminal_size_impl(PyObject *module, int fd) #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ /*[clinic input] -os._get_cpu_count_config +os._is_cpu_count_overridden() -> bool -Private function for get PyConfig.cpu_count +Private function to get PyConfig.cpu_count is overridden or not [clinic start generated code]*/ static PyObject * diff --git a/Python/initconfig.c b/Python/initconfig.c index 210a8294fd57c0..7ebb92798da843 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1625,7 +1625,7 @@ config_read_env_vars(PyConfig *config) static PyStatus config_init_cpu_count(PyConfig *config) { - const char *env = config_get_env(config, "PYTHON_CPU_COUNT"); + const char *env = config_get_env(config, "PYTHONCPUCOUNT"); if (env) { int cpu_count = -1; if (strcmp(env, "default") == 0) { From 57e82c57d3d896040a01d28a97223a80ac6aeb68 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 2 Oct 2023 20:10:59 +0900 Subject: [PATCH 63/68] fix --- Modules/clinic/posixmodule.c.h | 4 ++-- Modules/posixmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 11ca371514c809..2608929d8f554c 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10425,7 +10425,7 @@ PyDoc_STRVAR(os__get_cpu_count_config__doc__, "_get_cpu_count_config($module, /)\n" "--\n" "\n" -"Private function for get PyConfig.cpu_count"); +"Private function to get PyConfig.cpu_count is overridden or not"); #define OS__GET_CPU_COUNT_CONFIG_METHODDEF \ {"_get_cpu_count_config", (PyCFunction)os__get_cpu_count_config, METH_NOARGS, os__get_cpu_count_config__doc__}, @@ -12004,4 +12004,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=7c6a691f8d40bf1e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a38f7ef7a5ca4bd7 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 740650447c6e38..eeb8ed93de7e96 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14336,14 +14336,14 @@ os_get_terminal_size_impl(PyObject *module, int fd) #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ /*[clinic input] -os._is_cpu_count_overridden() -> bool +os._get_cpu_count_config Private function to get PyConfig.cpu_count is overridden or not [clinic start generated code]*/ static PyObject * os__get_cpu_count_config_impl(PyObject *module) -/*[clinic end generated code: output=3b0ba7e445c9c6b2 input=c52e5bc863732c21]*/ +/*[clinic end generated code: output=3b0ba7e445c9c6b2 input=312c6ed9a71c441c]*/ { const PyConfig *config = _Py_GetConfig(); if (config->cpu_count > 0) { From c37c8d0ce8c5b1ebaf5da353fdc7d940652c7339 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 3 Oct 2023 09:59:23 +0900 Subject: [PATCH 64/68] Use PYTHON_CPU_COUNT --- Doc/c-api/init_config.rst | 2 +- Doc/library/os.rst | 4 ++-- Doc/using/cmdline.rst | 4 ++-- Doc/whatsnew/3.13.rst | 2 +- Lib/test/test_cmd_line.py | 6 +++--- Python/initconfig.c | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 79feb3e502af08..c632e6a1e6a1b9 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -885,7 +885,7 @@ PyConfig into *cpu_count*. Configured by the :samp:`-X cpu_count={n|default}` command line - flag or the :envvar:`PYTHONCPUCOUNT` environment variable. + flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. Default: ``-1``. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7fd04cf08f21bc..15e5f5c951db51 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5196,7 +5196,7 @@ Miscellaneous System Information .. versionadded:: 3.4 .. versionchanged:: 3.13 - If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, :func:`cpu_count` returns the overridden value *n*. @@ -5218,7 +5218,7 @@ Miscellaneous System Information The :func:`cpu_count` function can be used to get the number of logical CPUs in the **system**. - If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHONCPUCOUNT` is set, + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, :func:`process_cpu_count` returns the overridden value *n*. See also the :func:`sched_getaffinity` functions. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 8814338087ad9a..aac0a5a85ae2ab 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -549,7 +549,7 @@ Miscellaneous options * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count` and :func:`os.process_cpu_count`. *n* must be greater than or equal to 1. This option is useful for users who need to limit CPU resources of a container system. - See also :envvar:`PYTHONCPUCOUNT`. + See also :envvar:`PYTHON_CPU_COUNT`. If *n* is ``default``, :func:`os.cpu_count` and :func:`os.process_cpu_count` are not overridden. It also allows passing arbitrary values and retrieving them through the @@ -1071,7 +1071,7 @@ conflict. .. versionadded:: 3.12 -.. envvar:: PYTHONCPUCOUNT +.. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides :func:`os.cpu_count` and and :func:`os.process_cpu_count` return result. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d75cd0f8c4dcd3..eb347e82ada4cf 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -171,7 +171,7 @@ os (Contributed by Victor Stinner in :gh:`109649`.) * :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overridden through - the new environment variable :envvar:`PYTHONCPUCOUNT` or the new command-line option + the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). (Contributed by Donghee Na in :gh:`109595`) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 5beb72096e2a3f..eaf19aa160e860 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -903,16 +903,16 @@ def test_cpu_count(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) self.assertEqual(self.res2int(res), (4321, 4321)) - res = assert_python_ok('-c', code, PYTHONCPUCOUNT='1234') + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (1234, 1234)) def test_cpu_count_default(self): code = "import os; print(os.cpu_count(), os.process_cpu_count())" res = assert_python_ok('-X', 'cpu_count=default', '-c', code) self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHONCPUCOUNT='1234') + res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - es = assert_python_ok('-c', code, PYTHONCPUCOUNT='default') + es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): diff --git a/Python/initconfig.c b/Python/initconfig.c index 7ebb92798da843..210a8294fd57c0 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1625,7 +1625,7 @@ config_read_env_vars(PyConfig *config) static PyStatus config_init_cpu_count(PyConfig *config) { - const char *env = config_get_env(config, "PYTHONCPUCOUNT"); + const char *env = config_get_env(config, "PYTHON_CPU_COUNT"); if (env) { int cpu_count = -1; if (strcmp(env, "default") == 0) { From c7726e5f57ba2cc596ffd6f064f2ac50fec62ab1 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 3 Oct 2023 11:00:51 +0900 Subject: [PATCH 65/68] Address Greg's review --- Lib/os.py | 3 +-- Modules/clinic/posixmodule.c.h | 20 +------------------- Modules/posixmodule.c | 18 ------------------ Python/clinic/sysmodule.c.h | 20 +++++++++++++++++++- Python/sysmodule.c | 18 ++++++++++++++++++ 5 files changed, 39 insertions(+), 40 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 6f7f97d3b18224..cefa2574a43182 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -55,7 +55,6 @@ def _get_exports_list(module): from posix import * try: from posix import _exit - from posix import _get_cpu_count_config __all__.append('_exit') except ImportError: pass @@ -1139,7 +1138,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and _get_cpu_count_config() != 'overridden': +if _exists('sched_getaffinity') and sys._get_cpu_count_config() != 'overridden': def process_cpu_count(): """ Get the number of CPUs of the current process. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 2608929d8f554c..fc39ab72bf2a51 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10421,24 +10421,6 @@ os_get_terminal_size(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* (defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL)) */ -PyDoc_STRVAR(os__get_cpu_count_config__doc__, -"_get_cpu_count_config($module, /)\n" -"--\n" -"\n" -"Private function to get PyConfig.cpu_count is overridden or not"); - -#define OS__GET_CPU_COUNT_CONFIG_METHODDEF \ - {"_get_cpu_count_config", (PyCFunction)os__get_cpu_count_config, METH_NOARGS, os__get_cpu_count_config__doc__}, - -static PyObject * -os__get_cpu_count_config_impl(PyObject *module); - -static PyObject * -os__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return os__get_cpu_count_config_impl(module); -} - PyDoc_STRVAR(os_cpu_count__doc__, "cpu_count($module, /)\n" "--\n" @@ -12004,4 +11986,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a38f7ef7a5ca4bd7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8b60de6ddb925bc3 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index eeb8ed93de7e96..3d94eab214f537 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14335,23 +14335,6 @@ os_get_terminal_size_impl(PyObject *module, int fd) } #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ -/*[clinic input] -os._get_cpu_count_config - -Private function to get PyConfig.cpu_count is overridden or not -[clinic start generated code]*/ - -static PyObject * -os__get_cpu_count_config_impl(PyObject *module) -/*[clinic end generated code: output=3b0ba7e445c9c6b2 input=312c6ed9a71c441c]*/ -{ - const PyConfig *config = _Py_GetConfig(); - if (config->cpu_count > 0) { - return PyUnicode_FromString("overridden"); - } - return PyUnicode_FromString("default"); -} - /*[clinic input] os.cpu_count @@ -15998,7 +15981,6 @@ static PyMethodDef posix_methods[] = { OS_LISTXATTR_METHODDEF OS_GET_TERMINAL_SIZE_METHODDEF - OS__GET_CPU_COUNT_CONFIG_METHODDEF OS_CPU_COUNT_METHODDEF OS_GET_INHERITABLE_METHODDEF OS_SET_INHERITABLE_METHODDEF diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 30691c3d08ae67..afdfe562c4afc9 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1380,6 +1380,24 @@ sys__getframemodulename(PyObject *module, PyObject *const *args, Py_ssize_t narg return return_value; } +PyDoc_STRVAR(sys__get_cpu_count_config__doc__, +"_get_cpu_count_config($module, /)\n" +"--\n" +"\n" +"Private function to get PyConfig.cpu_count is overridden or not"); + +#define SYS__GET_CPU_COUNT_CONFIG_METHODDEF \ + {"_get_cpu_count_config", (PyCFunction)sys__get_cpu_count_config, METH_NOARGS, sys__get_cpu_count_config__doc__}, + +static PyObject * +sys__get_cpu_count_config_impl(PyObject *module); + +static PyObject * +sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__get_cpu_count_config_impl(module); +} + #ifndef SYS_GETWINDOWSVERSION_METHODDEF #define SYS_GETWINDOWSVERSION_METHODDEF #endif /* !defined(SYS_GETWINDOWSVERSION_METHODDEF) */ @@ -1423,4 +1441,4 @@ sys__getframemodulename(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=549bb1f92a15f916 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dbea7a14012dd3b7 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b00301765e1890..1eb89f00d57bf1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2290,6 +2290,23 @@ sys__getframemodulename_impl(PyObject *module, int depth) return Py_NewRef(r); } +/*[clinic input] +sys._get_cpu_count_config +Private function to get PyConfig.cpu_count is overridden or not +[clinic start generated code]*/ + +static PyObject * +sys__get_cpu_count_config_impl(PyObject *module) +/*[clinic end generated code: output=56db9f22b9ef5580 input=bb27289c04f47ce2]*/ +{ + const PyConfig *config = _Py_GetConfig(); + if (config->cpu_count > 0) { + return PyUnicode_FromString("overridden"); + } + return PyUnicode_FromString("default"); +} + + static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { @@ -2424,6 +2441,7 @@ static PyMethodDef sys_methods[] = { SYS__STATS_CLEAR_METHODDEF SYS__STATS_DUMP_METHODDEF #endif + SYS__GET_CPU_COUNT_CONFIG_METHODDEF {NULL, NULL} // sentinel }; From de8bf53f5e2820ea855409265b22c9d61048b4e8 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 3 Oct 2023 11:23:43 +0900 Subject: [PATCH 66/68] Address Greg's code review --- Lib/os.py | 2 +- Python/clinic/sysmodule.c.h | 18 ++++++++++++++---- Python/sysmodule.c | 15 ++++++--------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index cefa2574a43182..a17946750ea7e7 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1138,7 +1138,7 @@ def add_dll_directory(path): ) -if _exists('sched_getaffinity') and sys._get_cpu_count_config() != 'overridden': +if _exists('sched_getaffinity') and sys._get_cpu_count_config() < 0: def process_cpu_count(): """ Get the number of CPUs of the current process. diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index afdfe562c4afc9..53410f2df7f28a 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1384,18 +1384,28 @@ PyDoc_STRVAR(sys__get_cpu_count_config__doc__, "_get_cpu_count_config($module, /)\n" "--\n" "\n" -"Private function to get PyConfig.cpu_count is overridden or not"); +"Private function for get PyConfig.cpu_count"); #define SYS__GET_CPU_COUNT_CONFIG_METHODDEF \ {"_get_cpu_count_config", (PyCFunction)sys__get_cpu_count_config, METH_NOARGS, sys__get_cpu_count_config__doc__}, -static PyObject * +static int sys__get_cpu_count_config_impl(PyObject *module); static PyObject * sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return sys__get_cpu_count_config_impl(module); + PyObject *return_value = NULL; + int _return_value; + + _return_value = sys__get_cpu_count_config_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; } #ifndef SYS_GETWINDOWSVERSION_METHODDEF @@ -1441,4 +1451,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=dbea7a14012dd3b7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c8da9442f1d43b25 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 1eb89f00d57bf1..7a4a3c8bf06544 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2291,22 +2291,19 @@ sys__getframemodulename_impl(PyObject *module, int depth) } /*[clinic input] -sys._get_cpu_count_config -Private function to get PyConfig.cpu_count is overridden or not +sys._get_cpu_count_config -> int + +Private function for get PyConfig.cpu_count [clinic start generated code]*/ -static PyObject * +static int sys__get_cpu_count_config_impl(PyObject *module) -/*[clinic end generated code: output=56db9f22b9ef5580 input=bb27289c04f47ce2]*/ +/*[clinic end generated code: output=36611bb5efad16dc input=80293caef314f45b]*/ { const PyConfig *config = _Py_GetConfig(); - if (config->cpu_count > 0) { - return PyUnicode_FromString("overridden"); - } - return PyUnicode_FromString("default"); + return config->cpu_count; } - static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { From 633914be017ca1fad816662572e20063f99855b6 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 3 Oct 2023 11:25:36 +0900 Subject: [PATCH 67/68] nit --- Python/clinic/sysmodule.c.h | 4 ++-- Python/sysmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 53410f2df7f28a..06105e221c1f89 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1384,7 +1384,7 @@ PyDoc_STRVAR(sys__get_cpu_count_config__doc__, "_get_cpu_count_config($module, /)\n" "--\n" "\n" -"Private function for get PyConfig.cpu_count"); +"Private function for getting PyConfig.cpu_count"); #define SYS__GET_CPU_COUNT_CONFIG_METHODDEF \ {"_get_cpu_count_config", (PyCFunction)sys__get_cpu_count_config, METH_NOARGS, sys__get_cpu_count_config__doc__}, @@ -1451,4 +1451,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=c8da9442f1d43b25 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3a7d3fbbcb281c22 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 7a4a3c8bf06544..a3b1438a75099a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2293,12 +2293,12 @@ sys__getframemodulename_impl(PyObject *module, int depth) /*[clinic input] sys._get_cpu_count_config -> int -Private function for get PyConfig.cpu_count +Private function for getting PyConfig.cpu_count [clinic start generated code]*/ static int sys__get_cpu_count_config_impl(PyObject *module) -/*[clinic end generated code: output=36611bb5efad16dc input=80293caef314f45b]*/ +/*[clinic end generated code: output=36611bb5efad16dc input=523e1ade2204084e]*/ { const PyConfig *config = _Py_GetConfig(); return config->cpu_count; From 37fbdfe123db097c6d47fe8f2c1719c65e048893 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Tue, 3 Oct 2023 04:57:44 +0000 Subject: [PATCH 68/68] Include multiprocessing in docs, reword. Adds PYTHON_CPU_COUNT help text. Mentions multiprocessing.cpu_count. rewords a few statements. --- Doc/c-api/init_config.rst | 6 +++--- Doc/library/multiprocessing.rst | 9 ++++++++- Doc/using/cmdline.rst | 13 +++++++------ Python/initconfig.c | 7 +++++-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index c632e6a1e6a1b9..0240e25b6f1607 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -880,9 +880,9 @@ PyConfig .. c:member:: int cpu_count - If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will override - the return value of :func:`os.cpu_count` and :func:`os.process_cpu_count` functions - into *cpu_count*. + If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will + override the return values of :func:`os.cpu_count`, + :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. Configured by the :samp:`-X cpu_count={n|default}` command line flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2f0f1f800fdc94..5cec9d87a01cc6 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -996,13 +996,20 @@ Miscellaneous This number is not equivalent to the number of CPUs the current process can use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` + :func:`os.process_cpu_count` (or ``len(os.sched_getaffinity(0))``). When the number of CPUs cannot be determined a :exc:`NotImplementedError` is raised. .. seealso:: :func:`os.cpu_count` + :func:`os.process_cpu_count` + + .. versionchanged:: 3.13 + + The return value can also be overridden using the + :option:`-X cpu_count <-X>` flag or :envvar:`PYTHON_CPU_COUNT` as this is + merely a wrapper around the :mod:`os` cpu count APIs. .. function:: current_process() diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index aac0a5a85ae2ab..5c9d4b106742d2 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -546,11 +546,12 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. - * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count` and :func:`os.process_cpu_count`. + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, + :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. *n* must be greater than or equal to 1. - This option is useful for users who need to limit CPU resources of a container system. - See also :envvar:`PYTHON_CPU_COUNT`. - If *n* is ``default``, :func:`os.cpu_count` and :func:`os.process_cpu_count` are not overridden. + This option may be useful for users who need to limit CPU resources of a + container system. See also :envvar:`PYTHON_CPU_COUNT`. + If *n* is ``default``, nothing is overridden. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -1073,8 +1074,8 @@ conflict. .. envvar:: PYTHON_CPU_COUNT - If this variable is set to a positive integer, it overrides - :func:`os.cpu_count` and and :func:`os.process_cpu_count` return result. + If this variable is set to a positive integer, it overrides the return + values of :func:`os.cpu_count` and :func:`os.process_cpu_count`. See also the :option:`-X cpu_count <-X>` command-line option. diff --git a/Python/initconfig.c b/Python/initconfig.c index 210a8294fd57c0..ae458dc702bd12 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -231,8 +231,9 @@ The following implementation-specific options are available:\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ The default is sys.int_info.default_max_str_digits. 0 disables.\n\ \n\ --X cpu_count=[n|default]: override CPU count of os.cpu_count() and os.process_cpu_count().\n\ - This helps for users who need to limit CPU resources of a container system." +-X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ + os.process_cpu_count(), and multiprocessing.cpu_count(). This can help users who need\n\ + to limit resources in a container." #ifdef Py_STATS "\n\ @@ -270,6 +271,8 @@ static const char usage_envvars[] = " locale coercion and locale compatibility warnings on stderr.\n" "PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" " debugger. It can be set to the callable of your debugger of choice.\n" +"PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" +" os.cpu_count(), and multiprocessing.cpu_count() if set to a positive integer.\n" "PYTHONDEVMODE: enable the development mode.\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n"