From cbfaad642045f1d55b8d177bd16646d042b311a9 Mon Sep 17 00:00:00 2001 From: rmorotti Date: Tue, 30 Apr 2024 12:00:10 +0100 Subject: [PATCH 1/2] gh-117151: increase default buffer size of shutil.copyfileobj() to 256k. it was set to 16k in the 1990s. it was raised to 64k in 2019. the discussion at the time mentioned another 5% improvement by raising to 128k and settled for a very conservative setting. it's 2024 now, I think it should be revisited to match modern hardware. I am measuring 0-15% performance improvement when raising to 256k on various types of disk. there is no downside as far as I can tell. this function is only intended for sequential copy of full files (or file like objects). it's the typical use case that benefits from larger operations. for reference, I came across this function while trying to profile pip that is using it to copy files when installing python packages. --- Lib/shutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 03a9d756030430..65ac4f31f3efb1 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -44,7 +44,7 @@ else: _winapi = None -COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 +COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 _USE_CP_SENDFILE = (hasattr(os, "sendfile") From 47c9fa70d88e6681254315d31328f0bb5bb1d79c Mon Sep 17 00:00:00 2001 From: rmorotti Date: Thu, 3 Oct 2024 12:57:12 +0100 Subject: [PATCH 2/2] add news --- .../Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst diff --git a/Misc/NEWS.d/next/Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst b/Misc/NEWS.d/next/Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst new file mode 100644 index 00000000000000..a7d6251f1e071f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst @@ -0,0 +1,3 @@ +The default buffer size used by :func:`shutil.copyfileobj` has been +increased from 64k to 256k on non-Windows platforms. It was already larger +on Windows.