From f10516b88f4da9d73248c975c213d5989acdfad8 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Tue, 27 Oct 2020 00:29:57 +0100 Subject: [PATCH 1/6] bpo-42160: reduce overhead in tempfile The _RandomSequence class in tempfile used to check the current pid every time its rng property was used. This commit replaces this code with os.register_at_fork to reduce the overhead. --- Lib/tempfile.py | 26 +++++++++++++------------- Lib/test/test_tempfile.py | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 770f72c25295cb..5c159db118ceb7 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -129,15 +129,19 @@ class _RandomNameSequence: _RandomNameSequence is an iterator.""" - characters = "abcdefghijklmnopqrstuvwxyz0123456789_" + def __init__(self, characters="abcdefghijklmnopqrstuvwxyz0123456789_", length=8, rng=None): + if rng is None: + rng = _Random() + if hasattr(_os, "fork"): + # prevent same state after fork + _os.register_at_fork(after_in_child=rng.seed) + self.rng = rng + self.characters = characters + self.length = length @property - def rng(self): - cur_pid = _os.getpid() - if cur_pid != getattr(self, '_rng_pid', None): - self._rng = _Random() - self._rng_pid = cur_pid - return self._rng + def variants(self): + return len(self.characters) ** self.length def __iter__(self): return self @@ -145,8 +149,7 @@ def __iter__(self): def __next__(self): c = self.characters choose = self.rng.choice - letters = [choose(c) for dummy in range(8)] - return ''.join(letters) + return ''.join(choose(c) for _ in range(self.length)) def _candidate_tempdir_list(): """Generate a list of candidate temporary directories which @@ -227,12 +230,9 @@ def _get_candidate_names(): global _name_sequence if _name_sequence is None: - _once_lock.acquire() - try: + with _once_lock: if _name_sequence is None: _name_sequence = _RandomNameSequence() - finally: - _once_lock.release() return _name_sequence diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 8ace883d74bb24..77d710efaf107b 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -153,8 +153,8 @@ def setUp(self): self.r = tempfile._RandomNameSequence() super().setUp() - def test_get_six_char_str(self): - # _RandomNameSequence returns a six-character string + def test_get_eight_char_str(self): + # _RandomNameSequence returns a eight-character string s = next(self.r) self.nameCheck(s, '', '', '') From a26da4f9e081f2a4abf6fb96f9572b8d21ff5154 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2020 00:42:09 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst diff --git a/Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst b/Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst new file mode 100644 index 00000000000000..3d443668b72cd5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst @@ -0,0 +1 @@ +replaced pid check in tempfile._RandomNameSequence with os.register_at_fork to reduce overhead \ No newline at end of file From 3090bec3049f19f654acaefebc1252fba7a6495d Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Tue, 27 Oct 2020 04:06:53 +0100 Subject: [PATCH 3/6] revert unnecessary change in _get_candidate_names --- Lib/tempfile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 5c159db118ceb7..4f4d342365be5e 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -230,9 +230,12 @@ def _get_candidate_names(): global _name_sequence if _name_sequence is None: - with _once_lock: + _once_lock.acquire() + try: if _name_sequence is None: _name_sequence = _RandomNameSequence() + finally: + _once_lock.release() return _name_sequence From faa415d894bea4d36766e5aeca043ff5ba31b7ef Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Thu, 29 Oct 2020 21:53:07 +0100 Subject: [PATCH 4/6] remove variants property from tempfile._RandomNameSequence --- Lib/tempfile.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 4f4d342365be5e..4d0b7e5b36020c 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -139,10 +139,6 @@ def __init__(self, characters="abcdefghijklmnopqrstuvwxyz0123456789_", length=8, self.characters = characters self.length = length - @property - def variants(self): - return len(self.characters) ** self.length - def __iter__(self): return self From 395887d798ed2289564e8d4a6d61cdbeb3c02b9c Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Thu, 29 Oct 2020 21:59:37 +0100 Subject: [PATCH 5/6] replace Random.choice in tempfile._RandomNameSequence with Random.choices --- Lib/tempfile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 4d0b7e5b36020c..1bc5c71fd03494 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -144,8 +144,7 @@ def __iter__(self): def __next__(self): c = self.characters - choose = self.rng.choice - return ''.join(choose(c) for _ in range(self.length)) + return ''.join(self.rng.choices(c, k=self.length)) def _candidate_tempdir_list(): """Generate a list of candidate temporary directories which From dcdde3985232aa96af6a8f0bb725d2065030e60d Mon Sep 17 00:00:00 2001 From: Eric W Date: Thu, 29 Oct 2020 22:02:51 +0100 Subject: [PATCH 6/6] fix typo in news entry Co-authored-by: Inada Naoki --- .../next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst b/Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst index 3d443668b72cd5..c5f3091283a87b 100644 --- a/Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst +++ b/Misc/NEWS.d/next/Library/2020-10-27-00-42-09.bpo-42160.eiLOCi.rst @@ -1 +1 @@ -replaced pid check in tempfile._RandomNameSequence with os.register_at_fork to reduce overhead \ No newline at end of file +Replaced pid check in ``tempfile._RandomNameSequence`` with ``os.register_at_fork`` to reduce overhead.