From 7613509e4402e95df4bb86fe4237a6e180bc36f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Jul 2019 12:45:56 +0200 Subject: [PATCH 1/2] bpo-37421: Fix multiprocessing get_temp_dir() finalizer Fix multiprocessing.util.get_temp_dir() finalizer: clear also the 'tempdir' configuration of the current process, so next call to get_temp_dir() will create a new temporary directory, rather than reusing the removed temporary directory. --- Lib/multiprocessing/util.py | 10 +++++++++- .../Library/2019-07-03-12-47-52.bpo-37421.gR5hC8.rst | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2019-07-03-12-47-52.bpo-37421.gR5hC8.rst diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 5674ad773f9764..9abeb173b748ca 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -106,6 +106,11 @@ def log_to_stderr(level=None): # Function returning a temp directory which will be removed on exit # +def _remove_temp_dir(rmtree, tempdir): + rmtree(tempdir) + process.current_process()._config['tempdir'] = None + + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') @@ -113,7 +118,10 @@ def get_temp_dir(): import shutil, tempfile tempdir = tempfile.mkdtemp(prefix='pymp-') info('created temp directory %s', tempdir) - Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) + # keep a strong reference to shutil.rmtree(), since the finalizer + # can be called late during Python shutdown + Finalize(None, _remove_temp_dir, args=(shutil.rmtree, tempdir), + exitpriority=-100) process.current_process()._config['tempdir'] = tempdir return tempdir diff --git a/Misc/NEWS.d/next/Library/2019-07-03-12-47-52.bpo-37421.gR5hC8.rst b/Misc/NEWS.d/next/Library/2019-07-03-12-47-52.bpo-37421.gR5hC8.rst new file mode 100644 index 00000000000000..450d76f072890c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-07-03-12-47-52.bpo-37421.gR5hC8.rst @@ -0,0 +1,4 @@ +Fix :func:`multiprocessing.util.get_temp_dir` finalizer: clear also the +'tempdir' configuration of the current process, so next call to +``get_temp_dir()`` will create a new temporary directory, rather than +reusing the removed temporary directory. From 08ad64cf73c60cc6563740c67e11164e4bc90ec4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Jul 2019 12:52:49 +0200 Subject: [PATCH 2/2] Fix if current_process() returs None --- Lib/multiprocessing/util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 9abeb173b748ca..19380917b6f5fa 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -108,8 +108,12 @@ def log_to_stderr(level=None): def _remove_temp_dir(rmtree, tempdir): rmtree(tempdir) - process.current_process()._config['tempdir'] = None + current_process = process.current_process() + # current_process() can be None if the finalizer is called + # late during Python finalization + if current_process is not None: + current_process._config['tempdir'] = None def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up