From ca76614a871554430689191e3737a6a453d6d30e Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 22 Jul 2019 12:14:50 +0100 Subject: [PATCH] Update logging cookbook to show multiple worker processes using the concurrent.futures module. (GH-14905) (cherry picked from commit d309352c6fd93a51f2b3011ca8c2125d3a5d394b) Co-authored-by: Vinay Sajip --- Doc/howto/logging-cookbook.rst | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 87ac79ef8072d6..e62308192d1655 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -948,6 +948,41 @@ This variant shows how you can e.g. apply configuration for particular loggers machinery in the main process (even though the logging events are generated in the worker processes) to direct the messages to the appropriate destinations. +Using concurrent.futures.ProcessPoolExecutor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to use :class:`concurrent.futures.ProcessPoolExecutor` to start +your worker processes, you need to create the queue slightly differently. +Instead of + +.. code-block:: python + + queue = multiprocessing.Queue(-1) + +you should use + +.. code-block:: python + + queue = multiprocessing.Manager().Queue(-1) # also works with the examples above + +and you can then replace the worker creation from this:: + + workers = [] + for i in range(10): + worker = multiprocessing.Process(target=worker_process, + args=(queue, worker_configurer)) + workers.append(worker) + worker.start() + for w in workers: + w.join() + +to this (remembering to first import :mod:`concurrent.futures`):: + + with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor: + for i in range(10): + executor.submit(worker_process, queue, worker_configurer) + + Using file rotation -------------------