From 3be08b4ac2b530b2cd00a6aea3259d29343b0316 Mon Sep 17 00:00:00 2001 From: pan324 Date: Thu, 12 Oct 2023 19:49:33 +0200 Subject: [PATCH 01/23] add track parameter to shared memory --- Doc/library/multiprocessing.shared_memory.rst | 20 ++++++++++++++----- Lib/multiprocessing/shared_memory.py | 6 +++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index f453e6403d932d..d3d9993456438b 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -36,7 +36,7 @@ or other communications requiring the serialization/deserialization and copying of data. -.. class:: SharedMemory(name=None, create=False, size=0) +.. class:: SharedMemory(name=None, create=False, size=0, track=True) Creates a new shared memory block or attaches to an existing shared memory block. Each shared memory block is assigned a unique name. @@ -64,13 +64,23 @@ copying of data. memory block may be larger or equal to the size requested. When attaching to an existing shared memory block, the ``size`` parameter is ignored. + *track*, when enabled, registers the shared memory block with the resource + tracker process. This process ensures proper cleanup of shared memory + blocks even when all other processes with access to the memory have failed + to do so (mainly due to being killed by signals). The resource tracker is + overzealous in certain situations and will delete a shared memory block + when any process with access to the shared memory has terminated. *track* + should be set to ``False`` if there is already another process in place + that does the bookkeeping. In most situations, this means that *track* + should be set to ``False`` when *create* is set to ``False``. + .. method:: close() Closes access to the shared memory from this instance. In order to - ensure proper cleanup of resources, all instances should call - ``close()`` once the instance is no longer needed. Note that calling - ``close()`` does not cause the shared memory block itself to be - destroyed. + ensure proper cleanup of resources, all instances with *track* set to + ``True`` should call ``close()`` once the instance is no longer needed. + Note that calling ``close()`` does not cause the shared memory block + itself to be destroyed. .. method:: unlink() diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 9a1e5aa17b87a2..4999b659260e23 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -72,7 +72,7 @@ class SharedMemory: _mode = 0o600 _prepend_leading_slash = True if _USE_POSIX else False - def __init__(self, name=None, create=False, size=0): + def __init__(self, name=None, create=False, size=0, track=True): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: @@ -116,8 +116,8 @@ def __init__(self, name=None, create=False, size=0): except OSError: self.unlink() raise - - resource_tracker.register(self._name, "shared_memory") + if track: + resource_tracker.register(self._name, "shared_memory") else: From f50812a84bb848304831dc40035d063c8a5f3e86 Mon Sep 17 00:00:00 2001 From: pan324 Date: Thu, 12 Oct 2023 20:11:01 +0200 Subject: [PATCH 02/23] phrasing --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index d3d9993456438b..fc924ef4ca3dd9 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -68,7 +68,7 @@ copying of data. tracker process. This process ensures proper cleanup of shared memory blocks even when all other processes with access to the memory have failed to do so (mainly due to being killed by signals). The resource tracker is - overzealous in certain situations and will delete a shared memory block + overzealous in certain situations and might delete a shared memory block when any process with access to the shared memory has terminated. *track* should be set to ``False`` if there is already another process in place that does the bookkeeping. In most situations, this means that *track* From cbc8431e88061a38c4fac3f1aee89bbe57d5b1c3 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:19:48 +0000 Subject: [PATCH 03/23] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst diff --git a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst new file mode 100644 index 00000000000000..30473448044155 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst @@ -0,0 +1 @@ +Add ``track`` parameter that allows using shared memory blocks without having to register with the resource tracker. From 7138447cd669529391d58114b66579d65bd8b44a Mon Sep 17 00:00:00 2001 From: pan324 Date: Fri, 13 Oct 2023 00:46:47 +0200 Subject: [PATCH 04/23] phrasing --- Doc/library/multiprocessing.shared_memory.rst | 36 +- Doc/library/result.html | 1151 +++++++++++++++++ Lib/multiprocessing/shared_memory.py | 4 + 3 files changed, 1172 insertions(+), 19 deletions(-) create mode 100644 Doc/library/result.html diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index fc924ef4ca3dd9..2b17ba0b66391b 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -65,35 +65,33 @@ copying of data. to an existing shared memory block, the ``size`` parameter is ignored. *track*, when enabled, registers the shared memory block with the resource - tracker process. This process ensures proper cleanup of shared memory + tracker process. This process ensures proper cleanup of shared memory blocks even when all other processes with access to the memory have failed - to do so (mainly due to being killed by signals). The resource tracker is + to do so (mainly due to being killed by signals). The resource tracker is overzealous in certain situations and might delete a shared memory block - when any process with access to the shared memory has terminated. *track* + when any process with access to the shared memory has terminated. *track* should be set to ``False`` if there is already another process in place - that does the bookkeeping. In most situations, this means that *track* + that does the bookkeeping. In most situations, this means that *track* should be set to ``False`` when *create* is set to ``False``. .. method:: close() - Closes access to the shared memory from this instance. In order to - ensure proper cleanup of resources, all instances with *track* set to - ``True`` should call ``close()`` once the instance is no longer needed. - Note that calling ``close()`` does not cause the shared memory block - itself to be destroyed. + Closes the file descriptor/handle to the shared memory from this + instance. All instances should call ``close()`` once the instance + is no longer needed. Depending on operating system, the underlying + memory may or may not be freed even if all handles to it have been + closed. To ensure proper cleanup, use the ``unlink()`` method. .. method:: unlink() - Requests that the underlying shared memory block be destroyed. In - order to ensure proper cleanup of resources, ``unlink()`` should be - called once (and only once) across all processes which have need - for the shared memory block. After requesting its destruction, a - shared memory block may or may not be immediately destroyed and - this behavior may differ across platforms. Attempts to access data - inside the shared memory block after ``unlink()`` has been called may - result in memory access errors. Note: the last process relinquishing - its hold on a shared memory block may call ``unlink()`` and - :meth:`close()` in either order. + Deletes the underlying shared memory block. This should be called only + once per shared memory block regardless of the number of handles to it. + After requesting its deletion, a shared memory block may or may not be + immediately destroyed and this behavior may differ across platforms. + Attempts to access data inside the shared memory block after + ``unlink()`` has been called may result in memory access errors. + To ensure proper bookkeeping, ``unlink()`` may only be called by + an instance with *track* enabled. .. attribute:: buf diff --git a/Doc/library/result.html b/Doc/library/result.html new file mode 100644 index 00000000000000..c57a8f5eda82cd --- /dev/null +++ b/Doc/library/result.html @@ -0,0 +1,1151 @@ + + + + + + +:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes + + + + +
+

:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes

+ + + +

Source code: :source:`Lib/multiprocessing/shared_memory.py`

+ + + +
+

This module provides a class, :class:`SharedMemory`, for the allocation +and management of shared memory to be accessed by one or more processes +on a multicore or symmetric multiprocessor (SMP) machine. To assist with +the life-cycle management of shared memory especially across distinct +processes, a :class:`~multiprocessing.managers.BaseManager` subclass, +:class:`SharedMemoryManager`, is also provided in the +multiprocessing.managers module.

+ + + +

In this module, shared memory refers to "System V style" shared memory blocks +(though is not necessarily implemented explicitly as such) and does not refer +to "distributed shared memory". This style of shared memory permits distinct +processes to potentially read and write to a common (or shared) region of +volatile memory. Processes are conventionally limited to only have access to +their own process memory space but shared memory permits the sharing +of data between processes, avoiding the need to instead send messages between +processes containing that data. Sharing data directly via memory can provide +significant performance benefits compared to sharing data via disk or socket +or other communications requiring the serialization/deserialization and +copying of data.

+

Creates a new shared memory block or attaches to an existing shared +memory block. Each shared memory block is assigned a unique name. +In this way, one process can create a shared memory block with a +particular name and a different process can attach to that same shared +memory block using that same name.

+

As a resource for sharing data across processes, shared memory blocks +may outlive the original process that created them. When one process +no longer needs access to a shared memory block that might still be +needed by other processes, the :meth:`close()` method should be called. +When a shared memory block is no longer needed by any process, the +:meth:`unlink()` method should be called to ensure proper cleanup.

+ + +

name is the unique name for the requested shared memory, specified as +a string. When creating a new shared memory block, if None (the +default) is supplied for the name, a novel name will be generated.

+

create controls whether a new shared memory block is created (True) +or an existing shared memory block is attached (False).

+

size specifies the requested number of bytes when creating a new shared +memory block. Because some platforms choose to allocate chunks of memory +based upon that platform's memory page size, the exact size of the shared +memory block may be larger or equal to the size requested. When attaching +to an existing shared memory block, the size parameter is ignored.

+

track, when enabled, registers the shared memory block with the resource +tracker process. This process ensures proper cleanup of shared memory +blocks even when all other processes with access to the memory have failed +to do so (mainly due to being killed by signals). The resource tracker is +overzealous in certain situations and will delete a shared memory block +when any process with access to the shared memory has terminated. track +should be set to False if there is already another process in place +that does the bookkeeping. In most situations, this means that track +should be set to False when create is set to False.

+ + + + + +

The following example demonstrates low-level use of :class:`SharedMemory` +instances:

+ +
>>> from multiprocessing import shared_memory
+>>> shm_a = shared_memory.SharedMemory(create=True, size=10)
+>>> type(shm_a.buf)
+<class 'memoryview'>
+>>> buffer = shm_a.buf
+>>> len(buffer)
+10
+>>> buffer[:4] = bytearray([22, 33, 44, 55])  # Modify multiple at once
+>>> buffer[4] = 100                           # Modify single byte at a time
+>>> # Attach to an existing shared memory block
+>>> shm_b = shared_memory.SharedMemory(shm_a.name)
+>>> import array
+>>> array.array('b', shm_b.buf[:5])  # Copy the data into a new array.array
+array('b', [22, 33, 44, 55, 100])
+>>> shm_b.buf[:5] = b'howdy'  # Modify via shm_b using bytes
+>>> bytes(shm_a.buf[:5])      # Access via shm_a
+b'howdy'
+>>> shm_b.close()   # Close each SharedMemory instance
+>>> shm_a.close()
+>>> shm_a.unlink()  # Call unlink only once to release the shared memory
+

The following example demonstrates a practical use of the :class:`SharedMemory` +class with NumPy arrays, accessing the +same numpy.ndarray from two distinct Python shells:

+ + +

A subclass of :class:`~multiprocessing.managers.BaseManager` which can be +used for the management of shared memory blocks across processes.

+ +

A call to :meth:`~multiprocessing.managers.BaseManager.start` on a +:class:`SharedMemoryManager` instance causes a new process to be started. +This new process's sole purpose is to manage the life cycle +of all shared memory blocks created through it. To trigger the release +of all shared memory blocks managed by that process, call +:meth:`~multiprocessing.managers.BaseManager.shutdown()` on the instance. +This triggers a :meth:`SharedMemory.unlink()` call on all of the +:class:`SharedMemory` objects managed by that process and then +stops the process itself. By creating SharedMemory instances +through a SharedMemoryManager, we avoid the need to manually track +and trigger the freeing of shared memory resources.

+ + + + + +

This class provides methods for creating and returning :class:`SharedMemory` +instances and for creating a list-like object (:class:`ShareableList`) +backed by shared memory.

+ + +

Refer to :class:`multiprocessing.managers.BaseManager` for a description +of the inherited address and authkey optional input arguments and how +they may be used to connect to an existing SharedMemoryManager service +from other processes.

+ + + +

The following example demonstrates the basic mechanisms of a +:class:`SharedMemoryManager`:

+ + +

The following example depicts a potentially more convenient pattern for using +:class:`SharedMemoryManager` objects via the :keyword:`with` statement to +ensure that all shared memory blocks are released after they are no longer +needed:

+ + + +

When using a :class:`SharedMemoryManager` in a :keyword:`with` statement, the +shared memory blocks created using that manager are all released when the +:keyword:`with` statement's code block finishes execution.

+ + + + +

The following example demonstrates basic use of a :class:`ShareableList` +instance:

+ +
+
>>> from multiprocessing import shared_memory
+>>> a = shared_memory.ShareableList(['howdy', b'HoWdY', -273.154, 100, None, True, 42])
+>>> [ type(entry) for entry in a ]
+[<class 'str'>, <class 'bytes'>, <class 'float'>, <class 'int'>, <class 'NoneType'>, <class 'bool'>, <class 'int'>]
+>>> a[2]
+-273.154
+>>> a[2] = -78.5
+>>> a[2]
+-78.5
+>>> a[2] = 'dry ice'  # Changing data types is supported as well
+>>> a[2]
+'dry ice'
+>>> a[2] = 'larger than previously allocated storage space'
+Traceback (most recent call last):
+  ...
+ValueError: exceeds available storage for existing str
+>>> a[2]
+'dry ice'
+>>> len(a)
+7
+>>> a.index(42)
+6
+>>> a.count(b'howdy')
+0
+>>> a.count(b'HoWdY')
+1
+>>> a.shm.close()
+>>> a.shm.unlink()
+>>> del a  # Use of a ShareableList after call to unlink() is unsupported
+
+
+

The following example depicts how one, two, or many processes may access the +same :class:`ShareableList` by supplying the name of the shared memory block +behind it:

+ +
+
>>> b = shared_memory.ShareableList(range(5))         # In a first process
+>>> c = shared_memory.ShareableList(name=b.shm.name)  # In a second process
+>>> c
+ShareableList([0, 1, 2, 3, 4], name='...')
+>>> c[-1] = -999
+>>> b[-1]
+-999
+>>> b.shm.close()
+>>> c.shm.close()
+>>> c.shm.unlink()
+
+
+

The following examples demonstrates that ShareableList +(and underlying SharedMemory) objects +can be pickled and unpickled if needed. +Note, that it will still be the same shared object. +This happens, because the deserialized object has +the same unique name and is just attached to an existing +object with the same name (if the object is still alive):

+
+
>>> import pickle
+>>> from multiprocessing import shared_memory
+>>> sl = shared_memory.ShareableList(range(10))
+>>> list(sl)
+[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+
>>> deserialized_sl = pickle.loads(pickle.dumps(sl))
+>>> list(deserialized_sl)
+[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+
>>> sl[0] = -1
+>>> deserialized_sl[1] = -2
+>>> list(sl)
+[-1, -2, 2, 3, 4, 5, 6, 7, 8, 9]
+>>> list(deserialized_sl)
+[-1, -2, 2, 3, 4, 5, 6, 7, 8, 9]
+
+
>>> sl.shm.close()
+>>> sl.shm.unlink()
+
+
+
+ + diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 4999b659260e23..5f42b41b3f0124 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -82,6 +82,7 @@ def __init__(self, name=None, create=False, size=0, track=True): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") + self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -239,6 +240,9 @@ def unlink(self): In order to ensure proper cleanup of resources, unlink should be called once (and only once) across all processes which have access to the shared memory block.""" + if not self._track: + raise TypeError("unlink() must be called by a tracking instance") + if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) resource_tracker.unregister(self._name, "shared_memory") From d44cb828ed17a9fa524390638e737fe6d7f32055 Mon Sep 17 00:00:00 2001 From: pan324 <103143968+pan324@users.noreply.github.com> Date: Fri, 13 Oct 2023 00:59:34 +0200 Subject: [PATCH 05/23] Delete Doc/library/result.html --- Doc/library/result.html | 1151 --------------------------------------- 1 file changed, 1151 deletions(-) delete mode 100644 Doc/library/result.html diff --git a/Doc/library/result.html b/Doc/library/result.html deleted file mode 100644 index c57a8f5eda82cd..00000000000000 --- a/Doc/library/result.html +++ /dev/null @@ -1,1151 +0,0 @@ - - - - - - -:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes - - - - -
-

:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes

- - - -

Source code: :source:`Lib/multiprocessing/shared_memory.py`

- - - -
-

This module provides a class, :class:`SharedMemory`, for the allocation -and management of shared memory to be accessed by one or more processes -on a multicore or symmetric multiprocessor (SMP) machine. To assist with -the life-cycle management of shared memory especially across distinct -processes, a :class:`~multiprocessing.managers.BaseManager` subclass, -:class:`SharedMemoryManager`, is also provided in the -multiprocessing.managers module.

- - - -

In this module, shared memory refers to "System V style" shared memory blocks -(though is not necessarily implemented explicitly as such) and does not refer -to "distributed shared memory". This style of shared memory permits distinct -processes to potentially read and write to a common (or shared) region of -volatile memory. Processes are conventionally limited to only have access to -their own process memory space but shared memory permits the sharing -of data between processes, avoiding the need to instead send messages between -processes containing that data. Sharing data directly via memory can provide -significant performance benefits compared to sharing data via disk or socket -or other communications requiring the serialization/deserialization and -copying of data.

-

Creates a new shared memory block or attaches to an existing shared -memory block. Each shared memory block is assigned a unique name. -In this way, one process can create a shared memory block with a -particular name and a different process can attach to that same shared -memory block using that same name.

-

As a resource for sharing data across processes, shared memory blocks -may outlive the original process that created them. When one process -no longer needs access to a shared memory block that might still be -needed by other processes, the :meth:`close()` method should be called. -When a shared memory block is no longer needed by any process, the -:meth:`unlink()` method should be called to ensure proper cleanup.

- - -

name is the unique name for the requested shared memory, specified as -a string. When creating a new shared memory block, if None (the -default) is supplied for the name, a novel name will be generated.

-

create controls whether a new shared memory block is created (True) -or an existing shared memory block is attached (False).

-

size specifies the requested number of bytes when creating a new shared -memory block. Because some platforms choose to allocate chunks of memory -based upon that platform's memory page size, the exact size of the shared -memory block may be larger or equal to the size requested. When attaching -to an existing shared memory block, the size parameter is ignored.

-

track, when enabled, registers the shared memory block with the resource -tracker process. This process ensures proper cleanup of shared memory -blocks even when all other processes with access to the memory have failed -to do so (mainly due to being killed by signals). The resource tracker is -overzealous in certain situations and will delete a shared memory block -when any process with access to the shared memory has terminated. track -should be set to False if there is already another process in place -that does the bookkeeping. In most situations, this means that track -should be set to False when create is set to False.

- - - - - -

The following example demonstrates low-level use of :class:`SharedMemory` -instances:

- -
>>> from multiprocessing import shared_memory
->>> shm_a = shared_memory.SharedMemory(create=True, size=10)
->>> type(shm_a.buf)
-<class 'memoryview'>
->>> buffer = shm_a.buf
->>> len(buffer)
-10
->>> buffer[:4] = bytearray([22, 33, 44, 55])  # Modify multiple at once
->>> buffer[4] = 100                           # Modify single byte at a time
->>> # Attach to an existing shared memory block
->>> shm_b = shared_memory.SharedMemory(shm_a.name)
->>> import array
->>> array.array('b', shm_b.buf[:5])  # Copy the data into a new array.array
-array('b', [22, 33, 44, 55, 100])
->>> shm_b.buf[:5] = b'howdy'  # Modify via shm_b using bytes
->>> bytes(shm_a.buf[:5])      # Access via shm_a
-b'howdy'
->>> shm_b.close()   # Close each SharedMemory instance
->>> shm_a.close()
->>> shm_a.unlink()  # Call unlink only once to release the shared memory
-

The following example demonstrates a practical use of the :class:`SharedMemory` -class with NumPy arrays, accessing the -same numpy.ndarray from two distinct Python shells:

- - -

A subclass of :class:`~multiprocessing.managers.BaseManager` which can be -used for the management of shared memory blocks across processes.

- -

A call to :meth:`~multiprocessing.managers.BaseManager.start` on a -:class:`SharedMemoryManager` instance causes a new process to be started. -This new process's sole purpose is to manage the life cycle -of all shared memory blocks created through it. To trigger the release -of all shared memory blocks managed by that process, call -:meth:`~multiprocessing.managers.BaseManager.shutdown()` on the instance. -This triggers a :meth:`SharedMemory.unlink()` call on all of the -:class:`SharedMemory` objects managed by that process and then -stops the process itself. By creating SharedMemory instances -through a SharedMemoryManager, we avoid the need to manually track -and trigger the freeing of shared memory resources.

- - - - - -

This class provides methods for creating and returning :class:`SharedMemory` -instances and for creating a list-like object (:class:`ShareableList`) -backed by shared memory.

- - -

Refer to :class:`multiprocessing.managers.BaseManager` for a description -of the inherited address and authkey optional input arguments and how -they may be used to connect to an existing SharedMemoryManager service -from other processes.

- - - -

The following example demonstrates the basic mechanisms of a -:class:`SharedMemoryManager`:

- - -

The following example depicts a potentially more convenient pattern for using -:class:`SharedMemoryManager` objects via the :keyword:`with` statement to -ensure that all shared memory blocks are released after they are no longer -needed:

- - - -

When using a :class:`SharedMemoryManager` in a :keyword:`with` statement, the -shared memory blocks created using that manager are all released when the -:keyword:`with` statement's code block finishes execution.

- - - - -

The following example demonstrates basic use of a :class:`ShareableList` -instance:

- -
-
>>> from multiprocessing import shared_memory
->>> a = shared_memory.ShareableList(['howdy', b'HoWdY', -273.154, 100, None, True, 42])
->>> [ type(entry) for entry in a ]
-[<class 'str'>, <class 'bytes'>, <class 'float'>, <class 'int'>, <class 'NoneType'>, <class 'bool'>, <class 'int'>]
->>> a[2]
--273.154
->>> a[2] = -78.5
->>> a[2]
--78.5
->>> a[2] = 'dry ice'  # Changing data types is supported as well
->>> a[2]
-'dry ice'
->>> a[2] = 'larger than previously allocated storage space'
-Traceback (most recent call last):
-  ...
-ValueError: exceeds available storage for existing str
->>> a[2]
-'dry ice'
->>> len(a)
-7
->>> a.index(42)
-6
->>> a.count(b'howdy')
-0
->>> a.count(b'HoWdY')
-1
->>> a.shm.close()
->>> a.shm.unlink()
->>> del a  # Use of a ShareableList after call to unlink() is unsupported
-
-
-

The following example depicts how one, two, or many processes may access the -same :class:`ShareableList` by supplying the name of the shared memory block -behind it:

- -
-
>>> b = shared_memory.ShareableList(range(5))         # In a first process
->>> c = shared_memory.ShareableList(name=b.shm.name)  # In a second process
->>> c
-ShareableList([0, 1, 2, 3, 4], name='...')
->>> c[-1] = -999
->>> b[-1]
--999
->>> b.shm.close()
->>> c.shm.close()
->>> c.shm.unlink()
-
-
-

The following examples demonstrates that ShareableList -(and underlying SharedMemory) objects -can be pickled and unpickled if needed. -Note, that it will still be the same shared object. -This happens, because the deserialized object has -the same unique name and is just attached to an existing -object with the same name (if the object is still alive):

-
-
>>> import pickle
->>> from multiprocessing import shared_memory
->>> sl = shared_memory.ShareableList(range(10))
->>> list(sl)
-[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-
-
>>> deserialized_sl = pickle.loads(pickle.dumps(sl))
->>> list(deserialized_sl)
-[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-
-
>>> sl[0] = -1
->>> deserialized_sl[1] = -2
->>> list(sl)
-[-1, -2, 2, 3, 4, 5, 6, 7, 8, 9]
->>> list(deserialized_sl)
-[-1, -2, 2, 3, 4, 5, 6, 7, 8, 9]
-
-
>>> sl.shm.close()
->>> sl.shm.unlink()
-
-
-
- - From dcda10f1b69776c36466733c7ceaa27a231a0a35 Mon Sep 17 00:00:00 2001 From: pan324 <103143968+pan324@users.noreply.github.com> Date: Sun, 15 Oct 2023 20:57:06 +0200 Subject: [PATCH 06/23] Update Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst Co-authored-by: Guido van Rossum --- .../next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst index 30473448044155..56031df9e5e4e7 100644 --- a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst +++ b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst @@ -1 +1 @@ -Add ``track`` parameter that allows using shared memory blocks without having to register with the resource tracker. +Add ``track`` parameter to :class:`multiprocessing.shared_memory.SharedMemory` that allows using shared memory blocks without having to register with the resource tracker. From 63d21d7938bcf7deab47dc07a7a0d33fca4ee54e Mon Sep 17 00:00:00 2001 From: pan324 <103143968+pan324@users.noreply.github.com> Date: Sun, 15 Oct 2023 21:19:12 +0200 Subject: [PATCH 07/23] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Guido van Rossum --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 2b17ba0b66391b..7599e46b8cf883 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -87,7 +87,7 @@ copying of data. Deletes the underlying shared memory block. This should be called only once per shared memory block regardless of the number of handles to it. After requesting its deletion, a shared memory block may or may not be - immediately destroyed and this behavior may differ across platforms. + immediately destroyed, and this behavior may differ across platforms. Attempts to access data inside the shared memory block after ``unlink()`` has been called may result in memory access errors. To ensure proper bookkeeping, ``unlink()`` may only be called by From e990e41823981c3993c320492f76a0872db08cf9 Mon Sep 17 00:00:00 2001 From: pan324 <103143968+pan324@users.noreply.github.com> Date: Sun, 15 Oct 2023 21:19:33 +0200 Subject: [PATCH 08/23] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Guido van Rossum --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 7599e46b8cf883..37dcac90d6d3b9 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -88,7 +88,7 @@ copying of data. once per shared memory block regardless of the number of handles to it. After requesting its deletion, a shared memory block may or may not be immediately destroyed, and this behavior may differ across platforms. - Attempts to access data inside the shared memory block after + Therefore, attempts to access data inside the shared memory block after ``unlink()`` has been called may result in memory access errors. To ensure proper bookkeeping, ``unlink()`` may only be called by an instance with *track* enabled. From 9c593bacaac2cca7e4b02df24838b9c1a76fd094 Mon Sep 17 00:00:00 2001 From: pan324 Date: Tue, 17 Oct 2023 14:09:54 +0200 Subject: [PATCH 09/23] Removed TypeError. Clarified documentation. --- Doc/library/multiprocessing.shared_memory.rst | 38 ++++++++++--------- Lib/multiprocessing/shared_memory.py | 3 -- Lib/test/_test_multiprocessing.py | 25 ++++++++++++ 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 37dcac90d6d3b9..e6b6dab3c237f3 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -64,34 +64,38 @@ copying of data. memory block may be larger or equal to the size requested. When attaching to an existing shared memory block, the ``size`` parameter is ignored. - *track*, when enabled, registers the shared memory block with the resource + *track*, when enabled, registers the shared memory block with a resource tracker process. This process ensures proper cleanup of shared memory blocks even when all other processes with access to the memory have failed - to do so (mainly due to being killed by signals). The resource tracker is - overzealous in certain situations and might delete a shared memory block - when any process with access to the shared memory has terminated. *track* - should be set to ``False`` if there is already another process in place - that does the bookkeeping. In most situations, this means that *track* - should be set to ``False`` when *create* is set to ``False``. + to do so (mainly due to being killed by signals). Unless a Python process + was created using any of the :mod:`multiprocessing` facilities (such as + :class:`multiprocessing.Process`), it will receive its own resource tracker + process when accessing shared memory with *track* enabled. This will cause + the shared memory to be deleted by the resource tracker of the first + process that terminates. To avoid this issue, users of :mod:`subprocess` or + standalone Python processes should set *track* to ``False`` when there is + already another process in place that does the bookkeeping. *track* has + an effect only on POSIX. Windows has its own tracking and does not use the + resource tracker. + + .. versionchanged:: 3.13 Added *track* parameter. .. method:: close() Closes the file descriptor/handle to the shared memory from this - instance. All instances should call ``close()`` once the instance - is no longer needed. Depending on operating system, the underlying - memory may or may not be freed even if all handles to it have been - closed. To ensure proper cleanup, use the ``unlink()`` method. + instance. :meth:`close()` should be called once access to the shared + memory block from this instance is no longer needed. Depending + on operating system, the underlying memory may or may not be freed + even if all handles to it have been closed. To ensure proper cleanup, + use the :meth:`unlink()` method. .. method:: unlink() Deletes the underlying shared memory block. This should be called only once per shared memory block regardless of the number of handles to it. - After requesting its deletion, a shared memory block may or may not be - immediately destroyed, and this behavior may differ across platforms. - Therefore, attempts to access data inside the shared memory block after - ``unlink()`` has been called may result in memory access errors. - To ensure proper bookkeeping, ``unlink()`` may only be called by - an instance with *track* enabled. + :meth:`unlink()` and :meth:`close()` can be called in any order, but + trying to access data inside a shared memory block after :meth:`unlink()` + may result in memory access errors, depending on platform. .. attribute:: buf diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 5f42b41b3f0124..3051c17f1402c4 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -82,7 +82,6 @@ def __init__(self, name=None, create=False, size=0, track=True): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") - self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -240,8 +239,6 @@ def unlink(self): In order to ensure proper cleanup of resources, unlink should be called once (and only once) across all processes which have access to the shared memory block.""" - if not self._track: - raise TypeError("unlink() must be called by a tracking instance") if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 0b333ca3b7e9dc..375dcfc33bbc10 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4440,6 +4440,31 @@ def test_shared_memory_cleaned_after_process_termination(self): "resource_tracker: There appear to be 1 leaked " "shared_memory objects to clean up at shutdown", err) + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_untracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=False, it must not register with the resource tracker. + cmd = '''if 1: + import sys + from unittest.mock import Mock + from multiprocessing import resource_tracker + from multiprocessing.shared_memory import SharedMemory + resource_tracker.register = Mock(side_effect=AssertionError) + mem = SharedMemory(create=False, name=sys.argv[1], track=False) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + try: + *_, err = test.support.script_helper.assert_python_ok("-c", cmd, + mem.name) + self.assertEqual(err, b'') + finally: + mem.close() + try: + mem.unlink() + except OSError: + pass + # # Test to verify that `Finalize` works. # From 9ef1ff357f2a6fe5373a97152de12a23c35e226d Mon Sep 17 00:00:00 2001 From: pan324 Date: Tue, 17 Oct 2023 14:32:05 +0200 Subject: [PATCH 10/23] untracking shmem can unlink now --- Lib/multiprocessing/shared_memory.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 3051c17f1402c4..989773a84232bd 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -71,6 +71,7 @@ class SharedMemory: _flags = os.O_RDWR _mode = 0o600 _prepend_leading_slash = True if _USE_POSIX else False + _track = True def __init__(self, name=None, create=False, size=0, track=True): if not size >= 0: @@ -82,6 +83,7 @@ def __init__(self, name=None, create=False, size=0, track=True): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") + self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -116,7 +118,7 @@ def __init__(self, name=None, create=False, size=0, track=True): except OSError: self.unlink() raise - if track: + if self._track: resource_tracker.register(self._name, "shared_memory") else: @@ -242,7 +244,8 @@ def unlink(self): if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) - resource_tracker.unregister(self._name, "shared_memory") + if self._track: + resource_tracker.unregister(self._name, "shared_memory") _encoding = "utf8" From e5fe67432746e12328a2da70b9133f83097873eb Mon Sep 17 00:00:00 2001 From: pan324 <103143968+pan324@users.noreply.github.com> Date: Thu, 19 Oct 2023 13:11:35 +0200 Subject: [PATCH 11/23] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Antoine Pitrou --- Doc/library/multiprocessing.shared_memory.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index e6b6dab3c237f3..febabe1210b26c 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -92,7 +92,8 @@ copying of data. .. method:: unlink() Deletes the underlying shared memory block. This should be called only - once per shared memory block regardless of the number of handles to it. + once per shared memory block regardless of the number of handles to it, + even in other processes. :meth:`unlink()` and :meth:`close()` can be called in any order, but trying to access data inside a shared memory block after :meth:`unlink()` may result in memory access errors, depending on platform. From 66acf900beb8b171156d61b7c23fceede46032f2 Mon Sep 17 00:00:00 2001 From: pan324 Date: Thu, 19 Oct 2023 19:56:02 +0200 Subject: [PATCH 12/23] removed unneeded try-except --- Lib/test/_test_multiprocessing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 375dcfc33bbc10..1b8d674eabba9b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4460,10 +4460,7 @@ def test_shared_memory_untracking(self): self.assertEqual(err, b'') finally: mem.close() - try: - mem.unlink() - except OSError: - pass + mem.unlink() # # Test to verify that `Finalize` works. From 3fdf625fd8b20720d8c6522c50c107956a3fefdb Mon Sep 17 00:00:00 2001 From: pan324 Date: Thu, 19 Oct 2023 20:31:21 +0200 Subject: [PATCH 13/23] phrasing of track parameter --- Doc/library/multiprocessing.shared_memory.rst | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index febabe1210b26c..8286fb848e5515 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -67,16 +67,19 @@ copying of data. *track*, when enabled, registers the shared memory block with a resource tracker process. This process ensures proper cleanup of shared memory blocks even when all other processes with access to the memory have failed - to do so (mainly due to being killed by signals). Unless a Python process - was created using any of the :mod:`multiprocessing` facilities (such as - :class:`multiprocessing.Process`), it will receive its own resource tracker - process when accessing shared memory with *track* enabled. This will cause - the shared memory to be deleted by the resource tracker of the first - process that terminates. To avoid this issue, users of :mod:`subprocess` or - standalone Python processes should set *track* to ``False`` when there is - already another process in place that does the bookkeeping. *track* has - an effect only on POSIX. Windows has its own tracking and does not use the - resource tracker. + to do so (mainly due to being killed by signals). + Python processes created from a common ancestor using :mod:`multiprocessing` + facilities share a single resource tracker process, and the lifetime of + shared memory segments is handled automatically among these processes. + Python processes created in any other way will receive their own own + resource tracker when accessing shared memory with *track* enabled. + This will cause the shared memory to be deleted by the resource tracker + of the first process that terminates. + To avoid this issue, users of :mod:`subprocess` or standalone Python + processes should set *track* to ``False`` when there is already another + process in place that does the bookkeeping. + *track* is ignored on Windows, which has its own tracking and + automatically deletes shared memory when all handles to it have been closed. .. versionchanged:: 3.13 Added *track* parameter. @@ -98,6 +101,9 @@ copying of data. trying to access data inside a shared memory block after :meth:`unlink()` may result in memory access errors, depending on platform. + This method has no effect on Windows, where the only way to delete a + shared memory block is to close all handles. + .. attribute:: buf A memoryview of contents of the shared memory block. From d65e3f8c5b77a189d5082d0098fcaf81c9554633 Mon Sep 17 00:00:00 2001 From: pan324 <103143968+pan324@users.noreply.github.com> Date: Tue, 24 Oct 2023 21:33:06 +0200 Subject: [PATCH 14/23] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Antoine Pitrou --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 8286fb848e5515..c940e7bb695967 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -71,7 +71,7 @@ copying of data. Python processes created from a common ancestor using :mod:`multiprocessing` facilities share a single resource tracker process, and the lifetime of shared memory segments is handled automatically among these processes. - Python processes created in any other way will receive their own own + Python processes created in any other way will receive their own resource tracker when accessing shared memory with *track* enabled. This will cause the shared memory to be deleted by the resource tracker of the first process that terminates. From a97c6d306761b86b12889218ae46fd6e31aefdd7 Mon Sep 17 00:00:00 2001 From: pan324 Date: Wed, 25 Oct 2023 00:42:57 +0200 Subject: [PATCH 15/23] untracking test --- Lib/test/_test_multiprocessing.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1b8d674eabba9b..a410f59ce46616 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4443,24 +4443,28 @@ def test_shared_memory_cleaned_after_process_termination(self): @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory - # with track=False, it must not register with the resource tracker. + # with track=False, it must not cause the memory to be deleted + # when terminating. cmd = '''if 1: import sys - from unittest.mock import Mock - from multiprocessing import resource_tracker from multiprocessing.shared_memory import SharedMemory - resource_tracker.register = Mock(side_effect=AssertionError) mem = SharedMemory(create=False, name=sys.argv[1], track=False) mem.close() ''' mem = shared_memory.SharedMemory(create=True, size=10) + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + # The resource tracker shares pipes with the subprocess, and so + # err existing means that the tracker process has terminated now. try: - *_, err = test.support.script_helper.assert_python_ok("-c", cmd, - mem.name) - self.assertEqual(err, b'') + self.assertEqual(rc, 0) + mem2 = shared_memory.SharedMemory(create=False, name=mem.name) + mem2.close() finally: mem.close() - mem.unlink() + try: + mem.unlink() + except OSError: + pass # # Test to verify that `Finalize` works. From 17c07f513e4b65a35508a0c6c04d37ec79158f0a Mon Sep 17 00:00:00 2001 From: pan324 Date: Wed, 25 Oct 2023 20:00:59 +0200 Subject: [PATCH 16/23] untrack tests both track=True and track=False --- Lib/test/_test_multiprocessing.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a410f59ce46616..751c15bac46292 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4440,11 +4440,13 @@ def test_shared_memory_cleaned_after_process_termination(self): "resource_tracker: There appear to be 1 leaked " "shared_memory objects to clean up at shutdown", err) +class MyTest(unittest.TestCase): @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=False, it must not cause the memory to be deleted - # when terminating. + # when terminating. When accessing with track=True, it must be + # deleted. cmd = '''if 1: import sys from multiprocessing.shared_memory import SharedMemory @@ -4465,6 +4467,26 @@ def test_shared_memory_untracking(self): mem.unlink() except OSError: pass + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=True) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + try: + self.assertEqual(rc, 0) + self.assertIn( + b"resource_tracker: There appear to be 1 leaked " + b"shared_memory objects to clean up at shutdown", err) + finally: + resource_tracker.unregister(mem._name, "shared_memory") + mem.close() + try: + mem.unlink() + except OSError: + pass # # Test to verify that `Finalize` works. From 13f3fb670b99eac30c1c11f5875a9e450cd2edac Mon Sep 17 00:00:00 2001 From: pan324 Date: Wed, 25 Oct 2023 20:01:31 +0200 Subject: [PATCH 17/23] untrack tests both track=True and track=False --- Lib/test/_test_multiprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 751c15bac46292..fa2255b4c32c54 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4440,7 +4440,7 @@ def test_shared_memory_cleaned_after_process_termination(self): "resource_tracker: There appear to be 1 leaked " "shared_memory objects to clean up at shutdown", err) -class MyTest(unittest.TestCase): +##class MyTest(unittest.TestCase): @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory From 7c7f0e71e584273e9ada9235354225097db1df33 Mon Sep 17 00:00:00 2001 From: pan324 Date: Wed, 25 Oct 2023 20:01:47 +0200 Subject: [PATCH 18/23] untrack tests both track=True and track=False --- Lib/test/_test_multiprocessing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index fa2255b4c32c54..4930e52eec7b5f 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4440,7 +4440,6 @@ def test_shared_memory_cleaned_after_process_termination(self): "resource_tracker: There appear to be 1 leaked " "shared_memory objects to clean up at shutdown", err) -##class MyTest(unittest.TestCase): @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory From 765afb7183bc2718e454c8ab6de12d12466ea2c2 Mon Sep 17 00:00:00 2001 From: pan324 Date: Thu, 30 Nov 2023 18:11:55 +0100 Subject: [PATCH 19/23] reliable test cleanup --- Doc/library/multiprocessing.shared_memory.rst | 2 +- Lib/multiprocessing/shared_memory.py | 14 ++++++++++---- Lib/test/_test_multiprocessing.py | 10 +++++----- .../2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index c940e7bb695967..750579067267ae 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -36,7 +36,7 @@ or other communications requiring the serialization/deserialization and copying of data. -.. class:: SharedMemory(name=None, create=False, size=0, track=True) +.. class:: SharedMemory(name=None, create=False, size=0, *, track=True) Creates a new shared memory block or attaches to an existing shared memory block. Each shared memory block is assigned a unique name. diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 989773a84232bd..67e70fdc27cf31 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -73,7 +73,7 @@ class SharedMemory: _prepend_leading_slash = True if _USE_POSIX else False _track = True - def __init__(self, name=None, create=False, size=0, track=True): + def __init__(self, name=None, create=False, size=0, *, track=True): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: @@ -238,9 +238,15 @@ def close(self): def unlink(self): """Requests that the underlying shared memory block be destroyed. - In order to ensure proper cleanup of resources, unlink should be - called once (and only once) across all processes which have access - to the shared memory block.""" + Unlink should be called once (and only once) across all handles + which have access to the shared memory block, even if these + handles belong to different processes. Closing and unlinking may + happen in any order, but trying to access data inside a shared + memory block after unlinking may result in memory errors, + depending on platform. + + This method has no effect on Windows, where the only way to + delete a shared memory block is to close all handles.""" if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4930e52eec7b5f..8bb3419374623d 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4453,19 +4453,19 @@ def test_shared_memory_untracking(self): mem.close() ''' mem = shared_memory.SharedMemory(create=True, size=10) - rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) # The resource tracker shares pipes with the subprocess, and so # err existing means that the tracker process has terminated now. try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) self.assertEqual(rc, 0) mem2 = shared_memory.SharedMemory(create=False, name=mem.name) mem2.close() finally: - mem.close() try: mem.unlink() except OSError: pass + mem.close() cmd = '''if 1: import sys from multiprocessing.shared_memory import SharedMemory @@ -4473,19 +4473,19 @@ def test_shared_memory_untracking(self): mem.close() ''' mem = shared_memory.SharedMemory(create=True, size=10) - rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) self.assertEqual(rc, 0) self.assertIn( b"resource_tracker: There appear to be 1 leaked " b"shared_memory objects to clean up at shutdown", err) finally: - resource_tracker.unregister(mem._name, "shared_memory") - mem.close() try: mem.unlink() except OSError: pass + resource_tracker.unregister(mem._name, "shared_memory") + mem.close() # # Test to verify that `Finalize` works. diff --git a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst index 56031df9e5e4e7..d7e6b225489b99 100644 --- a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst +++ b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst @@ -1 +1 @@ -Add ``track`` parameter to :class:`multiprocessing.shared_memory.SharedMemory` that allows using shared memory blocks without having to register with the resource tracker. +Add ``track`` parameter to :class:`multiprocessing.shared_memory.SharedMemory` that allows using shared memory blocks without having to register with the POSIX resource tracker that automatically releases them upon process exit. From a5848c1e5f6fc0ac15846944ecf33651743f4eb4 Mon Sep 17 00:00:00 2001 From: pan324 <103143968+pan324@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:33:25 +0100 Subject: [PATCH 20/23] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Gregory P. Smith --- Doc/library/multiprocessing.shared_memory.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 750579067267ae..671130d9b29fc0 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -65,9 +65,9 @@ copying of data. to an existing shared memory block, the ``size`` parameter is ignored. *track*, when enabled, registers the shared memory block with a resource - tracker process. This process ensures proper cleanup of shared memory - blocks even when all other processes with access to the memory have failed - to do so (mainly due to being killed by signals). + tracker process on platforms where the OS does not do this automatically. + The resource tracker ensures proper cleanup of the shared memory even + if all other processes with access to the memory exit without doing so. Python processes created from a common ancestor using :mod:`multiprocessing` facilities share a single resource tracker process, and the lifetime of shared memory segments is handled automatically among these processes. From 8255e015b61baeaca125e3196580c6c7721b91b6 Mon Sep 17 00:00:00 2001 From: pan324 Date: Fri, 1 Dec 2023 20:46:16 +0100 Subject: [PATCH 21/23] split tests --- Lib/test/_test_multiprocessing.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 8bb3419374623d..32b385474d6471 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4444,8 +4444,7 @@ def test_shared_memory_cleaned_after_process_termination(self): def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=False, it must not cause the memory to be deleted - # when terminating. When accessing with track=True, it must be - # deleted. + # when terminating. cmd = '''if 1: import sys from multiprocessing.shared_memory import SharedMemory @@ -4457,6 +4456,7 @@ def test_shared_memory_untracking(self): # err existing means that the tracker process has terminated now. try: rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertNotIn(b"resource_tracker", err) self.assertEqual(rc, 0) mem2 = shared_memory.SharedMemory(create=False, name=mem.name) mem2.close() @@ -4466,6 +4466,11 @@ def test_shared_memory_untracking(self): except OSError: pass mem.close() + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_tracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=True, it must cause the memory to be deleted when + # terminating. cmd = '''if 1: import sys from multiprocessing.shared_memory import SharedMemory @@ -4475,6 +4480,7 @@ def test_shared_memory_untracking(self): mem = shared_memory.SharedMemory(create=True, size=10) try: rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertNotIn(b"resource_tracker", err) self.assertEqual(rc, 0) self.assertIn( b"resource_tracker: There appear to be 1 leaked " From 5d89117595d3c183ef65bb0cf84e627f2c032305 Mon Sep 17 00:00:00 2001 From: pan324 Date: Fri, 1 Dec 2023 21:02:31 +0100 Subject: [PATCH 22/23] split tests --- Lib/test/_test_multiprocessing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 32b385474d6471..8e1c849b077c0e 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4480,7 +4480,6 @@ def test_shared_memory_tracking(self): mem = shared_memory.SharedMemory(create=True, size=10) try: rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) - self.assertNotIn(b"resource_tracker", err) self.assertEqual(rc, 0) self.assertIn( b"resource_tracker: There appear to be 1 leaked " From 66db5b8e201643125448c4d1ead98ef3422af2cb Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Fri, 1 Dec 2023 21:50:28 -0800 Subject: [PATCH 23/23] style: add a missing blank line --- Lib/test/_test_multiprocessing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 8e1c849b077c0e..89ab9b3eafb760 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4466,6 +4466,7 @@ def test_shared_memory_untracking(self): except OSError: pass mem.close() + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") def test_shared_memory_tracking(self): # gh-82300: When a separate Python process accesses shared memory