From e269c0997124b5c9aae983a2c240711a03b4d392 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Thu, 18 Apr 2024 22:23:55 +0300 Subject: [PATCH 01/59] Use custom loads & dumps instead of custom pickler & unpickler for Shelf --- Lib/shelve.py | 36 +++++++++++++++++---------- Lib/test/test_shelve.py | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 50584716e9ea64..73f84b87d5f8ba 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,13 +56,18 @@ the persistent dictionary on disk, if feasible). """ -from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler +from pickle import DEFAULT_PROTOCOL, Unpickler, dumps, loads from io import BytesIO import collections.abc __all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] + +class ShelveError(Exception): + pass + + class _ClosedDict(collections.abc.MutableMapping): 'Marker for a closed dict. Access attempts raise a ValueError.' @@ -82,7 +87,7 @@ class Shelf(collections.abc.MutableMapping): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): + keyencoding="utf-8", *, serializer=None, deserializer=None): self.dict = dict if protocol is None: protocol = DEFAULT_PROTOCOL @@ -91,6 +96,15 @@ def __init__(self, dict, protocol=None, writeback=False, self.cache = {} self.keyencoding = keyencoding + if serializer is None and deserializer is None: + self.serializer = dumps + self.deserializer = loads + elif (serializer is None) ^ (deserializer is None): + raise ShelveError("Serializer and deserializer must be defined together.") + else: + self.serializer = serializer + self.deserializer = deserializer + def __iter__(self): for k in self.dict.keys(): yield k.decode(self.keyencoding) @@ -110,8 +124,8 @@ def __getitem__(self, key): try: value = self.cache[key] except KeyError: - f = BytesIO(self.dict[key.encode(self.keyencoding)]) - value = Unpickler(f).load() + f = self.dict[key.encode(self.keyencoding)] + value = self.deserializer(f) if self.writeback: self.cache[key] = value return value @@ -119,10 +133,7 @@ def __getitem__(self, key): def __setitem__(self, key, value): if self.writeback: self.cache[key] = value - f = BytesIO() - p = Pickler(f, self._protocol) - p.dump(value) - self.dict[key.encode(self.keyencoding)] = f.getvalue() + self.dict[key.encode(self.keyencoding)] = self.serializer(value, self._protocol) def __delitem__(self, key): del self.dict[key.encode(self.keyencoding)] @@ -222,9 +233,9 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False): + def __init__(self, filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None): import dbm - Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, serializer=serializer, deserializer=deserializer) def clear(self): """Remove all items from the shelf.""" @@ -233,8 +244,7 @@ def clear(self): self.cache.clear() self.dict.clear() - -def open(filename, flag='c', protocol=None, writeback=False): +def open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None): """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying @@ -247,4 +257,4 @@ def open(filename, flag='c', protocol=None, writeback=False): See the module's __doc__ string for an overview of the interface. """ - return DbfilenameShelf(filename, flag, protocol, writeback) + return DbfilenameShelf(filename, flag, protocol, writeback, serializer, deserializer) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 08c6562f2a273e..3fdd9f36393416 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -3,6 +3,8 @@ import shelve import pickle import os +from io import BytesIO +from pydoc import locate from test.support import os_helper from collections.abc import MutableMapping @@ -165,6 +167,58 @@ def test_default_protocol(self): with shelve.Shelf({}) as s: self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) + def test_custom_serializer_and_deserializer(self): + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + value = BytesIO(data).read() + return locate(value.decode("utf-8")) + + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + num = 1 + s['number'] = num + self.assertEqual(s['number'], type(num)) + + with self.assertRaises(AssertionError): + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + pass + + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + s['number'] = 100 + self.assertEqual(s['number'], 100) + + with self.assertRaises(dbm.sqlite3.error): + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return BytesIO(data).read().decode("utf-8") + + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + s['number'] = 100 + self.assertEqual(s['number'], 100) + + def test_missing_custom_deserializer(self): + def serializer(obj, protocol=None): + pass + + with self.assertRaises(shelve.ShelveError): + shelve.Shelf({}, protocol=2, writeback=False, serializer=serializer) + + def test_missing_custom_serializer(self): + def deserializer(data): + pass + + with self.assertRaises(shelve.ShelveError): + shelve.Shelf({}, protocol=2, writeback=False, deserializer=deserializer) + class TestShelveBase: type2test = shelve.Shelf From 346514923a946725e8579149db0cee5bfa4983ad Mon Sep 17 00:00:00 2001 From: furkanonder Date: Thu, 18 Apr 2024 22:45:28 +0300 Subject: [PATCH 02/59] Allow custom loads & dumps instead of custom pickler & unpickler for BsdDbShelf --- Lib/shelve.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 73f84b87d5f8ba..3ee63404a687b1 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -197,7 +197,15 @@ class BsdDbShelf(Shelf): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): + keyencoding="utf-8", *, serializer=None, deserializer=None): + if serializer is None and deserializer is None: + self.serializer = dumps + self.deserializer = loads + elif (serializer is None) ^ (deserializer is None): + raise ShelveError("Serializer and deserializer must be defined together.") + else: + self.serializer = serializer + self.deserializer = deserializer Shelf.__init__(self, dict, protocol, writeback, keyencoding) def set_location(self, key): From 44b9fa1e9735f646b6dbc2a08b606eb8ac97ea5c Mon Sep 17 00:00:00 2001 From: furkanonder Date: Thu, 18 Apr 2024 22:49:05 +0300 Subject: [PATCH 03/59] Update documentation for serializer and deserializred functions --- Doc/library/shelve.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 95c54991887022..c8086478ac927b 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -17,7 +17,7 @@ This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. -.. function:: open(filename, flag='c', protocol=None, writeback=False) +.. function:: open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None) Open a persistent dictionary. The filename specified is the base filename for the underlying database. As a side-effect, an extension may be added to the @@ -41,6 +41,14 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). + By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + for serializing and deserializing. However *serializer* can be the function + that takes the :term:`bytes-like object` and returns the object. *deserializer* + can be the function that takes the object and returns :class:`bytes`. For example, + :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in + :mod:`shelve` using the serializer and deserializer functions, which do support + the :keyword:`lambda`. + .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. @@ -48,6 +56,9 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. + .. versionchanged:: 3.13 + Accepts *serializer* and *deserializer*. + .. note:: Do not rely on the shelf being closed automatically; always call @@ -150,7 +161,7 @@ Restrictions protocol. -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') +.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -164,7 +175,7 @@ Restrictions interpretation as for the :class:`Shelf` class. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By From f2eed32ec7d3395ba0aae175e9def8f69d2121f4 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 21 Apr 2024 00:11:47 +0300 Subject: [PATCH 04/59] Update Doc/library/shelve.rst Co-authored-by: Pieter Eendebak --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index c8086478ac927b..b25a5ae08e0ede 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -42,7 +42,7 @@ lots of shared sub-objects. The keys are ordinary strings. mutated). By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` - for serializing and deserializing. However *serializer* can be the function + for serializing and deserializing. However *deserializer* can be the function that takes the :term:`bytes-like object` and returns the object. *deserializer* can be the function that takes the object and returns :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in From b3e57236dbc44043e435b18125e3027360c7a7f7 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 21 Apr 2024 00:11:54 +0300 Subject: [PATCH 05/59] Update Doc/library/shelve.rst Co-authored-by: Pieter Eendebak --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index b25a5ae08e0ede..c99394adee1cc2 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -43,7 +43,7 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and returns the object. *deserializer* + that takes the :term:`bytes-like object` and returns the object. *serializer* can be the function that takes the object and returns :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in :mod:`shelve` using the serializer and deserializer functions, which do support From 53d5557de5eaa7d8cf7c34d719424528701dc903 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 22 Apr 2024 00:03:22 +0300 Subject: [PATCH 06/59] Update documentation for serializer and deserializer functions --- Doc/library/shelve.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index c99394adee1cc2..f36e46d8c61afe 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -43,11 +43,11 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and returns the object. *serializer* - can be the function that takes the object and returns :class:`bytes`. For example, - :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in - :mod:`shelve` using the serializer and deserializer functions, which do support - the :keyword:`lambda`. + that takes the :term:`bytes-like object` and the *protocol* parameter and returns + the object. *serializer* can be the function that takes the object and returns + :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not + support, can be used in :mod:`shelve` using the serializer and deserializer + functions, which do support the :keyword:`lambda`. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle @@ -57,7 +57,7 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.13 - Accepts *serializer* and *deserializer*. + Accepts *serializer* and *deserializer* as parameters. .. note:: @@ -128,7 +128,7 @@ Restrictions which can cause hard crashes when trying to read from the database. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') +.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -146,6 +146,11 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. + The *deserializer* parameter can be the function that takes the + :term:`bytes-like object` and the *protocol* parameter and returns the + object. *serializer* parameter can be the function that takes the object + and returns :class:`bytes`. + A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. @@ -160,6 +165,8 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. + .. versionchanged:: 3.13 + Accepts *serializer* and *deserializer* as parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) @@ -171,8 +178,8 @@ Restrictions modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The - optional *protocol*, *writeback*, and *keyencoding* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* + parameters have the same interpretation as for the :class:`Shelf` class. .. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None) @@ -181,8 +188,8 @@ Restrictions object. The underlying file will be opened using :func:`dbm.open`. By default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol* and *writeback* parameters have the same - interpretation as for the :class:`Shelf` class. + function. The optional *protocol*, *writeback*, *serializer* and *deserializer* + parameters have the same interpretation as for the :class:`Shelf` class. .. _shelve-example: From 1e295bad4fa551cae44e1c2f09b13527066ff746 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 22 Apr 2024 00:35:57 +0300 Subject: [PATCH 07/59] Fix lines according to PEP-8 --- Lib/shelve.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 3ee63404a687b1..5532d3103cc7e7 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -100,7 +100,8 @@ def __init__(self, dict, protocol=None, writeback=False, self.serializer = dumps self.deserializer = loads elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be defined together.") + raise ShelveError("Serializer and deserializer must be" + "defined together.") else: self.serializer = serializer self.deserializer = deserializer @@ -133,7 +134,8 @@ def __getitem__(self, key): def __setitem__(self, key, value): if self.writeback: self.cache[key] = value - self.dict[key.encode(self.keyencoding)] = self.serializer(value, self._protocol) + serialized_value = self.serializer(value, self._protocol) + self.dict[key.encode(self.keyencoding)] = serialized_value def __delitem__(self, key): del self.dict[key.encode(self.keyencoding)] @@ -202,7 +204,8 @@ def __init__(self, dict, protocol=None, writeback=False, self.serializer = dumps self.deserializer = loads elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be defined together.") + raise ShelveError("Serializer and deserializer must be" + "defined together.") else: self.serializer = serializer self.deserializer = deserializer @@ -241,9 +244,11 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None): + def __init__(self, filename, flag='c', protocol=None, writeback=False, + serializer=None, deserializer=None): import dbm - Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, serializer=serializer, deserializer=deserializer) + Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, + serializer=serializer, deserializer=deserializer) def clear(self): """Remove all items from the shelf.""" @@ -252,7 +257,8 @@ def clear(self): self.cache.clear() self.dict.clear() -def open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None): +def open(filename, flag='c', protocol=None, writeback=False, *, + serializer=None, deserializer=None): """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying @@ -265,4 +271,5 @@ def open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, See the module's __doc__ string for an overview of the interface. """ - return DbfilenameShelf(filename, flag, protocol, writeback, serializer, deserializer) + return DbfilenameShelf(filename, flag, protocol, writeback, + serializer, deserializer) From 3cbabe94ae5d920b30c1f8b0a17b6fdb9b863378 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 22 Apr 2024 00:43:07 +0300 Subject: [PATCH 08/59] Fix doc according to line 80 --- Doc/library/shelve.rst | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f36e46d8c61afe..f138c632f5a3de 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -17,7 +17,8 @@ This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. -.. function:: open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None) +.. function:: open(filename, flag='c', protocol=None, writeback=False, *, \ + serializer=None, deserializer=None) Open a persistent dictionary. The filename specified is the base filename for the underlying database. As a side-effect, an extension may be added to the @@ -43,11 +44,12 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and the *protocol* parameter and returns - the object. *serializer* can be the function that takes the object and returns - :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not - support, can be used in :mod:`shelve` using the serializer and deserializer - functions, which do support the :keyword:`lambda`. + that takes the :term:`bytes-like object` and the *protocol* parameter and + returns the object. *serializer* can be the function that takes the object + and returns :class:`bytes`. For example, :keyword:`lambda`, which + the :mod:`pickle` does notsupport, can be used in :mod:`shelve` using + theserializer and deserializer functions, which do support the + :keyword:`lambda`. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle @@ -128,7 +130,8 @@ Restrictions which can cause hard crashes when trying to read from the database. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) +.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', \ + serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -168,7 +171,8 @@ Restrictions .. versionchanged:: 3.13 Accepts *serializer* and *deserializer* as parameters. -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) +.. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -182,14 +186,16 @@ Restrictions parameters have the same interpretation as for the :class:`Shelf` class. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, \ + serializer=None, deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By default, the file will be created and opened for both read and write. The - optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol*, *writeback*, *serializer* and *deserializer* - parameters have the same interpretation as for the :class:`Shelf` class. + optional *flag* parameter has the same interpretation as for the + :func:`.open` function. The optional *protocol*, *writeback*, *serializer* + and *deserializer*parameters have the same interpretation as for the + :class:`Shelf` class. .. _shelve-example: From c6b43e23a0b211c4f8cb52fe88040892c1e6f586 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Tue, 23 Apr 2024 12:14:04 +0300 Subject: [PATCH 09/59] Fix inline emphasis issue in docs --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f138c632f5a3de..46746dd0d31ab0 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -194,7 +194,7 @@ Restrictions default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* - and *deserializer*parameters have the same interpretation as for the + and *deserializer* parameters have the same interpretation as for the :class:`Shelf` class. From 4f79cf607efea150fab4aa3d29827b86c9d0e888 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 01:31:46 +0300 Subject: [PATCH 10/59] Update the definition of the open function. Co-authored-by: Petr Viktorin --- Doc/library/shelve.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 46746dd0d31ab0..7553cde9fd933a 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -43,13 +43,12 @@ lots of shared sub-objects. The keys are ordinary strings. mutated). By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` - for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and the *protocol* parameter and - returns the object. *serializer* can be the function that takes the object - and returns :class:`bytes`. For example, :keyword:`lambda`, which - the :mod:`pickle` does notsupport, can be used in :mod:`shelve` using - theserializer and deserializer functions, which do support the - :keyword:`lambda`. + for serializing and deserializing. This can be changed by supplying + *serializer* and *deserializer*, respectively. The *serializer* argument + should be a function that takes an object and returns its representation + as a :term:`bytes-like object`; *deserializer* should be a function that + takes :class:`bytes` and returns the corresponding object. + If one of these is given, the other must be given as well. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle From 798fdb227a6b64575456d12d626c1b8f3cba034d Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 03:37:36 +0300 Subject: [PATCH 11/59] Pass the serializer and serializer arguments of Shelf.__init__ of BsdDbShelf --- Lib/shelve.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 5532d3103cc7e7..0564df6d3108e0 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -200,16 +200,8 @@ class BsdDbShelf(Shelf): def __init__(self, dict, protocol=None, writeback=False, keyencoding="utf-8", *, serializer=None, deserializer=None): - if serializer is None and deserializer is None: - self.serializer = dumps - self.deserializer = loads - elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be" - "defined together.") - else: - self.serializer = serializer - self.deserializer = deserializer - Shelf.__init__(self, dict, protocol, writeback, keyencoding) + Shelf.__init__(self, dict, protocol, writeback, keyencoding, + serializer=serializer, deserializer=deserializer) def set_location(self, key): (key, value) = self.dict.set_location(key) From bb1150d5e4ea12fe6ddbda67102b1a55a20d5e03 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 03:38:29 +0300 Subject: [PATCH 12/59] Add unittests for BsdDbShelf --- Lib/test/test_shelve.py | 77 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 3fdd9f36393416..87fc85f54ed5e2 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -6,7 +6,7 @@ from io import BytesIO from pydoc import locate -from test.support import os_helper +from test.support import os_helper, import_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -178,7 +178,9 @@ def deserializer(data): os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + with shelve.open(self.fn, + serializer=serializer, + deserializer=deserializer) as s: num = 1 s['number'] = num self.assertEqual(s['number'], type(num)) @@ -190,7 +192,9 @@ def serializer(obj, protocol=None): def deserializer(data): pass - with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + with shelve.open(self.fn, + serializer=serializer, + deserializer=deserializer) as s: s['number'] = 100 self.assertEqual(s['number'], 100) @@ -201,23 +205,84 @@ def serializer(obj, protocol=None): def deserializer(data): return BytesIO(data).read().decode("utf-8") - with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + with shelve.open(self.fn, + serializer=serializer, + deserializer=deserializer) as s: s['number'] = 100 self.assertEqual(s['number'], 100) + def test_custom_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module('berkeleydb') + + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + value = BytesIO(data).read() + return locate(value.decode("utf-8")) + + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + num = 1 + s['number'] = num + self.assertEqual(s['number'], type(num)) + + with self.assertRaises(AssertionError): + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + pass + + with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s['number'] = 100 + self.assertEqual(s['number'], 100) + + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return BytesIO(data).read().decode("utf-8") + + with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s['number'] = 100 + self.assertNotEqual(s['number'], 100) + self.assertEqual(s['number'], "") + def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): pass with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, protocol=2, writeback=False, serializer=serializer) + shelve.Shelf({}, + protocol=2, writeback=False, serializer=serializer) + + with self.assertRaises(shelve.ShelveError): + shelve.BsdDbShelf({}, + protocol=2, + writeback=False, serializer=serializer) def test_missing_custom_serializer(self): def deserializer(data): pass with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, protocol=2, writeback=False, deserializer=deserializer) + shelve.Shelf({}, + protocol=2, + writeback=False, deserializer=deserializer) + + with self.assertRaises(shelve.ShelveError): + shelve.BsdDbShelf({}, + protocol=2, + writeback=False, deserializer=deserializer) class TestShelveBase: From 1159bb6072e7405bed98dfd28c94ba6dba7c1fd1 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 04:44:05 +0300 Subject: [PATCH 13/59] Update BsdDbShelf's set_location, last and first functions --- Lib/shelve.py | 9 +++------ Lib/test/test_shelve.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 0564df6d3108e0..c51d1d0dedfed0 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -205,8 +205,7 @@ def __init__(self, dict, protocol=None, writeback=False, def set_location(self, key): (key, value) = self.dict.set_location(key) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def next(self): (key, value) = next(self.dict) @@ -220,13 +219,11 @@ def previous(self): def first(self): (key, value) = self.dict.first() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def last(self): (key, value) = self.dict.last() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) class DbfilenameShelf(Shelf): diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 87fc85f54ed5e2..e9877a91a87e57 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -224,13 +224,27 @@ def deserializer(data): os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: num = 1 s['number'] = num + num2 = 2 + s['number2'] = num2 self.assertEqual(s['number'], type(num)) + key, value = s.set_location(b'number') + self.assertEqual("number", key) + self.assertEqual(value, type(num)) + + key, value = s.first() + self.assertEqual("number", key) + self.assertEqual(s['number'], value) + + key, value = s.last() + self.assertEqual("number2", key) + self.assertEqual(s['number2'], value) + with self.assertRaises(AssertionError): def serializer(obj, protocol=None): return bytes(f"{type(obj).__name__}", 'utf-8') @@ -238,7 +252,7 @@ def serializer(obj, protocol=None): def deserializer(data): pass - with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: s['number'] = 100 @@ -250,7 +264,7 @@ def serializer(obj, protocol=None): def deserializer(data): return BytesIO(data).read().decode("utf-8") - with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: s['number'] = 100 From 4b4f1b65668d7de8c502b26a443f1c17f4392055 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 04:54:23 +0300 Subject: [PATCH 14/59] Update BsdDbShelf's next and previous functions --- Lib/shelve.py | 6 ++---- Lib/test/test_shelve.py | 8 ++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index c51d1d0dedfed0..9e8f68a6216250 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -209,13 +209,11 @@ def set_location(self, key): def next(self): (key, value) = next(self.dict) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def previous(self): (key, value) = self.dict.previous() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def first(self): (key, value) = self.dict.first() diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index e9877a91a87e57..43acbbec41bd9b 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -233,10 +233,18 @@ def deserializer(data): s['number2'] = num2 self.assertEqual(s['number'], type(num)) + key, value = s.previous() + self.assertEqual("number2", key) + self.assertEqual(value, type(num)) + key, value = s.set_location(b'number') self.assertEqual("number", key) self.assertEqual(value, type(num)) + key, value = s.next() + self.assertEqual("number2", key) + self.assertEqual(value, type(num)) + key, value = s.first() self.assertEqual("number", key) self.assertEqual(s['number'], value) From 41448d3b47da311983e02f993073d6ab1a2417b7 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 01:43:29 +0300 Subject: [PATCH 15/59] Refer to shelve.open function for the deserializer and serializer arguments on the shelf. Co-authored-by: Petr Viktorin --- Doc/library/shelve.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 3712cfbb514569..9bc79244141b46 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -148,10 +148,7 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. - The *deserializer* parameter can be the function that takes the - :term:`bytes-like object` and the *protocol* parameter and returns the - object. *serializer* parameter can be the function that takes the object - and returns :class:`bytes`. + The *deserializer* and *serializer* are as in :func:`~shelve.open`. A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. From fbbe5ea7d86956dc22aa982e3a9213923a5449f9 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 02:56:49 +0300 Subject: [PATCH 16/59] Refer to shelve.open function for the deserializer and serializer arguments on the BsdDbShelf and DbfilenameShelf --- Doc/library/shelve.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 9bc79244141b46..00a9a6779a121f 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -57,7 +57,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Accepts *serializer* and *deserializer* as parameters. .. note:: @@ -164,7 +164,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Accepts *serializer* and *deserializer* as parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ @@ -179,7 +179,7 @@ Restrictions methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* - parameters have the same interpretation as for the :class:`Shelf` class. + parameters have the same interpretation as for the :func:`~shelve.open`. .. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, \ @@ -191,7 +191,7 @@ Restrictions optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* and *deserializer* parameters have the same interpretation as for the - :class:`Shelf` class. + :func:`~shelve.open` .. _shelve-example: From 6823ef2c5f7c6425774cda4e84f36a73f905b3b3 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 00:01:08 +0000 Subject: [PATCH 17/59] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst diff --git a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst new file mode 100644 index 00000000000000..63f8d34728e03f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst @@ -0,0 +1 @@ +The :mod:`shelve` now accepts custom serializer and deserializer functions. From 2affece27e8dc9ed58ff33f86e5fbd0a34955a98 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 03:12:37 +0300 Subject: [PATCH 18/59] Update the versionchanged statements --- Doc/library/shelve.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 00a9a6779a121f..f996f691272a99 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -58,7 +58,8 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.14 - Accepts *serializer* and *deserializer* as parameters. + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. .. note:: @@ -165,7 +166,8 @@ Restrictions protocol. .. versionchanged:: 3.14 - Accepts *serializer* and *deserializer* as parameters. + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ keyencoding='utf-8', serializer=None, deserializer=None) From 6bfebee0a70419b0f98e6e8712e0b96327fa3e67 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 03:28:35 +0300 Subject: [PATCH 19/59] change type of num2 --- Lib/test/test_shelve.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 43acbbec41bd9b..ca64793f17b333 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -229,13 +229,13 @@ def deserializer(data): deserializer=deserializer) as s: num = 1 s['number'] = num - num2 = 2 + num2 = 2.3 s['number2'] = num2 self.assertEqual(s['number'], type(num)) key, value = s.previous() self.assertEqual("number2", key) - self.assertEqual(value, type(num)) + self.assertEqual(value, type(num2)) key, value = s.set_location(b'number') self.assertEqual("number", key) @@ -243,7 +243,7 @@ def deserializer(data): key, value = s.next() self.assertEqual("number2", key) - self.assertEqual(value, type(num)) + self.assertEqual(value, type(num2)) key, value = s.first() self.assertEqual("number", key) From 82d58a76f86aff004ddeda03c0ea887323d878e5 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 18 Jul 2024 00:45:01 +0300 Subject: [PATCH 20/59] Add test_custom_incomplete_serializer_and_deserializer case --- Lib/test/test_shelve.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index ca64793f17b333..919ef1d9830c60 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -198,7 +198,12 @@ def deserializer(data): s['number'] = 100 self.assertEqual(s['number'], 100) - with self.assertRaises(dbm.sqlite3.error): + def test_custom_incomplete_serializer_and_deserializer(self): + dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with self.assertRaises(dbm_sqlite3.error): def serializer(obj, protocol=None): pass From 5f97676dee2b6c85d7518a9a746ff9553cb28183 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:53:09 +0300 Subject: [PATCH 21/59] Specify that the Shelf, DbfilenameShelf and BsdDbShelf class's takes only keywords in documentation --- Doc/library/shelve.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f996f691272a99..dd007762a666e3 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -130,8 +130,8 @@ Restrictions which can cause hard crashes when trying to read from the database. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', \ - serializer=None, deserializer=None) +.. class:: Shelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', * serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -170,7 +170,8 @@ Restrictions :func:`pickle.dumps` and :func:`pickle.loads`. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ - keyencoding='utf-8', serializer=None, deserializer=None) + keyencoding='utf-8', *, \, + serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -184,8 +185,9 @@ Restrictions parameters have the same interpretation as for the :func:`~shelve.open`. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, \ - serializer=None, deserializer=None) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ + writeback=False, *, serializer=None, \ + deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By From 048daee641e09e4ae7d93beab142c6efa2ab7537 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:55:48 +0300 Subject: [PATCH 22/59] And and update the versionchanged's text --- Doc/library/shelve.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index dd007762a666e3..8c25460602d871 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -58,8 +58,7 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.14 - Accepts custom *serializer* and *deserializer* functions in place of - :func:`pickle.dumps` and :func:`pickle.loads`. + Added the *serializer* and *deserializer* parameters. .. note:: @@ -166,8 +165,7 @@ Restrictions protocol. .. versionchanged:: 3.14 - Accepts custom *serializer* and *deserializer* functions in place of - :func:`pickle.dumps` and :func:`pickle.loads`. + Added the *serializer* and *deserializer* parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ keyencoding='utf-8', *, \, @@ -184,6 +182,8 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as for the :func:`~shelve.open`. + .. versionchanged:: 3.14 + Added the *serializer* and *deserializer* parameters. .. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ writeback=False, *, serializer=None, \ @@ -197,6 +197,8 @@ Restrictions and *deserializer* parameters have the same interpretation as for the :func:`~shelve.open` + .. versionchanged:: 3.14 + Added the *serializer* and *deserializer* parameters. .. _shelve-example: From 00837d0f6c69a65b2e338bf4e2c1398550a52346 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:58:45 +0300 Subject: [PATCH 23/59] Update the news entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst index 63f8d34728e03f..735249b4dae224 100644 --- a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst +++ b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst @@ -1 +1,2 @@ -The :mod:`shelve` now accepts custom serializer and deserializer functions. +The :mod:`shelve` module now accepts custom serialization +and deserialization functions. From 12929637e18d7202447d2057265a1279909f71ec Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:58:54 +0300 Subject: [PATCH 24/59] Update the versionchanged's text --- Doc/library/shelve.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 8c25460602d871..1dab989ae3b2b7 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -58,7 +58,8 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.14 - Added the *serializer* and *deserializer* parameters. + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. .. note:: @@ -195,7 +196,7 @@ Restrictions optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* and *deserializer* parameters have the same interpretation as for the - :func:`~shelve.open` + :func:`~shelve.open`. .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. From 97a6d7cc5b4bf7e48b119d60fc0444abec4305ac Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 01:19:44 +0300 Subject: [PATCH 25/59] Add new testcases to other bytes objects --- Lib/test/test_shelve.py | 57 +++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 919ef1d9830c60..f268d5fc52c04a 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,3 +1,5 @@ +import array + import unittest import dbm import shelve @@ -168,22 +170,57 @@ def test_default_protocol(self): self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) def test_custom_serializer_and_deserializer(self): - def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + def serializer(obj, protocol): + if isinstance(obj, (bytes, bytearray, memoryview, int)): + return f"{type(obj).__name__}" + elif isinstance(obj, array.array): + return obj.tobytes() + else: + raise TypeError( + f"Unsupported type for serialization: {type(obj)}" + ) def deserializer(data): - value = BytesIO(data).read() - return locate(value.decode("utf-8")) + if isinstance(data, (bytes, bytearray, memoryview, int)): + value = BytesIO(data).read() + return value.decode("utf-8") + elif isinstance(data, array.array): + return array.array("b", data) + else: + raise TypeError( + f"Unsupported type for deserialization: {type(data)}" + ) os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.open(self.fn, - serializer=serializer, - deserializer=deserializer) as s: - num = 1 - s['number'] = num - self.assertEqual(s['number'], type(num)) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + with shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + num = 1 + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + memoryview_data = memoryview(b"abcdefgh") + + s["number"] = num + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + s["memoryview_data"] = memoryview_data + + self.assertEqual(s["number"], "int") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From 3becbc84ff8ebfcd8658934914d9bcffc642e51a Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 01:27:38 +0300 Subject: [PATCH 26/59] Add new testcases to test custom serializer protocl --- Lib/test/test_shelve.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index f268d5fc52c04a..a0427ec401372e 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -172,7 +172,10 @@ def test_default_protocol(self): def test_custom_serializer_and_deserializer(self): def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, memoryview, int)): - return f"{type(obj).__name__}" + if protocol == 5: + return obj + else: + return f"{type(obj).__name__}" elif isinstance(obj, array.array): return obj.tobytes() else: @@ -214,13 +217,24 @@ def deserializer(data): s["array_data"] = array_data s["memoryview_data"] = memoryview_data - self.assertEqual(s["number"], "int") - self.assertEqual(s["bytes_data"], "bytes") - self.assertEqual(s["bytearray_data"], "bytearray") - self.assertEqual( - s["array_data"], array_data.tobytes().decode() - ) - self.assertEqual(s["memoryview_data"], "memoryview") + if proto == 5: + self.assertEqual(s["number"], str(num)) + self.assertEqual(s["bytes_data"], "Hello, world!") + self.assertEqual( + s["bytearray_data"], bytearray_data.decode() + ) + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + self.assertEqual(s["memoryview_data"], "abcdefgh") + else: + self.assertEqual(s["number"], "int") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From 3a5d6ed31c1537f504dbc810d643789b0338b7fb Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 02:01:13 +0300 Subject: [PATCH 27/59] Add new testcases to other bytes objects --- Lib/test/test_shelve.py | 162 ++++++++++++++++++++++++++++++++-------- 1 file changed, 130 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index a0427ec401372e..83343c07540784 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -6,7 +6,6 @@ import pickle import os from io import BytesIO -from pydoc import locate from test.support import os_helper, import_helper from collections.abc import MutableMapping @@ -268,46 +267,145 @@ def deserializer(data): self.assertEqual(s['number'], 100) def test_custom_serializer_and_deserializer_bsd_db_shelf(self): - berkeleydb = import_helper.import_module('berkeleydb') + berkeleydb = import_helper.import_module("berkeleydb") def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + if protocol == 5: + return bytes(f"{len(type(obj).__name__)}", encoding="utf-8") + else: + return bytes(f"{type(obj).__name__}", "utf-8") def deserializer(data): value = BytesIO(data).read() - return locate(value.decode("utf-8")) + return value.decode("utf-8") os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), - serializer=serializer, - deserializer=deserializer) as s: - num = 1 - s['number'] = num - num2 = 2.3 - s['number2'] = num2 - self.assertEqual(s['number'], type(num)) - - key, value = s.previous() - self.assertEqual("number2", key) - self.assertEqual(value, type(num2)) - - key, value = s.set_location(b'number') - self.assertEqual("number", key) - self.assertEqual(value, type(num)) - - key, value = s.next() - self.assertEqual("number2", key) - self.assertEqual(value, type(num2)) - - key, value = s.first() - self.assertEqual("number", key) - self.assertEqual(s['number'], value) - - key, value = s.last() - self.assertEqual("number2", key) - self.assertEqual(s['number2'], value) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=5): + with shelve.BsdDbShelf( + berkeleydb.btopen(self.fn), + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + num = 1 + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + memoryview_data = memoryview(b"abcdefgh") + + s["number"] = num + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + s["memoryview_data"] = memoryview_data + + if proto == 5: + self.assertEqual( + s["number"], f"{len(type(num).__name__)}" + ) + self.assertEqual( + s["bytes_data"], + f"{len(type(bytes_data).__name__)}", + ) + self.assertEqual( + s["bytearray_data"], + f"{len(type(bytearray_data).__name__)}", + ) + self.assertEqual( + s["array_data"], + f"{len(type(array_data).__name__)}", + ) + self.assertEqual( + s["memoryview_data"], + f"{len(type(memoryview_data).__name__)}", + ) + + key, value = s.set_location(b"number") + self.assertEqual("number", key) + self.assertEqual(value, f"{len(type(num).__name__)}") + + key, value = s.previous() + self.assertEqual("memoryview_data", key) + self.assertEqual( + value, f"{len(type(memoryview_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + + key, value = s.last() + self.assertEqual("number", key) + self.assertEqual( + s["number"], f"{len(type(num).__name__)}" + ) + else: + key, value = s.set_location(b"number") + self.assertEqual("number", key) + self.assertEqual(value, "int") + + key, value = s.previous() + self.assertEqual("memoryview_data", key) + self.assertEqual(value, "memoryview") + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + key, value = s.last() + self.assertEqual("number", key) + self.assertEqual(s["number"], value) + + self.assertEqual(s["number"], "int") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual(s["array_data"], "array") + self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From fb74832a5be03c30c1b890db39773469bf41dca4 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 02:10:20 +0300 Subject: [PATCH 28/59] Delete comma from document --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 1dab989ae3b2b7..c4929f1eac82a0 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -169,7 +169,7 @@ Restrictions Added the *serializer* and *deserializer* parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ - keyencoding='utf-8', *, \, + keyencoding='utf-8', *, \ serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, From d670c95fc50fc83b64a55b8cb25aa7692df22d45 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 23:17:37 +0300 Subject: [PATCH 29/59] Update the description of open function --- Doc/library/shelve.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index c4929f1eac82a0..628226c735403c 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -45,10 +45,12 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. This can be changed by supplying *serializer* and *deserializer*, respectively. The *serializer* argument - should be a function that takes an object and returns its representation - as a :term:`bytes-like object`; *deserializer* should be a function that - takes :class:`bytes` and returns the corresponding object. - If one of these is given, the other must be given as well. + should be a function that takes an object and the *protocol* argument passed + to the open function and returns its representation as a + :term:`bytes-like object`; *protocol* argument that may be ignored by the + function. *deserializer* should be a function that takes :class:`bytes` and + returns the corresponding object. If one of these is given, the other must + be given as well. Otherwise :mod:`shelve` will raise a :exc:`ShelveError`. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle From f2e22eb7dafb9703484d8838e0a8753a2912598b Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 15:59:26 +0300 Subject: [PATCH 30/59] sort the imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 83343c07540784..8a6c65dd8b9098 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -7,7 +7,7 @@ import os from io import BytesIO -from test.support import os_helper, import_helper +from test.support import import_helper, os_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator From e00a52ff7205b78f39302641129e28d5452501a5 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:01:13 +0300 Subject: [PATCH 31/59] add white space MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8a6c65dd8b9098..3aede496d8d680 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,5 +1,5 @@ -import array +import array import unittest import dbm import shelve From 3af3f9774650723b4c995f7de62d99d77ab04cc7 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:01:53 +0300 Subject: [PATCH 32/59] Don't use f-string in type(obj).__name__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 3aede496d8d680..260bd828229b91 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -174,7 +174,7 @@ def serializer(obj, protocol): if protocol == 5: return obj else: - return f"{type(obj).__name__}" + return type(obj).__name__ elif isinstance(obj, array.array): return obj.tobytes() else: From 9d232e5f5324d1126d0412a626bd869a18f68280 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:02:04 +0300 Subject: [PATCH 33/59] Don't use f-string in type(obj).__name__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 260bd828229b91..dd254d9a698b6c 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -237,7 +237,7 @@ def deserializer(data): with self.assertRaises(AssertionError): def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + return bytes(type(obj).__name__, 'utf-8') def deserializer(data): pass From 26fc95958e09ff58221dd26159c3e68bf78f08eb Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:02:27 +0300 Subject: [PATCH 34/59] Don't use f-string in type(obj).__name__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index dd254d9a698b6c..8551d40ae03230 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -409,7 +409,7 @@ def deserializer(data): with self.assertRaises(AssertionError): def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + return bytes(type(obj).__name__, 'utf-8') def deserializer(data): pass From ab005aa269ebfd730a51eeeeb6a889473d29e2d6 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:03:36 +0300 Subject: [PATCH 35/59] Set shelve class argument only serializer and deserializer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 9e8f68a6216250..9343cae0e47c52 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -231,7 +231,7 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False, + def __init__(self, filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, From 60523091fa7bdbe75ffeb927c8eacc4b39859add Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:04:39 +0300 Subject: [PATCH 36/59] Update shelveError message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/shelve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 9343cae0e47c52..dec1c586b13410 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -100,8 +100,8 @@ def __init__(self, dict, protocol=None, writeback=False, self.serializer = dumps self.deserializer = loads elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be" - "defined together.") + raise ShelveError("serializer and deserializer must be " + "defined together") else: self.serializer = serializer self.deserializer = deserializer From 87b66d5419d2f4985cd594da10c3272806b83152 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:05:20 +0300 Subject: [PATCH 37/59] pass serializer and deserializer as keyword argument to DbfilenameShelf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index dec1c586b13410..2dcea5efe7877f 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -259,4 +259,4 @@ def open(filename, flag='c', protocol=None, writeback=False, *, """ return DbfilenameShelf(filename, flag, protocol, writeback, - serializer, deserializer) + serializer=serializer, deserializer=deserializer) From 0c2f2551321ce805f51ddb6e9a5d12f66783db8f Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 19:13:52 +0300 Subject: [PATCH 38/59] Remove unused import --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 2dcea5efe7877f..aeb4dcb76e02f7 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,7 +56,7 @@ the persistent dictionary on disk, if feasible). """ -from pickle import DEFAULT_PROTOCOL, Unpickler, dumps, loads +from pickle import DEFAULT_PROTOCOL, dumps, loads from io import BytesIO import collections.abc From 3db0c8e1fd4e735c0a7e414922b681035837f45c Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 19:14:47 +0300 Subject: [PATCH 39/59] Update shelve testcases --- Lib/test/test_shelve.py | 78 ++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8551d40ae03230..eea1d5f46a8c49 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -170,7 +170,7 @@ def test_default_protocol(self): def test_custom_serializer_and_deserializer(self): def serializer(obj, protocol): - if isinstance(obj, (bytes, bytearray, memoryview, int)): + if isinstance(obj, (bytes, bytearray, memoryview, str)): if protocol == 5: return obj else: @@ -183,7 +183,7 @@ def serializer(obj, protocol): ) def deserializer(data): - if isinstance(data, (bytes, bytearray, memoryview, int)): + if isinstance(data, (bytes, bytearray, memoryview, str)): value = BytesIO(data).read() return value.decode("utf-8") elif isinstance(data, array.array): @@ -204,20 +204,20 @@ def deserializer(data): serializer=serializer, deserializer=deserializer, ) as s: - num = 1 + bar = "bar" bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) memoryview_data = memoryview(b"abcdefgh") - s["number"] = num + s["foo"] = bar s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data s["memoryview_data"] = memoryview_data if proto == 5: - self.assertEqual(s["number"], str(num)) + self.assertEqual(s["foo"], str(bar)) self.assertEqual(s["bytes_data"], "Hello, world!") self.assertEqual( s["bytearray_data"], bytearray_data.decode() @@ -227,7 +227,7 @@ def deserializer(data): ) self.assertEqual(s["memoryview_data"], "abcdefgh") else: - self.assertEqual(s["number"], "int") + self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual( @@ -245,8 +245,8 @@ def deserializer(data): with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertEqual(s['number'], 100) + s["foo"] = "bar" + self.assertEqual(s["foo"], "bar") def test_custom_incomplete_serializer_and_deserializer(self): dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") @@ -263,8 +263,8 @@ def deserializer(data): with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertEqual(s['number'], 100) + s["foo"] = "bar" + self.assertEqual(s["foo"], "bar") def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") @@ -290,13 +290,13 @@ def deserializer(data): serializer=serializer, deserializer=deserializer, ) as s: - num = 1 + bar = "bar" bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) memoryview_data = memoryview(b"abcdefgh") - s["number"] = num + s["foo"] = "bar" s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data @@ -304,7 +304,7 @@ def deserializer(data): if proto == 5: self.assertEqual( - s["number"], f"{len(type(num).__name__)}" + s["foo"], f"{len(type(bar).__name__)}" ) self.assertEqual( s["bytes_data"], @@ -323,15 +323,9 @@ def deserializer(data): f"{len(type(memoryview_data).__name__)}", ) - key, value = s.set_location(b"number") - self.assertEqual("number", key) - self.assertEqual(value, f"{len(type(num).__name__)}") - - key, value = s.previous() - self.assertEqual("memoryview_data", key) - self.assertEqual( - value, f"{len(type(memoryview_data).__name__)}" - ) + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, f"{len(type(bar).__name__)}") key, value = s.previous() self.assertEqual("bytes_data", key) @@ -357,6 +351,12 @@ def deserializer(data): value, f"{len(type(bytearray_data).__name__)}" ) + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + key, value = s.first() self.assertEqual("array_data", key) self.assertEqual( @@ -364,18 +364,14 @@ def deserializer(data): ) key, value = s.last() - self.assertEqual("number", key) + self.assertEqual("memoryview_data", key) self.assertEqual( - s["number"], f"{len(type(num).__name__)}" + s["memoryview_data"], f"{len(type(memoryview_data).__name__)}" ) else: - key, value = s.set_location(b"number") - self.assertEqual("number", key) - self.assertEqual(value, "int") - - key, value = s.previous() - self.assertEqual("memoryview_data", key) - self.assertEqual(value, "memoryview") + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, "str") key, value = s.previous() self.assertEqual("bytes_data", key) @@ -393,15 +389,19 @@ def deserializer(data): self.assertEqual("bytearray_data", key) self.assertEqual(value, "bytearray") + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + key, value = s.first() self.assertEqual("array_data", key) self.assertEqual(value, "array") key, value = s.last() - self.assertEqual("number", key) - self.assertEqual(s["number"], value) + self.assertEqual("memoryview_data", key) + self.assertEqual(s["memoryview_data"], value) - self.assertEqual(s["number"], "int") + self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual(s["array_data"], "array") @@ -417,8 +417,8 @@ def deserializer(data): with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertEqual(s['number'], 100) + s["foo"] = "bar" + self.assertEqual(s["foo"], "bar") def serializer(obj, protocol=None): pass @@ -429,9 +429,9 @@ def deserializer(data): with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertNotEqual(s['number'], 100) - self.assertEqual(s['number'], "") + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertEqual(s["foo"], "") def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): From 5c39d94b282b71d2f5cf45942d1500580134f645 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 01:30:42 +0300 Subject: [PATCH 40/59] Remove memoryview testcases --- Lib/test/test_shelve.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index eea1d5f46a8c49..af8109962ab82e 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -170,7 +170,7 @@ def test_default_protocol(self): def test_custom_serializer_and_deserializer(self): def serializer(obj, protocol): - if isinstance(obj, (bytes, bytearray, memoryview, str)): + if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: return obj else: @@ -183,7 +183,7 @@ def serializer(obj, protocol): ) def deserializer(data): - if isinstance(data, (bytes, bytearray, memoryview, str)): + if isinstance(data, (bytes, bytearray, str)): value = BytesIO(data).read() return value.decode("utf-8") elif isinstance(data, array.array): @@ -208,13 +208,11 @@ def deserializer(data): bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) - memoryview_data = memoryview(b"abcdefgh") s["foo"] = bar s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data - s["memoryview_data"] = memoryview_data if proto == 5: self.assertEqual(s["foo"], str(bar)) @@ -225,7 +223,6 @@ def deserializer(data): self.assertEqual( s["array_data"], array_data.tobytes().decode() ) - self.assertEqual(s["memoryview_data"], "abcdefgh") else: self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") @@ -233,7 +230,6 @@ def deserializer(data): self.assertEqual( s["array_data"], array_data.tobytes().decode() ) - self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): @@ -294,13 +290,11 @@ def deserializer(data): bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) - memoryview_data = memoryview(b"abcdefgh") s["foo"] = "bar" s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data - s["memoryview_data"] = memoryview_data if proto == 5: self.assertEqual( @@ -318,10 +312,6 @@ def deserializer(data): s["array_data"], f"{len(type(array_data).__name__)}", ) - self.assertEqual( - s["memoryview_data"], - f"{len(type(memoryview_data).__name__)}", - ) key, value = s.set_location(b"foo") self.assertEqual("foo", key) @@ -362,12 +352,6 @@ def deserializer(data): self.assertEqual( value, f"{len(type(array_data).__name__)}" ) - - key, value = s.last() - self.assertEqual("memoryview_data", key) - self.assertEqual( - s["memoryview_data"], f"{len(type(memoryview_data).__name__)}" - ) else: key, value = s.set_location(b"foo") self.assertEqual("foo", key) @@ -397,15 +381,10 @@ def deserializer(data): self.assertEqual("array_data", key) self.assertEqual(value, "array") - key, value = s.last() - self.assertEqual("memoryview_data", key) - self.assertEqual(s["memoryview_data"], value) - self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual(s["array_data"], "array") - self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From 1ca1801712b40f06b0f47127195b98fd02f3e943 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 02:07:50 +0300 Subject: [PATCH 41/59] Add ShelveError to shelve's __all__ --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index aeb4dcb76e02f7..69f775ee99bc41 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -61,7 +61,7 @@ import collections.abc -__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] +__all__ = ["ShelveError", "Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] class ShelveError(Exception): From 5a42de1145a27a4d7ff1f19cde115571cc41b194 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 02:08:28 +0300 Subject: [PATCH 42/59] Add ShelveError to shelve documentation --- Doc/library/shelve.rst | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 628226c735403c..da6b3147b9dc60 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -133,7 +133,7 @@ Restrictions .. class:: Shelf(dict, protocol=None, writeback=False, \ - keyencoding='utf-8', * serializer=None, deserializer=None) + keyencoding='utf-8', *, serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -151,7 +151,8 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. - The *deserializer* and *serializer* are as in :func:`~shelve.open`. + The *serializer* and *deserializer* parameters have the same interpretation + as in :func:`~shelve.open`. A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. @@ -183,7 +184,7 @@ Restrictions methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* - parameters have the same interpretation as for the :func:`~shelve.open`. + parameters have the same interpretation as in :func:`~shelve.open`. .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. @@ -197,7 +198,7 @@ Restrictions default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* - and *deserializer* parameters have the same interpretation as for the + and *deserializer* parameters have the same interpretation as in the :func:`~shelve.open`. .. versionchanged:: 3.14 @@ -242,6 +243,19 @@ object):: d.close() # close it +Exceptions +---------- + +.. exception:: ShelveError + + Exception raised when one of the arguments *deserializer* and *serializer* + is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf` + and :class:`DbfilenameShelf` + + The *deserializer* and *serializer* arguments must be given together. + + .. versionadded:: 3.14 + .. seealso:: Module :mod:`dbm` From b0a5ee385977b3284fa1d36201019bc5b8348fcc Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 11:39:06 +0300 Subject: [PATCH 43/59] Add blank lines after versionadded and versionchanged --- Doc/library/shelve.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index da6b3147b9dc60..7b4c62af3897c2 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -171,6 +171,7 @@ Restrictions .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. + .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ keyencoding='utf-8', *, \ serializer=None, deserializer=None) @@ -189,6 +190,7 @@ Restrictions .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. + .. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ writeback=False, *, serializer=None, \ deserializer=None) @@ -204,6 +206,7 @@ Restrictions .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. + .. _shelve-example: Example @@ -256,6 +259,7 @@ Exceptions .. versionadded:: 3.14 + .. seealso:: Module :mod:`dbm` From 4202ede4dcf7c3257e99cf8f7ed6dc5d2edf8398 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 11:40:35 +0300 Subject: [PATCH 44/59] Remove white space in test_shelve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index af8109962ab82e..cd63d1c90c9f4f 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,4 +1,3 @@ - import array import unittest import dbm From 2827eb483c1066f442caa653c0a0a3e73221fca2 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 12:46:45 +0300 Subject: [PATCH 45/59] Add test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf --- Lib/test/test_shelve.py | 52 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index af8109962ab82e..2d3a750b4139c5 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -231,19 +231,6 @@ def deserializer(data): s["array_data"], array_data.tobytes().decode() ) - with self.assertRaises(AssertionError): - def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') - - def deserializer(data): - pass - - with shelve.open(self.fn, - serializer=serializer, - deserializer=deserializer) as s: - s["foo"] = "bar" - self.assertEqual(s["foo"], "bar") - def test_custom_incomplete_serializer_and_deserializer(self): dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") os.mkdir(self.dirname) @@ -256,11 +243,21 @@ def serializer(obj, protocol=None): def deserializer(data): return BytesIO(data).read().decode("utf-8") - with shelve.open(self.fn, - serializer=serializer, + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" - self.assertEqual(s["foo"], "bar") + + def serializer(obj, protocol=None): + return bytes(type(obj).__name__, 'utf-8') + + def deserializer(data): + pass + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertEqual(s["foo"], None) def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") @@ -386,18 +383,21 @@ def deserializer(data): self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual(s["array_data"], "array") - with self.assertRaises(AssertionError): - def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') + def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") - def deserializer(data): - pass + def serializer(obj, protocol=None): + return bytes(type(obj).__name__, 'utf-8') - with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), - serializer=serializer, - deserializer=deserializer) as s: - s["foo"] = "bar" - self.assertEqual(s["foo"], "bar") + def deserializer(data): + pass + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertEqual(s["foo"], None) def serializer(obj, protocol=None): pass From 54188bd2bbf0ec5ce4d72c582c477d5d70588b80 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 13:01:51 +0300 Subject: [PATCH 46/59] Update the serializer and deserializer functions --- Lib/test/test_shelve.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 4cb3c8448319cc..a9550d06da4955 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -4,7 +4,6 @@ import shelve import pickle import os -from io import BytesIO from test.support import import_helper, os_helper from collections.abc import MutableMapping @@ -183,8 +182,7 @@ def serializer(obj, protocol): def deserializer(data): if isinstance(data, (bytes, bytearray, str)): - value = BytesIO(data).read() - return value.decode("utf-8") + return data.decode("utf-8") elif isinstance(data, array.array): return array.array("b", data) else: @@ -240,14 +238,14 @@ def serializer(obj, protocol=None): pass def deserializer(data): - return BytesIO(data).read().decode("utf-8") + return data.decode("utf-8") with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') + return type(obj).__name__.encode("utf-8") def deserializer(data): pass @@ -262,14 +260,13 @@ def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") def serializer(obj, protocol=None): + data = obj.__class__.__name__ if protocol == 5: - return bytes(f"{len(type(obj).__name__)}", encoding="utf-8") - else: - return bytes(f"{type(obj).__name__}", "utf-8") + data = str(len(data)) + return data.encode("utf-8") def deserializer(data): - value = BytesIO(data).read() - return value.decode("utf-8") + return data.decode("utf-8") os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) @@ -384,9 +381,11 @@ def deserializer(data): def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') + return type(obj).__name__.encode("utf-8") def deserializer(data): pass @@ -402,7 +401,7 @@ def serializer(obj, protocol=None): pass def deserializer(data): - return BytesIO(data).read().decode("utf-8") + return data.decode("utf-8") with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, From 786a248848a08e23d840a01bc9af3b545a1094f8 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 13:06:02 +0300 Subject: [PATCH 47/59] Move os.mkdir and addCleanup functions beginning of the testcases --- Lib/test/test_shelve.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index a9550d06da4955..dc9ced404a8903 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -167,6 +167,9 @@ def test_default_protocol(self): self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) def test_custom_serializer_and_deserializer(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: @@ -190,9 +193,6 @@ def deserializer(data): f"Unsupported type for deserialization: {type(data)}" ) - os.mkdir(self.dirname) - self.addCleanup(os_helper.rmtree, self.dirname) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): with shelve.open( @@ -258,6 +258,8 @@ def deserializer(data): def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) def serializer(obj, protocol=None): data = obj.__class__.__name__ @@ -268,9 +270,6 @@ def serializer(obj, protocol=None): def deserializer(data): return data.decode("utf-8") - os.mkdir(self.dirname) - self.addCleanup(os_helper.rmtree, self.dirname) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=5): with shelve.BsdDbShelf( From 20c245005035f3567d7d45df9b3eb885bedf0d7c Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 13:17:06 +0300 Subject: [PATCH 48/59] Use self.assertIsNone when checking None types --- Lib/test/test_shelve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index dc9ced404a8903..90793b842ca034 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -254,7 +254,7 @@ def deserializer(data): deserializer=deserializer) as s: s["foo"] = "bar" self.assertNotEqual(s["foo"], "bar") - self.assertEqual(s["foo"], None) + self.assertIsNone(s["foo"]) def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") @@ -394,7 +394,7 @@ def deserializer(data): deserializer=deserializer) as s: s["foo"] = "bar" self.assertNotEqual(s["foo"], "bar") - self.assertEqual(s["foo"], None) + self.assertIsNone(s["foo"]) def serializer(obj, protocol=None): pass From b3770ae34aa40fa74b5b82a85c007d0011e9acb2 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 15:12:37 +0300 Subject: [PATCH 49/59] change the test order --- Lib/test/test_shelve.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 90793b842ca034..151f4108700ccb 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -283,7 +283,7 @@ def deserializer(data): bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) - s["foo"] = "bar" + s["foo"] = bar s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data @@ -393,8 +393,8 @@ def deserializer(data): serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" - self.assertNotEqual(s["foo"], "bar") self.assertIsNone(s["foo"]) + self.assertNotEqual(s["foo"], "bar") def serializer(obj, protocol=None): pass @@ -406,8 +406,9 @@ def deserializer(data): serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" - self.assertNotEqual(s["foo"], "bar") self.assertEqual(s["foo"], "") + self.assertNotEqual(s["foo"], "bar") + def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): From 4d9599b672e5b8819ff45ab935ff58ae50093bb5 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 15:28:24 +0300 Subject: [PATCH 50/59] Update shelve module version references from 3.14 to 3.15 --- Doc/library/shelve.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 7b4c62af3897c2..f7e8496d646d10 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -59,7 +59,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Accepts custom *serializer* and *deserializer* functions in place of :func:`pickle.dumps` and :func:`pickle.loads`. @@ -168,7 +168,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -187,7 +187,7 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -203,7 +203,7 @@ Restrictions and *deserializer* parameters have the same interpretation as in the :func:`~shelve.open`. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -257,7 +257,7 @@ Exceptions The *deserializer* and *serializer* arguments must be given together. - .. versionadded:: 3.14 + .. versionadded:: 3.15 .. seealso:: From 8b06918f0c08e8eb3c63ee98daa16f3f7248829a Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:24:05 +0300 Subject: [PATCH 51/59] Change shelve module version references from 3.15 to next modified: Doc/library/shelve.rst --- Doc/library/shelve.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f7e8496d646d10..9e7c8af0dbdc17 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -59,7 +59,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: 3.15 + .. versionchanged:: next Accepts custom *serializer* and *deserializer* functions in place of :func:`pickle.dumps` and :func:`pickle.loads`. @@ -168,7 +168,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: 3.15 + .. versionchanged:: next Added the *serializer* and *deserializer* parameters. @@ -187,7 +187,7 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: 3.15 + .. versionchanged:: next Added the *serializer* and *deserializer* parameters. @@ -200,10 +200,10 @@ Restrictions default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* - and *deserializer* parameters have the same interpretation as in the + and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: 3.15 + .. versionchanged:: next Added the *serializer* and *deserializer* parameters. From b0f0bbc44186f520ec9182af3a78fde78a5f9bc6 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:26:24 +0300 Subject: [PATCH 52/59] Change shelve module version references from 3.15 to next --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 9e7c8af0dbdc17..01f6c3308f7fff 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -257,7 +257,7 @@ Exceptions The *deserializer* and *serializer* arguments must be given together. - .. versionadded:: 3.15 + .. versionadded:: next .. seealso:: From d1bb227ca24054235030004513d43707a6e88f16 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:36:53 +0300 Subject: [PATCH 53/59] refactor nested context managers for better readability --- Lib/test/test_shelve.py | 65 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 151f4108700ccb..0e5f695c9c9619 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -194,39 +194,38 @@ def deserializer(data): ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - with shelve.open( - self.fn, - protocol=proto, - serializer=serializer, - deserializer=deserializer, - ) as s: - bar = "bar" - bytes_data = b"Hello, world!" - bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") - array_data = array.array("i", [1, 2, 3, 4, 5]) - - s["foo"] = bar - s["bytes_data"] = bytes_data - s["bytearray_data"] = bytearray_data - s["array_data"] = array_data - - if proto == 5: - self.assertEqual(s["foo"], str(bar)) - self.assertEqual(s["bytes_data"], "Hello, world!") - self.assertEqual( - s["bytearray_data"], bytearray_data.decode() - ) - self.assertEqual( - s["array_data"], array_data.tobytes().decode() - ) - else: - self.assertEqual(s["foo"], "str") - self.assertEqual(s["bytes_data"], "bytes") - self.assertEqual(s["bytearray_data"], "bytearray") - self.assertEqual( - s["array_data"], array_data.tobytes().decode() - ) + with self.subTest(proto=proto), shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer + ) as s: + bar = "bar" + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + + if proto == 5: + self.assertEqual(s["foo"], str(bar)) + self.assertEqual(s["bytes_data"], "Hello, world!") + self.assertEqual( + s["bytearray_data"], bytearray_data.decode() + ) + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + else: + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) def test_custom_incomplete_serializer_and_deserializer(self): dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") From 791743b85b46cf8238f528a1e11d28d3e6aeb769 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:40:06 +0300 Subject: [PATCH 54/59] simplify assertRaises calls in test_missing_custom_deserializer & test_missing_custom_serializer --- Lib/test/test_shelve.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 0e5f695c9c9619..8c75b9839a3da9 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -413,28 +413,17 @@ def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): pass - with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, - protocol=2, writeback=False, serializer=serializer) - - with self.assertRaises(shelve.ShelveError): - shelve.BsdDbShelf({}, - protocol=2, - writeback=False, serializer=serializer) + kwargs = dict(protocol=2, writeback=False, serializer=serializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) def test_missing_custom_serializer(self): def deserializer(data): pass - with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, - protocol=2, - writeback=False, deserializer=deserializer) - - with self.assertRaises(shelve.ShelveError): - shelve.BsdDbShelf({}, - protocol=2, - writeback=False, deserializer=deserializer) + kwargs = dict(protocol=2, writeback=False, deserializer=deserializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) class TestShelveBase: From 6b4be8b7d1bf957ccc37a3561356bd6f9fa6f3dc Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:45:23 +0300 Subject: [PATCH 55/59] refactor nested context managers for better readability --- Lib/test/test_shelve.py | 211 ++++++++++++++++++++-------------------- 1 file changed, 105 insertions(+), 106 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8c75b9839a3da9..8e00b9e103f3e5 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -270,112 +270,111 @@ def deserializer(data): return data.decode("utf-8") for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=5): - with shelve.BsdDbShelf( - berkeleydb.btopen(self.fn), - protocol=proto, - serializer=serializer, - deserializer=deserializer, - ) as s: - bar = "bar" - bytes_data = b"Hello, world!" - bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") - array_data = array.array("i", [1, 2, 3, 4, 5]) - - s["foo"] = bar - s["bytes_data"] = bytes_data - s["bytearray_data"] = bytearray_data - s["array_data"] = array_data - - if proto == 5: - self.assertEqual( - s["foo"], f"{len(type(bar).__name__)}" - ) - self.assertEqual( - s["bytes_data"], - f"{len(type(bytes_data).__name__)}", - ) - self.assertEqual( - s["bytearray_data"], - f"{len(type(bytearray_data).__name__)}", - ) - self.assertEqual( - s["array_data"], - f"{len(type(array_data).__name__)}", - ) - - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, f"{len(type(bar).__name__)}") - - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) - else: - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, "str") - - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") - - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") - - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") - - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") - - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") - - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") - - self.assertEqual(s["foo"], "str") - self.assertEqual(s["bytes_data"], "bytes") - self.assertEqual(s["bytearray_data"], "bytearray") - self.assertEqual(s["array_data"], "array") + with self.subTest(proto=proto), shelve.BsdDbShelf( + berkeleydb.btopen(self.fn), + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + bar = "bar" + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + + if proto == 5: + self.assertEqual( + s["foo"], f"{len(type(bar).__name__)}" + ) + self.assertEqual( + s["bytes_data"], + f"{len(type(bytes_data).__name__)}", + ) + self.assertEqual( + s["bytearray_data"], + f"{len(type(bytearray_data).__name__)}", + ) + self.assertEqual( + s["array_data"], + f"{len(type(array_data).__name__)}", + ) + + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, f"{len(type(bar).__name__)}") + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + else: + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, "str") + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual(s["array_data"], "array") def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") From 34a32b9842ca2bd35b975ab81e91e404358b604f Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 19:03:07 +0300 Subject: [PATCH 56/59] Add type_name_len helper and use shorter variable names to reduce line wrapping --- Lib/test/test_shelve.py | 146 +++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 83 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8e00b9e103f3e5..bb4caf3c89a17a 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -269,6 +269,9 @@ def serializer(obj, protocol=None): def deserializer(data): return data.decode("utf-8") + def type_name_len(obj): + return f"{(len(type(obj).__name__))}" + for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto), shelve.BsdDbShelf( berkeleydb.btopen(self.fn), @@ -277,99 +280,77 @@ def deserializer(data): deserializer=deserializer, ) as s: bar = "bar" - bytes_data = b"Hello, world!" - bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") - array_data = array.array("i", [1, 2, 3, 4, 5]) + bytes_obj = b"Hello, world!" + bytearray_obj = bytearray(b"\x00\x01\x02\x03\x04") + arr_obj = array.array("i", [1, 2, 3, 4, 5]) s["foo"] = bar - s["bytes_data"] = bytes_data - s["bytearray_data"] = bytearray_data - s["array_data"] = array_data + s["bytes_data"] = bytes_obj + s["bytearray_data"] = bytearray_obj + s["array_data"] = arr_obj if proto == 5: - self.assertEqual( - s["foo"], f"{len(type(bar).__name__)}" - ) - self.assertEqual( - s["bytes_data"], - f"{len(type(bytes_data).__name__)}", - ) - self.assertEqual( - s["bytearray_data"], - f"{len(type(bytearray_data).__name__)}", - ) - self.assertEqual( - s["array_data"], - f"{len(type(array_data).__name__)}", - ) - - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, f"{len(type(bar).__name__)}") - - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) + self.assertEqual(s["foo"], type_name_len(bar)) + self.assertEqual(s["bytes_data"], type_name_len(bytes_obj)) + self.assertEqual(s["bytearray_data"], + type_name_len(bytearray_obj)) + self.assertEqual(s["array_data"], type_name_len(arr_obj)) + + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, type_name_len(bar)) + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) else: - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, "str") + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, "str") - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") @@ -407,7 +388,6 @@ def deserializer(data): self.assertEqual(s["foo"], "") self.assertNotEqual(s["foo"], "bar") - def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): pass From 4b000cd99d9398a1f36989e2c01540e0ef37bfa9 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 2 Jun 2025 20:05:48 +0300 Subject: [PATCH 57/59] Improve the description of the open function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/shelve.rst | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 01f6c3308f7fff..84fe97cf7f9212 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -44,13 +44,18 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. This can be changed by supplying - *serializer* and *deserializer*, respectively. The *serializer* argument - should be a function that takes an object and the *protocol* argument passed - to the open function and returns its representation as a - :term:`bytes-like object`; *protocol* argument that may be ignored by the - function. *deserializer* should be a function that takes :class:`bytes` and - returns the corresponding object. If one of these is given, the other must - be given as well. Otherwise :mod:`shelve` will raise a :exc:`ShelveError`. + *serializer* and *deserializer*, respectively. + + The *serializer* argument must be a callable which takes an object ``obj`` + and the *protocol* as inputs and returns the representation ``obj`` as a + :term:`bytes-like object`; the *protocol* value may be ignored by the + serializer. + + The *deserializer* argument must be callable which takes a serialized object + given as a :class:`bytes` object and returns the corresponding object. + + A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* + is not, or vice-versa. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle From 2dcda2a1dc3badcd8b76acef9a43ef1cd462b20d Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 2 Jun 2025 20:07:20 +0300 Subject: [PATCH 58/59] Update the description of ShelveError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 84fe97cf7f9212..fc99f4590cc480 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -258,7 +258,7 @@ Exceptions Exception raised when one of the arguments *deserializer* and *serializer* is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf` - and :class:`DbfilenameShelf` + and :class:`DbfilenameShelf`. The *deserializer* and *serializer* arguments must be given together. From 00bfb014052d375cece488bae287a83752eadecc Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 2 Jun 2025 20:14:25 +0300 Subject: [PATCH 59/59] Simplify conditional branches in serializer and deserializer functions --- Lib/test/test_shelve.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index bb4caf3c89a17a..64609ab9dd9a62 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -174,24 +174,19 @@ def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: return obj - else: - return type(obj).__name__ + return type(obj).__name__ elif isinstance(obj, array.array): return obj.tobytes() - else: - raise TypeError( - f"Unsupported type for serialization: {type(obj)}" - ) + raise TypeError(f"Unsupported type for serialization: {type(obj)}") def deserializer(data): if isinstance(data, (bytes, bytearray, str)): return data.decode("utf-8") elif isinstance(data, array.array): return array.array("b", data) - else: - raise TypeError( - f"Unsupported type for deserialization: {type(data)}" - ) + raise TypeError( + f"Unsupported type for deserialization: {type(data)}" + ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto), shelve.open(