gh-99631: Add custom loads and dumps support for the shelve module

Doc/library/shelve.rst

@@ -17,7 +17,8 @@ This includes most class instances, recursive data types, and objects containing
 lots of shared sub-objects. The keys are ordinary strings.
 
 
-.. function:: open(filename, flag='c', protocol=None, writeback=False)
+.. function:: open(filename, flag='c', protocol=None, writeback=False, *, \
+ serializer=None, deserializer=None)
 
  Open a persistent dictionary. The filename specified is the base filename for
  the underlying database. As a side-effect, an extension may be added to the
@@ -41,13 +42,32 @@ lots of shared sub-objects. The keys are ordinary strings.
  determine which accessed entries are mutable, nor which ones were actually
  mutated).
 
+ By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads`
+ for serializing and deserializing. This can be changed by supplying
+ *serializer* and *deserializer*, respectively.
+
+ The *serializer* argument must be a callable which takes an object ``obj``
+ and the *protocol* as inputs and returns the representation ``obj`` as a
+ :term:`bytes-like object`; the *protocol* value may be ignored by the
+ serializer.
+
+ The *deserializer* argument must be callable which takes a serialized object
+ given as a :class:`bytes` object and returns the corresponding object.
+
+ A :exc:`ShelveError` is raised if *serializer* is given but *deserializer*
+ is not, or vice-versa.
+
  .. versionchanged:: 3.10
  :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle
  protocol.
 
  .. versionchanged:: 3.11
  Accepts :term:`path-like object` for filename.
 
+ .. versionchanged:: next
+ Accepts custom *serializer* and *deserializer* functions in place of
+ :func:`pickle.dumps` and :func:`pickle.loads`.
+
  .. note::
 
  Do not rely on the shelf being closed automatically; always call
@@ -129,7 +149,8 @@ Restrictions
  explicitly.
 
 
-.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8')
+.. class:: Shelf(dict, protocol=None, writeback=False, \
+ keyencoding='utf-8', *, serializer=None, deserializer=None)
 
  A subclass of :class:`collections.abc.MutableMapping` which stores pickled
  values in the *dict* object.
@@ -147,6 +168,9 @@ Restrictions
  The *keyencoding* parameter is the encoding used to encode keys before they
  are used with the underlying dict.
 
+ The *serializer* and *deserializer* parameters have the same interpretation
+ as in :func:`~shelve.open`.
+
  A :class:`Shelf` object can also be used as a context manager, in which
  case it will be automatically closed when the :keyword:`with` block ends.
 
@@ -161,8 +185,13 @@ Restrictions
  :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle
  protocol.
 
+ .. versionchanged:: next
+ Added the *serializer* and *deserializer* parameters.
 
-.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8')
+
+.. class:: BsdDbShelf(dict, protocol=None, writeback=False, \
+ keyencoding='utf-8', *, \
+ serializer=None, deserializer=None)
 
  A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`,
  :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods.
@@ -172,18 +201,27 @@ Restrictions
  modules. The *dict* object passed to the constructor must support those
  methods. This is generally accomplished by calling one of
  :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The
- optional *protocol*, *writeback*, and *keyencoding* parameters have the same
- interpretation as for the :class:`Shelf` class.
+ optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer*
+ parameters have the same interpretation as in :func:`~shelve.open`.
+
+ .. versionchanged:: next
+ Added the *serializer* and *deserializer* parameters.
 
 
-.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False)
+.. class:: DbfilenameShelf(filename, flag='c', protocol=None, \
+ writeback=False, *, serializer=None, \
+ deserializer=None)
 
  A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like
  object. The underlying file will be opened using :func:`dbm.open`. By
  default, the file will be created and opened for both read and write. The
- optional *flag* parameter has the same interpretation as for the :func:`.open`
- function. The optional *protocol* and *writeback* parameters have the same
- interpretation as for the :class:`Shelf` class.
+ optional *flag* parameter has the same interpretation as for the
+ :func:`.open` function. The optional *protocol*, *writeback*, *serializer*
+ and *deserializer* parameters have the same interpretation as in
+ :func:`~shelve.open`.
+
+ .. versionchanged:: next
+ Added the *serializer* and *deserializer* parameters.
 
 
 .. _shelve-example:
@@ -225,6 +263,20 @@ object)::
  d.close() # close it
 
 
+Exceptions
+----------
+
+.. exception:: ShelveError
+
+ Exception raised when one of the arguments *deserializer* and *serializer*
+ is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf`
+ and :class:`DbfilenameShelf`.
+
+ The *deserializer* and *serializer* arguments must be given together.
+
+ .. versionadded:: next
+
+
 .. seealso::
 
  Module :mod:`dbm`

Lib/shelve.py

@@ -56,12 +56,17 @@
 the persistent dictionary on disk, if feasible).
 """
 
-from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler
+from pickle import DEFAULT_PROTOCOL, dumps, loads
 from io import BytesIO
 
 import collections.abc
 
-__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"]
+__all__ = ["ShelveError", "Shelf", "BsdDbShelf", "DbfilenameShelf", "open"]
+
+
+class ShelveError(Exception):
+ pass
+
 
 class _ClosedDict(collections.abc.MutableMapping):
  'Marker for a closed dict. Access attempts raise a ValueError.'
@@ -82,7 +87,7 @@ class Shelf(collections.abc.MutableMapping):
  """
 
  def __init__(self, dict, protocol=None, writeback=False,
- keyencoding="utf-8"):
+ keyencoding="utf-8", *, serializer=None, deserializer=None):
  self.dict = dict
  if protocol is None:
  protocol = DEFAULT_PROTOCOL
@@ -91,6 +96,16 @@ def __init__(self, dict, protocol=None, writeback=False,
  self.cache = {}
  self.keyencoding = keyencoding
 
+ if serializer is None and deserializer is None:
+ self.serializer = dumps
+ self.deserializer = loads
+ elif (serializer is None) ^ (deserializer is None):
+ raise ShelveError("serializer and deserializer must be "
+ "defined together")
+ else:
+ self.serializer = serializer
+ self.deserializer = deserializer
+
  def __iter__(self):
  for k in self.dict.keys():
  yield k.decode(self.keyencoding)
@@ -110,19 +125,17 @@ def __getitem__(self, key):
  try:
  value = self.cache[key]
  except KeyError:
- f = BytesIO(self.dict[key.encode(self.keyencoding)])
- value = Unpickler(f).load()
+ f = self.dict[key.encode(self.keyencoding)]
+ value = self.deserializer(f)
  if self.writeback:
  self.cache[key] = value
  return value
 
  def __setitem__(self, key, value):
  if self.writeback:
  self.cache[key] = value
- f = BytesIO()
- p = Pickler(f, self._protocol)
- p.dump(value)
- self.dict[key.encode(self.keyencoding)] = f.getvalue()
+ serialized_value = self.serializer(value, self._protocol)
+ self.dict[key.encode(self.keyencoding)] = serialized_value
 
  def __delitem__(self, key):
  del self.dict[key.encode(self.keyencoding)]
@@ -191,33 +204,29 @@ class BsdDbShelf(Shelf):
  """
 
  def __init__(self, dict, protocol=None, writeback=False,
- keyencoding="utf-8"):
- Shelf.__init__(self, dict, protocol, writeback, keyencoding)
+ keyencoding="utf-8", *, serializer=None, deserializer=None):
+ Shelf.__init__(self, dict, protocol, writeback, keyencoding,
+ serializer=serializer, deserializer=deserializer)
 
  def set_location(self, key):
  (key, value) = self.dict.set_location(key)
- f = BytesIO(value)
- return (key.decode(self.keyencoding), Unpickler(f).load())
+ return (key.decode(self.keyencoding), self.deserializer(value))
 
  def next(self):
  (key, value) = next(self.dict)
- f = BytesIO(value)
- return (key.decode(self.keyencoding), Unpickler(f).load())
+ return (key.decode(self.keyencoding), self.deserializer(value))
 
  def previous(self):
  (key, value) = self.dict.previous()
- f = BytesIO(value)
- return (key.decode(self.keyencoding), Unpickler(f).load())
+ return (key.decode(self.keyencoding), self.deserializer(value))
 
  def first(self):
  (key, value) = self.dict.first()
- f = BytesIO(value)
- return (key.decode(self.keyencoding), Unpickler(f).load())
+ return (key.decode(self.keyencoding), self.deserializer(value))
 
  def last(self):
  (key, value) = self.dict.last()
- f = BytesIO(value)
- return (key.decode(self.keyencoding), Unpickler(f).load())
+ return (key.decode(self.keyencoding), self.deserializer(value))
 
 
 class DbfilenameShelf(Shelf):
@@ -227,9 +236,11 @@ class DbfilenameShelf(Shelf):
  See the module's __doc__ string for an overview of the interface.
  """
 
- def __init__(self, filename, flag='c', protocol=None, writeback=False):
+ def __init__(self, filename, flag='c', protocol=None, writeback=False, *,
+ serializer=None, deserializer=None):
  import dbm
- Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
+ Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback,
+ serializer=serializer, deserializer=deserializer)
 
  def clear(self):
  """Remove all items from the shelf."""
@@ -238,8 +249,8 @@ def clear(self):
  self.cache.clear()
  self.dict.clear()
 
-
-def open(filename, flag='c', protocol=None, writeback=False):
+def open(filename, flag='c', protocol=None, writeback=False, *,
+ serializer=None, deserializer=None):
  """Open a persistent dictionary for reading and writing.
 
  The filename parameter is the base filename for the underlying
@@ -252,4 +263,5 @@ def open(filename, flag='c', protocol=None, writeback=False):
  See the module's __doc__ string for an overview of the interface.
  """
 
- return DbfilenameShelf(filename, flag, protocol, writeback)
+ return DbfilenameShelf(filename, flag, protocol, writeback,
+ serializer=serializer, deserializer=deserializer)