Add maxlen parameter to diskcache.Deque (grantjenks#191)

grantjenks · web-flow · commit 35dbeabd283b · 2023-04-17T20:38:04.000-07:00
* Add maxlen parameter to diskcache.Deque
diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py
@@ -44,14 +44,15 @@ def cache(self, name):
         """
         return self._cache.cache(name)
 
-    def deque(self, name):
+    def deque(self, name, maxlen=None):
         """Return Deque with given `name` in subdirectory.
 
         :param str name: subdirectory name for Deque
+        :param maxlen: max length (default None, no max)
         :return: Deque with given name
 
         """
-        return self._cache.deque(name)
+        return self._cache.deque(name, maxlen=maxlen)
 
     def index(self, name):
         """Return Index with given `name` in subdirectory.
diff --git a/diskcache/fanout.py b/diskcache/fanout.py
@@ -612,7 +612,7 @@ def cache(self, name, timeout=60, disk=None, **settings):
             _caches[name] = temp
             return temp
 
-    def deque(self, name):
+    def deque(self, name, maxlen=None):
         """Return Deque with given `name` in subdirectory.
 
         >>> cache = FanoutCache()
@@ -626,6 +626,7 @@ def deque(self, name):
         1
 
         :param str name: subdirectory name for Deque
+        :param maxlen: max length (default None, no max)
         :return: Deque with given name
 
         """
@@ -641,7 +642,7 @@ def deque(self, name):
                 disk=self._disk,
                 eviction_policy='none',
             )
-            deque = Deque.fromcache(cache)
+            deque = Deque.fromcache(cache, maxlen=maxlen)
             _deques[name] = deque
             return deque
 
diff --git a/diskcache/persistent.py b/diskcache/persistent.py
@@ -75,7 +75,7 @@ class Deque(Sequence):
 
     """
 
-    def __init__(self, iterable=(), directory=None):
+    def __init__(self, iterable=(), directory=None, maxlen=None):
         """Initialize deque instance.
 
         If directory is None then temporary directory created. The directory
@@ -86,10 +86,11 @@ def __init__(self, iterable=(), directory=None):
 
         """
         self._cache = Cache(directory, eviction_policy='none')
-        self.extend(iterable)
+        self._maxlen = float('inf') if maxlen is None else maxlen
+        self._extend(iterable)
 
     @classmethod
-    def fromcache(cls, cache, iterable=()):
+    def fromcache(cls, cache, iterable=(), maxlen=None):
         """Initialize deque using `cache`.
 
         >>> cache = Cache()
@@ -111,7 +112,8 @@ def fromcache(cls, cache, iterable=()):
         # pylint: disable=no-member,protected-access
         self = cls.__new__(cls)
         self._cache = cache
-        self.extend(iterable)
+        self._maxlen = float('inf') if maxlen is None else maxlen
+        self._extend(iterable)
         return self
 
     @property
@@ -124,6 +126,31 @@ def directory(self):
         """Directory path where deque is stored."""
         return self._cache.directory
 
+    @property
+    def maxlen(self):
+        """Max length of the deque."""
+        return self._maxlen
+
+    @maxlen.setter
+    def maxlen(self, value):
+        """Set max length of the deque.
+
+        Pops items from left while length greater than max.
+
+        >>> deque = Deque()
+        >>> deque.extendleft('abcde')
+        >>> deque.maxlen = 3
+        >>> list(deque)
+        ['c', 'd', 'e']
+
+        :param value: max length
+
+        """
+        self._maxlen = value
+        with self._cache.transact(retry=True):
+            while len(self._cache) > self._maxlen:
+                self._popleft()
+
     def _index(self, index, func):
         len_self = len(self)
 
@@ -244,7 +271,7 @@ def __iadd__(self, iterable):
         :return: deque with added items
 
         """
-        self.extend(iterable)
+        self._extend(iterable)
         return self
 
     def __iter__(self):
@@ -292,10 +319,11 @@ def __reversed__(self):
                 pass
 
     def __getstate__(self):
-        return self.directory
+        return self.directory, self.maxlen
 
     def __setstate__(self, state):
-        self.__init__(directory=state)
+        directory, maxlen = state
+        self.__init__(directory=directory, maxlen=maxlen)
 
     def append(self, value):
         """Add `value` to back of deque.
@@ -310,7 +338,12 @@ def append(self, value):
         :param value: value to add to back of deque
 
         """
-        self._cache.push(value, retry=True)
+        with self._cache.transact(retry=True):
+            self._cache.push(value, retry=True)
+            if len(self._cache) > self._maxlen:
+                self._popleft()
+
+    _append = append
 
     def appendleft(self, value):
         """Add `value` to front of deque.
@@ -325,7 +358,12 @@ def appendleft(self, value):
         :param value: value to add to front of deque
 
         """
-        self._cache.push(value, side='front', retry=True)
+        with self._cache.transact(retry=True):
+            self._cache.push(value, side='front', retry=True)
+            if len(self._cache) > self._maxlen:
+                self._pop()
+
+    _appendleft = appendleft
 
     def clear(self):
         """Remove all elements from deque.
@@ -340,6 +378,13 @@ def clear(self):
         """
         self._cache.clear(retry=True)
 
+    _clear = clear
+
+    def copy(self):
+        """Copy deque with same directory and max length."""
+        TypeSelf = type(self)
+        return TypeSelf(directory=self.directory, maxlen=self.maxlen)
+
     def count(self, value):
         """Return number of occurrences of `value` in deque.
 
@@ -365,7 +410,9 @@ def extend(self, iterable):
 
         """
         for value in iterable:
-            self.append(value)
+            self._append(value)
+
+    _extend = extend
 
     def extendleft(self, iterable):
         """Extend front side of deque with value from `iterable`.
@@ -379,7 +426,7 @@ def extendleft(self, iterable):
 
         """
         for value in iterable:
-            self.appendleft(value)
+            self._appendleft(value)
 
     def peek(self):
         """Peek at value at back of deque.
@@ -459,6 +506,8 @@ def pop(self):
             raise IndexError('pop from an empty deque')
         return value
 
+    _pop = pop
+
     def popleft(self):
         """Remove and return value at front of deque.
 
@@ -483,6 +532,8 @@ def popleft(self):
             raise IndexError('pop from an empty deque')
         return value
 
+    _popleft = popleft
+
     def remove(self, value):
         """Remove first occurrence of `value` in deque.
 
@@ -537,8 +588,8 @@ def reverse(self):
         # forward iterator and a reverse iterator, the reverse method could
         # avoid making copies of the values.
         temp = Deque(iterable=reversed(self))
-        self.clear()
-        self.extend(temp)
+        self._clear()
+        self._extend(temp)
         directory = temp.directory
         temp._cache.close()
         del temp
@@ -575,22 +626,22 @@ def rotate(self, steps=1):
 
             for _ in range(steps):
                 try:
-                    value = self.pop()
+                    value = self._pop()
                 except IndexError:
                     return
                 else:
-                    self.appendleft(value)
+                    self._appendleft(value)
         else:
             steps *= -1
             steps %= len_self
 
             for _ in range(steps):
                 try:
-                    value = self.popleft()
+                    value = self._popleft()
                 except IndexError:
                     return
                 else:
-                    self.append(value)
+                    self._append(value)
 
     __hash__ = None  # type: ignore
 
@@ -669,7 +720,9 @@ def __init__(self, *args, **kwargs):
                 args = args[1:]
             directory = None
         self._cache = Cache(directory, eviction_policy='none')
-        self.update(*args, **kwargs)
+        self._update(*args, **kwargs)
+
+    _update = MutableMapping.update
 
     @classmethod
     def fromcache(cls, cache, *args, **kwargs):
@@ -695,7 +748,7 @@ def fromcache(cls, cache, *args, **kwargs):
         # pylint: disable=no-member,protected-access
         self = cls.__new__(cls)
         self._cache = cache
-        self.update(*args, **kwargs)
+        self._update(*args, **kwargs)
         return self
 
     @property
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
@@ -565,6 +565,9 @@ access and editing at both front and back sides. :class:`Deque
     4
     >>> other.popleft()
     'foo'
+    >>> thing = Deque('abcde', maxlen=3)
+    >>> list(thing)
+    ['c', 'd', 'e']
 
 :class:`Deque <diskcache.Deque>` objects provide an efficient and safe means of
 cross-thread and cross-process communication. :class:`Deque <diskcache.Deque>`
diff --git a/tests/test_deque.py b/tests/test_deque.py
@@ -77,6 +77,20 @@ def test_getsetdel(deque):
     assert len(deque) == 0
 
 
+def test_append(deque):
+    deque.maxlen = 3
+    for item in 'abcde':
+        deque.append(item)
+    assert deque == 'cde'
+
+
+def test_appendleft(deque):
+    deque.maxlen = 3
+    for item in 'abcde':
+        deque.appendleft(item)
+    assert deque == 'edc'
+
+
 def test_index_positive(deque):
     cache = mock.MagicMock()
     cache.__len__.return_value = 3
@@ -131,9 +145,12 @@ def test_state(deque):
     sequence = list('abcde')
     deque.extend(sequence)
     assert deque == sequence
+    deque.maxlen = 3
+    assert list(deque) == sequence[-3:]
     state = pickle.dumps(deque)
     values = pickle.loads(state)
-    assert values == sequence
+    assert values == sequence[-3:]
+    assert values.maxlen == 3
 
 
 def test_compare(deque):
@@ -161,6 +178,14 @@ def test_repr():
     assert repr(deque) == 'Deque(directory=%r)' % directory
 
 
+def test_copy(deque):
+    sequence = list('abcde')
+    deque.extend(sequence)
+    temp = deque.copy()
+    assert deque == sequence
+    assert temp == sequence
+
+
 def test_count(deque):
     deque += 'abbcccddddeeeee'
 

Original file line number	Diff line number	Diff line change
@@ -565,6 +565,9 @@ access and editing at both front and back sides. :class:`Deque
`565`	`565`	`4`
`566`	`566`	`>>> other.popleft()`
`567`	`567`	`'foo'`
	`568`	`+ >>> thing = Deque('abcde', maxlen=3)`
	`569`	`+ >>> list(thing)`
	`570`	`+ ['c', 'd', 'e']`
`568`	`571`
`569`	`572`	:class:`Deque <diskcache.Deque>` objects provide an efficient and safe means of
`570`	`573`	cross-thread and cross-process communication. :class:`Deque <diskcache.Deque>`