LimitedSets support on faster systems refs #3879 and #3891 (#3892)

* LimitedSets support on faster systems refs #3879 and #3891

* corrected reference (ported from #3879)

* handle update() calls where time is specified

* added monotonic() replacement to time() in LimitSets; refs #3891

* using monotonic() from celery.vine; refs #3891

* oops forgot to remove some old references to code; sorry

* references to time in LimitSet tests updated

* removed unnessisary inline comments

celery/utils/collections.py

@@ -3,7 +3,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import sys
-import time
+from celery.five import monotonic
 
 from collections import (
     Callable, Mapping, MutableMapping, MutableSet, Sequence,
@@ -526,7 +526,7 @@ class LimitedSet(object):
         False
         >>> len(s)  # maxlen is reached
         50000
-        >>> s.purge(now=time.time() + 7200)  # clock + 2 hours
+        >>> s.purge(now=monotonic() + 7200)  # clock + 2 hours
         >>> len(s)  # now only minlen items are cached
         4000
         >>>> 57000 in s  # even this item is gone now
@@ -573,7 +573,7 @@ class LimitedSet(object):
     def add(self, item, now=None):
         # type: (Any, float) -> None
         """Add a new item, or reset the expiry time of an existing item."""
-        now = now or time.time()
+        now = now or monotonic()
         if item in self._data:
             self.discard(item)
         entry = (now, item)
@@ -624,7 +624,7 @@ class LimitedSet(object):
             now (float): Time of purging -- by default right now.
                 This can be useful for unit testing.
         """
-        now = now or time.time()
+        now = now or monotonic()
         now = now() if isinstance(now, Callable) else now
         if self.maxlen:
             while len(self._data) > self.maxlen:

t/unit/utils/test_collections.py

@@ -5,7 +5,7 @@ import pytest
 
 from collections import Mapping
 from itertools import count
-from time import time
+from celery.five import monotonic
 
 from case import skip
 from billiard.einfo import ExceptionInfo
@@ -198,21 +198,21 @@ class test_LimitedSet:
         s = LimitedSet(maxlen=10, expires=1)
         [s.add(i) for i in range(10)]
         s.maxlen = 2
-        s.purge(now=time() + 100)
+        s.purge(now=monotonic() + 100)
         assert len(s) == 0
 
         # not expired
         s = LimitedSet(maxlen=None, expires=1)
         [s.add(i) for i in range(10)]
         s.maxlen = 2
-        s.purge(now=lambda: time() - 100)
+        s.purge(now=lambda: monotonic() - 100)
         assert len(s) == 2
 
         # expired -> minsize
         s = LimitedSet(maxlen=10, minlen=10, expires=1)
         [s.add(i) for i in range(20)]
         s.minlen = 3
-        s.purge(now=time() + 3)
+        s.purge(now=monotonic() + 3)
         assert s.minlen == len(s)
         assert len(s._heap) <= s.maxlen * (
             100. + s.max_heap_percent_overload) / 100
@@ -293,8 +293,6 @@ class test_LimitedSet:
 
     def test_iterable_and_ordering(self):
         s = LimitedSet(maxlen=35, expires=None)
-        # we use a custom clock here, as time.time() does not have enough
-        # precision when called quickly (can return the same value twice).
         clock = count(1)
         for i in reversed(range(15)):
             s.add(i, now=next(clock))