Fixes bugs in saferepr on Python3

celery/tests/utils/test_saferepr.py

@@ -5,7 +5,7 @@ import re
 from decimal import Decimal
 from pprint import pprint
 
-from celery.five import items, long_t, values
+from celery.five import items, long_t, text_t, values
 
 from celery.utils.saferepr import saferepr
 
@@ -45,13 +45,28 @@ D_ALL = {
 
 D_D_TEXT = {b'rest': D_TEXT}
 
-RE_OLD_SET_REPR = re.compile(r'(?:frozen)?set\d?\(\[(.+?)\]\)')
+RE_OLD_SET_REPR = re.compile(r'(?<!frozen)set\([\[|\{](.+?)[\}\]]\)')
 RE_OLD_SET_REPR_REPLACE = r'{\1}'
-
-
-def from_old_repr(s):
-    return RE_OLD_SET_REPR.sub(
-        RE_OLD_SET_REPR_REPLACE, s).replace("u'", "'")
+RE_OLD_SET_CUSTOM_REPR = re.compile(r'((?:frozen)?set\d?\()\[(.+?)\](\))')
+RE_OLD_SET_CUSTOM_REPR_REPLACE = r'\1{\2}\3'
+RE_EMPTY_SET_REPR = re.compile(r'((?:frozen)?set\d?)\(\[\]\)')
+RE_EMPTY_SET_REPR_REPLACE = r'\1()'
+RE_LONG_SUFFIX = re.compile(r'(\d)+L')
+
+
+def old_repr(s):
+    return text_t(RE_LONG_SUFFIX.sub(
+        r'\1',
+        RE_EMPTY_SET_REPR.sub(
+            RE_EMPTY_SET_REPR_REPLACE,
+            RE_OLD_SET_REPR.sub(
+                RE_OLD_SET_REPR_REPLACE,
+                RE_OLD_SET_CUSTOM_REPR.sub(
+                    RE_OLD_SET_CUSTOM_REPR_REPLACE, repr(s).replace("u'", "'"))
+                ),
+            ),
+        ),
+    ).replace('set([])', 'set()')
 
 
 class list2(list):
@@ -114,23 +129,24 @@ class test_saferepr(Case):
 
     def test_safe_types(self):
         for value in values(D_NUMBERS):
-            self.assertEqual(saferepr(value), repr(value))
+            self.assertEqual(saferepr(value), old_repr(value))
 
     def test_numbers_dict(self):
-        self.assertEqual(saferepr(D_NUMBERS), repr(D_NUMBERS))
+        self.assertEqual(saferepr(D_NUMBERS), old_repr(D_NUMBERS))
 
     def test_numbers_list(self):
-        self.assertEqual(saferepr(L_NUMBERS), repr(L_NUMBERS))
+        self.assertEqual(saferepr(L_NUMBERS), old_repr(L_NUMBERS))
 
     def test_numbers_keys(self):
-        self.assertEqual(saferepr(D_INT_KEYS), repr(D_INT_KEYS))
+        self.assertEqual(saferepr(D_INT_KEYS), old_repr(D_INT_KEYS))
 
     def test_text(self):
-        self.assertEqual(saferepr(D_TEXT), repr(D_TEXT).replace("u'", "'"))
+        self.assertEqual(saferepr(D_TEXT), old_repr(D_TEXT).replace("u'", "'"))
 
     def test_text_maxlen(self):
-        self.assertEqual(saferepr(D_D_TEXT, 100),
-                from_old_repr(repr(D_D_TEXT)[:99] + "...', ...}}"))
+        self.assertTrue(
+            saferepr(D_D_TEXT, 100).endswith("...', ...}}")
+        )
 
     def test_same_as_repr(self):
         # Simple objects, small containers and classes that overwrite __repr__
@@ -159,5 +175,5 @@ class test_saferepr(Case):
             range(10, -11, -1)
         )
         for simple in types:
-            native = from_old_repr(repr(simple))
+            native = old_repr(simple)
             self.assertEqual(saferepr(simple), native)

celery/utils/saferepr.py

@@ -9,21 +9,38 @@
 
     - Sets are represented the Python 3 way: ``{1, 2}`` vs ``set([1, 2])``.
     - Unicode strings does not have the ``u'`` prefix, even on Python 2.
+    - Empty set formatted as ``set()`` (Python3), not ``set([])`` (Python2).
+    - Longs do not have the ``L`` suffix.
 
     Very slow with no limits, super quick with limits.
 
 """
+from __future__ import absolute_import, unicode_literals
+
+import sys
+
 from collections import Iterable, Mapping, deque, namedtuple
 
+from decimal import Decimal
 from itertools import chain
 from numbers import Number
 from pprint import _recursion
 
+from kombu.utils.encoding import bytes_to_str
+
 from celery.five import items, text_t
 
-from .text import truncate
+from .text import truncate, truncate_bytes
 
-__all__ = ['saferepr']
+__all__ = ['saferepr', 'reprstream']
+
+IS_PY3 = sys.version_info[0] == 3
+
+if IS_PY3:
+    range_t = (range, )
+else:
+    class range_t(object):  # noqa
+        pass
 
 _literal = namedtuple('_literal', ('value', 'truncate', 'direction'))
 _key = namedtuple('_key', ('value',))
@@ -75,7 +92,7 @@ def _chainlist(it, LIT_LIST_SEP=LIT_LIST_SEP):
 
 
 def _repr_empty_set(s):
-    return '%s([])' % (type(s).__name__,)
+    return '%s()' % (type(s).__name__,)
 
 
 def _saferepr(o, maxlen=None, maxlevels=3, seen=None):
@@ -88,11 +105,15 @@ def _saferepr(o, maxlen=None, maxlevels=3, seen=None):
             stack.append(it)
             break
         if isinstance(token, _literal):
-            val = str(token.value)
+            val = token.value
         elif isinstance(token, _key):
             val = repr(token.value).replace("u'", "'")
         elif isinstance(token, _quoted):
-            val = "'%s'" % (truncate(token.value, maxlen),)
+            val = token.value
+            if IS_PY3 and isinstance(val, bytes):
+                val = "b'%s'" % (bytes_to_str(truncate_bytes(val, maxlen)),)
+            else:
+                val = "'%s'" % (truncate(val, maxlen),)
         else:
             val = truncate(token, maxlen)
         yield val
@@ -105,6 +126,16 @@ def _saferepr(o, maxlen=None, maxlevels=3, seen=None):
                 yield rest2.value
 
 
+def _reprseq(val, lit_start, lit_end, builtin_type, chainer):
+    if type(val) is builtin_type:  # noqa
+        return lit_start, lit_end, chainer(val)
+    return (
+        _literal('%s(%s' % (type(val).__name__, lit_start.value), False, +1),
+        _literal('%s)' % (lit_end.value,), False, -1),
+        chainer(val)
+    )
+
+
 def reprstream(stack, seen=None, maxlevels=3, level=0, isinstance=isinstance):
     seen = seen or set()
     append = stack.append
@@ -126,17 +157,22 @@ def reprstream(stack, seen=None, maxlevels=3, level=0, isinstance=isinstance):
                 yield val, it
             elif isinstance(val, _key):
                 yield val, it
-            elif isinstance(val, safe_t):
+            elif isinstance(val, Decimal):
                 yield repr(val), it
+            elif isinstance(val, safe_t):
+                yield text_t(val), it
             elif isinstance(val, chars_t):
                 yield _quoted(val), it
+            elif isinstance(val, range_t):
+                yield repr(val), it
             else:
                 if isinstance(val, set_t):
                     if not val:
                         yield _repr_empty_set(val), it
                         continue
-                    lit_start, lit_end, val = (
-                        LIT_SET_START, LIT_SET_END, _chainlist(val))
+                    lit_start, lit_end, val = _reprseq(
+                        val, LIT_SET_START, LIT_SET_END, set, _chainlist,
+                    )
                 elif isinstance(val, tuple):
                     lit_start, lit_end, val = (
                         LIT_TUPLE_START,

celery/utils/text.py

@@ -62,11 +62,17 @@ def indent(t, indent=0, sep='\n'):
     return sep.join(' ' * indent + p for p in t.split(sep))
 
 
-def truncate(text, maxlen=128, suffix='...'):
+def truncate(s, maxlen=128, suffix='...'):
     """Truncates text to a maximum number of characters."""
-    if maxlen and len(text) >= maxlen:
-        return text[:maxlen].rsplit(' ', 1)[0] + suffix
-    return text
+    if maxlen and len(s) >= maxlen:
+        return s[:maxlen].rsplit(' ', 1)[0] + suffix
+    return s
+
+
+def truncate_bytes(s, maxlen=128, suffix=b'...'):
+    if maxlen and len(s) >= maxlen:
+        return s[:maxlen].rsplit(b' ', 1)[0] + suffix
+    return s
 
 
 def pluralize(n, text, suffix='s'):