Merge branch '4.0'

Changelog

@@ -8,6 +8,128 @@ This document contains change notes for bugfix releases in
 the 4.0.x series (latentcall), please see :ref:`whatsnew-4.0` for
 an overview of what's new in Celery 4.0.
 
+.. _version-4.0.1:
+
+4.0.1
+=====
+:release-date: TBA
+:release-by: Ask Solem
+
+- **Tasks**: Added new method to register class-based tasks (Issue #3615).
+
+    To register a class based task you should now call ``app.register_task``:
+
+    .. code-block:: python
+
+        from celery import Celery, Task
+
+        app = Celery()
+
+        class CustomTask(Task):
+
+            def run(self):
+                return 'hello'
+
+        app.register_task(CustomTask())
+
+- **Tasks**: The ``task-sent`` event was not being sent even if
+  configured to do so (Issue #3646).
+
+- **Worker**: Fixed AMQP heartbeat support for eventlet/gevent pools
+  (Issue #3649).
+
+- **App**: ``app.conf.humanize()`` would not work if configuration
+  not finalized (Issue #3652).
+
+- **Utils**: ``saferepr`` attempted to show iterables as lists
+  and mappings as dicts.
+
+- **Utils**: ``saferepr`` did not handle unicode-errors
+  when attempting to format ``bytes`` on Python 3 (Issue #3610).
+
+- **Utils**: ``saferepr`` should now properly represent byte strings
+  with non-ascii characters (Issue #3600).
+
+- **Results**: Fixed bug in elasticsearch where _index method missed
+  the body argument (Issue #3606).
+
+    Fix contributed by **何翔宇** (Sean Ho).
+
+- **Canvas**: Fixed :exc:`ValueError` in chord with single task header
+  (Issue #3608).
+
+    Fix contributed by **Viktor Holmqvist**.
+
+- **Task**: Ensure class-based task has name prior to registration
+  (Issue #3616).
+
+    Fix contributed by **Rick Wargo**.
+
+- **Beat**: Fixed problem with strings in shelve (Issue #3644).
+
+    Fix contributed by **Alli**.
+
+- **Worker**: Fixed :exc:`KeyError` in ``inspect stats`` when ``-O`` argument
+  set to something other than ``fast`` or ``fair`` (Issue #3621).
+
+- **Task**: Retried tasks were no longer sent to the original queue
+  (Issue #3622).
+
+- **Worker**: Python 3: Fixed None/int type comparison in
+  :file:`apps/worker.py` (Issue #3631).
+
+- **Results**: Redis has a new :setting:`redis_socket_connect_timeout`
+  setting.
+
+- **Results**: Redis result backend passed the ``socket_connect_timeout``
+  argument to UNIX socket based connections by mistake, causing a crash.
+
+- **Worker**: Fixed missing logo in worker splash screen when running on
+  Python 3.x (Issue #3627).
+
+    Fix contributed by **Brian Luan**.
+
+- **Deps**: Fixed ``celery[redis]`` bundle installation (Issue #3643).
+
+    Fix contributed by **Rémi Marenco**.
+
+- **Deps**: Bundle ``celery[sqs]`` now also requires :pypi:`pycurl`
+  (Issue #3619).
+
+- **Worker**: Hard time limits were no longer being respected (Issue #3618).
+
+- **Worker**: Soft time limit log showed ``Trues`` instead of the number
+  of seconds.
+
+- **App**: ``registry_cls`` argument no longer had any effect (Issue #3613).
+
+- **Worker**: Event producer now uses ``connection_for_Write`` (Issue #3525).
+
+- **Results**: Redis/memcache backends now uses :setting:`result_expires`
+  to expire chord counter (Issue #3573).
+
+    Contributed by **Tayfun Sen**.
+
+- **Django**: Fixed command for upgrading settings with Django (Issue #3563).
+
+    Fix contributed by **François Voron**.
+
+- **Testing**: Added a ``celery_parameters`` test fixture to be able to use
+  customized ``Celery`` init parameters. (#3626)
+
+    Contributed by **Steffen Allner**.
+
+- Documentation improvements contributed by
+
+    - :github_user:`csfeathers`
+    - **Moussa Taifi**
+    - **Yuhannaa**
+    - **Laurent Peuch**
+    - **Christian**
+    - **Bruno Alla**
+    - **Steven Johns**
+    - :github_user:`tnir`
+    - **GDR!**
 
 .. _version-4.0.0:
 

celery/utils/saferepr.py

@@ -13,6 +13,7 @@ Very slow with no limits, super quick with limits.
 from __future__ import absolute_import, unicode_literals
 
 import sys
+import traceback
 
 from collections import deque, namedtuple
 
@@ -38,13 +39,33 @@ else:
     class range_t(object):  # noqa
         pass
 
+#: Node representing literal text.
+#:   - .value: is the literal text value
+#:   - .truncate: specifies if this text can be truncated, for things like
+#:                LIT_DICT_END this will be False, as we always display
+#:                the ending brackets, e.g:  [[[1, 2, 3, ...,], ..., ]]
+#:   - .direction: If +1 the current level is increment by one,
+#:                 if -1 the current level is decremented by one, and
+#:                 if 0 the current level is unchanged.
 _literal = namedtuple('_literal', ('value', 'truncate', 'direction'))
+
+#: Node representing a dictionary key.
 _key = namedtuple('_key', ('value',))
+
+#: Node representing quoted text, e.g. a string value.
 _quoted = namedtuple('_quoted', ('value',))
+
+
+#: Recursion protection.
 _dirty = namedtuple('_dirty', ('objid',))
 
+#: Types that are repsented as chars.
 chars_t = (bytes, text_t)
+
+#: Types that are regarded as safe to call repr on.
 safe_t = (Number,)
+
+#: Set types.
 set_t = (frozenset, set)
 
 LIT_DICT_START = _literal('{', False, +1)
@@ -61,6 +82,7 @@ LIT_TUPLE_END_SV = _literal(',)', False, -1)
 
 
 def saferepr(o, maxlen=None, maxlevels=3, seen=None):
+    # type: (Any, int, int, Set) -> str
     """Safe version of :func:`repr`.
 
     Warning:
@@ -76,6 +98,7 @@ def saferepr(o, maxlen=None, maxlevels=3, seen=None):
 def _chaindict(mapping,
                LIT_DICT_KVSEP=LIT_DICT_KVSEP,
                LIT_LIST_SEP=LIT_LIST_SEP):
+    # type: (Dict, _literal, _literal) -> Iterator[Any]
     size = len(mapping)
     for i, (k, v) in enumerate(items(mapping)):
         yield _key(k)
@@ -86,6 +109,7 @@ def _chaindict(mapping,
 
 
 def _chainlist(it, LIT_LIST_SEP=LIT_LIST_SEP):
+    # type: (List) -> Iterator[Any]
     size = len(it)
     for i, v in enumerate(it):
         yield v
@@ -94,10 +118,12 @@ def _chainlist(it, LIT_LIST_SEP=LIT_LIST_SEP):
 
 
 def _repr_empty_set(s):
+    # type: (Set) -> str
     return '%s()' % (type(s).__name__,)
 
 
 def _safetext(val):
+    # type: (AnyStr) -> str
     if isinstance(val, bytes):
         try:
             val.encode('utf-8')
@@ -109,6 +135,7 @@ def _safetext(val):
 
 
 def _format_binary_bytes(val, maxlen, ellipsis='...'):
+    # type: (bytes, int, str) -> str
     if maxlen and len(val) > maxlen:
         # we don't want to copy all the data, just take what we need.
         chunk = memoryview(val)[:maxlen].tobytes()
@@ -117,6 +144,7 @@ def _format_binary_bytes(val, maxlen, ellipsis='...'):
 
 
 def _repr_binary_bytes(val):
+    # type: (bytes) -> str
     try:
         return val.decode('utf-8')
     except UnicodeDecodeError:
@@ -132,13 +160,24 @@ def _repr_binary_bytes(val):
 
 
 def _format_chars(val, maxlen):
+    # type: (AnyStr, int) -> str
     if IS_PY3 and isinstance(val, bytes):  # pragma: no cover
         return _format_binary_bytes(val, maxlen)
     else:
         return "'{0}'".format(truncate(val, maxlen))
 
 
+def _repr(obj):
+    # type: (Any) -> str
+    try:
+        return repr(obj)
+    except Exception as exc:
+        return '<Unrepresentable {0!r}{1:#x}: {2!r} {3!r}>'.format(
+            type(obj), id(obj), exc, '\n'.join(traceback.format_stack()))
+
+
 def _saferepr(o, maxlen=None, maxlevels=3, seen=None):
+    # type: (Any, int, int, Set) -> str
     stack = deque([iter([o])])
     for token, it in reprstream(stack, seen=seen, maxlevels=maxlevels):
         if maxlen is not None and maxlen <= 0:
@@ -166,6 +205,7 @@ def _saferepr(o, maxlen=None, maxlevels=3, seen=None):
 
 
 def _reprseq(val, lit_start, lit_end, builtin_type, chainer):
+    # type: (Sequence, _literal, _literal, Any, Any) -> Tuple[Any, ...]
     if type(val) is builtin_type:  # noqa
         return lit_start, lit_end, chainer(val)
     return (
@@ -177,6 +217,7 @@ def _reprseq(val, lit_start, lit_end, builtin_type, chainer):
 
 def reprstream(stack, seen=None, maxlevels=3, level=0, isinstance=isinstance):
     """Streaming repr, yielding tokens."""
+    # type: (deque, Set, int, int, Callable) -> Iterator[Any]
     seen = seen or set()
     append = stack.append
     popleft = stack.popleft
@@ -198,13 +239,13 @@ def reprstream(stack, seen=None, maxlevels=3, level=0, isinstance=isinstance):
             elif isinstance(val, _key):
                 yield val, it
             elif isinstance(val, Decimal):
-                yield repr(val), it
+                yield _repr(val), it
             elif isinstance(val, safe_t):
                 yield text_t(val), it
             elif isinstance(val, chars_t):
                 yield _quoted(val), it
             elif isinstance(val, range_t):  # pragma: no cover
-                yield repr(val), it
+                yield _repr(val), it
             else:
                 if isinstance(val, set_t):
                     if not val:
@@ -226,7 +267,7 @@ def reprstream(stack, seen=None, maxlevels=3, level=0, isinstance=isinstance):
                         LIT_LIST_START, LIT_LIST_END, _chainlist(val))
                 else:
                     # other type of object
-                    yield repr(val), it
+                    yield _repr(val), it
                     continue
 
                 if maxlevels and level >= maxlevels:

t/unit/utils/test_saferepr.py

@@ -222,3 +222,9 @@ class test_saferepr:
             assert result.endswith("...'")
         else:  # Python 3.4
             assert result
+
+    def test_repr_raises(self):
+        class O(object):
+            def __repr__(self):
+                raise KeyError('foo')
+        assert 'Unrepresentable' in saferepr(O())