bugfix for non-serializable exception arguments when JSON is the selected serializer (#4864)

CONTRIBUTORS.txt

@@ -260,3 +260,4 @@ Igor Kasianov, 2018/01/20
 Derek Harland, 2018/02/15
 Chris Mitchell, 2018/02/27
 Josue Balandrano Coronel, 2018/05/24
+Tom Booth, 2018/07/06

celery/backends/base.py

@@ -33,6 +33,7 @@ from celery.utils.collections import BufferMap
 from celery.utils.functional import LRUCache, arity_greater
 from celery.utils.log import get_logger
 from celery.utils.serialization import (create_exception_cls,
+                                        ensure_serializable,
                                         get_pickleable_exception,
                                         get_pickled_exception)
 
@@ -236,7 +237,7 @@ class Backend(object):
         if serializer in EXCEPTION_ABLE_CODECS:
             return get_pickleable_exception(exc)
         return {'exc_type': type(exc).__name__,
-                'exc_message': exc.args,
+                'exc_message': ensure_serializable(exc.args, self.encode),
                 'exc_module': type(exc).__module__}
 
     def exception_to_python(self, exc):

celery/utils/serialization.py

@@ -56,6 +56,8 @@ def find_pickleable_exception(exc, loads=pickle.loads,
 
     Arguments:
         exc (BaseException): An exception instance.
+        loads: decoder to use.
+        dumps: encoder to use
 
     Returns:
         Exception: Nearest pickleable parent exception class
@@ -84,6 +86,26 @@ def create_exception_cls(name, module, parent=None):
     return subclass_exception(name, parent, module)
 
 
+def ensure_serializable(items, encoder):
+    """Ensure items will serialize.
+
+    For a given list of arbitrary objects, return the object
+    or a string representation, safe for serialization.
+
+    Arguments:
+        items (Iterable[Any]): Objects to serialize.
+        encoder (Callable): Callable function to serialize with.
+    """
+    safe_exc_args = []
+    for arg in items:
+        try:
+            encoder(arg)
+            safe_exc_args.append(arg)
+        except Exception:  # pylint: disable=broad-except
+            safe_exc_args.append(safe_repr(arg))
+    return tuple(safe_exc_args)
+
+
 @python_2_unicode_compatible
 class UnpickleableExceptionWrapper(Exception):
     """Wraps unpickleable exceptions.
@@ -116,13 +138,7 @@ class UnpickleableExceptionWrapper(Exception):
     exc_args = None
 
     def __init__(self, exc_module, exc_cls_name, exc_args, text=None):
-        safe_exc_args = []
-        for arg in exc_args:
-            try:
-                pickle.dumps(arg)
-                safe_exc_args.append(arg)
-            except Exception:  # pylint: disable=broad-except
-                safe_exc_args.append(safe_repr(arg))
+        safe_exc_args = ensure_serializable(exc_args, pickle.dumps)
         self.exc_module = exc_module
         self.exc_cls_name = exc_cls_name
         self.exc_args = safe_exc_args

t/unit/backends/test_base.py

@@ -145,6 +145,17 @@ class test_prepare_exception:
         y = self.b.exception_to_python(x)
         assert isinstance(y, KeyError)
 
+    def test_json_exception_arguments(self):
+        self.b.serializer = 'json'
+        x = self.b.prepare_exception(Exception(object))
+        assert x == {
+            'exc_message': serialization.ensure_serializable(
+                (object,), self.b.encode),
+            'exc_type': Exception.__name__,
+            'exc_module': Exception.__module__}
+        y = self.b.exception_to_python(x)
+        assert isinstance(y, Exception)
+
     def test_impossible(self):
         self.b.serializer = 'pickle'
         x = self.b.prepare_exception(Impossible())

t/unit/utils/test_serialization.py

@@ -1,14 +1,17 @@
 from __future__ import absolute_import, unicode_literals
 
+import json
+import pickle
 import sys
 from datetime import date, datetime, time, timedelta
 
 import pytest
 import pytz
-from case import Mock, mock
+from case import Mock, mock, skip
 from kombu import Queue
 
 from celery.utils.serialization import (UnpickleableExceptionWrapper,
+                                        ensure_serializable,
                                         get_pickleable_etype, jsonify)
 
 
@@ -25,6 +28,23 @@ class test_AAPickle:
             sys.modules['celery.utils.serialization'] = prev
 
 
+class test_ensure_serializable:
+
+    @skip.unless_python3()
+    def test_json_py3(self):
+        assert (1, "<class 'object'>") == \
+            ensure_serializable([1, object], encoder=json.dumps)
+
+    @skip.if_python3()
+    def test_json_py2(self):
+        assert (1, "<type 'object'>") == \
+            ensure_serializable([1, object], encoder=json.dumps)
+
+    def test_pickle(self):
+        assert (1, object) == \
+            ensure_serializable((1, object), encoder=pickle.dumps)
+
+
 class test_UnpickleExceptionWrapper:
 
     def test_init(self):