json bytes decoding for Python3 (Issue #2033)

celery/app/amqp.py

@@ -9,6 +9,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import numbers
+import sys
 
 from collections import Mapping, namedtuple
 from datetime import timedelta
@@ -31,8 +32,10 @@ from . import routes as _routes
 
 __all__ = ['AMQP', 'Queues', 'task_message']
 
+PY3 = sys.version_info[0] == 3
+
 # json in Python2.7 borks if dict contains byte keys.
-JSON_NEEDS_UNICODE_KEYS = not try_import('simplejson')
+JSON_NEEDS_UNICODE_KEYS = not PY3 and not try_import('simplejson')
 
 #: Human readable queue declaration.
 QUEUE_FORMAT = """
@@ -45,7 +48,8 @@ task_message = namedtuple('task_message',
 
 
 def utf8dict(d, encoding='utf-8'):
-    return {k.encode(encoding): v for k, v in items(d)}
+    return {k.decode(encoding) if isinstance(k, bytes) else k: v
+            for k, v in items(d)}
 
 
 class Queues(dict):

celery/canvas.py

@@ -12,6 +12,8 @@
 """
 from __future__ import absolute_import, unicode_literals
 
+import sys
+
 from collections import MutableSequence, deque
 from copy import deepcopy
 from functools import partial as _partial, reduce
@@ -21,6 +23,7 @@ from itertools import chain as _chain
 from kombu.utils import cached_property, fxrange, reprcall, uuid
 
 from celery._state import current_app, get_current_worker_task
+from celery.local import try_import
 from celery.result import GroupResult
 from celery.utils import abstract
 from celery.utils.functional import (
@@ -32,6 +35,11 @@ from celery.utils.text import truncate
 __all__ = ['Signature', 'chain', 'xmap', 'xstarmap', 'chunks',
            'group', 'chord', 'signature', 'maybe_signature']
 
+PY3 = sys.version_info[0] == 3
+
+# json in Python2.7 borks if dict contains byte keys.
+JSON_NEEDS_UNICODE_KEYS = PY3 and not try_import('simplejson')
+
 
 class _getitem_property(object):
     """Attribute -> dict key descriptor.
@@ -323,6 +331,11 @@ class Signature(dict):
     def __repr__(self):
         return self.reprcall()
 
+    if JSON_NEEDS_UNICODE_KEYS:
+        def items(self):
+            for k, v in dict.items(self):
+                yield k.decode() if isinstance(k, bytes) else k, v
+
     @property
     def name(self):
         # for duck typing compatibility with Task.name