Use kombu.utils.encoding instead

celery/backends/base.py

@@ -8,16 +8,18 @@ import sys
 from datetime import timedelta
 
 from kombu import serialization
+from kombu.utils.encoding import bytes_to_str, ensure_bytes, from_utf8
 
 from .. import states
 from ..app import current_task
 from ..datastructures import LRUCache
 from ..exceptions import TimeoutError, TaskRevokedError
 from ..utils import timeutils
-from ..utils.encoding import bytes_to_str, ensure_bytes, from_utf8
-from ..utils.serialization import (get_pickled_exception,
-                                   get_pickleable_exception,
-                                   create_exception_cls)
+from ..utils.serialization import (
+        get_pickled_exception,
+        get_pickleable_exception,
+        create_exception_cls,
+)
 
 EXCEPTION_ABLE_CODECS = frozenset(["pickle", "yaml"])
 is_py3k = sys.version_info >= (3, 0)

celery/bin/celeryd_multi.py

@@ -101,11 +101,12 @@ from collections import defaultdict
 from subprocess import Popen
 from time import sleep
 
+from kombu.utils.encoding import from_utf8
+
 from .. import __version__
 from ..platforms import shellsplit
 from ..utils import term
 from ..utils import pluralize
-from ..utils.encoding import from_utf8
 
 SIGNAMES = set(sig for sig in dir(signal)
                         if sig.startswith("SIG") and "_" not in sig)

celery/concurrency/base.py

@@ -5,9 +5,10 @@ import logging
 import os
 import time
 
+from kombu.utils.encoding import safe_repr
+
 from .. import log
 from ..utils import timer2
-from ..utils.encoding import safe_repr
 
 
 def apply_target(target, args=(), kwargs={}, callback=None,

celery/loaders/base.py

@@ -20,12 +20,13 @@ import warnings
 from anyjson import deserialize
 from datetime import datetime
 
+from kombu.utils.encoding import safe_str
+
 from ..datastructures import DictAttribute
 from ..exceptions import ImproperlyConfigured
 from ..utils import (cached_property, get_cls_by_name,
                      import_from_cwd as _import_from_cwd)
 from ..utils.functional import maybe_list
-from ..utils.encoding import safe_str
 
 BUILTIN_MODULES = frozenset()
 

celery/security/serialization.py

@@ -4,9 +4,9 @@ import base64
 import sys
 
 from kombu.serialization import registry, encode, decode
+from kombu.utils.encoding import bytes_to_str, str_to_bytes
 
 from ..exceptions import SecurityError
-from ..utils.encoding import bytes_to_str, str_to_bytes
 
 from .certificate import Certificate, FSCertStore
 from .key import PrivateKey

celery/tests/test_backends/test_cache.py

@@ -6,6 +6,7 @@ import types
 
 from contextlib import contextmanager
 
+from kombu.utils.encoding import str_to_bytes
 from mock import Mock, patch
 
 from celery import current_app
@@ -15,7 +16,6 @@ from celery.exceptions import ImproperlyConfigured
 from celery.result import AsyncResult
 from celery.task import subtask
 from celery.utils import uuid
-from celery.utils.encoding import str_to_bytes
 
 from celery.tests.utils import Case, mask_modules, reset_modules
 

celery/tests/test_task/test_task_http.py

@@ -12,11 +12,11 @@ except ImportError:  # py3k
     from urllib.request import addinfourl  # noqa
 
 from anyjson import serialize
+from kombu.utils.encoding import from_utf8
 
 from celery.task import http
 from celery.tests.utils import Case, eager_tasks
 from celery.utils.compat import StringIO
-from celery.utils.encoding import from_utf8
 
 
 @contextmanager

celery/tests/test_worker/test_worker_job.py

@@ -11,6 +11,7 @@ import time
 from datetime import datetime, timedelta
 
 from kombu.transport.base import Message
+from kombu.utils.encoding import from_utf8, default_encode
 from mock import Mock
 from nose import SkipTest
 
@@ -27,7 +28,6 @@ from celery.result import AsyncResult
 from celery.task import task as task_dec
 from celery.task.base import Task
 from celery.utils import uuid
-from celery.utils.encoding import from_utf8, default_encode
 from celery.worker.job import Request, TaskRequest, execute_and_trace
 from celery.worker.state import revoked
 

celery/utils/encoding.py

@@ -3,9 +3,7 @@
     celery.utils.encoding
     ~~~~~~~~~~~~~~~~~~~~~
 
-    Utilities to encode text, and to safely emit text from running
-    applications without crashing with the infamous :exc:`UnicodeDecodeError`
-    exception.
+    This module has moved to :mod:`kombu.utils.encoding`.
 
     :copyright: (c) 2009 - 2012 by Ask Solem.
     :license: BSD, see LICENSE for more details.
@@ -13,91 +11,15 @@
 """
 from __future__ import absolute_import
 
-import sys
-import traceback
-
-is_py3k = sys.version_info >= (3, 0)
-
-
-if is_py3k:
-
-    def str_to_bytes(s):
-        if isinstance(s, str):
-            return s.encode()
-        return s
-
-    def bytes_to_str(s):
-        if isinstance(s, bytes):
-            return s.decode()
-        return s
-
-    def from_utf8(s, *args, **kwargs):
-        return s
-
-    def ensure_bytes(s):
-        if not isinstance(s, bytes):
-            return str_to_bytes(s)
-        return s
-
-    def default_encode(obj):
-        return obj
-
-    str_t = str
-    bytes_t = bytes
-
-else:
-
-    def str_to_bytes(s):                # noqa
-        if isinstance(s, unicode):
-            return s.encode()
-        return s
-
-    def bytes_to_str(s):                # noqa
-        return s
-
-    def from_utf8(s, *args, **kwargs):  # noqa
-        return s.encode("utf-8", *args, **kwargs)
-
-    def default_encode(obj):            # noqa
-        return unicode(obj, default_encoding())
-
-    str_t = unicode
-    bytes_t = str
-    ensure_bytes = str_to_bytes
-
-
-if sys.platform.startswith("java"):
-
-    def default_encoding():
-        return "utf-8"
-else:
-
-    def default_encoding():       # noqa
-        return sys.getfilesystemencoding()
-
-
-def safe_str(s, errors="replace"):
-    s = bytes_to_str(s)
-    if not isinstance(s, basestring):
-        return safe_repr(s, errors)
-    return _safe_str(s, errors)
-
-
-def _safe_str(s, errors="replace"):
-    if is_py3k:
-        return s
-    encoding = default_encoding()
-    try:
-        if isinstance(s, unicode):
-            return s.encode(encoding, errors)
-        return unicode(s, encoding, errors)
-    except Exception, exc:
-        return "<Unrepresentable %r: %r %r>" % (
-                type(s), exc, "\n".join(traceback.format_stack()))
-
-
-def safe_repr(o, errors="replace"):
-    try:
-        return repr(o)
-    except Exception:
-        return _safe_str(o, errors)
+from kombu.utils.encoding import (
+        default_encode,
+        default_encoding,
+        bytes_t,
+        bytes_to_str,
+        str_t,
+        str_to_bytes,
+        ensure_bytes,
+        from_utf8,
+        safe_str,
+        safe_repr,
+)

celery/worker/consumer.py

@@ -81,13 +81,15 @@ import socket
 import threading
 import warnings
 
+from kombu.utils.encoding import safe_repr
+
 from ..abstract import StartStopComponent
 from ..app import app_or_default
 from ..datastructures import AttributeDict
 from ..exceptions import InvalidTaskError
 from ..utils import noop
 from ..utils import timer2
-from ..utils.encoding import safe_repr
+
 from . import state
 from .control import Panel
 from .heartbeat import Heart

celery/worker/control.py

@@ -13,10 +13,11 @@ from __future__ import absolute_import
 
 from datetime import datetime
 
+from kombu.utils.encoding import safe_repr
+
 from ..platforms import signals as _signals
 from ..utils import timeutils
 from ..utils.compat import UserDict
-from ..utils.encoding import safe_repr
 
 from . import state
 from .state import revoked

celery/worker/job.py

@@ -19,6 +19,8 @@ import sys
 
 from datetime import datetime
 
+from kombu.utils.encoding import safe_repr, safe_str
+
 from .. import current_app
 from .. import exceptions
 from ..app import app_or_default
@@ -26,7 +28,6 @@ from ..datastructures import ExceptionInfo
 from ..execute.trace import build_tracer, trace_task, report_internal_error
 from ..platforms import set_mp_process_title as setps
 from ..utils import noop, kwdict, fun_takes_kwargs, truncate_text
-from ..utils.encoding import safe_repr, safe_str
 from ..utils.timeutils import maybe_iso8601, timezone
 
 from . import state

contrib/release/doc4allmods

@@ -24,7 +24,8 @@ SKIP_FILES="celery.backends.pyredis.rst
             celery.conf.rst
             celery.decorators.rst
             celery.db.a805d4bd.rst
-            celery.db.dfd042c7.rst"
+            celery.db.dfd042c7.rst
+            celery.utils.encoding.rst"
 
 modules=$(find "$PACKAGE" -name "*.py")
 

docs/internals/reference/celery.utils.encoding.rst

@@ -1,11 +0,0 @@
-=====================================================
- celery.utils.encoding
-=====================================================
-
-.. contents::
-    :local:
-.. currentmodule:: celery.utils.encoding
-
-.. automodule:: celery.utils.encoding
-    :members:
-    :undoc-members:

docs/internals/reference/index.rst

@@ -55,7 +55,6 @@
     celery.utils.functional
     celery.utils.term
     celery.utils.timeutils
-    celery.utils.encoding
     celery.utils.compat
     celery.utils.patch
     celery.utils.serialization