Code now works on both Python 3 and Python 2 (without using 2to3)

celery/__init__.py

@@ -27,8 +27,8 @@ VERSION_BANNER = '{0} ({1})'.format(__version__, SERIES)
 import os
 if os.environ.get('C_IMPDEBUG'):
     import sys
-    import __builtin__
-    real_import = __builtin__.__import__
+    from .five import builtins
+    real_import = builtins.__import__
 
     def debug_import(name, locals=None, globals=None, fromlist=None,
             level=-1):
@@ -36,7 +36,7 @@ if os.environ.get('C_IMPDEBUG'):
         importer_name = glob and glob.get('__name__') or 'unknown'
         print('-- {0} imports {1}'.format(importer_name, name))
         return real_import(name, locals, globals, fromlist, level)
-    __builtin__.__import__ = debug_import
+    builtins.__import__ = debug_import
 
 STATICA_HACK = True
 globals()['kcah_acitats'[::-1].upper()] = False
@@ -54,7 +54,7 @@ if STATICA_HACK:
     from celery.utils import uuid                       # noqa
 
 # Lazy loading
-from .__compat__ import recreate_module
+from .five import recreate_module
 
 old_module, new_module = recreate_module(__name__,  # pragma: no cover
     by_module={

celery/app/__init__.py

@@ -10,6 +10,8 @@ from __future__ import absolute_import
 
 import os
 
+from collections import Callable
+
 from celery.local import Proxy
 from celery import _state
 from celery._state import (  # noqa
@@ -127,6 +129,6 @@ def shared_task(*args, **kwargs):
             return Proxy(task_by_cons)
         return __inner
 
-    if len(args) == 1 and callable(args[0]):
+    if len(args) == 1 and isinstance(args[0], Callable):
         return create_shared_task(**kwargs)(args[0])
     return create_shared_task(**kwargs)

celery/app/abstract.py

@@ -9,6 +9,8 @@
 """
 from __future__ import absolute_import
 
+from celery.five import items, with_metaclass
+
 
 class from_config(object):
 
@@ -23,7 +25,7 @@ class _configurated(type):
 
     def __new__(cls, name, bases, attrs):
         attrs['__confopts__'] = dict((attr, spec.get_key(attr))
-                                          for attr, spec in attrs.iteritems()
+                                          for attr, spec in items(attrs)
                                               if isinstance(spec, from_config))
         inherit_from = attrs.get('inherit_confopts', ())
         for subcls in bases:
@@ -34,18 +36,18 @@ class _configurated(type):
         for subcls in inherit_from:
             attrs['__confopts__'].update(subcls.__confopts__)
         attrs = dict((k, v if not isinstance(v, from_config) else None)
-                        for k, v in attrs.iteritems())
+                        for k, v in items(attrs))
         return super(_configurated, cls).__new__(cls, name, bases, attrs)
 
 
+@with_metaclass(_configurated)
 class configurated(object):
-    __metaclass__ = _configurated
 
     def setup_defaults(self, kwargs, namespace='celery'):
         confopts = self.__confopts__
         app, find = self.app, self.app.conf.find_value_for_key
 
-        for attr, keyname in confopts.iteritems():
+        for attr, keyname in items(confopts):
             try:
                 value = kwargs[attr]
             except KeyError:
@@ -55,7 +57,7 @@ class configurated(object):
                     value = find(keyname, namespace)
             setattr(self, attr, value)
 
-        for attr_name, attr_value in kwargs.iteritems():
+        for attr_name, attr_value in items(kwargs):
             if attr_name not in confopts and attr_value is not None:
                 setattr(self, attr_name, attr_value)
 

celery/app/amqp.py

@@ -18,6 +18,7 @@ from kombu.utils import cached_property, uuid
 from kombu.utils.encoding import safe_repr
 
 from celery import signals
+from celery.five import items
 from celery.utils.text import indent as textindent
 
 from . import app_or_default
@@ -55,7 +56,7 @@ class Queues(dict):
         self.ha_policy = ha_policy
         if isinstance(queues, (tuple, list)):
             queues = dict((q.name, q) for q in queues)
-        for name, q in (queues or {}).iteritems():
+        for name, q in items(queues or {}):
             self.add(q) if isinstance(q, Queue) else self.add_compat(name, **q)
 
     def __getitem__(self, name):
@@ -119,7 +120,7 @@ class Queues(dict):
         if not active:
             return ''
         info = [QUEUE_FORMAT.strip().format(q)
-                    for _, q in sorted(active.iteritems())]
+                    for _, q in sorted(items(active))]
         if indent_first:
             return textindent('\n'.join(info), indent)
         return info[0] + '\n' + textindent('\n'.join(info[1:]), indent)
@@ -270,7 +271,8 @@ class TaskConsumer(Consumer):
     def __init__(self, channel, queues=None, app=None, **kw):
         self.app = app or self.app
         super(TaskConsumer, self).__init__(channel,
-                queues or self.app.amqp.queues.consume_from.values(), **kw)
+                queues or list(self.app.amqp.queues.consume_from.values()),
+                **kw)
 
 
 class AMQP(object):

celery/app/annotations.py

@@ -12,6 +12,7 @@
 """
 from __future__ import absolute_import
 
+from celery.five import string_t
 from celery.utils.functional import firstmethod, mpromise
 from celery.utils.imports import instantiate
 
@@ -44,7 +45,7 @@ def prepare(annotations):
     def expand_annotation(annotation):
         if isinstance(annotation, dict):
             return MapAnnotation(annotation)
-        elif isinstance(annotation, basestring):
+        elif isinstance(annotation, string_t):
             return mpromise(instantiate, annotation)
         return annotation
 

celery/app/base.py

@@ -11,7 +11,7 @@ from __future__ import absolute_import
 import threading
 import warnings
 
-from collections import defaultdict, deque
+from collections import Callable, defaultdict, deque
 from contextlib import contextmanager
 from copy import deepcopy
 from functools import wraps
@@ -22,10 +22,11 @@ from kombu.clocks import LamportClock
 from kombu.utils import cached_property
 
 from celery import platforms
+from celery._state import _task_stack, _tls, get_current_app, _register_app
 from celery.exceptions import AlwaysEagerIgnored
+from celery.five import items, values
 from celery.loaders import get_loader_cls
 from celery.local import PromiseProxy, maybe_evaluate
-from celery._state import _task_stack, _tls, get_current_app, _register_app
 from celery.utils.functional import first
 from celery.utils.imports import instantiate, symbol_by_name
 
@@ -153,7 +154,7 @@ class Celery(object):
 
             return _create_task_cls
 
-        if len(args) == 1 and callable(args[0]):
+        if len(args) == 1 and isinstance(args[0], Callable):
             return inner_create_task_cls(**opts)(*args)
         return inner_create_task_cls(**opts)
 
@@ -180,11 +181,11 @@ class Celery(object):
                 while pending:
                     maybe_evaluate(pending.popleft())
 
-                for task in self._tasks.itervalues():
+                for task in values(self._tasks):
                     task.bind(self)
 
     def add_defaults(self, fun):
-        if not callable(fun):
+        if not isinstance(fun, Callable):
             d, fun = fun, lambda: d
         if self.configured:
             return self.conf.add_defaults(fun())
@@ -334,7 +335,7 @@ class Celery(object):
         while pending:
             s.add_defaults(pending.popleft()())
         if self._preconf:
-            for key, value in self._preconf.iteritems():
+            for key, value in items(self._preconf):
                 setattr(s, key, value)
         return s
 

celery/app/builtins.py

@@ -10,7 +10,6 @@
 from __future__ import absolute_import
 
 from collections import deque
-from itertools import imap, izip, starmap
 
 from celery._state import get_current_worker_task
 from celery.utils import uuid
@@ -91,7 +90,7 @@ def add_map_task(app):
     @app.task(name='celery.map')
     def xmap(task, it):
         task = subtask(task).type
-        return list(imap(task, it))
+        return [task(item) for item in it]
     return xmap
 
 
@@ -102,7 +101,7 @@ def add_starmap_task(app):
     @app.task(name='celery.starmap')
     def xstarmap(task, it):
         task = subtask(task).type
-        return list(starmap(task, it))
+        return [task(*item) for item in it]
     return xstarmap
 
 
@@ -160,7 +159,7 @@ def add_group_task(app):
                 return task, AsyncResult(tid)
 
             try:
-                tasks, res = list(izip(*[prepare_member(task)
+                tasks, res = list(zip(*[prepare_member(task)
                                                 for task in tasks]))
             except ValueError:  # tasks empty
                 tasks, res = [], []
@@ -323,8 +322,8 @@ def add_chord_task(app):
                 opt_value = options.pop(opt_name, None)
                 if opt_value:
                     body.set(**{opt_name: opt_value})
-            map(body.link, options.pop('link', []))
-            map(body.link_error, options.pop('link_error', []))
+            [body.link(s) for s in options.pop('link', [])]
+            [body.link_error(s) for s in options.pop('link_error', [])]
             callback_id = body.options.setdefault('task_id', task_id or uuid())
             parent = super(Chord, self).apply_async((header, body, args),
                                                      kwargs, **options)

celery/app/defaults.py

@@ -13,6 +13,7 @@ import sys
 from collections import deque
 from datetime import timedelta
 
+from celery.five import items
 from celery.utils import strtobool
 from celery.utils.functional import memoize
 
@@ -51,7 +52,7 @@ class Option(object):
     def __init__(self, default=None, *args, **kwargs):
         self.default = default
         self.type = kwargs.get('type') or 'string'
-        for attr, value in kwargs.iteritems():
+        for attr, value in items(kwargs):
             setattr(self, attr, value)
 
     def to_python(self, value):
@@ -215,7 +216,7 @@ def flatten(d, ns=''):
     stack = deque([(ns, d)])
     while stack:
         name, space = stack.popleft()
-        for key, value in space.iteritems():
+        for key, value in items(space):
             if isinstance(value, dict):
                 stack.append((name + key + '_', value))
             else:
@@ -242,7 +243,7 @@ def find(name, namespace='celery'):
         return namespace, name.upper(), NAMESPACES[namespace][name.upper()]
     except KeyError:
         # - Try all the other namespaces.
-        for ns, keys in NAMESPACES.iteritems():
+        for ns, keys in items(NAMESPACES):
             if ns.upper() == name.upper():
                 return None, ns, keys
             elif isinstance(keys, dict):

celery/app/log.py

@@ -22,6 +22,7 @@ from kombu.log import NullHandler
 
 from celery import signals
 from celery._state import get_current_task
+from celery.five import string_t
 from celery.utils import isatty
 from celery.utils.log import (
     get_logger, mlevel,
@@ -31,7 +32,7 @@ from celery.utils.log import (
 )
 from celery.utils.term import colored
 
-is_py3k = sys.version_info[0] == 3
+PY3 = sys.version_info[0] == 3
 
 
 class TaskFormatter(ColorFormatter):
@@ -85,7 +86,7 @@ class Logging(object):
         format = format or self.format
         colorize = self.supports_color(colorize, logfile)
         reset_multiprocessing_logger()
-        if not is_py3k:
+        if not PY3:
             ensure_process_aware_logger()
         receivers = signals.setup_logging.send(sender=None,
                         loglevel=loglevel, logfile=logfile,
@@ -110,7 +111,7 @@ class Logging(object):
 
         # This is a hack for multiprocessing's fork+exec, so that
         # logging before Process.run works.
-        logfile_name = logfile if isinstance(logfile, basestring) else ''
+        logfile_name = logfile if isinstance(logfile, string_t) else ''
         os.environ.update(_MP_FORK_LOGLEVEL_=str(loglevel),
                           _MP_FORK_LOGFILE_=logfile_name,
                           _MP_FORK_LOGFORMAT_=format)

celery/app/registry.py

@@ -14,6 +14,7 @@ from importlib import import_module
 
 from celery._state import get_current_app
 from celery.exceptions import NotRegistered
+from celery.five import items
 
 
 class TaskRegistry(dict):
@@ -54,7 +55,7 @@ class TaskRegistry(dict):
         return self.filter_types('periodic')
 
     def filter_types(self, type):
-        return dict((name, task) for name, task in self.iteritems()
+        return dict((name, task) for name, task in items(self)
                                 if getattr(task, 'type', 'regular') == type)
 
 

celery/app/routes.py

@@ -10,6 +10,7 @@
 from __future__ import absolute_import
 
 from celery.exceptions import QueueNotFound
+from celery.five import string_t
 from celery.utils import lpmerge
 from celery.utils.functional import firstmethod, mpromise
 from celery.utils.imports import instantiate
@@ -51,7 +52,7 @@ class Router(object):
 
     def expand_destination(self, route):
         # Route can be a queue name: convenient for direct exchanges.
-        if isinstance(route, basestring):
+        if isinstance(route, string_t):
             queue, route = route, {}
         else:
             # can use defaults from configured queue, but override specific
@@ -84,7 +85,7 @@ def prepare(routes):
     def expand_route(route):
         if isinstance(route, dict):
             return MapRoute(route)
-        if isinstance(route, basestring):
+        if isinstance(route, string_t):
             return mpromise(instantiate, route)
         return route
 

celery/app/task.py

@@ -12,10 +12,10 @@ import sys
 
 from celery import current_app
 from celery import states
-from celery.__compat__ import class_property
 from celery._state import get_current_worker_task, _task_stack
 from celery.datastructures import ExceptionInfo
 from celery.exceptions import MaxRetriesExceededError, RetryTaskError
+from celery.five import class_property, items, with_metaclass
 from celery.result import EagerResult
 from celery.utils import gen_task_name, fun_takes_kwargs, uuid, maybe_reraise
 from celery.utils.functional import mattrgetter, maybe_list
@@ -134,6 +134,7 @@ class TaskType(type):
         return '<unbound {0.__name__}>'.format(cls)
 
 
+@with_metaclass(TaskType)
 class Task(object):
     """Task base class.
 
@@ -142,7 +143,6 @@ class Task(object):
     is overridden).
 
     """
-    __metaclass__ = TaskType
     __trace__ = None
     __v2_compat__ = False  # set by old base in celery.task.base
 
@@ -306,7 +306,7 @@ class Task(object):
     @classmethod
     def annotate(self):
         for d in resolve_all_annotations(self.app.annotations, self):
-            for key, value in d.iteritems():
+            for key, value in items(d):
                 if key.startswith('@'):
                     self.add_around(key[1:], value)
                 else:
@@ -631,7 +631,7 @@ class Task(object):
                               'delivery_info': {'is_eager': True}}
             supported_keys = fun_takes_kwargs(task.run, default_kwargs)
             extend_with = dict((key, val)
-                                    for key, val in default_kwargs.items()
+                                    for key, val in items(default_kwargs)
                                         if key in supported_keys)
             kwargs.update(extend_with)
 

celery/app/utils.py

@@ -13,6 +13,7 @@ import platform as _platform
 
 from celery import datastructures
 from celery import platforms
+from celery.five import items
 from celery.utils.text import pretty
 from celery.utils.imports import qualname
 
@@ -96,7 +97,7 @@ class Settings(datastructures.ConfigurationView):
         """Returns a human readable string showing changes to the
         configuration."""
         return '\n'.join('{0}: {1}'.format(key, pretty(value, width=50))
-                        for key, value in self.without_defaults().iteritems())
+                        for key, value in items(self.without_defaults()))
 
 
 class AppPickler(object):

celery/apps/worker.py

@@ -24,6 +24,7 @@ from billiard import current_process
 from celery import VERSION_BANNER, platforms, signals
 from celery.app.abstract import from_config
 from celery.exceptions import SystemTerminate
+from celery.five import string, string_t
 from celery.loaders.app import AppLoader
 from celery.task import trace
 from celery.utils import cry, isatty
@@ -161,7 +162,7 @@ class Worker(WorkController):
 
     def startup_info(self):
         app = self.app
-        concurrency = unicode(self.concurrency)
+        concurrency = string(self.concurrency)
         appr = '{0}:0x{1:x}'.format(app.main or '__main__', id(app))
         if not isinstance(app.loader, AppLoader):
             loader = qualname(app.loader)
@@ -172,7 +173,7 @@ class Worker(WorkController):
             max, min = self.autoscale
             concurrency = '{{min={0}, max={1}}}'.format(min, max)
         pool = self.pool_cls
-        if not isinstance(pool, basestring):
+        if not isinstance(pool, string_t):
             pool = pool.__module__
         concurrency += ' ({0})'.format(pool.split('.')[-1])
         events = 'ON'

celery/backends/__init__.py

@@ -14,6 +14,7 @@ from kombu.utils.url import _parse_url
 
 from celery.local import Proxy
 from celery._state import current_app
+from celery.five import reraise
 from celery.utils.imports import symbol_by_name
 from celery.utils.functional import memoize
 
@@ -45,8 +46,8 @@ def get_backend_cls(backend=None, loader=None):
     try:
         return symbol_by_name(backend, aliases)
     except ValueError as exc:
-        raise ValueError, ValueError(UNKNOWN_BACKEND.format(
-                    backend, exc)), sys.exc_info()[2]
+        reraise(ValueError, ValueError(UNKNOWN_BACKEND.format(
+                    backend, exc)), sys.exc_info()[2])
 
 
 def get_backend_by_url(backend=None, loader=None):

celery/backends/amqp.py

@@ -21,6 +21,7 @@ from kombu.messaging import Consumer, Producer
 
 from celery import states
 from celery.exceptions import TimeoutError
+from celery.five import range
 from celery.utils.log import get_logger
 
 from .base import BaseBackend
@@ -149,7 +150,7 @@ class AMQPBackend(BaseBackend):
             binding = self._create_binding(task_id)(channel)
             binding.declare()
             latest, acc = None, None
-            for i in xrange(backlog_limit):
+            for i in range(backlog_limit):
                 latest, acc = acc, binding.get(no_ack=True)
                 if not acc:  # no more messages
                     break

celery/backends/base.py

@@ -17,7 +17,6 @@ import time
 import sys
 
 from datetime import timedelta
-from itertools import imap
 
 from kombu import serialization
 from kombu.utils.encoding import bytes_to_str, ensure_bytes, from_utf8
@@ -26,6 +25,7 @@ from celery import states
 from celery.app import current_task
 from celery.datastructures import LRUCache
 from celery.exceptions import TimeoutError, TaskRevokedError
+from celery.five import items
 from celery.result import from_serializable, GroupResult
 from celery.utils import timeutils
 from celery.utils.serialization import (
@@ -35,7 +35,7 @@ from celery.utils.serialization import (
 )
 
 EXCEPTION_ABLE_CODECS = frozenset(['pickle', 'yaml'])
-is_py3k = sys.version_info >= (3, 0)
+PY3 = sys.version_info >= (3, 0)
 
 
 def unpickle_backend(cls, args, kwargs):
@@ -116,7 +116,7 @@ class BaseBackend(object):
         return payload
 
     def decode(self, payload):
-        payload = is_py3k and payload or str(payload)
+        payload = PY3 and payload or str(payload)
         return serialization.decode(payload,
                                     content_type=self.content_type,
                                     content_encoding=self.content_encoding)
@@ -326,7 +326,7 @@ class KeyValueStoreBackend(BaseBackend):
         if hasattr(values, 'items'):
             # client returns dict so mapping preserved.
             return dict((self._strip_prefix(k), self.decode(v))
-                            for k, v in values.iteritems()
+                            for k, v in items(values)
                                 if v is not None)
         else:
             # client returns list so need to recreate mapping.
@@ -354,8 +354,8 @@ class KeyValueStoreBackend(BaseBackend):
             r = self._mget_to_results(self.mget([self.get_key_for_task(k)
                                                     for k in keys]), keys)
             self._cache.update(r)
-            ids.difference_update(set(imap(bytes_to_str, r)))
-            for key, value in r.iteritems():
+            ids.difference_update(set(map(bytes_to_str, r)))
+            for key, value in items(r):
                 yield bytes_to_str(key), value
             if timeout and iterations * interval >= timeout:
                 raise TimeoutError('Operation timed out ({0})'.format(timeout))

celery/backends/cassandra.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# -* coding: utf-8 -*-
 """
     celery.backends.cassandra
     ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -108,7 +108,7 @@ class CassandraBackend(BaseBackend):
                     pycassa.UnavailableException,
                     socket.error,
                     socket.timeout,
-                    Thrift.TException), exc:
+                    Thrift.TException) as exc:
                 if time.time() > ts:
                     raise
                 logger.warn('Cassandra error: %r. Retrying...', exc)
@@ -158,7 +158,7 @@ class CassandraBackend(BaseBackend):
             try:
                 if self.detailed_mode:
                     row = cf.get(task_id, column_reversed=True, column_count=1)
-                    meta = self.decode(row.values()[0])
+                    meta = self.decode(list(row.values())[0])
                     meta['task_id'] = task_id
                 else:
                     obj = cf.get(task_id)

celery/backends/database/__init__.py

@@ -12,6 +12,7 @@ from functools import wraps
 
 from celery import states
 from celery.exceptions import ImproperlyConfigured
+from celery.five import range
 from celery.utils.timeutils import maybe_timedelta
 
 from celery.backends.base import BaseBackend
@@ -39,7 +40,7 @@ def retry(fun):
     def _inner(*args, **kwargs):
         max_retries = kwargs.pop('max_retries', 3)
 
-        for retries in xrange(max_retries + 1):
+        for retries in range(max_retries + 1):
             try:
                 return fun(*args, **kwargs)
             except (DatabaseError, OperationalError):

celery/backends/mongodb.py

@@ -27,6 +27,7 @@ from kombu.utils import cached_property
 
 from celery import states
 from celery.exceptions import ImproperlyConfigured
+from celery.five import string_t
 from celery.utils.timeutils import maybe_timedelta
 
 from .base import BaseBackend
@@ -92,7 +93,7 @@ class MongoBackend(BaseBackend):
             # This enables the use of replica sets and sharding.
             # See pymongo.Connection() for more info.
             args = [self.mongodb_host]
-            if isinstance(self.mongodb_host, basestring) \
+            if isinstance(self.mongodb_host, string_t) \
                     and not self.mongodb_host.startswith('mongodb://'):
                 args.append(self.mongodb_port)
 

celery/beat.py

@@ -26,6 +26,7 @@ from . import platforms
 from . import signals
 from . import current_app
 from .app import app_or_default
+from .five import items, reraise, values
 from .schedules import maybe_schedule, crontab
 from .utils.imports import instantiate
 from .utils.timeutils import humanize_seconds
@@ -116,7 +117,7 @@ class ScheduleEntry(object):
         return self.schedule.is_due(self.last_run_at)
 
     def __iter__(self):
-        return vars(self).iteritems()
+        return iter(items(vars(self)))
 
     def __repr__(self):
         return '<Entry: {0.name} {call} {0.schedule}'.format(self,
@@ -189,7 +190,7 @@ class Scheduler(object):
         """
         remaining_times = []
         try:
-            for entry in self.schedule.itervalues():
+            for entry in values(self.schedule):
                 next_time_to_run = self.maybe_due(entry, self.publisher)
                 if next_time_to_run:
                     remaining_times.append(next_time_to_run)
@@ -223,9 +224,9 @@ class Scheduler(object):
                                         publisher=publisher,
                                         **entry.options)
         except Exception as exc:
-            raise SchedulingError, SchedulingError(
+            reraise(SchedulingError, SchedulingError(
                 "Couldn't apply scheduled task {0.name}: {exc}".format(
-                    entry, exc)), sys.exc_info()[2]
+                    entry, exc)), sys.exc_info()[2])
         finally:
             if self.should_sync():
                 self._do_sync()
@@ -262,7 +263,7 @@ class Scheduler(object):
 
     def update_from_dict(self, dict_):
         self.schedule.update(dict((name, self._maybe_entry(name, entry))
-                                for name, entry in dict_.items()))
+                                for name, entry in items(dict_)))
 
     def merge_inplace(self, b):
         schedule = self.schedule
@@ -365,7 +366,7 @@ class PersistentScheduler(Scheduler):
         self._store.update(__version__=__version__, tz=tz, utc_enabled=utc)
         self.sync()
         debug('Current schedule:\n' + '\n'.join(repr(entry)
-                                    for entry in entries.itervalues()))
+                                    for entry in values(entries)))
 
     def get_schedule(self):
         return self._store['entries']

celery/bin/base.py

@@ -65,12 +65,12 @@ import sys
 import warnings
 
 from collections import defaultdict
-from itertools import izip
 from optparse import OptionParser, IndentedHelpFormatter, make_option as Option
 from types import ModuleType
 
 import celery
 from celery.exceptions import CDeprecationWarning, CPendingDeprecationWarning
+from celery.five import items, string_t
 from celery.platforms import EX_FAILURE, EX_USAGE, maybe_patch_concurrency
 from celery.utils import text
 from celery.utils.imports import symbol_by_name, import_from_cwd
@@ -203,7 +203,7 @@ class Command(object):
         return self.option_list
 
     def expanduser(self, value):
-        if isinstance(value, basestring):
+        if isinstance(value, string_t):
             return os.path.expanduser(value)
         return value
 
@@ -224,7 +224,7 @@ class Command(object):
     def prepare_args(self, options, args):
         if options:
             options = dict((k, self.expanduser(v))
-                            for k, v in vars(options).iteritems()
+                            for k, v in items(vars(options))
                                 if not k.startswith('_'))
         args = [self.expanduser(arg) for arg in args]
         self.check_args(args)
@@ -263,7 +263,7 @@ class Command(object):
     def prepare_parser(self, parser):
         docs = [self.parse_doc(doc) for doc in (self.doc, __doc__) if doc]
         for doc in docs:
-            for long_opt, help in doc.iteritems():
+            for long_opt, help in items(doc):
                 option = parser.get_option(long_opt)
                 if option is not None:
                     option.help = ' '.join(help).format(default=option.default)
@@ -338,7 +338,7 @@ class Command(object):
         opts = {}
         for opt in self.preload_options:
             for t in (opt._long_opts, opt._short_opts):
-                opts.update(dict(izip(t, [opt.dest] * len(t))))
+                opts.update(dict(zip(t, [opt.dest] * len(t))))
         index = 0
         length = len(args)
         while index < length:

celery/bin/camqadm.py

@@ -12,15 +12,17 @@ import sys
 import shlex
 import pprint
 
+from collections import Callable
 from functools import partial
 from itertools import count
 
-from amqplib import client_0_8 as amqp
+from amqp import Message
 
 from celery.app import app_or_default
 from celery.utils.functional import padlist
 
 from celery.bin.base import Command
+from celery.five import string_t
 from celery.utils import strtobool
 
 # Map to coerce strings to other types.
@@ -97,7 +99,7 @@ class Spec(object):
             if response is None:
                 return 'ok.'
             return response
-        if callable(self.returns):
+        if isinstance(self.returns, Callable):
             return self.returns(response)
         return self.returns.format(response)
 
@@ -148,7 +150,7 @@ class AMQShell(cmd.Cmd):
     identchars = cmd.IDENTCHARS = '.'
     needs_reconnect = False
     counter = 1
-    inc_counter = count(2).next
+    inc_counter = count(2)
 
     builtins = {'EOF': 'do_exit',
                 'exit': 'do_exit',
@@ -181,7 +183,7 @@ class AMQShell(cmd.Cmd):
         'basic.get': Spec(('queue', str),
                           ('no_ack', bool, 'off'),
                           returns=dump_message),
-        'basic.publish': Spec(('msg', amqp.Message),
+        'basic.publish': Spec(('msg', Message),
                               ('exchange', str),
                               ('routing_key', str),
                               ('mandatory', bool, 'no'),
@@ -215,7 +217,7 @@ class AMQShell(cmd.Cmd):
 
             >>> get_amqp_api_command('queue.delete', ['pobox', 'yes', 'no'])
             (<bound method Channel.queue_delete of
-             <amqplib.client_0_8.channel.Channel object at 0x...>>,
+             <amqp.channel.Channel object at 0x...>>,
              ('testfoo', True, False))
 
         """
@@ -300,7 +302,7 @@ class AMQShell(cmd.Cmd):
         if cmd == '':
             return self.default(line)
         else:
-            self.counter = self.inc_counter()
+            self.counter = next(self.inc_counter)
             try:
                 self.respond(self.dispatch(cmd, arg))
             except (AttributeError, KeyError) as exc:
@@ -312,7 +314,7 @@ class AMQShell(cmd.Cmd):
     def respond(self, retval):
         """What to do with the return value of a command."""
         if retval is not None:
-            if isinstance(retval, basestring):
+            if isinstance(retval, string_t):
                 self.say(retval)
             else:
                 self.say(pprint.pformat(retval))

celery/bin/celery.py

@@ -19,6 +19,7 @@ from operator import itemgetter
 from pprint import pformat
 
 from celery.datastructures import DependencyGraph, GraphFormatter
+from celery.five import items, string, string_t, values
 from celery.platforms import EX_OK, EX_FAILURE, EX_UNAVAILABLE, EX_USAGE
 from celery.utils import term
 from celery.utils import text
@@ -200,7 +201,7 @@ class Command(BaseCommand):
 
     def say_remote_command_reply(self, replies):
         c = self.colored
-        node = iter(replies).next()  # <-- take first.
+        node = next(iter(replies))  # <-- take first.
         reply = replies[node]
         status, preply = self.prettify(reply)
         self.say_chat('->', c.cyan(node, ': ') + status,
@@ -213,8 +214,8 @@ class Command(BaseCommand):
         if isinstance(n, dict):
             if 'ok' in n or 'error' in n:
                 return self.prettify_dict_ok_error(n)
-        if isinstance(n, basestring):
-            return OK, unicode(n)
+        if isinstance(n, string_t):
+            return OK, string(n)
         return OK, pformat(n)
 
     def say_chat(self, direction, title, body=''):
@@ -406,12 +407,12 @@ class call(Command):
     def run(self, name, *_, **kw):
         # Positional args.
         args = kw.get('args') or ()
-        if isinstance(args, basestring):
+        if isinstance(args, string_t):
             args = anyjson.loads(args)
 
         # Keyword args.
         kwargs = kw.get('kwargs') or {}
-        if isinstance(kwargs, basestring):
+        if isinstance(kwargs, string_t):
             kwargs = anyjson.loads(kwargs)
 
         # Expires can be int/float.
@@ -554,7 +555,7 @@ class _RemoteControl(Command):
 
         destination = kwargs.get('destination')
         timeout = kwargs.get('timeout') or self.choices[method][0]
-        if destination and isinstance(destination, basestring):
+        if destination and isinstance(destination, string_t):
             destination = [dest.strip() for dest in destination.split(',')]
 
         try:
@@ -807,7 +808,7 @@ class shell(Command):  # pragma: no cover
 
         if not without_tasks:
             self.locals.update(dict((task.__name__, task)
-                                for task in self.app.tasks.itervalues()
+                                for task in values(self.app.tasks)
                                     if not task.name.startswith('celery.')))
 
         if force_python:
@@ -999,7 +1000,7 @@ class graph(Command):
         def maybe_list(l, sep=','):
             return (l[0], l[1].split(sep) if sep in l[1] else l[1])
 
-        args = dict(map(simplearg, args))
+        args = dict(simplearg(arg) for arg in args)
         generic = 'generic' in args
 
         def generic_label(node):
@@ -1099,7 +1100,7 @@ class graph(Command):
         except KeyError:
             replies = self.app.control.inspect().stats()
             workers, threads = [], []
-            for worker, reply in replies.iteritems():
+            for worker, reply in items(replies):
                 workers.append(worker)
                 threads.append(reply['pool']['max-concurrency'])
 

celery/bin/celeryd.py

@@ -121,6 +121,7 @@ import sys
 from celery import concurrency
 from celery.bin.base import Command, Option, daemon_options
 from celery.bin.celeryd_detach import detached_celeryd
+from celery.five import string_t
 from celery.utils.log import LOG_LEVELS, mlevel
 
 
@@ -158,7 +159,7 @@ class WorkerCommand(Command):
             except KeyError:  # pragma: no cover
                 self.die('Unknown level {0!r}. Please use one of {1}.'.format(
                     loglevel, '|'.join(l for l in LOG_LEVELS
-                      if isinstance(l, basestring))))
+                      if isinstance(l, string_t))))
         return self.app.Worker(
             hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, **kwargs
         ).start()

celery/bin/celeryd_multi.py

@@ -98,7 +98,6 @@ import socket
 import sys
 
 from collections import defaultdict
-from itertools import imap
 from subprocess import Popen
 from time import sleep
 
@@ -106,6 +105,7 @@ from kombu.utils import cached_property
 from kombu.utils.encoding import from_utf8
 
 from celery import VERSION_BANNER
+from celery.five import items
 from celery.platforms import Pidfile, IS_WINDOWS
 from celery.utils import term
 from celery.utils.text import pluralize
@@ -428,7 +428,7 @@ def multi_args(p, cmd='celeryd', append='', prefix='', suffix=''):
         except ValueError:
             pass
         else:
-            names = list(imap(str, range(1, noderange + 1)))
+            names = list(map(str, range(1, noderange + 1)))
             prefix = 'celery'
     cmd = options.pop('--cmd', cmd)
     append = options.pop('--append', append)
@@ -439,7 +439,7 @@ def multi_args(p, cmd='celeryd', append='', prefix='', suffix=''):
     if suffix in ('""', "''"):
         suffix = ''
 
-    for ns_name, ns_opts in p.namespaces.items():
+    for ns_name, ns_opts in list(items(p.namespaces)):
         if ',' in ns_name or (ranges and '-' in ns_name):
             for subns in parse_ns_range(ns_name, ranges):
                 p.namespaces[subns].update(ns_opts)
@@ -451,7 +451,7 @@ def multi_args(p, cmd='celeryd', append='', prefix='', suffix=''):
                                 '%n': name})
         argv = ([expand(cmd)] +
                 [format_opt(opt, expand(value))
-                        for opt, value in p.optmerge(name, options).items()] +
+                        for opt, value in items(p.optmerge(name, options))] +
                 [passthrough])
         if append:
             argv.append(expand(append))
@@ -529,7 +529,7 @@ def parse_ns_range(ns, ranges=False):
     for space in ',' in ns and ns.split(',') or [ns]:
         if ranges and '-' in space:
             start, stop = space.split('-')
-            x = list(imap(str, range(int(start), int(stop) + 1)))
+            x = list(map(str, range(int(start), int(stop) + 1)))
             ret.extend(x)
         else:
             ret.append(space)
@@ -541,8 +541,8 @@ def abbreviations(mapping):
     def expand(S):
         ret = S
         if S is not None:
-            for short, long in mapping.items():
-                ret = ret.replace(short, long)
+            for short_opt, long_opt in items(mapping):
+                ret = ret.replace(short_opt, long_opt)
         return ret
 
     return expand

celery/bootsteps.py

@@ -16,6 +16,7 @@ from kombu.common import ignore_errors
 from kombu.utils import symbol_by_name
 
 from .datastructures import DependencyGraph, GraphFormatter
+from .five import values, with_metaclass
 from .utils.imports import instantiate, qualname
 from .utils.log import get_logger
 from .utils.threads import default_socket_timeout
@@ -198,12 +199,12 @@ class Namespace(object):
         return self.steps[name]
 
     def _find_last(self):
-        for C in self.steps.itervalues():
+        for C in values(self.steps):
             if C.last:
                 return C
 
     def _firstpass(self, steps):
-        stream = deque(step.requires for step in steps.itervalues())
+        stream = deque(step.requires for step in values(steps))
         while stream:
             for node in stream.popleft():
                 node = symbol_by_name(node)
@@ -214,7 +215,7 @@ class Namespace(object):
     def _finalize_steps(self, steps):
         last = self._find_last()
         self._firstpass(steps)
-        it = ((C, C.requires) for C in steps.itervalues())
+        it = ((C, C.requires) for C in values(steps))
         G = self.graph = DependencyGraph(it,
             formatter=self.GraphFormatter(root=last),
         )
@@ -265,6 +266,7 @@ class StepType(type):
         return 'step:{0.name}{{{0.requires!r}}}'.format(self)
 
 
+@with_metaclass(StepType)
 class Step(object):
     """A Bootstep.
 
@@ -274,7 +276,6 @@ class Step(object):
     parent instantiation-time.
 
     """
-    __metaclass__ = StepType
 
     #: Optional step name, will use qualname if not specified.
     name = None

celery/canvas.py

@@ -13,7 +13,7 @@ from __future__ import absolute_import
 
 from copy import deepcopy
 from operator import itemgetter
-from itertools import chain as _chain, imap
+from itertools import chain as _chain
 
 from kombu.utils import cached_property, fxrange, kwdict, reprcall, uuid
 
@@ -230,7 +230,7 @@ class chain(Signature):
         return chain(*d['kwargs']['tasks'], **kwdict(d['options']))
 
     def __repr__(self):
-        return ' | '.join(imap(repr, self.tasks))
+        return ' | '.join(map(repr, self.tasks))
 Signature.register_type(chain)
 
 
@@ -329,9 +329,9 @@ class group(Signature):
         return self.type(tasks, result, gid, args)
 
     def skew(self, start=1.0, stop=None, step=1.0):
-        _next_skew = fxrange(start, stop, step, repeatlast=True).next
+        it = fxrange(start, stop, step, repeatlast=True)
         for task in self.tasks:
-            task.set(countdown=_next_skew())
+            task.set(countdown=next(it))
         return self
 
     def __iter__(self):

celery/concurrency/processes.py

@@ -20,6 +20,7 @@ from celery import platforms
 from celery import signals
 from celery._state import set_default_app
 from celery.concurrency.base import BasePool
+from celery.five import items
 from celery.task import trace
 
 #: List of signals to reset when a child process starts.
@@ -53,7 +54,7 @@ def process_initializer(app, hostname):
     app.finalize()
 
     from celery.task.trace import build_tracer
-    for name, task in app.tasks.iteritems():
+    for name, task in items(app.tasks):
         task.__trace__ = build_tracer(name, task, app.loader, hostname)
     signals.worker_process_init.send(sender=None)
 
@@ -121,7 +122,7 @@ class TaskPool(BasePool):
                 'timeouts': (self._pool.soft_timeout, self._pool.timeout)}
 
     def init_callbacks(self, **kwargs):
-        for k, v in kwargs.iteritems():
+        for k, v in items(kwargs):
             setattr(self._pool, k, v)
 
     def handle_timeouts(self):

celery/concurrency/threads.py

@@ -8,7 +8,7 @@
 """
 from __future__ import absolute_import
 
-from celery.utils.compat import UserDict
+from celery.five import UserDict
 
 from .base import apply_target, BasePool
 

celery/contrib/abortable.py

@@ -37,7 +37,7 @@ In the consumer:
 
        def run(self, **kwargs):
            results = []
-           for x in xrange(100):
+           for x in range(100):
                # Check after every 5 loops..
                if x % 5 == 0:  # alternatively, check when some timer is due
                    if self.is_aborted(**kwargs):

celery/contrib/batches.py

@@ -40,9 +40,9 @@ Registering the click is done as follows:
 from __future__ import absolute_import
 
 from itertools import count
-from Queue import Empty, Queue
 
 from celery.task import Task
+from celery.five import Empty, Queue
 from celery.utils.log import get_logger
 from celery.worker.job import Request
 
@@ -132,7 +132,7 @@ class Batches(Task):
 
     def __init__(self):
         self._buffer = Queue()
-        self._count = count(1).next
+        self._count = count(1)
         self._tref = None
         self._pool = None
 
@@ -160,7 +160,7 @@ class Batches(Task):
                 self._tref = timer.apply_interval(self.flush_interval * 1000.0,
                                                   flush_buffer)
 
-            if not self._count() % self.flush_every:
+            if not next(self._count) % self.flush_every:
                 flush_buffer()
 
         return task_message_handler

celery/contrib/migrate.py

@@ -6,7 +6,7 @@
     Migration tools.
 
 """
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import, print_function, unicode_literals
 
 import socket
 
@@ -19,6 +19,7 @@ from kombu.exceptions import StdChannelError
 from kombu.utils.encoding import ensure_bytes
 
 from celery.app import app_or_default
+from celery.five import string, string_t
 from celery.utils import worker_direct
 
 
@@ -40,8 +41,8 @@ class State(object):
     @property
     def strtotal(self):
         if not self.total_apx:
-            return u'?'
-        return unicode(self.total_apx)
+            return '?'
+        return string(self.total_apx)
 
     def __repr__(self):
         if self.filtered:
@@ -113,7 +114,7 @@ def migrate_tasks(source, dest, migrate=migrate_task, app=None,
 
 
 def _maybe_queue(app, q):
-    if isinstance(q, basestring):
+    if isinstance(q, string_t):
         return app.amqp.queues[q]
     return q
 
@@ -166,7 +167,7 @@ def move(predicate, connection=None, exchange=None, routing_key=None,
     .. code-block:: python
 
         def transform(value):
-            if isinstance(value, basestring):
+            if isinstance(value, string_t):
                 return Queue(value, Exchange(value), value)
             return value
 
@@ -225,7 +226,7 @@ def task_id_in(ids, body, message):
 
 
 def prepare_queues(queues):
-    if isinstance(queues, basestring):
+    if isinstance(queues, string_t):
         queues = queues.split(',')
     if isinstance(queues, list):
         queues = dict(tuple(islice(cycle(q.split(':')), None, 2))
@@ -241,7 +242,7 @@ def start_filter(app, conn, filter, limit=None, timeout=1.0,
         consume_from=None, state=None, **kwargs):
     state = state or State()
     queues = prepare_queues(queues)
-    if isinstance(tasks, basestring):
+    if isinstance(tasks, string_t):
         tasks = set(tasks.split(','))
     if tasks is None:
         tasks = set([])

celery/contrib/rdb.py

@@ -41,11 +41,11 @@ import os
 import socket
 import sys
 
-from itertools import imap
 from pdb import Pdb
 
 from billiard import current_process
 
+from celery.five import range
 from celery.platforms import ignore_errno
 
 default_port = 6899
@@ -97,7 +97,7 @@ class Rdb(Pdb):
         self.say(BANNER.format(self=self))
 
         self._client, address = self._sock.accept()
-        self.remote_addr = ':'.join(imap(str, address))
+        self.remote_addr = ':'.join(map(str, address))
         self.say(SESSION_STARTED.format(self=self))
         self._handle = sys.stdin = sys.stdout = self._client.makefile('rw')
         Pdb.__init__(self, completekey='tab',
@@ -110,7 +110,7 @@ class Rdb(Pdb):
         except ValueError:
             pass
         this_port = None
-        for i in xrange(search_limit):
+        for i in range(search_limit):
             _sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             this_port = port + skew + i
             try:

celery/datastructures.py

@@ -14,11 +14,13 @@ import time
 from collections import defaultdict
 from functools import partial
 from itertools import chain
+from operator import itemgetter
 
 from billiard.einfo import ExceptionInfo  # noqa
 from kombu.utils.encoding import safe_str
 from kombu.utils.limits import TokenBucket  # noqa
 
+from .five import items
 from .utils.functional import LRUCache, first, uniq  # noqa
 
 DOT_HEAD = """
@@ -75,7 +77,7 @@ class GraphFormatter(object):
     def attrs(self, d, scheme=None):
         d = dict(self.scheme, **dict(scheme, **d or {}) if scheme else d)
         return self._attrsep.join(
-            safe_str(self.attr(k, v)) for k, v in d.iteritems()
+            safe_str(self.attr(k, v)) for k, v in items(d)
         )
 
     def head(self, **attrs):
@@ -152,7 +154,7 @@ class DependencyGraph(object):
         self[A].append(B)
 
     def find_last(self, g):
-        for obj in g.adjacent.keys():
+        for obj in g.adjacent:
             if obj.last:
                 return obj
 
@@ -205,7 +207,7 @@ class DependencyGraph(object):
 
     def edges(self):
         """Returns generator that yields for all edges in the graph."""
-        return (obj for obj, adj in self.iteritems() if adj)
+        return (obj for obj, adj in items(self) if adj)
 
     def _khan62(self):
         """Khans simple topological sort algorithm from '62
@@ -280,7 +282,7 @@ class DependencyGraph(object):
                 seen.add(draw.label(obj))
 
         P(draw.head())
-        for obj, adjacent in self.iteritems():
+        for obj, adjacent in items(self):
             if not adjacent:
                 if_not_seen(draw.terminal_node, obj)
             for req in adjacent:
@@ -304,7 +306,7 @@ class DependencyGraph(object):
         return obj in self.adjacent
 
     def _iterate_items(self):
-        return self.adjacent.iteritems()
+        return items(self.adjacent)
     items = iteritems = _iterate_items
 
     def __repr__(self):
@@ -471,7 +473,7 @@ class ConfigurationView(AttributeDictMixin):
         return False
 
     def __repr__(self):
-        return repr(dict(self.iteritems()))
+        return repr(dict(items(self)))
 
     def __iter__(self):
         return self._iterate_keys()
@@ -571,7 +573,7 @@ class LimitedSet(object):
 
     @property
     def chronologically(self):
-        return sorted(self._data.items(), key=lambda (value, when): when)
+        return sorted(self._data.items(), key=itemgetter(1))
 
     @property
     def first(self):

celery/events/cursesmon.py

@@ -21,6 +21,7 @@ from math import ceil
 from celery import VERSION_BANNER
 from celery import states
 from celery.app import app_or_default
+from celery.five import items, values
 from celery.utils.text import abbr, abbrtask
 
 BORDER_SPACING = 4
@@ -160,11 +161,12 @@ class CursesMonitor(object):  # pragma: no cover
     def alert(self, callback, title=None):
         self.win.erase()
         my, mx = self.win.getmaxyx()
-        y = blank_line = count(2).next
+        y = blank_line = count(2)
         if title:
-            self.win.addstr(y(), 3, title, curses.A_BOLD | curses.A_UNDERLINE)
-            blank_line()
-        callback(my, mx, y())
+            self.win.addstr(next(y), 3, title,
+                            curses.A_BOLD | curses.A_UNDERLINE)
+            next(blank_line)
+        callback(my, mx, next(y))
         self.win.addstr(my - 1, 0, 'Press any key to continue...',
                         curses.A_BOLD)
         self.win.refresh()
@@ -195,16 +197,17 @@ class CursesMonitor(object):  # pragma: no cover
     def alert_remote_control_reply(self, reply):
 
         def callback(my, mx, xs):
-            y = count(xs).next
+            y = count(xs)
             if not reply:
-                self.win.addstr(y(), 3, 'No replies received in 1s deadline.',
+                self.win.addstr(next(y), 3,
+                        'No replies received in 1s deadline.',
                         curses.A_BOLD + curses.color_pair(2))
                 return
 
             for subreply in reply:
-                curline = y()
+                curline = next(y)
 
-                host, response = subreply.items()[0]
+                host, response = next(items(subreply))
                 host = '{0}: '.format(host)
                 self.win.addstr(curline, 3, host, curses.A_BOLD)
                 attr = curses.A_NORMAL
@@ -250,16 +253,17 @@ class CursesMonitor(object):  # pragma: no cover
 
         def alert_callback(mx, my, xs):
             my, mx = self.win.getmaxyx()
-            y = count(xs).next
+            y = count(xs)
             task = self.state.tasks[self.selected_task]
             info = task.info(extra=['state'])
             infoitems = [('args', info.pop('args', None)),
-                         ('kwargs', info.pop('kwargs', None))] + info.items()
+                         ('kwargs', info.pop('kwargs', None))
+                        ] + list(info.items())
             for key, value in infoitems:
                 if key is None:
                     continue
                 value = str(value)
-                curline = y()
+                curline = next(y)
                 keys = key + ': '
                 self.win.addstr(curline, 3, keys, curses.A_BOLD)
                 wrapped = wrap(value, mx - 2)
@@ -269,7 +273,7 @@ class CursesMonitor(object):  # pragma: no cover
                                  self.screen_width - (len(keys) + 3)))
                 else:
                     for subline in wrapped:
-                        nexty = y()
+                        nexty = next(y)
                         if nexty >= my - 1:
                             subline = ' ' * 4 + '[...]'
                         elif nexty >= my:
@@ -289,9 +293,9 @@ class CursesMonitor(object):  # pragma: no cover
             return curses.beep()
 
         def alert_callback(my, mx, xs):
-            y = count(xs).next
+            y = count(xs)
             for line in task.traceback.split('\n'):
-                self.win.addstr(y(), 3, line)
+                self.win.addstr(next(y), 3, line)
 
         return self.alert(alert_callback,
                 'Task Exception Traceback for {0.selected_task}'.format(self))
@@ -301,12 +305,12 @@ class CursesMonitor(object):  # pragma: no cover
             return
 
         def alert_callback(my, mx, xs):
-            y = count(xs).next
+            y = count(xs)
             task = self.state.tasks[self.selected_task]
             result = getattr(task, 'result', None) or getattr(task,
                     'exception', None)
             for line in wrap(result, mx - 2):
-                self.win.addstr(y(), 3, line)
+                self.win.addstr(next(y), 3, line)
 
         return self.alert(alert_callback,
                 'Task Result for {0.selected_task}'.format(self))
@@ -334,14 +338,14 @@ class CursesMonitor(object):  # pragma: no cover
         win = self.win
         self.handle_keypress()
         x = LEFT_BORDER_OFFSET
-        y = blank_line = count(2).next
+        y = blank_line = count(2)
         my, mx = win.getmaxyx()
         win.erase()
         win.bkgd(' ', curses.color_pair(1))
         win.border()
         win.addstr(1, x, self.greet, curses.A_DIM | curses.color_pair(5))
-        blank_line()
-        win.addstr(y(), x, self.format_row('UUID', 'TASK',
+        next(blank_line)
+        win.addstr(next(y), x, self.format_row('UUID', 'TASK',
                                            'WORKER', 'TIME', 'STATE'),
                 curses.A_BOLD | curses.A_UNDERLINE)
         tasks = self.tasks
@@ -351,11 +355,11 @@ class CursesMonitor(object):  # pragma: no cover
                     break
 
                 if task.uuid:
-                    lineno = y()
+                    lineno = next(y)
                 self.display_task_row(lineno, task)
 
         # -- Footer
-        blank_line()
+        next(blank_line)
         win.hline(my - 6, x, curses.ACS_HLINE, self.screen_width - 4)
 
         # Selected Task Info
@@ -374,7 +378,7 @@ class CursesMonitor(object):  # pragma: no cover
                 if 'result' in info:
                     info['result'] = abbr(info['result'], 16)
                 info = ' '.join('{0}={1}'.format(key, value)
-                            for key, value in info.items())
+                            for key, value in items(info))
                 detail = '... -> key i'
             infowin = abbr(info,
                            self.screen_width - len(self.selected_str) - 2,
@@ -400,7 +404,7 @@ class CursesMonitor(object):  # pragma: no cover
         win.addstr(my - 3, x, self.info_str, curses.A_BOLD)
         win.addstr(my - 3, x + len(self.info_str),
                 STATUS_SCREEN.format(s=self.state,
-                    w_alive=len([w for w in self.state.workers.values()
+                    w_alive=len([w for w in values(self.state.workers)
                                     if w.alive]),
                     w_all=len(self.state.workers)),
                 curses.A_DIM)
@@ -457,7 +461,7 @@ class CursesMonitor(object):  # pragma: no cover
     @property
     def workers(self):
         return [hostname
-                    for hostname, w in self.state.workers.items()
+                    for hostname, w in items(self.state.workers)
                         if w.alive]
 
 

celery/events/state.py

@@ -29,6 +29,7 @@ from kombu.utils import kwdict
 
 from celery import states
 from celery.datastructures import AttributeDict, LRUCache
+from celery.five import items, values
 from celery.utils.log import get_logger
 
 # The window (in percentage) is added to the workers heartbeat
@@ -191,8 +192,9 @@ class Task(Element):
         :param fields: Event data.
 
         """
-        if self.worker:
-            self.worker.update_heartbeat(fields['local_received'], timestamp)
+        time_received = fields.get('local_received') or 0
+        if self.worker and time_received:
+            self.worker.update_heartbeat(time_received, timestamp)
         if state != states.RETRY and self.state != states.RETRY and \
                 states.state(state) < states.state(self.state):
             # this state logically happens-before the current state, so merge.
@@ -363,7 +365,7 @@ class State(object):
         task.worker = worker
 
         taskheap = self._taskheap
-        timestamp = fields['timestamp']
+        timestamp = fields.get('timestamp') or 0
         clock = 0 if type == 'sent' else fields.get('clock')
         heappush(taskheap, _lamportinfo(clock, timestamp, worker.id, task))
         curcount = len(self.tasks)
@@ -392,7 +394,7 @@ class State(object):
             self.event_callback(self, event)
 
     def itertasks(self, limit=None):
-        for index, row in enumerate(self.tasks.iteritems()):
+        for index, row in enumerate(items(self.tasks)):
             yield row
             if limit and index + 1 >= limit:
                 break
@@ -428,11 +430,11 @@ class State(object):
 
     def task_types(self):
         """Returns a list of all seen task types."""
-        return list(sorted(set(task.name for task in self.tasks.itervalues())))
+        return list(sorted(set(task.name for task in values(self.tasks))))
 
     def alive_workers(self):
         """Returns a list of (seemingly) alive workers."""
-        return [w for w in self.workers.values() if w.alive]
+        return [w for w in values(self.workers) if w.alive]
 
     def __repr__(self):
         return '<State: events={0.event_count} tasks={0.task_count}>' \

celery/exceptions.py

@@ -8,6 +8,8 @@
 """
 from __future__ import absolute_import
 
+from .five import string_t
+
 from billiard.exceptions import (  # noqa
     SoftTimeLimitExceeded, TimeLimitExceeded, WorkerLostError, Terminated,
 )
@@ -66,7 +68,7 @@ class RetryTaskError(Exception):
     def __init__(self, message=None, exc=None, when=None, **kwargs):
         from kombu.utils.encoding import safe_repr
         self.message = message
-        if isinstance(exc, basestring):
+        if isinstance(exc, string_t):
             self.exc, self.excs = None, exc
         else:
             self.exc, self.excs = exc, safe_repr(exc) if exc else None

celery/__compat__.py → celery/five.py

@@ -1,26 +1,173 @@
 # -*- coding: utf-8 -*-
 """
-    celery.__compat__
-    ~~~~~~~~~~~~~~~~~
+    celery.five
+    ~~~~~~~~~~~
+
+    Compatibility implementations of features
+    only available in newer Python versions.
 
-    This module contains utilities to dynamically
-    recreate modules, either for lazy loading or
-    to create old modules at runtime instead of
-    having them litter the source tree.
 
 """
 from __future__ import absolute_import
 
+############## py3k #########################################################
+import sys
+PY3 = sys.version_info[0] == 3
+
+try:
+    reload = reload                         # noqa
+except NameError:                           # pragma: no cover
+    from imp import reload                  # noqa
+
+try:
+    from UserList import UserList           # noqa
+except ImportError:                         # pragma: no cover
+    from collections import UserList        # noqa
+
+try:
+    from UserDict import UserDict           # noqa
+except ImportError:                         # pragma: no cover
+    from collections import UserDict        # noqa
+
+
+if PY3:
+    import builtins
+
+    from queue import Queue, Empty
+    from itertools import zip_longest
+    from io import StringIO, BytesIO
+
+    map = map
+    string = str
+    string_t = str
+    long_t = int
+    text_t = str
+    range = range
+
+    open_fqdn = 'builtins.open'
+
+    def items(d):
+        return d.items()
+
+    def keys(d):
+        return d.keys()
+
+    def values(d):
+        return d.values()
+
+    def nextfun(it):
+        return it.__next__
+
+    exec_ = getattr(builtins, 'exec')
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+    class WhateverIO(StringIO):
+
+        def write(self, data):
+            if isinstance(data, bytes):
+                data = data.encode()
+            StringIO.write(self, data)
+
+else:
+    import __builtin__ as builtins  # noqa
+    from Queue import Queue, Empty  # noqa
+    from itertools import imap as map, izip_longest as zip_longest  # noqa
+    from StringIO import StringIO   # noqa
+    string = unicode                # noqa
+    string_t = basestring           # noqa
+    text_t = unicode
+    long_t = long                   # noqa
+    range = xrange
+
+    open_fqdn = '__builtin__.open'
+
+    def items(d):                   # noqa
+        return d.iteritems()
+
+    def keys(d):                    # noqa
+        return d.iterkeys()
+
+    def values(d):                  # noqa
+        return d.itervalues()
+
+    def nextfun(it):                # noqa
+        return it.next
+
+    def exec_(code, globs=None, locs=None):
+        """Execute code in a namespace."""
+        if globs is None:
+            frame = sys._getframe(1)
+            globs = frame.f_globals
+            if locs is None:
+                locs = frame.f_locals
+            del frame
+        elif locs is None:
+            locs = globs
+        exec("""exec code in globs, locs""")
+
+    exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""")
+
+    BytesIO = WhateverIO = StringIO         # noqa
+
+def with_metaclass(Type, skip_attrs=set(['__dict__', '__weakref__'])):
+    """Class decorator to set metaclass.
+
+    Works with both Python 3 and Python 3 and it does not add
+    an extra class in the lookup order like ``six.with_metaclass`` does
+    (that is -- it copies the original class instead of using inheritance).
+
+    """
+
+    def _clone_with_metaclass(Class):
+        attrs = dict((key, value) for key, value in items(vars(Class))
+                        if key not in skip_attrs)
+        return Type(Class.__name__, Class.__bases__, attrs)
+
+    return _clone_with_metaclass
+
+
+############## collections.OrderedDict ######################################
+# was moved to kombu
+from kombu.utils.compat import OrderedDict  # noqa
+
+############## threading.TIMEOUT_MAX #######################################
+try:
+    from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX
+except ImportError:
+    THREAD_TIMEOUT_MAX = 1e10  # noqa
+
+############## format(int, ',d') ##########################
+
+if sys.version_info >= (2, 7):  # pragma: no cover
+    def format_d(i):
+        return format(i, ',d')
+else:  # pragma: no cover
+    def format_d(i):  # noqa
+        s = '%d' % i
+        groups = []
+        while s and s[-1].isdigit():
+            groups.append(s[-3:])
+            s = s[:-3]
+        return s + ','.join(reversed(groups))
+
+
+############## Module Generation ##########################
+
+# Utilities to dynamically
+# recreate modules, either for lazy loading or
+# to create old modules at runtime instead of
+# having them litter the source tree.
 import operator
 import sys
 
 from functools import reduce
 from importlib import import_module
-from itertools import imap
 from types import ModuleType
 
-from .local import Proxy
-
 MODULE_DEPRECATED = """
 The module %s is deprecated and will be removed in a future version.
 """
@@ -154,7 +301,7 @@ def create_module(name, attrs, cls_attrs=None, pkg=None, base=MagicModule,
     cls_attrs = {} if cls_attrs is None else cls_attrs
 
     attrs = dict((attr_name, prepare_attr(attr) if prepare_attr else attr)
-                    for attr_name, attr in attrs.iteritems())
+                    for attr_name, attr in attrs.items())
     module = sys.modules[fqdn] = type(name, (base, ), cls_attrs)(fqdn)
     module.__dict__.update(attrs)
     return module
@@ -169,7 +316,7 @@ def recreate_module(name, compat_modules=(), by_module={}, direct={},
     cattrs = dict(_compat_modules=compat_modules,
                   _all_by_module=by_module, _direct=direct,
                   _object_origins=origins,
-                  __all__=tuple(set(reduce(operator.add, imap(tuple, [
+                  __all__=tuple(set(reduce(operator.add, map(tuple, [
                                 compat_modules, origins, direct, attrs])))))
     new_module = create_module(name, attrs, cls_attrs=cattrs, base=base)
     new_module.__dict__.update(dict((mod, get_compat_module(new_module, mod))
@@ -178,14 +325,15 @@ def recreate_module(name, compat_modules=(), by_module={}, direct={},
 
 
 def get_compat_module(pkg, name):
+    from .local import Proxy
 
     def prepare(attr):
-        if isinstance(attr, basestring):
+        if isinstance(attr, string_t):
             return Proxy(getappattr, (attr, ))
         return attr
 
     attrs = COMPAT_MODULES[pkg.__name__][name]
-    if isinstance(attrs, basestring):
+    if isinstance(attrs, string_t):
         fqdn = '.'.join([pkg.__name__, name])
         module = sys.modules[fqdn] = import_module(attrs)
         return module
@@ -195,6 +343,6 @@ def get_compat_module(pkg, name):
 
 def get_origins(defs):
     origins = {}
-    for module, items in defs.iteritems():
+    for module, items in defs.items():
         origins.update(dict((item, module) for item in items))
     return origins

celery/fixups/django.py

@@ -168,7 +168,7 @@ class DjangoFixup(object):
         for close in funs:
             try:
                 close()
-            except self.database_errors, exc:
+            except self.database_errors as exc:
                 str_exc = str(exc)
                 if 'closed' not in str_exc and 'not connected' not in str_exc:
                     raise

celery/loaders/base.py

@@ -16,17 +16,17 @@ import re
 import sys
 
 from datetime import datetime
-from itertools import imap
 
 from kombu.utils import cached_property
 from kombu.utils.encoding import safe_str
 
 from celery.datastructures import DictAttribute
 from celery.exceptions import ImproperlyConfigured
+from celery.five import reraise, string_t
+from celery.utils.functional import maybe_list
 from celery.utils.imports import (
     import_from_cwd, symbol_by_name, NotAPackage, find_module,
 )
-from celery.utils.functional import maybe_list
 
 BUILTIN_MODULES = frozenset()
 
@@ -146,7 +146,7 @@ class BaseLoader(object):
         return self.config_from_object(module_name, silent=silent)
 
     def config_from_object(self, obj, silent=False):
-        if isinstance(obj, basestring):
+        if isinstance(obj, string_t):
             try:
                 if '.' in obj:
                     obj = symbol_by_name(obj, imp=self.import_from_cwd)
@@ -166,13 +166,13 @@ class BaseLoader(object):
             self.find_module(name)
         except NotAPackage:
             if name.endswith('.py'):
-                raise NotAPackage, NotAPackage(
+                reraise(NotAPackage, NotAPackage(
                         CONFIG_WITH_SUFFIX.format(
                             module=name,
-                            suggest=name[:-3])), sys.exc_info()[2]
-            raise NotAPackage, NotAPackage(
+                            suggest=name[:-3])), sys.exc_info()[2])
+            reraise(NotAPackage, NotAPackage(
                     CONFIG_INVALID_NAME.format(
-                        module=name)), sys.exc_info()[2]
+                        module=name)), sys.exc_info()[2])
         else:
             return self.import_from_cwd(name)
 
@@ -223,7 +223,7 @@ class BaseLoader(object):
                     raise ValueError('{0!r}: {1}'.format(ns_key, exc))
             return ns_key, value
 
-        return dict(imap(getarg, args))
+        return dict(getarg(arg) for arg in args)
 
     def mail_admins(self, subject, body, fail_silently=False,
             sender=None, to=None, host=None, port=None,

celery/local.py

@@ -13,7 +13,8 @@
 from __future__ import absolute_import
 
 import importlib
-import sys
+
+from .five import long_t, string, string_t
 
 
 def symbol_by_name(name, aliases={}, imp=None, package=None,
@@ -54,7 +55,7 @@ def symbol_by_name(name, aliases={}, imp=None, package=None,
     if imp is None:
         imp = importlib.import_module
 
-    if not isinstance(name, basestring):
+    if not isinstance(name, string_t):
         return name                                 # already a class
 
     name = aliases.get(name) or name
@@ -65,9 +66,8 @@ def symbol_by_name(name, aliases={}, imp=None, package=None,
     try:
         try:
             module = imp(module_name, package=package, **kwargs)
-        except ValueError, exc:
-            raise ValueError, ValueError(
-                    "Couldn't import %r: %s" % (name, exc)), sys.exc_info()[2]
+        except ValueError as exc:
+            raise ValueError("Couldn't import %r: %s" % (name, exc))
         return getattr(module, cls_name) if cls_name else module
     except (ImportError, AttributeError):
         if default is None:
@@ -88,7 +88,7 @@ class Proxy(object):
     """Proxy to another object."""
 
     # Code stolen from werkzeug.local.Proxy.
-    __slots__ = ('__local', '__args', '__kwargs', '__dict__', '__name__')
+    __slots__ = ('__local', '__args', '__kwargs', '__dict__')
 
     def __init__(self, local, args=None, kwargs=None, name=None):
         object.__setattr__(self, '_Proxy__local', local)
@@ -145,15 +145,16 @@ class Proxy(object):
             return '<{0} unbound>'.format(self.__class__.__name__)
         return repr(obj)
 
-    def __nonzero__(self):
+    def __bool__(self):
         try:
             return bool(self._get_current_object())
         except RuntimeError:  # pragma: no cover
             return False
+    __nonzero__ = __bool__  # Py2
 
     def __unicode__(self):
         try:
-            return unicode(self._get_current_object())
+            return string(self._get_current_object())
         except RuntimeError:  # pragma: no cover
             return repr(self)
 
@@ -217,7 +218,7 @@ class Proxy(object):
     __invert__ = lambda x: ~(x._get_current_object())
     __complex__ = lambda x: complex(x._get_current_object())
     __int__ = lambda x: int(x._get_current_object())
-    __long__ = lambda x: long(x._get_current_object())
+    __long__ = lambda x: long_t(x._get_current_object())
     __float__ = lambda x: float(x._get_current_object())
     __oct__ = lambda x: oct(x._get_current_object())
     __hex__ = lambda x: hex(x._get_current_object())

celery/platforms.py

@@ -18,9 +18,9 @@ import sys
 
 from billiard import current_process
 from contextlib import contextmanager
-from itertools import imap
 
 from .local import try_import
+from .five import items, map, reraise, string_t
 
 _setproctitle = try_import('setproctitle')
 resource = try_import('resource')
@@ -46,6 +46,12 @@ PIDFILE_MODE = ((os.R_OK | os.W_OK) << 6) | ((os.R_OK) << 3) | ((os.R_OK))
 PIDLOCKED = """ERROR: Pidfile ({0}) already exists.
 Seems we're already running? (pid: {1})"""
 
+try:
+    from io import UnsupportedOperation
+    FILENO_ERRORS = (AttributeError, UnsupportedOperation)
+except ImportError:  # Py2
+    FILENO_ERRORS = (AttributeError, )  # noqa
+
 
 def pyimplementation():
     """Returns string identifying the current Python implementation."""
@@ -54,9 +60,9 @@ def pyimplementation():
     elif sys.platform.startswith('java'):
         return 'Jython ' + sys.platform
     elif hasattr(sys, 'pypy_version_info'):
-        v = '.'.join(imap(str, sys.pypy_version_info[:3]))
+        v = '.'.join(map(str, sys.pypy_version_info[:3]))
         if sys.pypy_version_info[3:]:
-            v += '-' + ''.join(imap(str, sys.pypy_version_info[3:]))
+            v += '-' + ''.join(map(str, sys.pypy_version_info[3:]))
         return 'PyPy ' + v
     else:
         return 'CPython'
@@ -135,7 +141,7 @@ class Pidfile(object):
         try:
             self.write_pid()
         except OSError as exc:
-            raise LockFailed, LockFailed(str(exc)), sys.exc_info()[2]
+            reraise(LockFailed, LockFailed(str(exc)), sys.exc_info()[2])
         return self
     __enter__ = acquire
 
@@ -254,7 +260,7 @@ def fileno(f):
     """Get object fileno, or :const:`None` if not defined."""
     try:
         return f.fileno()
-    except AttributeError:
+    except FILENO_ERRORS:
         pass
 
 
@@ -541,7 +547,7 @@ class Signals(object):
         """Get signal number from signal name."""
         if isinstance(signal_name, int):
             return signal_name
-        if not isinstance(signal_name, basestring) \
+        if not isinstance(signal_name, string_t) \
                 or not signal_name.isupper():
             raise TypeError('signal name must be uppercase string.')
         if not signal_name.startswith('SIG'):
@@ -583,7 +589,7 @@ class Signals(object):
 
     def update(self, _d_=None, **sigmap):
         """Set signal handlers from a mapping."""
-        for signal_name, handler in dict(_d_ or {}, **sigmap).iteritems():
+        for signal_name, handler in items(dict(_d_ or {}, **sigmap)):
             self[signal_name] = handler
 
 
@@ -634,7 +640,7 @@ else:
 
 def get_errno(n):
     """Get errno for string, e.g. ``ENOENT``."""
-    if isinstance(n, basestring):
+    if isinstance(n, string_t):
         return getattr(errno, n)
     return n
 
@@ -660,7 +666,7 @@ def ignore_errno(*errnos, **kwargs):
     errnos = [get_errno(errno) for errno in errnos]
     try:
         yield
-    except types, exc:
+    except types as exc:
         if not hasattr(exc, 'errno'):
             raise
         if exc.errno not in errnos:

celery/result.py

@@ -12,7 +12,6 @@ import time
 
 from collections import deque
 from copy import copy
-from itertools import imap
 
 from kombu.utils import cached_property
 from kombu.utils.compat import OrderedDict
@@ -22,6 +21,7 @@ from . import states
 from .app import app_or_default
 from .datastructures import DependencyGraph, GraphFormatter
 from .exceptions import IncompleteStream, TimeoutError
+from .five import items, map, range, string_t
 
 
 def from_serializable(r):
@@ -31,7 +31,8 @@ def from_serializable(r):
         id = parent = None
         res, nodes = r
         if nodes:
-            return GroupResult(res, [AsyncResult(id) for id, _ in nodes])
+            return GroupResult(res,
+                        [from_serializable(child) for child in nodes])
         if isinstance(res, (list, tuple)):
             id, parent = res[0], res[1]
         return AsyncResult(id, parent=parent)
@@ -143,7 +144,7 @@ class AsyncResult(ResultBase):
 
             @task()
             def A(how_many):
-                return group(B.s(i) for i in xrange(how_many))
+                return group(B.s(i) for i in range(how_many))
 
             @task()
             def B(i):
@@ -224,7 +225,7 @@ class AsyncResult(ResultBase):
     def __eq__(self, other):
         if isinstance(other, AsyncResult):
             return other.id == self.id
-        elif isinstance(other, basestring):
+        elif isinstance(other, string_t):
             return other == self.id
         return NotImplemented
 
@@ -339,7 +340,7 @@ class ResultSet(ResultBase):
         :raises KeyError: if the result is not a member.
 
         """
-        if isinstance(result, basestring):
+        if isinstance(result, string_t):
             result = AsyncResult(result)
         try:
             self.results.remove(result)
@@ -408,7 +409,7 @@ class ResultSet(ResultBase):
         :returns: the number of tasks completed.
 
         """
-        return sum(imap(int, (result.successful() for result in self.results)))
+        return sum(map(int, (result.successful() for result in self.results)))
 
     def forget(self):
         """Forget about (and possible remove the result of) all the tasks."""
@@ -441,7 +442,7 @@ class ResultSet(ResultBase):
 
         while results:
             removed = set()
-            for task_id, result in results.iteritems():
+            for task_id, result in items(results):
                 if result.ready():
                     yield result.get(timeout=timeout and timeout - elapsed,
                                      propagate=propagate)
@@ -543,7 +544,7 @@ class ResultSet(ResultBase):
 
         """
         results = self.results
-        acc = [None for _ in xrange(len(self))]
+        acc = [None for _ in range(len(self))]
         for task_id, meta in self.iter_native(timeout=timeout,
                                               interval=interval):
             acc[results.index(task_id)] = meta['result']

celery/schedules.py

@@ -16,6 +16,7 @@ from datetime import datetime, timedelta
 from kombu.utils import cached_property
 
 from . import current_app
+from .five import string_t
 from .utils import is_iterable
 from .utils.timeutils import (
     timedelta_seconds, weekday, maybe_timedelta, remaining,
@@ -30,7 +31,7 @@ Invalid crontab pattern. Valid range is {min}-{max}. \
 
 CRON_INVALID_TYPE = """\
 Argument cronspec needs to be of any of the following types: \
-int, basestring, or an iterable type. {type!r} was given.\
+int, str, or an iterable type. {type!r} was given.\
 """
 
 
@@ -222,7 +223,7 @@ class crontab_parser(object):
         return range(self.min_, self.max_ + self.min_)
 
     def _expand_number(self, s):
-        if isinstance(s, basestring) and s[0] == '-':
+        if isinstance(s, string_t) and s[0] == '-':
             raise self.ParseException('negative numbers not supported')
         try:
             i = int(s)
@@ -310,13 +311,13 @@ class crontab(schedule):
         """Takes the given cronspec argument in one of the forms::
 
             int         (like 7)
-            basestring  (like '3-5,*/15', '*', or 'monday')
+            str         (like '3-5,*/15', '*', or 'monday')
             set         (like set([0,15,30,45]))
             list        (like [8-17])
 
         And convert it to an (expanded) set representing all time unit
         values on which the crontab triggers.  Only in case of the base
-        type being 'basestring', parsing occurs.  (It is fast and
+        type being 'str', parsing occurs.  (It is fast and
         happens only once for each crontab instance, so there is no
         significant performance overhead involved.)
 
@@ -332,7 +333,7 @@ class crontab(schedule):
         """
         if isinstance(cronspec, int):
             result = set([cronspec])
-        elif isinstance(cronspec, basestring):
+        elif isinstance(cronspec, string_t):
             result = crontab_parser(max_, min_).parse(cronspec)
         elif isinstance(cronspec, set):
             result = cronspec

celery/security/certificate.py

@@ -11,7 +11,10 @@ from __future__ import absolute_import
 import glob
 import os
 
+from kombu.utils.encoding import bytes_to_str
+
 from celery.exceptions import SecurityError
+from celery.five import values
 
 from .utils import crypto, reraise_errors
 
@@ -34,7 +37,7 @@ class Certificate(object):
 
     def get_issuer(self):
         """Returns issuer (CA) as a string"""
-        return ' '.join(x[1] for x in
+        return ' '.join(bytes_to_str(x[1]) for x in
                         self._cert.get_issuer().get_components())
 
     def get_id(self):
@@ -55,7 +58,7 @@ class CertStore(object):
 
     def itercerts(self):
         """an iterator over the certificates"""
-        for c in self._certs.itervalues():
+        for c in values(self._certs):
             yield c
 
     def __getitem__(self, id):

celery/security/key.py

@@ -8,6 +8,8 @@
 """
 from __future__ import absolute_import
 
+from kombu.utils.encoding import ensure_bytes
+
 from .utils import crypto, reraise_errors
 
 
@@ -20,4 +22,4 @@ class PrivateKey(object):
     def sign(self, data, digest):
         """sign string containing data."""
         with reraise_errors('Unable to sign data: {0!r}'):
-            return crypto.sign(self._key, data, digest)
+            return crypto.sign(self._key, ensure_bytes(data), digest)

celery/security/serialization.py

@@ -10,9 +10,8 @@ from __future__ import absolute_import
 
 import base64
 
-from itertools import izip
 from kombu.serialization import registry, encode, decode
-from kombu.utils.encoding import bytes_to_str, str_to_bytes
+from kombu.utils.encoding import bytes_to_str, str_to_bytes, ensure_bytes
 
 from .certificate import Certificate, FSCertStore
 from .key import PrivateKey
@@ -48,6 +47,7 @@ class SecureSerializer(object):
             # this way the receiver doesn't have to decode the contents
             # to verify the signature (and thus avoiding potential flaws
             # in the decoding step).
+            body = ensure_bytes(body)
             return self._pack(body, content_type, content_encoding,
                               signature=self._key.sign(body, self._digest),
                               signer=self._cert.get_id())
@@ -61,18 +61,23 @@ class SecureSerializer(object):
                                        payload['signer'],
                                        payload['body'])
             self._cert_store[signer].verify(body, signature, self._digest)
-        return decode(body, payload['content_type'],
+        return decode(bytes_to_str(body), payload['content_type'],
                             payload['content_encoding'], force=True)
 
     def _pack(self, body, content_type, content_encoding, signer, signature,
-            sep='\x00\x01'):
-        return b64encode(sep.join([signer, signature,
-                                   content_type, content_encoding, body]))
-
-    def _unpack(self, payload, sep='\x00\x01',
-            fields=('signer', 'signature', 'content_type',
-                    'content_encoding', 'body')):
-        return dict(izip(fields, b64decode(payload).split(sep)))
+            sep=str_to_bytes('\x00\x01')):
+        fields = sep.join(ensure_bytes(s)
+                for s in [signer, signature, content_type,
+                          content_encoding, body])
+        return b64encode(fields)
+
+    def _unpack(self, payload, sep=str_to_bytes('\x00\x01')):
+        values = b64decode(ensure_bytes(payload)).split(sep)
+        return {'signer': bytes_to_str(values[0]),
+                'signature': ensure_bytes(values[1]),
+                'content_type': bytes_to_str(values[2]),
+                'content_encoding': bytes_to_str(values[3]),
+                'body': ensure_bytes(values[4])}
 
 
 def register_auth(key=None, cert=None, store=None, digest='sha1',

celery/security/utils.py

@@ -13,6 +13,7 @@ import sys
 from contextlib import contextmanager
 
 from celery.exceptions import SecurityError
+from celery.five import reraise
 
 try:
     from OpenSSL import crypto
@@ -27,4 +28,6 @@ def reraise_errors(msg='{0!r}', errors=None):
     try:
         yield
     except errors as exc:
-        raise SecurityError, SecurityError(msg.format(exc)), sys.exc_info()[2]
+        reraise(SecurityError,
+                SecurityError(msg.format(exc)),
+                sys.exc_info()[2])

celery/task/__init__.py

@@ -12,7 +12,7 @@
 from __future__ import absolute_import
 
 from celery._state import current_app, current_task as current
-from celery.__compat__ import MagicModule, recreate_module
+from celery.five import MagicModule, recreate_module
 from celery.local import Proxy
 
 __all__ = [

celery/task/base.py

@@ -14,8 +14,8 @@ from __future__ import absolute_import
 from kombu import Exchange
 
 from celery import current_app
-from celery.__compat__ import class_property, reclassmethod
 from celery.app.task import Context, TaskType, Task as BaseTask  # noqa
+from celery.five import class_property, reclassmethod
 from celery.schedules import maybe_schedule
 from celery.utils.log import get_task_logger
 

celery/task/http.py

@@ -10,16 +10,15 @@ from __future__ import absolute_import
 
 import anyjson
 import sys
-import urllib2
 
-from urllib import urlencode
-from urlparse import urlparse
 try:
-    from urlparse import parse_qsl
+    from urllib.parse import parse_qsl, urlencode, urlparse   # Py3
 except ImportError:  # pragma: no cover
-    from cgi import parse_qsl  # noqa
+    from urllib import urlencode              # noqa
+    from urlparse import urlparse, parse_qsl  # noqa
 
 from celery import __version__ as celery_version
+from celery.five import items, reraise
 from celery.utils.log import get_task_logger
 from .base import Task as BaseTask
 
@@ -27,33 +26,26 @@ GET_METHODS = frozenset(['GET', 'HEAD'])
 logger = get_task_logger(__name__)
 
 
-class InvalidResponseError(Exception):
-    """The remote server gave an invalid response."""
-
-
-class RemoteExecuteError(Exception):
-    """The remote task gave a custom error."""
-
-
-class UnknownStatusError(InvalidResponseError):
-    """The remote server gave an unknown status."""
-
-
-def maybe_utf8(value):
-    """Encode to utf-8, only if the value is Unicode."""
-    if isinstance(value, unicode):
-        return value.encode('utf-8')
-    return value
-
-
 if sys.version_info[0] == 3:  # pragma: no cover
 
+    from urllib.request import Request, urlopen
+
     def utf8dict(tup):
         if not isinstance(tup, dict):
             return dict(tup)
         return tup
+
 else:
 
+    from urllib2 import Request, urlopen  # noqa
+
+    def maybe_utf8(value):  # noqa
+        """Encode to utf-8, only if the value is Unicode."""
+        if isinstance(value, unicode):
+            return value.encode('utf-8')
+        return value
+
+
     def utf8dict(tup):  # noqa
         """With a dict's items() tuple return a new dict with any utf-8
         keys/values encoded."""
@@ -61,6 +53,18 @@ else:
                         for key, value in tup)
 
 
+class InvalidResponseError(Exception):
+    """The remote server gave an invalid response."""
+
+
+class RemoteExecuteError(Exception):
+    """The remote task gave a custom error."""
+
+
+class UnknownStatusError(InvalidResponseError):
+    """The remote server gave an unknown status."""
+
+
 def extract_response(raw_response, loads=anyjson.loads):
     """Extract the response text from a raw JSON response."""
     if not raw_response:
@@ -68,8 +72,8 @@ def extract_response(raw_response, loads=anyjson.loads):
     try:
         payload = loads(raw_response)
     except ValueError as exc:
-        raise InvalidResponseError, InvalidResponseError(
-                str(exc)), sys.exc_info()[2]
+        reraise(InvalidResponseError, InvalidResponseError(
+                str(exc)), sys.exc_info()[2])
 
     status = payload['status']
     if status == 'success':
@@ -106,7 +110,7 @@ class MutableURL(object):
 
     def __str__(self):
         scheme, netloc, path, params, query, fragment = self.parts
-        query = urlencode(utf8dict(self.query.items()))
+        query = urlencode(utf8dict(items(self.query)))
         components = [scheme + '://', netloc, path or '/',
                       ';{0}'.format(params)   if params   else '',
                       '?{0}'.format(query)    if query    else '',
@@ -138,10 +142,10 @@ class HttpDispatch(object):
 
     def make_request(self, url, method, params):
         """Makes an HTTP request and returns the response."""
-        request = urllib2.Request(url, params)
-        for key, val in self.http_headers.items():
+        request = Request(url, params)
+        for key, val in items(self.http_headers):
             request.add_header(key, val)
-        response = urllib2.urlopen(request)         # user catches errors.
+        response = urlopen(request)  # user catches errors.
         return response.read()
 
     def dispatch(self):
@@ -151,7 +155,7 @@ class HttpDispatch(object):
         if self.method in GET_METHODS:
             url.query.update(self.task_kwargs)
         else:
-            params = urlencode(utf8dict(self.task_kwargs.items()))
+            params = urlencode(utf8dict(items(self.task_kwargs)))
         raw_response = self.make_request(str(url), self.method, params)
         return extract_response(raw_response)
 

celery/tests/app/test_app.py

@@ -11,6 +11,7 @@ from celery import Celery
 from celery import app as _app
 from celery import _state
 from celery.app import defaults
+from celery.five import items
 from celery.loaders.base import BaseLoader
 from celery.platforms import pyimplementation
 from celery.utils.serialization import pickle
@@ -27,7 +28,7 @@ THIS_IS_A_KEY = 'this is a value'
 class Object(object):
 
     def __init__(self, **kwargs):
-        for key, value in kwargs.items():
+        for key, value in items(kwargs):
             setattr(self, key, value)
 
 
@@ -380,13 +381,13 @@ class test_App(Case):
                                        'userid': 'guest',
                                        'password': 'guest',
                                        'virtual_host': '/'},
-                            self.app.connection('amqplib://').info())
+                        self.app.connection('pyamqp://').info())
         self.app.conf.BROKER_PORT = 1978
         self.app.conf.BROKER_VHOST = 'foo'
         self.assertDictContainsSubset({'port': 1978,
                                        'virtual_host': 'foo'},
-                    self.app.connection('amqplib://:1978/foo').info())
-        conn = self.app.connection('amqplib:////value')
+        self.app.connection('pyamqp://:1978/foo').info())
+        conn = self.app.connection('pyamqp:////value')
         self.assertDictContainsSubset({'virtual_host': '/value'},
                                       conn.info())
 

celery/tests/app/test_beat.py

@@ -8,6 +8,7 @@ from nose import SkipTest
 
 from celery import beat
 from celery import task
+from celery.five import keys, string_t
 from celery.result import AsyncResult
 from celery.schedules import schedule
 from celery.task.base import Task
@@ -189,7 +190,7 @@ class test_Scheduler(Case):
 
     def test_info(self):
         scheduler = mScheduler()
-        self.assertIsInstance(scheduler.info, basestring)
+        self.assertIsInstance(scheduler.info, string_t)
 
     def test_maybe_entry(self):
         s = mScheduler()
@@ -368,8 +369,8 @@ class test_Service(Case):
         schedule = s.scheduler.schedule
         self.assertIsInstance(schedule, dict)
         self.assertIsInstance(s.scheduler, beat.Scheduler)
-        scheduled = schedule.keys()
-        for task_name in sh['entries'].keys():
+        scheduled = list(schedule.keys())
+        for task_name in keys(sh['entries']):
             self.assertIn(task_name, scheduled)
 
         s.sync()

celery/tests/app/test_builtins.py

@@ -4,6 +4,7 @@ from mock import Mock, patch
 
 from celery import current_app as app, group, task, chord
 from celery.app import builtins
+from celery.five import range
 from celery._state import _task_stack
 from celery.tests.utils import Case
 
@@ -142,18 +143,18 @@ class test_chord(Case):
         app.tasks['celery.chord'] = self.prev
 
     def test_apply_async(self):
-        x = chord([add.s(i, i) for i in xrange(10)], body=xsum.s())
+        x = chord([add.s(i, i) for i in range(10)], body=xsum.s())
         r = x.apply_async()
         self.assertTrue(r)
         self.assertTrue(r.parent)
 
     def test_run_header_not_group(self):
-        self.task([add.s(i, i) for i in xrange(10)], xsum.s())
+        self.task([add.s(i, i) for i in range(10)], xsum.s())
 
     def test_apply_eager(self):
         app.conf.CELERY_ALWAYS_EAGER = True
         try:
-            x = chord([add.s(i, i) for i in xrange(10)], body=xsum.s())
+            x = chord([add.s(i, i) for i in range(10)], body=xsum.s())
             r = x.apply_async()
             self.assertEqual(r.get(), 90)
 

celery/tests/app/test_control.py

@@ -189,7 +189,7 @@ class test_Broadcast(Case):
     @with_mock_broadcast
     def test_revoke_from_resultset(self):
         r = self.app.GroupResult(uuid(),
-                                 map(self.app.AsyncResult,
-                                        [uuid() for i in range(10)]))
+                                 [self.app.AsyncResult(x)
+                                     for x in [uuid() for i in range(10)]])
         r.revoke()
         self.assertIn('revoke', MockMailbox.sent)

celery/tests/app/test_loaders.py

@@ -13,6 +13,7 @@ from celery.exceptions import (
         ImproperlyConfigured,
         CPendingDeprecationWarning,
 )
+from celery.five import items
 from celery.loaders import base
 from celery.loaders import default
 from celery.loaders.app import AppLoader
@@ -33,7 +34,7 @@ dict_config = dict(FOO=10, BAR=20)
 class Object(object):
 
     def __init__(self, **kwargs):
-        for k, v in kwargs.items():
+        for k, v in items(kwargs):
             setattr(self, k, v)
 
 

celery/tests/backends/test_amqp.py

@@ -3,7 +3,6 @@ from __future__ import absolute_import
 import socket
 
 from datetime import timedelta
-from Queue import Empty, Queue
 
 from mock import patch
 
@@ -13,6 +12,7 @@ from celery.app import app_or_default
 from celery.backends.amqp import AMQPBackend
 from celery.datastructures import ExceptionInfo
 from celery.exceptions import TimeoutError
+from celery.five import Empty, Queue, range
 from celery.utils import uuid
 
 from celery.tests.utils import AppCase, sleepdeprived
@@ -216,7 +216,7 @@ class test_AMQPBackend(AppCase):
         b = self.create_backend()
 
         tids = []
-        for i in xrange(10):
+        for i in range(10):
             tid = uuid()
             b.store_result(tid, i, states.SUCCESS)
             tids.append(tid)

celery/tests/backends/test_base.py

@@ -7,6 +7,7 @@ from mock import Mock
 from nose import SkipTest
 
 from celery import current_app
+from celery.five import items, range
 from celery.result import AsyncResult, GroupResult
 from celery.utils import serialization
 from celery.utils.serialization import subclass_exception
@@ -66,7 +67,7 @@ class test_BaseBackend_interface(Case):
         p, current_app.tasks[unlock] = current_app.tasks.get(unlock), Mock()
         try:
             b.on_chord_apply('dakj221', 'sdokqweok',
-                             result=map(AsyncResult, [1, 2, 3]))
+                             result=[AsyncResult(x) for x in [1, 2, 3]])
             self.assertTrue(current_app.tasks[unlock].apply_async.call_count)
         finally:
             current_app.tasks[unlock] = p
@@ -229,14 +230,14 @@ class test_KeyValueStoreBackend(Case):
     def test_get_many(self):
         for is_dict in True, False:
             self.b.mget_returns_dict = is_dict
-            ids = dict((uuid(), i) for i in xrange(10))
-            for id, i in ids.items():
+            ids = dict((uuid(), i) for i in range(10))
+            for id, i in items(ids):
                 self.b.mark_as_done(id, i)
-            it = self.b.get_many(ids.keys())
+            it = self.b.get_many(list(ids))
             for i, (got_id, got_state) in enumerate(it):
                 self.assertEqual(got_state['result'], ids[got_id])
             self.assertEqual(i, 9)
-            self.assertTrue(list(self.b.get_many(ids.keys())))
+            self.assertTrue(list(self.b.get_many(list(ids))))
 
     def test_get_missing_meta(self):
         self.assertIsNone(self.b.get_result('xxx-missing'))

celery/tests/backends/test_cache.py

@@ -12,6 +12,7 @@ from celery import current_app
 from celery import states
 from celery.backends.cache import CacheBackend, DummyClient
 from celery.exceptions import ImproperlyConfigured
+from celery.five import items, string, text_t
 from celery.result import AsyncResult
 from celery.task import subtask
 from celery.utils import uuid
@@ -119,9 +120,9 @@ class MyMemcachedStringEncodingError(Exception):
 class MemcachedClient(DummyClient):
 
     def set(self, key, value, *args, **kwargs):
-        if isinstance(key, unicode):
+        if isinstance(key, text_t):
             raise MyMemcachedStringEncodingError(
-                    'Keys must be str, not unicode.  Convert your unicode '
+                    'Keys must be bytes, not string.  Convert your '
                     'strings using mystring.encode(charset)!')
         return super(MemcachedClient, self).set(key, value, *args, **kwargs)
 
@@ -188,7 +189,7 @@ class test_get_best_memcache(Case, MockCacheMixin):
 
     def test_backends(self):
         from celery.backends.cache import backends
-        for name, fun in backends.items():
+        for name, fun in items(backends):
             self.assertTrue(fun())
 
 
@@ -200,7 +201,7 @@ class test_memcache_key(Case, MockCacheMixin):
                 with mask_modules('pylibmc'):
                     from celery.backends import cache
                     cache._imp = [None]
-                    task_id, result = unicode(uuid()), 42
+                    task_id, result = string(uuid()), 42
                     b = cache.CacheBackend(backend='memcache')
                     b.store_result(task_id, result, status=states.SUCCESS)
                     self.assertEqual(b.get_result(task_id), result)
@@ -221,7 +222,7 @@ class test_memcache_key(Case, MockCacheMixin):
             with self.mock_pylibmc():
                 from celery.backends import cache
                 cache._imp = [None]
-                task_id, result = unicode(uuid()), 42
+                task_id, result = string(uuid()), 42
                 b = cache.CacheBackend(backend='memcache')
                 b.store_result(task_id, result, status=states.SUCCESS)
                 self.assertEqual(b.get_result(task_id), result)

celery/tests/backends/test_database.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 from datetime import datetime
 
@@ -152,7 +152,7 @@ class test_DatabaseBackend(Case):
         tb = DatabaseBackend()
 
         tid = uuid()
-        res = {u'something': 'special'}
+        res = {'something': 'special'}
         self.assertEqual(tb.save_group(tid, res), res)
 
         res2 = tb.restore_group(tid)

celery/tests/backends/test_mongodb.py

@@ -196,7 +196,7 @@ class test_MongoBackend(AppCase):
         self.assertEquals(
             ['status', 'task_id', 'date_done', 'traceback', 'result',
              'children'],
-            ret_val.keys())
+            list(ret_val.keys()))
 
     @patch('celery.backends.mongodb.MongoBackend._get_database')
     def test_get_task_meta_for_no_result(self, mock_get_database):
@@ -250,7 +250,8 @@ class test_MongoBackend(AppCase):
         mock_database.__getitem__.assert_called_once_with(MONGODB_COLLECTION)
         mock_collection.find_one.assert_called_once_with(
             {'_id': sentinel.taskset_id})
-        self.assertEquals(['date_done', 'result', 'task_id'], ret_val.keys())
+        self.assertEquals(['date_done', 'result', 'task_id'],
+                list(ret_val.keys()))
 
     @patch('celery.backends.mongodb.MongoBackend._get_database')
     def test_delete_group(self, mock_get_database):

celery/tests/backends/test_redis.py

@@ -139,7 +139,7 @@ class test_RedisBackend(Case):
 
     def test_on_chord_apply(self):
         self.Backend().on_chord_apply('group_id', {},
-                                      result=map(AsyncResult, [1, 2, 3]))
+                result=[AsyncResult(x) for x in [1, 2, 3]])
 
     def test_mget(self):
         b = self.MockBackend()

celery/tests/bin/test_camqadm.py

@@ -71,8 +71,8 @@ class test_AMQShell(AppCase):
     def test_completenames(self):
         self.assertEqual(self.shell.completenames('queue.dec'),
                 ['queue.declare'])
-        self.assertEqual(self.shell.completenames('declare'),
-                ['queue.declare', 'exchange.declare'])
+        self.assertEqual(sorted(self.shell.completenames('declare')),
+                sorted(['queue.declare', 'exchange.declare']))
 
     def test_empty_line(self):
         self.shell.emptyline = Mock()

celery/tests/bin/test_celeryd_multi.py

@@ -87,27 +87,39 @@ class test_multi_args(Case):
         it = multi_args(p, cmd='COMMAND', append='*AP*',
                 prefix='*P*', suffix='*S*')
         names = list(it)
-        self.assertEqual(names[0][0:2], ('*P*jerry*S*',
+
+        def assert_line_in(name, args):
+            self.assertIn(name, [tup[0] for tup in names])
+            argv = None
+            for item in names:
+                if item[0] == name:
+                    argv = item[1]
+            self.assertTrue(argv)
+            for arg in args:
+                self.assertIn(arg, argv)
+
+
+        assert_line_in('*P*jerry*S*',
             [
                 'COMMAND', '-n *P*jerry*S*', '-Q bar',
                 '-c 5', '--flag', '--logfile=foo',
                 '-- .disable_rate_limits=1', '*AP*',
             ]
-        ))
-        self.assertEqual(names[1][0:2], ('*P*elaine*S*',
+        )
+        assert_line_in('*P*elaine*S*',
             [
                 'COMMAND', '-n *P*elaine*S*', '-Q bar',
                 '-c 5', '--flag', '--logfile=foo',
                 '-- .disable_rate_limits=1', '*AP*',
             ]
-        ))
-        self.assertEqual(names[2][0:2], ('*P*kramer*S*',
+        )
+        assert_line_in('*P*kramer*S*',
             [
                 'COMMAND', '--loglevel=DEBUG', '-n *P*kramer*S*',
                 '-Q bar', '--flag', '--logfile=foo',
                 '-- .disable_rate_limits=1', '*AP*',
             ]
-        ))
+        )
         expand = names[0][2]
         self.assertEqual(expand('%h'), '*P*jerry*S*')
         self.assertEqual(expand('%n'), 'jerry')
@@ -284,20 +296,25 @@ class test_MultiTool(Case):
 
         p = NamespacedOptionParser(['foo', 'bar', 'baz'])
         nodes = self.t.getpids(p, 'celeryd', callback=callback)
-        self.assertEqual(nodes, [
-            ('foo.e.com',
-              ('celeryd', '--pidfile=celeryd@foo.pid', '-n foo.e.com', ''),
-             10),
-            ('bar.e.com',
-              ('celeryd', '--pidfile=celeryd@bar.pid', '-n bar.e.com', ''),
-             11),
-        ])
+        node_0, node_1 = nodes
+        self.assertEqual(node_0[0], 'foo.e.com')
+        self.assertEqual(sorted(node_0[1]),
+            sorted(('celeryd', '--pidfile=celeryd@foo.pid',
+                    '-n foo.e.com', '')))
+        self.assertEqual(node_0[2], 10)
+
+        self.assertEqual(node_1[0], 'bar.e.com')
+        self.assertEqual(sorted(node_1[1]),
+            sorted(('celeryd', '--pidfile=celeryd@bar.pid',
+                    '-n bar.e.com', '')))
+        self.assertEqual(node_1[2], 11)
         self.assertTrue(callback.called)
-        callback.assert_called_with(
-            'baz.e.com',
+        cargs, _ = callback.call_args
+        self.assertEqual(cargs[0], 'baz.e.com')
+        self.assertItemsEqual(cargs[1],
             ['celeryd', '--pidfile=celeryd@baz.pid', '-n baz.e.com', ''],
-            None,
         )
+        self.assertIsNone(cargs[2])
         self.assertIn('DOWN', self.fh.getvalue())
 
         # without callback, should work
@@ -316,10 +333,12 @@ class test_MultiTool(Case):
 
         callback = Mock()
         self.t.stop(['foo', 'bar', 'baz'], 'celeryd', callback=callback)
-        sigs = self.t.signal_node.call_args_list
+        sigs = sorted(self.t.signal_node.call_args_list)
         self.assertEqual(len(sigs), 2)
-        self.assertEqual(sigs[0][0], ('foo.e.com', 10, signal.SIGTERM))
-        self.assertEqual(sigs[1][0], ('bar.e.com', 11, signal.SIGTERM))
+        self.assertIn(('foo.e.com', 10, signal.SIGTERM),
+                [tup[0] for tup in sigs])
+        self.assertIn(('bar.e.com', 11, signal.SIGTERM),
+                [tup[0] for tup in sigs])
         self.t.signal_node.return_value = False
         self.assertTrue(callback.called)
         self.t.stop(['foo', 'bar', 'baz'], 'celeryd', callback=None)

celery/tests/concurrency/test_concurrency.py

@@ -13,12 +13,12 @@ class test_BasePool(Case):
     def test_apply_target(self):
 
         scratch = {}
-        counter = count(0).next
+        counter = count(0)
 
         def gen_callback(name, retval=None):
 
             def callback(*args):
-                scratch[name] = (counter(), args)
+                scratch[name] = (next(counter), args)
                 return retval
 
             return callback

celery/tests/concurrency/test_pool.py

@@ -43,10 +43,10 @@ class test_TaskPool(Case):
         p = self.TaskPool(2)
         p.start()
         scratchpad = {}
-        proc_counter = itertools.count().next
+        proc_counter = itertools.count()
 
         def mycallback(ret_value):
-            process = proc_counter()
+            process = next(proc_counter)
             scratchpad[process] = {}
             scratchpad[process]['ret_value'] = ret_value
 

celery/tests/concurrency/test_processes.py

@@ -7,6 +7,7 @@ from itertools import cycle
 from mock import Mock
 from nose import SkipTest
 
+from celery.five import items, range
 from celery.utils.functional import noop
 from celery.tests.utils import Case
 try:
@@ -38,7 +39,7 @@ except ImportError:
 class Object(object):   # for writeable attributes.
 
     def __init__(self, **kwargs):
-        [setattr(self, k, v) for k, v in kwargs.items()]
+        [setattr(self, k, v) for k, v in items(kwargs)]
 
 
 class MockResult(object):
@@ -69,7 +70,7 @@ class MockPool(object):
         self._state = mp.RUN
         self._processes = kwargs.get('processes')
         self._pool = [Object(pid=i) for i in range(self._processes)]
-        self._current_proc = cycle(xrange(self._processes)).next
+        self._current_proc = cycle(range(self._processes))
 
     def close(self):
         self.closed = True
@@ -97,7 +98,7 @@ class ExeMockPool(MockPool):
         from threading import Timer
         res = target(*args, **kwargs)
         Timer(0.1, callback, (res, )).start()
-        return MockResult(res, self._current_proc())
+        return MockResult(res, next(self._current_proc))
 
 
 class TaskPool(mp.TaskPool):

celery/tests/contrib/test_migrate.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 from kombu import Connection, Producer, Queue, Exchange
 from kombu.exceptions import StdChannelError
@@ -27,9 +27,9 @@ class test_State(Case):
 
     def test_strtotal(self):
         x = State()
-        self.assertEqual(x.strtotal, u'?')
+        self.assertEqual(x.strtotal, '?')
         x.total_apx = 100
-        self.assertEqual(x.strtotal, u'100')
+        self.assertEqual(x.strtotal, '100')
 
 
 class test_migrate_task(Case):

celery/tests/contrib/test_rdb.py

@@ -13,6 +13,10 @@ from celery.contrib.rdb import (
 from celery.tests.utils import Case, WhateverIO, skip_if_pypy
 
 
+class SockErr(socket.error):
+    errno = None
+
+
 class test_Rdb(Case):
 
     @patch('celery.contrib.rdb.Rdb')
@@ -49,11 +53,11 @@ class test_Rdb(Case):
             with patch('celery.contrib.rdb._frame'):
                 rdb.set_trace()
                 rdb.set_trace(Mock())
-                pset.side_effect = socket.error
+                pset.side_effect = SockErr
                 pset.side_effect.errno = errno.ECONNRESET
                 rdb.set_trace()
                 pset.side_effect.errno = errno.ENOENT
-                with self.assertRaises(socket.error):
+                with self.assertRaises(SockErr):
                     rdb.set_trace()
 
         # _close_session
@@ -80,9 +84,9 @@ class test_Rdb(Case):
             curproc.return_value.name = 'PoolWorker-10'
             Rdb(out=out)
 
-        err = sock.return_value.bind.side_effect = socket.error()
+        err = sock.return_value.bind.side_effect = SockErr()
         err.errno = errno.ENOENT
-        with self.assertRaises(socket.error):
+        with self.assertRaises(SockErr):
             Rdb(out=out)
         err.errno = errno.EADDRINUSE
         with self.assertRaises(Exception):

celery/tests/events/test_events.py

@@ -41,7 +41,7 @@ class test_EventDispatcher(AppCase):
 
     def test_send(self):
         producer = MockProducer()
-        eventer = self.app.events.Dispatcher(object(), enabled=False)
+        eventer = self.app.events.Dispatcher(Mock(), enabled=False)
         eventer.publisher = producer
         eventer.enabled = True
         eventer.send('World War II', ended=True)

celery/tests/events/test_state.py

@@ -21,18 +21,23 @@ class replay(object):
     def setup(self):
         pass
 
+    def next_event(self):
+        ev = self.events[next(self.position)]
+        ev['local_received'] = ev['timestamp']
+        return ev
+
     def __iter__(self):
         return self
 
     def __next__(self):
         try:
-            self.state.event(self.events[self.position()])
+            self.state.event(self.next_event())
         except IndexError:
             raise StopIteration()
     next = __next__
 
     def rewind(self):
-        self.position = count(0).next
+        self.position = count(0)
         return self
 
     def play(self):
@@ -305,13 +310,13 @@ class test_State(Case):
     def test_tasks_by_timestamp(self):
         r = ev_snapshot(State())
         r.play()
-        self.assertEqual(len(r.state.tasks_by_timestamp()), 20)
+        self.assertEqual(len(list(r.state.tasks_by_timestamp())), 20)
 
     def test_tasks_by_type(self):
         r = ev_snapshot(State())
         r.play()
-        self.assertEqual(len(r.state.tasks_by_type('task1')), 10)
-        self.assertEqual(len(r.state.tasks_by_type('task2')), 10)
+        self.assertEqual(len(list(r.state.tasks_by_type('task1'))), 10)
+        self.assertEqual(len(list(r.state.tasks_by_type('task2'))), 10)
 
     def test_alive_workers(self):
         r = ev_snapshot(State())
@@ -321,8 +326,8 @@ class test_State(Case):
     def test_tasks_by_worker(self):
         r = ev_snapshot(State())
         r.play()
-        self.assertEqual(len(r.state.tasks_by_worker('utest1')), 10)
-        self.assertEqual(len(r.state.tasks_by_worker('utest2')), 10)
+        self.assertEqual(len(list(r.state.tasks_by_worker('utest1'))), 10)
+        self.assertEqual(len(list(r.state.tasks_by_worker('utest2'))), 10)
 
     def test_survives_unknown_worker_event(self):
         s = State()

celery/tests/functional/case.py

@@ -36,7 +36,7 @@ def try_while(fun, reason='Timed out', timeout=10, interval=0.5):
 
 class Worker(object):
     started = False
-    next_worker_id = count(1).next
+    worker_ids = count(1)
     _shutdown_called = False
 
     def __init__(self, hostname, loglevel='error'):
@@ -87,7 +87,7 @@ class Worker(object):
         if caller:
             hostname = '.'.join([qualname(caller), hostname])
         else:
-            hostname += str(cls.next_worker_id())
+            hostname += str(next(cls.worker_ids()))
         worker = cls(hostname)
         worker.ensure_started()
         stack = traceback.format_stack()
@@ -109,7 +109,7 @@ class WorkerCase(Case):
 
     @classmethod
     def setUpClass(cls):
-        logging.getLogger('amqplib').setLevel(logging.ERROR)
+        logging.getLogger('amqp').setLevel(logging.ERROR)
         cls.worker = Worker.managed(cls.hostname, caller=cls)
 
     @classmethod

celery/tests/security/test_security.py

@@ -16,12 +16,11 @@ Generated with:
 """
 from __future__ import absolute_import
 
-import __builtin__
-
 from mock import Mock, patch
 
 from celery import current_app
 from celery.exceptions import ImproperlyConfigured
+from celery.five import builtins
 from celery.security import setup_security, disable_untrusted_serializers
 from kombu.serialization import registry
 
@@ -85,13 +84,13 @@ class test_security(SecurityCase):
 
         self.assertRaises(ImproperlyConfigured, setup_security)
 
-        _import = __builtin__.__import__
+        _import = builtins.__import__
 
         def import_hook(name, *args, **kwargs):
             if name == 'OpenSSL':
                 raise ImportError
             return _import(name, *args, **kwargs)
 
-        __builtin__.__import__ = import_hook
+        builtins.__import__ = import_hook
         self.assertRaises(ImproperlyConfigured, setup_security)
-        __builtin__.__import__ = _import
+        builtins.__import__ = _import

celery/tests/security/test_serialization.py

@@ -1,11 +1,12 @@
 from __future__ import absolute_import
 
-from celery.exceptions import SecurityError
+from kombu.serialization import registry
+from kombu.utils.encoding import ensure_bytes
 
+from celery.exceptions import SecurityError
 from celery.security.serialization import SecureSerializer, register_auth
 from celery.security.certificate import Certificate, CertStore
 from celery.security.key import PrivateKey
-from kombu.serialization import registry
 
 from . import CERT1, CERT2, KEY1, KEY2
 from .case import SecurityCase

celery/tests/slow/test_buckets.py

@@ -4,12 +4,12 @@ import sys
 import time
 
 from functools import partial
-from itertools import chain, izip
-from Queue import Empty
+from itertools import chain
 
 from mock import Mock, patch
 
 from celery.app.registry import TaskRegistry
+from celery.five import Empty, range
 from celery.task.base import Task
 from celery.utils import timeutils
 from celery.utils import uuid
@@ -65,8 +65,8 @@ class test_TokenBucketQueue(Case):
         x = buckets.TokenBucketQueue(fill_rate=10)
         # 20 items should take at least one second to complete
         time_start = time.time()
-        [x.put(str(i)) for i in xrange(20)]
-        for i in xrange(20):
+        [x.put(str(i)) for i in range(20)]
+        for i in range(20):
             sys.stderr.write('.')
             x.wait()
         self.assertGreater(time.time() - time_start, 1.5)
@@ -210,10 +210,10 @@ class test_TaskBucket(Case):
     def test_auto_add_on_missing(self):
         b = buckets.TaskBucket(task_registry=self.registry)
         for task_cls in self.task_classes:
-            self.assertIn(task_cls.name, b.buckets.keys())
+            self.assertIn(task_cls.name, list(b.buckets.keys()))
         self.registry.register(TaskD)
         self.assertTrue(b.get_bucket_for_type(TaskD.name))
-        self.assertIn(TaskD.name, b.buckets.keys())
+        self.assertIn(TaskD.name, list(b.buckets.keys()))
         self.registry.unregister(TaskD)
 
     @skip_if_disabled
@@ -249,7 +249,7 @@ class test_TaskBucket(Case):
         b = buckets.TaskBucket(task_registry=self.registry)
 
         cjob = lambda i: MockJob(uuid(), TaskA.name, [i], {})
-        jobs = [cjob(i) for i in xrange(20)]
+        jobs = [cjob(i) for i in range(20)]
         [b.put(job) for job in jobs]
 
         self.assertEqual(b.qsize(), 20)
@@ -266,14 +266,14 @@ class test_TaskBucket(Case):
         b = buckets.TaskBucket(task_registry=self.registry)
 
         cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
-        ajobs = [cjob(i, TaskA) for i in xrange(10)]
-        bjobs = [cjob(i, TaskB) for i in xrange(20)]
-        jobs = list(chain(*izip(bjobs, ajobs)))
+        ajobs = [cjob(i, TaskA) for i in range(10)]
+        bjobs = [cjob(i, TaskB) for i in range(20)]
+        jobs = list(chain(*zip(bjobs, ajobs)))
         for job in jobs:
             b.put(job)
 
         got_ajobs = 0
-        for job in (b.get() for i in xrange(20)):
+        for job in (b.get() for i in range(20)):
             if job.name == TaskA.name:
                 got_ajobs += 1
 
@@ -287,13 +287,13 @@ class test_TaskBucket(Case):
 
             cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
 
-            ajobs = [cjob(i, TaskA) for i in xrange(10)]
-            bjobs = [cjob(i, TaskB) for i in xrange(10)]
-            cjobs = [cjob(i, TaskC) for i in xrange(10)]
-            djobs = [cjob(i, TaskD) for i in xrange(10)]
+            ajobs = [cjob(i, TaskA) for i in range(10)]
+            bjobs = [cjob(i, TaskB) for i in range(10)]
+            cjobs = [cjob(i, TaskC) for i in range(10)]
+            djobs = [cjob(i, TaskD) for i in range(10)]
 
             # Spread the jobs around.
-            jobs = list(chain(*izip(ajobs, bjobs, cjobs, djobs)))
+            jobs = list(chain(*zip(ajobs, bjobs, cjobs, djobs)))
 
             [b.put(job) for job in jobs]
             for i, job in enumerate(jobs):

celery/tests/tasks/test_chord.py

@@ -6,6 +6,7 @@ from contextlib import contextmanager
 from celery import canvas
 from celery import current_app
 from celery import result
+from celery.five import range
 from celery.result import AsyncResult, GroupResult
 from celery.task import task, TaskSet
 from celery.tests.utils import AppCase, Mock
@@ -67,7 +68,7 @@ class test_unlock_chord_task(AppCase):
                 subtask, canvas.maybe_subtask = canvas.maybe_subtask, passthru
                 try:
                     unlock('group_id', callback_s,
-                           result=map(AsyncResult, [1, 2, 3]))
+                           result=[AsyncResult(r) for r in [1, 2, 3]])
                 finally:
                     canvas.maybe_subtask = subtask
                 callback.apply_async.assert_called_with(([2, 4, 8, 6], ), {})
@@ -87,7 +88,7 @@ class test_unlock_chord_task(AppCase):
             try:
                 callback = Mock()
                 unlock('group_id', callback, interval=10, max_retries=30,
-                            result=map(AsyncResult, [1, 2, 3]))
+                            result=[AsyncResult(x) for x in [1, 2, 3]])
                 self.assertFalse(callback.delay.call_count)
                 # did retry
                 unlock.retry.assert_called_with(countdown=10, max_retries=30)
@@ -113,10 +114,10 @@ class test_chord(AppCase):
 
         self.app.conf.CELERY_ALWAYS_EAGER = True
         try:
-            x = chord(addX.s(i, i) for i in xrange(10))
+            x = chord(addX.s(i, i) for i in range(10))
             body = sumX.s()
             result = x(body)
-            self.assertEqual(result.get(), sum(i + i for i in xrange(10)))
+            self.assertEqual(result.get(), sum(i + i for i in range(10)))
         finally:
             self.app.conf.CELERY_ALWAYS_EAGER = False
 
@@ -129,7 +130,7 @@ class test_chord(AppCase):
         m.AsyncResult = AsyncResult
         prev, chord.Chord = chord.Chord, m
         try:
-            x = chord(add.s(i, i) for i in xrange(10))
+            x = chord(add.s(i, i) for i in range(10))
             body = add.s(2)
             result = x(body)
             self.assertTrue(result.id)
@@ -151,8 +152,8 @@ class test_Chord_task(AppCase):
             Chord = current_app.tasks['celery.chord']
 
             body = dict()
-            Chord(TaskSet(add.subtask((i, i)) for i in xrange(5)), body)
-            Chord([add.subtask((i, i)) for i in xrange(5)], body)
+            Chord(TaskSet(add.subtask((i, i)) for i in range(5)), body)
+            Chord([add.subtask((i, i)) for i in range(5)], body)
             self.assertEqual(current_app.backend.on_chord_apply.call_count, 2)
         finally:
             current_app.backend = prev

celery/tests/tasks/test_context.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-'
 from __future__ import absolute_import
 
+from collections import Callable
+
 from celery.task.base import Context
 from celery.tests.utils import Case
 
@@ -13,7 +15,7 @@ def get_context_as_dict(ctx, getter=getattr):
         if attr_name.startswith('_'):
             continue   # Ignore pseudo-private attributes
         attr = getter(ctx, attr_name)
-        if callable(attr):
+        if isinstance(attr, Callable):
             continue   # Ignore methods and other non-trivial types
         defaults[attr_name] = attr
     return defaults

celery/tests/tasks/test_http.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 from contextlib import contextmanager
 from functools import wraps
@@ -11,27 +11,26 @@ except ImportError:  # py3k
 from anyjson import dumps
 from kombu.utils.encoding import from_utf8
 
+from celery.five import StringIO, items
 from celery.task import http
 from celery.tests.utils import Case, eager_tasks
-from celery.utils.compat import StringIO
 
 
 @contextmanager
 def mock_urlopen(response_method):
 
-    import urllib2
-    urlopen = urllib2.urlopen
+    urlopen = http.urlopen
 
     @wraps(urlopen)
     def _mocked(url, *args, **kwargs):
         response_data, headers = response_method(url)
         return addinfourl(StringIO(response_data), headers, url)
 
-    urllib2.urlopen = _mocked
+    http.urlopen = _mocked
 
     yield True
 
-    urllib2.urlopen = urlopen
+    http.urlopen = urlopen
 
 
 def _response(res):
@@ -54,10 +53,10 @@ class test_encodings(Case):
 
     def test_utf8dict(self):
         uk = 'foobar'
-        d = {u'følelser ær langé': u'ærbadægzaååÆØÅ',
+        d = {'følelser ær langé': 'ærbadægzaååÆØÅ',
              from_utf8(uk): from_utf8('xuzzybaz')}
 
-        for key, value in http.utf8dict(d.items()).items():
+        for key, value in items(http.utf8dict(items(d))):
             self.assertIsInstance(key, str)
             self.assertIsInstance(value, str)
 

celery/tests/tasks/test_result.py

@@ -5,9 +5,8 @@ from mock import Mock
 
 from celery import states
 from celery.app import app_or_default
-from celery.exceptions import IncompleteStream
-from celery.utils import uuid
-from celery.utils.serialization import pickle
+from celery.exceptions import IncompleteStream, TimeoutError
+from celery.five import range
 from celery.result import (
     AsyncResult,
     EagerResult,
@@ -16,9 +15,10 @@ from celery.result import (
     ResultSet,
     from_serializable,
 )
-from celery.exceptions import TimeoutError
 from celery.task import task
 from celery.task.base import Task
+from celery.utils import uuid
+from celery.utils.serialization import pickle
 
 from celery.tests.utils import AppCase
 from celery.tests.utils import skip_if_quick
@@ -47,7 +47,7 @@ def save_result(task):
 
 
 def make_mock_group(size=10):
-    tasks = [mock_task('ts%d' % i, states.SUCCESS, i) for i in xrange(size)]
+    tasks = [mock_task('ts%d' % i, states.SUCCESS, i) for i in range(size)]
     [save_result(task) for task in tasks]
     return [AsyncResult(task['id']) for task in tasks]
 
@@ -81,7 +81,7 @@ class test_AsyncResult(AppCase):
     def test_get_children(self):
         tid = uuid()
         x = AsyncResult(tid)
-        child = [AsyncResult(uuid()).serializable() for i in xrange(10)]
+        child = [AsyncResult(uuid()).serializable() for i in range(10)]
         x.backend._cache[tid] = {'children': child}
         self.assertTrue(x.children)
         self.assertEqual(len(x.children), 10)
@@ -247,14 +247,15 @@ class test_AsyncResult(AppCase):
 class test_ResultSet(AppCase):
 
     def test_resultset_repr(self):
-        self.assertTrue(repr(ResultSet(map(AsyncResult, ['1', '2', '3']))))
+        self.assertTrue(repr(ResultSet([AsyncResult(t)
+                                            for t in ['1', '2', '3']])))
 
     def test_eq_other(self):
         self.assertFalse(ResultSet([1, 3, 3]) == 1)
         self.assertTrue(ResultSet([1]) == ResultSet([1]))
 
     def test_get(self):
-        x = ResultSet(map(AsyncResult, [1, 2, 3]))
+        x = ResultSet([AsyncResult(t) for t in [1, 2, 3]])
         b = x.results[0].backend = Mock()
         b.supports_native_join = False
         x.join_native = Mock()
@@ -416,7 +417,7 @@ class test_GroupResult(AppCase):
         ts = GroupResult(uuid(), subtasks)
         backend.ids = [subtask.id for subtask in subtasks]
         res = ts.join_native()
-        self.assertEqual(res, range(10))
+        self.assertEqual(res, list(range(10)))
 
     def test_iter_native(self):
         backend = SimpleBackend()
@@ -453,11 +454,11 @@ class test_GroupResult(AppCase):
     def test___iter__(self):
         it = iter(self.ts)
         results = sorted(list(it))
-        self.assertListEqual(results, list(xrange(self.size)))
+        self.assertListEqual(results, list(range(self.size)))
 
     def test_join(self):
         joined = self.ts.join()
-        self.assertListEqual(joined, list(xrange(self.size)))
+        self.assertListEqual(joined, list(range(self.size)))
 
     def test_successful(self):
         self.assertTrue(self.ts.successful())

celery/tests/tasks/test_tasks.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+from collections import Callable
 from datetime import datetime, timedelta
 from functools import wraps
 from mock import patch
@@ -18,6 +19,7 @@ from celery import current_app
 from celery.app import app_or_default
 from celery.exceptions import RetryTaskError
 from celery.execute import send_task
+from celery.five import items, range, string_t
 from celery.result import EagerResult
 from celery.schedules import crontab, crontab_parser, ParseException
 from celery.utils import uuid
@@ -250,14 +252,14 @@ class test_tasks(Case):
         self.assertEqual(task_data['task'], task_name)
         task_kwargs = task_data.get('kwargs', {})
         if test_eta:
-            self.assertIsInstance(task_data.get('eta'), basestring)
+            self.assertIsInstance(task_data.get('eta'), string_t)
             to_datetime = parse_iso8601(task_data.get('eta'))
             self.assertIsInstance(to_datetime, datetime)
         if test_expires:
-            self.assertIsInstance(task_data.get('expires'), basestring)
+            self.assertIsInstance(task_data.get('expires'), string_t)
             to_datetime = parse_iso8601(task_data.get('expires'))
             self.assertIsInstance(to_datetime, datetime)
-        for arg_name, arg_value in kwargs.items():
+        for arg_name, arg_value in items(kwargs):
             self.assertEqual(task_kwargs.get(arg_name), arg_value)
 
     def test_incomplete_task_cls(self):
@@ -280,7 +282,7 @@ class test_tasks(Case):
         T1 = self.createTask('c.unittest.t.t1')
         self.assertIsInstance(T1, BaseTask)
         self.assertTrue(T1.run())
-        self.assertTrue(callable(T1),
+        self.assertTrue(isinstance(T1, Callable),
                 'Task class is callable()')
         self.assertTrue(T1(),
                 'Task class runs run() when called')
@@ -495,7 +497,7 @@ class test_TaskSet(Case):
                                            'id': subtask.id}, m)
             increment_counter(
                     increment_by=m.get('kwargs', {}).get('increment_by'))
-        self.assertEqual(increment_counter.count, sum(xrange(1, 10)))
+        self.assertEqual(increment_counter.count, sum(range(1, 10)))
 
     def test_named_taskset(self):
         prefix = 'test_named_taskset-'
@@ -686,15 +688,15 @@ class test_crontab_parser(Case):
         self.assertEqual(crontab_parser(8).parse('*/2'),
                           set([0, 2, 4, 6]))
         self.assertEqual(crontab_parser().parse('*/2'),
-                          set(i * 2 for i in xrange(30)))
+                          set(i * 2 for i in range(30)))
         self.assertEqual(crontab_parser().parse('*/3'),
-                          set(i * 3 for i in xrange(20)))
+                          set(i * 3 for i in range(20)))
         self.assertEqual(crontab_parser(8, 1).parse('*/2'),
                           set([1, 3, 5, 7]))
         self.assertEqual(crontab_parser(min_=1).parse('*/2'),
-                          set(i * 2 + 1 for i in xrange(30)))
+                          set(i * 2 + 1 for i in range(30)))
         self.assertEqual(crontab_parser(min_=1).parse('*/3'),
-                          set(i * 3 + 1 for i in xrange(20)))
+                          set(i * 3 + 1 for i in range(20)))
 
     def test_parse_composite(self):
         self.assertEqual(crontab_parser(8).parse('*/2'), set([0, 2, 4, 6]))
@@ -1072,7 +1074,7 @@ class test_crontab_is_due(Case):
                                                    ffwd=relativedelta)
         if not isinstance(d1, relativedelta):
             self.assertEqual(l1, l2)
-            for field, value in d1._fields().iteritems():
+            for field, value in items(d1._fields()):
                 self.assertEqual(getattr(d1, field), value)
             self.assertFalse(d2.years)
             self.assertFalse(d2.months)

celery/tests/utilities/test_datastructures.py

@@ -9,7 +9,7 @@ from celery.datastructures import (
     ConfigurationView,
     DependencyGraph,
 )
-from celery.utils.compat import THREAD_TIMEOUT_MAX
+from celery.five import THREAD_TIMEOUT_MAX, items, range
 from celery.tests.utils import Case, WhateverIO
 
 
@@ -85,11 +85,12 @@ class test_ConfigurationView(Case):
         expected = {'changed_key': 1,
                     'default_key': 1,
                     'both': 2}
-        self.assertDictEqual(dict(self.view.items()), expected)
+        self.assertDictEqual(dict(items(self.view)), expected)
         self.assertItemsEqual(list(iter(self.view)),
-                              expected.keys())
-        self.assertItemsEqual(self.view.keys(), expected.keys())
-        self.assertItemsEqual(self.view.values(), expected.values())
+                              list(expected.keys()))
+        self.assertItemsEqual(list(self.view.keys()), list(expected.keys()))
+        self.assertItemsEqual(list(self.view.values()),
+                list(expected.values()))
 
 
 class test_ExceptionInfo(Case):
@@ -173,33 +174,33 @@ class test_LRUCache(Case):
     def test_expires(self):
         limit = 100
         x = LRUCache(limit=limit)
-        slots = list(xrange(limit * 2))
+        slots = list(range(limit * 2))
         for i in slots:
             x[i] = i
-        self.assertListEqual(x.keys(), list(slots[limit:]))
+        self.assertListEqual(list(x.keys()), list(slots[limit:]))
 
     def test_least_recently_used(self):
         x = LRUCache(3)
 
         x[1], x[2], x[3] = 1, 2, 3
-        self.assertEqual(x.keys(), [1, 2, 3])
+        self.assertEqual(list(x.keys()), [1, 2, 3])
 
         x[4], x[5] = 4, 5
-        self.assertEqual(x.keys(), [3, 4, 5])
+        self.assertEqual(list(x.keys()), [3, 4, 5])
 
         # access 3, which makes it the last used key.
         x[3]
         x[6] = 6
-        self.assertEqual(x.keys(), [5, 3, 6])
+        self.assertEqual(list(x.keys()), [5, 3, 6])
 
         x[7] = 7
-        self.assertEqual(x.keys(), [3, 6, 7])
+        self.assertEqual(list(x.keys()), [3, 6, 7])
 
     def assertSafeIter(self, method, interval=0.01, size=10000):
         from threading import Thread, Event
         from time import sleep
         x = LRUCache(size)
-        x.update(zip(xrange(size), xrange(size)))
+        x.update(zip(range(size), range(size)))
 
         class Burglar(Thread):
 
@@ -242,7 +243,7 @@ class test_LRUCache(Case):
     def test_items(self):
         c = LRUCache()
         c.update(a=1, b=2, c=3)
-        self.assertTrue(c.items())
+        self.assertTrue(list(items(c)))
 
 
 class test_AttributeDict(Case):
@@ -279,11 +280,11 @@ class test_DependencyGraph(Case):
         self.assertLess(order.index('A'), order.index('C'))
 
     def test_edges(self):
-        self.assertListEqual(list(self.graph1().edges()),
+        self.assertItemsEqual(list(self.graph1().edges()),
                              ['C', 'D'])
 
     def test_items(self):
-        self.assertDictEqual(dict(self.graph1().items()),
+        self.assertDictEqual(dict(items(self.graph1())),
                 {'A': [], 'B': [],
                  'C': ['A'], 'D': ['C', 'B']})
 

celery/tests/utilities/test_encoding.py

@@ -1,9 +1,10 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 import sys
 
 from nose import SkipTest
 
+from celery.five import string
 from celery.utils import encoding
 from celery.tests.utils import Case
 
@@ -13,13 +14,12 @@ class test_encoding(Case):
     def test_safe_str(self):
         self.assertTrue(encoding.safe_str(object()))
         self.assertTrue(encoding.safe_str('foo'))
-        self.assertTrue(encoding.safe_str(u'foo'))
 
     def test_safe_str_UnicodeDecodeError(self):
         if sys.version_info >= (3, 0):
             raise SkipTest('py3k: not relevant')
 
-        class foo(unicode):
+        class foo(string):
 
             def encode(self, *args, **kwargs):
                 raise UnicodeDecodeError('foo')

celery/tests/utilities/test_local.py

@@ -1,5 +1,6 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
+from celery.five import string, long_t
 from celery.local import Proxy, PromiseProxy, maybe_evaluate, try_import
 from celery.tests.utils import Case
 
@@ -34,12 +35,13 @@ class test_Proxy(Case):
         self.assertEqual(x.__dict__, real.__dict__)
         self.assertEqual(repr(x), repr(real))
 
-    def test_nonzero(self):
+    def test_bool(self):
 
         class X(object):
 
-            def __nonzero__(self):
+            def __bool__(self):
                 return False
+            __nonzero__ = __bool__
 
         x = Proxy(lambda: X())
         self.assertFalse(x)
@@ -58,15 +60,17 @@ class test_Proxy(Case):
         class X(object):
 
             def __unicode__(self):
-                return u'UNICODE'
+                return 'UNICODE'
+            __str__ = __unicode__
 
             def __repr__(self):
                 return 'REPR'
 
         x = Proxy(lambda: X())
-        self.assertEqual(unicode(x), u'UNICODE')
+        self.assertEqual(string(x), 'UNICODE')
         del(X.__unicode__)
-        self.assertEqual(unicode(x), 'REPR')
+        del(X.__str__)
+        self.assertEqual(string(x), 'REPR')
 
     def test_dir(self):
 
@@ -198,7 +202,7 @@ class test_Proxy(Case):
         x = Proxy(lambda: 10)
         self.assertEqual(type(x.__float__()), float)
         self.assertEqual(type(x.__int__()), int)
-        self.assertEqual(type(x.__long__()), long)
+        self.assertEqual(type(x.__long__()), long_t)
         self.assertTrue(hex(x))
         self.assertTrue(oct(x))
 

celery/tests/utilities/test_platforms.py

@@ -9,6 +9,7 @@ from mock import Mock, patch
 
 from celery import current_app
 from celery import platforms
+from celery.five import open_fqdn
 from celery.platforms import (
     get_fdmax,
     ignore_errno,
@@ -262,7 +263,7 @@ if not current_app.IS_WINDOWS:
         @patch('celery.platforms.signals')
         @patch('celery.platforms.maybe_drop_privileges')
         @patch('os.geteuid')
-        @patch('__builtin__.open')
+        @patch(open_fqdn)
         def test_default(self, open, geteuid, maybe_drop,
                 signals, pidlock):
             geteuid.return_value = 0
@@ -493,7 +494,7 @@ if not current_app.IS_WINDOWS:
         @patch('os.getpid')
         @patch('os.open')
         @patch('os.fdopen')
-        @patch('__builtin__.open')
+        @patch(open_fqdn)
         def test_write_pid(self, open_, fdopen, osopen, getpid, fsync):
             getpid.return_value = 1816
             osopen.return_value = 13
@@ -519,7 +520,7 @@ if not current_app.IS_WINDOWS:
         @patch('os.getpid')
         @patch('os.open')
         @patch('os.fdopen')
-        @patch('__builtin__.open')
+        @patch(open_fqdn)
         def test_write_reread_fails(self, open_, fdopen,
                 osopen, getpid, fsync):
             getpid.return_value = 1816
@@ -545,11 +546,11 @@ if not current_app.IS_WINDOWS:
                     return
                 raise ValueError()
             setgroups.side_effect = on_setgroups
-            _setgroups_hack(range(400))
+            _setgroups_hack(list(range(400)))
 
             setgroups.side_effect = ValueError()
             with self.assertRaises(ValueError):
-                _setgroups_hack(range(400))
+                _setgroups_hack(list(range(400)))
 
         @patch('os.setgroups', create=True)
         def test_setgroups_hack_OSError(self, setgroups):
@@ -563,31 +564,31 @@ if not current_app.IS_WINDOWS:
                 raise exc
             setgroups.side_effect = on_setgroups
 
-            _setgroups_hack(range(400))
+            _setgroups_hack(list(range(400)))
 
             setgroups.side_effect = exc
             with self.assertRaises(OSError):
-                _setgroups_hack(range(400))
+                _setgroups_hack(list(range(400)))
 
             exc2 = OSError()
             exc.errno = errno.ESRCH
             setgroups.side_effect = exc2
             with self.assertRaises(OSError):
-                _setgroups_hack(range(400))
+                _setgroups_hack(list(range(400)))
 
         @patch('os.sysconf')
         @patch('celery.platforms._setgroups_hack')
         def test_setgroups(self, hack, sysconf):
             sysconf.return_value = 100
-            setgroups(range(400))
-            hack.assert_called_with(range(100))
+            setgroups(list(range(400)))
+            hack.assert_called_with(list(range(100)))
 
         @patch('os.sysconf')
         @patch('celery.platforms._setgroups_hack')
         def test_setgroups_sysconf_raises(self, hack, sysconf):
             sysconf.side_effect = ValueError()
-            setgroups(range(400))
-            hack.assert_called_with(range(400))
+            setgroups(list(range(400)))
+            hack.assert_called_with(list(range(400)))
 
         @patch('os.getgroups')
         @patch('os.sysconf')
@@ -598,7 +599,7 @@ if not current_app.IS_WINDOWS:
             esrch.errno = errno.ESRCH
             hack.side_effect = esrch
             with self.assertRaises(OSError):
-                setgroups(range(400))
+                setgroups(list(range(400)))
 
         @patch('os.getgroups')
         @patch('os.sysconf')
@@ -608,11 +609,11 @@ if not current_app.IS_WINDOWS:
             eperm = OSError()
             eperm.errno = errno.EPERM
             hack.side_effect = eperm
-            getgroups.return_value = range(400)
-            setgroups(range(400))
+            getgroups.return_value = list(range(400))
+            setgroups(list(range(400)))
             getgroups.assert_called_with()
 
             getgroups.return_value = [1000]
             with self.assertRaises(OSError):
-                setgroups(range(400))
+                setgroups(list(range(400)))
             getgroups.assert_called_with()

celery/tests/utilities/test_saferef.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+from celery.five import range
 from celery.utils.dispatch.saferef import safe_ref
 from celery.tests.utils import Case
 
@@ -25,14 +26,14 @@ class SaferefTests(Case):
     def setUp(self):
         ts = []
         ss = []
-        for x in xrange(5000):
+        for x in range(5000):
             t = Class1()
             ts.append(t)
             s = safe_ref(t.x, self._closure)
             ss.append(s)
         ts.append(fun)
         ss.append(safe_ref(fun, self._closure))
-        for x in xrange(30):
+        for x in range(30):
             t = Class2()
             ts.append(t)
             s = safe_ref(t, self._closure)

celery/tests/utilities/test_term.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 from celery.utils import term
 from celery.utils.term import colored, fg
@@ -38,31 +38,31 @@ class test_colored(Case):
         self.assertTrue(str(colored().iwhite('f')))
         self.assertTrue(str(colored().reset('f')))
 
-        self.assertTrue(str(colored().green(u'∂bar')))
+        self.assertTrue(str(colored().green('∂bar')))
 
         self.assertTrue(
-            colored().red(u'éefoo') + colored().green(u'∂bar'))
+            colored().red('éefoo') + colored().green('∂bar'))
 
         self.assertEqual(
             colored().red('foo').no_color(), 'foo')
 
         self.assertTrue(
-            repr(colored().blue(u'åfoo')))
+            repr(colored().blue('åfoo')))
 
-        self.assertEqual(repr(colored()), "''")
+        self.assertIn("''", repr(colored()))
 
         c = colored()
         s = c.red('foo', c.blue('bar'), c.green('baz'))
         self.assertTrue(s.no_color())
 
-        c._fold_no_color(s, u'øfoo')
-        c._fold_no_color(u'fooå', s)
+        c._fold_no_color(s, 'øfoo')
+        c._fold_no_color('fooå', s)
 
-        c = colored().red(u'åfoo')
-        self.assertEqual(c._add(c, u'baræ'),
-            u'\x1b[1;31m\xe5foo\x1b[0mbar\xe6')
+        c = colored().red('åfoo')
+        self.assertEqual(c._add(c, 'baræ'),
+            '\x1b[1;31m\xe5foo\x1b[0mbar\xe6')
 
-        c2 = colored().blue(u'ƒƒz')
+        c2 = colored().blue('ƒƒz')
         c3 = c._add(c, c2)
         self.assertEqual(c3,
-            u'\x1b[1;31m\xe5foo\x1b[0m\x1b[1;34m\u0192\u0192z\x1b[0m')
+            '\x1b[1;31m\xe5foo\x1b[0m\x1b[1;34m\u0192\u0192z\x1b[0m')

celery/tests/utilities/test_utils.py

@@ -5,6 +5,7 @@ from kombu.utils.functional import promise
 from mock import patch
 
 from celery import utils
+from celery.five import nextfun, range
 from celery.utils import text
 from celery.utils import functional
 from celery.utils.functional import mpromise, maybe_list
@@ -89,11 +90,11 @@ class test_utils(Case):
                 return True
             return False
 
-        self.assertEqual(5, functional.first(predicate, xrange(10)))
+        self.assertEqual(5, functional.first(predicate, range(10)))
         self.assertEqual(iterations[0], 6)
 
         iterations[0] = 0
-        self.assertIsNone(functional.first(predicate, xrange(10, 20)))
+        self.assertIsNone(functional.first(predicate, range(10, 20)))
         self.assertEqual(iterations[0], 10)
 
     def test_truncate_text(self):
@@ -141,8 +142,8 @@ class test_mpromise(Case):
 
     def test_is_memoized(self):
 
-        it = iter(xrange(20, 30))
-        p = mpromise(it.next)
+        it = iter(range(20, 30))
+        p = mpromise(nextfun(it))
         self.assertEqual(p(), 20)
         self.assertTrue(p.evaluated)
         self.assertEqual(p(), 20)

celery/tests/utils.py

@@ -16,10 +16,6 @@ import re
 import sys
 import time
 import warnings
-try:
-    import __builtin__ as builtins
-except ImportError:  # py3k
-    import builtins  # noqa
 
 from contextlib import contextmanager
 from functools import partial, wraps
@@ -30,9 +26,12 @@ from nose import SkipTest
 from kombu.log import NullHandler
 from kombu.utils import nested
 
-from ..app import app_or_default
-from ..utils.compat import WhateverIO
-from ..utils.functional import noop
+from celery.app import app_or_default
+from celery.five import (
+    WhateverIO, builtins, items, reraise,
+    string_t, values, open_fqdn,
+)
+from celery.utils.functional import noop
 
 
 class Mock(mock.Mock):
@@ -40,7 +39,7 @@ class Mock(mock.Mock):
     def __init__(self, *args, **kwargs):
         attrs = kwargs.pop('attrs', None) or {}
         super(Mock, self).__init__(*args, **kwargs)
-        for attr_name, attr_value in attrs.items():
+        for attr_name, attr_value in items(attrs):
             setattr(self, attr_name, attr_value)
 
 
@@ -70,7 +69,7 @@ class _AssertRaisesBaseContext(object):
         self.expected = expected
         self.failureException = test_case.failureException
         self.obj_name = None
-        if isinstance(expected_regex, basestring):
+        if isinstance(expected_regex, string_t):
             expected_regex = re.compile(expected_regex)
         self.expected_regex = expected_regex
 
@@ -82,7 +81,7 @@ class _AssertWarnsContext(_AssertRaisesBaseContext):
         # The __warningregistry__'s need to be in a pristine state for tests
         # to work properly.
         warnings.resetwarnings()
-        for v in sys.modules.values():
+        for v in list(values(sys.modules)):
             if getattr(v, '__warningregistry__', None):
                 v.__warningregistry__ = {}
         self.warnings_manager = warnings.catch_warnings(record=True)
@@ -138,7 +137,7 @@ class Case(unittest.TestCase):
     def assertDictContainsSubset(self, expected, actual, msg=None):
         missing, mismatched = [], []
 
-        for key, value in expected.iteritems():
+        for key, value in items(expected):
             if key not in actual:
                 missing.append(key)
             elif value != actual[key]:
@@ -454,7 +453,7 @@ def patch_modules(*modules):
     for mod in modules:
         prev[mod], sys.modules[mod] = sys.modules[mod], ModuleType(mod)
     yield
-    for name, mod in prev.iteritems():
+    for name, mod in items(prev):
         if mod is None:
             sys.modules.pop(name, None)
         else:
@@ -500,7 +499,7 @@ def mock_context(mock, typ=Mock):
 
     def on_exit(*x):
         if x[0]:
-            raise x[0], x[1], x[2]
+            reraise(x[0], x[1], x[2])
     context.__exit__.side_effect = on_exit
     context.__enter__.return_value = context
     yield context
@@ -509,7 +508,7 @@ def mock_context(mock, typ=Mock):
 
 @contextmanager
 def mock_open(typ=WhateverIO, side_effect=None):
-    with mock.patch('__builtin__.open') as open_:
+    with mock.patch(open_fqdn) as open_:
         with mock_context(open_) as context:
             if side_effect is not None:
                 context.__enter__.side_effect = side_effect
@@ -528,7 +527,7 @@ def patch_settings(app=None, **config):
         from celery import current_app
         app = current_app
     prev = {}
-    for key, value in config.iteritems():
+    for key, value in items(config):
         try:
             prev[key] = getattr(app.conf, key)
         except AttributeError:
@@ -537,7 +536,7 @@ def patch_settings(app=None, **config):
 
     yield app.conf
 
-    for key, value in prev.iteritems():
+    for key, value in items(prev):
         setattr(app.conf, key, value)
 
 

celery/tests/worker/test_control.py

@@ -6,7 +6,7 @@ import socket
 from datetime import datetime, timedelta
 
 from kombu import pidbox
-from mock import Mock, patch
+from mock import Mock, patch, call
 
 from celery import current_app
 from celery.datastructures import AttributeDict
@@ -71,23 +71,21 @@ class test_ControlPanel(Case):
     def test_enable_events(self):
         consumer = Consumer()
         panel = self.create_panel(consumer=consumer)
-        consumer.event_dispatcher.enabled = False
+        evd = consumer.event_dispatcher
+        evd.groups = set()
         panel.handle('enable_events')
-        self.assertTrue(consumer.event_dispatcher.enable.call_count)
-        self.assertIn(('worker-online', ),
-                consumer.event_dispatcher.send.call_args)
-        consumer.event_dispatcher.enabled = True
+        self.assertIn('task', evd.groups)
+        evd.groups = set(['task'])
         self.assertIn('already enabled', panel.handle('enable_events')['ok'])
 
     def test_disable_events(self):
         consumer = Consumer()
         panel = self.create_panel(consumer=consumer)
-        consumer.event_dispatcher.enabled = True
+        evd = consumer.event_dispatcher
+        evd.enabled = True
+        evd.groups = set(['task'])
         panel.handle('disable_events')
-        self.assertTrue(consumer.event_dispatcher.disable.call_count)
-        self.assertIn(('worker-offline', ),
-                      consumer.event_dispatcher.send.call_args)
-        consumer.event_dispatcher.enabled = False
+        self.assertNotIn('task', evd.groups)
         self.assertIn('already disabled', panel.handle('disable_events')['ok'])
 
     def test_heartbeat(self):
@@ -436,7 +434,7 @@ class test_ControlPanel(Case):
 
         self.assertTrue(consumer.controller.pool.restart.called)
         self.assertFalse(_reload.called)
-        self.assertEqual([(('foo',), {}), (('bar',), {})],
+        self.assertEqual([call('bar'), call('foo')],
                           _import.call_args_list)
 
     def test_pool_restart_reload_modules(self):

celery/tests/worker/test_hub.py

@@ -8,6 +8,7 @@ from celery.worker.hub import (
 
 from mock import Mock, call, patch
 
+from celery.five import range
 from celery.tests.utils import Case
 
 
@@ -24,6 +25,9 @@ class File(object):
             return self.fd == other.fd
         return NotImplemented
 
+    def __hash__(self):
+        return hash(self.fd)
+
 
 class test_DummyLock(Case):
 
@@ -59,7 +63,7 @@ class test_BoundedSemaphore(Case):
 
     def test_bounded(self):
         x = BoundedSemaphore(2)
-        for i in xrange(100):
+        for i in range(100):
             x.release()
         self.assertEqual(x.value, 2)
 
@@ -88,24 +92,24 @@ class test_BoundedSemaphore(Case):
 
         self.assertFalse(x._waiting)
         x.grow(3)
-        for i in xrange(x.initial_value):
+        for i in range(x.initial_value):
             self.assertTrue(x.acquire(Mock()))
         self.assertFalse(x.acquire(Mock()))
         x.clear()
 
         x.shrink(3)
-        for i in xrange(x.initial_value):
+        for i in range(x.initial_value):
             self.assertTrue(x.acquire(Mock()))
         self.assertFalse(x.acquire(Mock()))
         self.assertEqual(x.value, 0)
 
-        for i in xrange(100):
+        for i in range(100):
             x.release()
         self.assertEqual(x.value, x.initial_value)
 
     def test_clear(self):
         x = BoundedSemaphore(10)
-        for i in xrange(11):
+        for i in range(11):
             x.acquire(Mock())
         self.assertTrue(x._waiting)
         self.assertEqual(x.value, 0)
@@ -158,13 +162,13 @@ class test_Hub(Case):
         hub.scheduler = se()
 
         self.assertEqual(hub.fire_timers(max_timers=10), 3.982)
-        hub.timer.apply_entry.assert_has_calls(map(call, [e3, e2, e1]))
+        hub.timer.apply_entry.assert_has_calls([call(x) for x in [e3, e2, e1]])
 
-        entries[:] = [Mock() for _ in xrange(11)]
+        entries[:] = [Mock() for _ in range(11)]
         keep = list(entries)
         self.assertEqual(hub.fire_timers(max_timers=10, min_delay=1.13), 1.13)
-        hub.timer.apply_entry.assert_has_calls(map(call,
-            reversed(keep[1:])))
+        hub.timer.apply_entry.assert_has_calls([call(x)
+            for x in reversed(keep[1:])])
         self.assertEqual(hub.fire_timers(max_timers=10), 3.982)
         hub.timer.apply_entry.assert_has_calls(call(keep[0]))
 

celery/tests/worker/test_mediator.py

@@ -2,10 +2,9 @@ from __future__ import absolute_import
 
 import sys
 
-from Queue import Queue
-
 from mock import Mock, patch
 
+from celery.five import Queue
 from celery.worker.mediator import Mediator
 from celery.worker.state import revoked as revoked_tasks
 from celery.tests.utils import Case

celery/tests/worker/test_request.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 import anyjson
 import os
@@ -25,6 +25,7 @@ from celery.exceptions import (
     InvalidTaskError,
     TaskRevokedError,
 )
+from celery.five import keys
 from celery.task.trace import (
     trace_task,
     _trace_task_ret,
@@ -127,16 +128,16 @@ class test_default_encode(Case):
     def test_jython(self):
         prev, sys.platform = sys.platform, 'java 1.6.1'
         try:
-            self.assertEqual(default_encode('foo'), 'foo')
+            self.assertEqual(default_encode(bytes('foo')), 'foo')
         finally:
             sys.platform = prev
 
-    def test_cython(self):
+    def test_cpython(self):
         prev, sys.platform = sys.platform, 'darwin'
         gfe, sys.getfilesystemencoding = sys.getfilesystemencoding, \
                                          lambda: 'utf-8'
         try:
-            self.assertEqual(default_encode('foo'), 'foo')
+            self.assertEqual(default_encode(bytes('foo')), 'foo')
         finally:
             sys.platform = prev
             sys.getfilesystemencoding = gfe
@@ -668,7 +669,7 @@ class test_TaskRequest(AppCase):
         self.assertTrue(x)
 
     def test_from_message(self):
-        us = u'æØåveéðƒeæ'
+        us = 'æØåveéðƒeæ'
         body = {'task': mytask.name, 'id': uuid(),
                 'args': [2], 'kwargs': {us: 'bar'}}
         m = Message(None, body=anyjson.dumps(body), backend='foo',
@@ -681,8 +682,8 @@ class test_TaskRequest(AppCase):
         self.assertEqual(tw.args, body['args'])
         us = from_utf8(us)
         if sys.version_info < (2, 6):
-            self.assertEqual(tw.kwargs.keys()[0], us)
-            self.assertIsInstance(tw.kwargs.keys()[0], str)
+            self.assertEqual(next(keys(tw.kwargs)), us)
+            self.assertIsInstance(next(keys(tw.kwargs)), str)
 
     def test_from_message_empty_args(self):
         body = {'task': mytask.name, 'id': uuid()}
@@ -704,7 +705,7 @@ class test_TaskRequest(AppCase):
 
     def test_from_message_nonexistant_task(self):
         body = {'task': 'cu.mytask.doesnotexist', 'id': uuid(),
-                'args': [2], 'kwargs': {u'æØåveéðƒeæ': 'bar'}}
+                'args': [2], 'kwargs': {'æØåveéðƒeæ': 'bar'}}
         m = Message(None, body=anyjson.dumps(body), backend='foo',
                           content_type='application/json',
                           content_encoding='utf-8')
@@ -825,8 +826,8 @@ class test_TaskRequest(AppCase):
         self._test_on_failure(Exception('Inside unit tests'))
 
     def test_on_failure_unicode_exception(self):
-        self._test_on_failure(Exception(u'Бобры атакуют'))
+        self._test_on_failure(Exception('Бобры атакуют'))
 
     def test_on_failure_utf8_exception(self):
         self._test_on_failure(Exception(
-            from_utf8(u'Бобры атакуют')))
+            from_utf8('Бобры атакуют')))

celery/tests/worker/test_worker.py

@@ -5,7 +5,6 @@ import socket
 from collections import deque
 from datetime import datetime, timedelta
 from threading import Event
-from Queue import Empty
 
 from billiard.exceptions import WorkerLostError
 from kombu import Connection
@@ -21,6 +20,7 @@ from celery.bootsteps import RUN, CLOSE, TERMINATE, StartStopStep
 from celery.concurrency.base import BasePool
 from celery.datastructures import AttributeDict
 from celery.exceptions import SystemTerminate
+from celery.five import Empty, range
 from celery.task import task as task_dec
 from celery.task import periodic_task as periodic_task_dec
 from celery.utils import uuid
@@ -29,7 +29,7 @@ from celery.worker import components
 from celery.worker.buckets import FastQueue
 from celery.worker.job import Request
 from celery.worker import consumer
-from celery.worker.consumer import Consumer
+from celery.worker.consumer import Consumer as __Consumer
 from celery.utils.serialization import pickle
 from celery.utils.timer2 import Timer
 
@@ -52,6 +52,15 @@ def find_step(obj, typ):
     return obj.namespace.steps[typ.name]
 
 
+class Consumer(__Consumer):
+
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault('enable_mingle', False)  # disable Mingle step
+        kwargs.setdefault('enable_gossip', False)  # disable Gossip step
+        kwargs.setdefault('enable_heartbeat', False)  # disable Heart step
+        super(Consumer, self).__init__(*args, **kwargs)
+
+
 class _MyKombuConsumer(Consumer):
     broadcast_consumer = Mock()
     task_consumer = Mock()
@@ -150,11 +159,11 @@ class test_QoS(Case):
         qos = self._QoS(10)
 
         def add():
-            for i in xrange(1000):
+            for i in range(1000):
                 qos.increment_eventually()
 
         def sub():
-            for i in xrange(1000):
+            for i in range(1000):
                 qos.decrement_eventually()
 
         def threaded(funs):
@@ -355,7 +364,8 @@ class test_Consumer(Case):
         self.assertIn("Can't decode message body", crit.call_args[0][0])
 
     def _get_on_message(self, l):
-        l.qos = Mock()
+        if l.qos is None:
+            l.qos = Mock()
         l.event_dispatcher = Mock()
         l.task_consumer = Mock()
         l.connection = Mock()
@@ -380,7 +390,7 @@ class test_Consumer(Case):
         self.assertEqual(in_bucket.execute(), 2 * 4 * 8)
         self.assertTrue(self.timer.empty())
 
-    def test_start_connection_error(self):
+    def test_start_channel_error(self):
 
         class MockConsumer(Consumer):
             iterations = 0
@@ -393,15 +403,12 @@ class test_Consumer(Case):
 
         l = MockConsumer(self.ready_queue, timer=self.timer,
                              send_events=False, pool=BasePool())
-        l.connection_errors = (KeyError, )
-        with self.assertRaises(SyntaxError):
+        l.channel_errors = (KeyError, )
+        with self.assertRaises(KeyError):
             l.start()
-        l.heart.stop()
         l.timer.stop()
 
-    def test_start_channel_error(self):
-        # Regression test for AMQPChannelExceptions that can occur within the
-        # consumer. (i.e. 404 errors)
+    def test_start_connection_error(self):
 
         class MockConsumer(Consumer):
             iterations = 0
@@ -415,9 +422,8 @@ class test_Consumer(Case):
         l = MockConsumer(self.ready_queue, timer=self.timer,
                              send_events=False, pool=BasePool())
 
-        l.channel_errors = (KeyError, )
+        l.connection_errors = (KeyError, )
         self.assertRaises(SyntaxError, l.start)
-        l.heart.stop()
         l.timer.stop()
 
     def test_loop_ignores_socket_timeout(self):
@@ -485,7 +491,7 @@ class test_Consumer(Case):
 
     def test_ignore_errors(self):
         l = MyKombuConsumer(self.ready_queue, timer=self.timer)
-        l.connection_errors = (KeyError, )
+        l.connection_errors = (AttributeError, KeyError, )
         l.channel_errors = (SyntaxError, )
         ignore_errors(l, Mock(side_effect=AttributeError('foo')))
         ignore_errors(l, Mock(side_effect=KeyError('foo')))
@@ -513,7 +519,7 @@ class test_Consumer(Case):
                            args=[2, 4, 8], kwargs={})
 
         l.task_consumer = Mock()
-        l.qos = QoS(l.task_consumer.qos, 1)
+        qos = l.qos = QoS(l.task_consumer.qos, 1)
         current_pcount = l.qos.value
         l.event_dispatcher = Mock()
         l.enabled = False
@@ -600,7 +606,7 @@ class test_Consumer(Case):
         m.reject.assert_called_with()
         self.assertTrue(logger.critical.call_count)
 
-    def test_receieve_message_eta(self):
+    def test_receive_message_eta(self):
         l = _MyKombuConsumer(self.ready_queue, timer=self.timer)
         l.steps.pop()
         l.event_dispatcher = Mock()

celery/utils/__init__.py

@@ -12,7 +12,6 @@ import os
 import sys
 import traceback
 import warnings
-import types
 import datetime
 
 from functools import partial, wraps
@@ -22,7 +21,7 @@ from pprint import pprint
 from kombu.entity import Exchange, Queue
 
 from celery.exceptions import CPendingDeprecationWarning, CDeprecationWarning
-from .compat import StringIO
+from celery.five import StringIO, items, reraise, string_t
 
 from .functional import noop
 
@@ -95,7 +94,7 @@ def lpmerge(L, R):
 
     Keeps values from `L`, if the value in `R` is :const:`None`."""
     set = L.__setitem__
-    [set(k, v) for k, v in R.iteritems() if v is not None]
+    [set(k, v) for k, v in items(R) if v is not None]
     return L
 
 
@@ -162,7 +161,7 @@ def cry():  # pragma: no cover
     out = StringIO()
     P = partial(print, file=out)
     sep = '=' * 49
-    for tid, frame in sys._current_frames().iteritems():
+    for tid, frame in items(sys._current_frames()):
         thread = tmap.get(tid, main_thread)
         if not thread:
             # skip old junk (left-overs from a fork)
@@ -184,7 +183,7 @@ def maybe_reraise():
     exc_info = sys.exc_info()
     try:
         if exc_info[2]:
-            raise exc_info[0], exc_info[1], exc_info[2]
+            reraise(exc_info[0], exc_info[1], exc_info[2])
     finally:
         # see http://docs.python.org/library/sys.html#sys.exc_info
         del(exc_info)
@@ -193,7 +192,7 @@ def maybe_reraise():
 def strtobool(term, table={'false': False, 'no': False, '0': False,
                              'true':  True, 'yes': True,  '1': True,
                              'on':    True, 'off': False}):
-    if isinstance(term, basestring):
+    if isinstance(term, string_t):
         try:
             return table[term.lower()]
         except KeyError:
@@ -203,12 +202,12 @@ def strtobool(term, table={'false': False, 'no': False, '0': False,
 
 def jsonify(obj):
     """Transforms object making it suitable for json serialization"""
-    if isinstance(obj, (int, float, basestring, types.NoneType)):
+    if isinstance(obj, (int, float, string_t, type(None))):
         return obj
     elif isinstance(obj, (tuple, list)):
         return [jsonify(o) for o in obj]
     elif isinstance(obj, dict):
-        return dict((k, jsonify(v)) for k, v in obj.iteritems())
+        return dict((k, jsonify(v)) for k, v in items(obj))
     # See "Date Time String Format" in the ECMA-262 specification.
     elif isinstance(obj, datetime.datetime):
         r = obj.isoformat()

celery/utils/compat.py

@@ -1,67 +1 @@
-# -*- coding: utf-8 -*-
-"""
-    celery.utils.compat
-    ~~~~~~~~~~~~~~~~~~~
-
-    Compatibility implementations of features
-    only available in newer Python versions.
-
-
-"""
-from __future__ import absolute_import
-
-############## py3k #########################################################
-import sys
-is_py3k = sys.version_info[0] == 3
-
-try:
-    reload = reload                         # noqa
-except NameError:                           # pragma: no cover
-    from imp import reload                  # noqa
-
-try:
-    from UserList import UserList           # noqa
-except ImportError:                         # pragma: no cover
-    from collections import UserList        # noqa
-
-try:
-    from UserDict import UserDict           # noqa
-except ImportError:                         # pragma: no cover
-    from collections import UserDict        # noqa
-
-if is_py3k:                                 # pragma: no cover
-    from io import StringIO, BytesIO
-    from .encoding import bytes_to_str
-
-    class WhateverIO(StringIO):
-
-        def write(self, data):
-            StringIO.write(self, bytes_to_str(data))
-else:
-    from StringIO import StringIO           # noqa
-    BytesIO = WhateverIO = StringIO         # noqa
-
-
-############## collections.OrderedDict ######################################
-# was moved to kombu
-from kombu.utils.compat import OrderedDict  # noqa
-
-############## threading.TIMEOUT_MAX #######################################
-try:
-    from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX
-except ImportError:
-    THREAD_TIMEOUT_MAX = 1e10  # noqa
-
-############## format(int, ',d') ##########################
-
-if sys.version_info >= (2, 7):  # pragma: no cover
-    def format_d(i):
-        return format(i, ',d')
-else:  # pragma: no cover
-    def format_d(i):  # noqa
-        s = '%d' % i
-        groups = []
-        while s and s[-1].isdigit():
-            groups.append(s[-3:])
-            s = s[:-3]
-        return s + ','.join(reversed(groups))
+from celery.five import *  # noqa

celery/utils/debug.py

@@ -10,7 +10,7 @@ from __future__ import absolute_import
 
 import os
 
-from .compat import format_d
+from celery.five import format_d, range
 
 try:
     from psutil import Process
@@ -61,7 +61,7 @@ def sample(x, n, k=0):
 
     """
     j = len(x) // n
-    for _ in xrange(n):
+    for _ in range(n):
         yield x[k]
         k += j
 

celery/utils/dispatch/saferef.py

@@ -10,6 +10,8 @@ from __future__ import absolute_import
 import weakref
 import traceback
 
+from collections import Callable
+
 
 def safe_ref(target, on_delete=None):  # pragma: no cover
     """Return a *safe* weak reference to a callable target
@@ -23,15 +25,15 @@ def safe_ref(target, on_delete=None):  # pragma: no cover
         goes out of scope with the reference object, (either a
         :class:`weakref.ref` or a :class:`BoundMethodWeakref`) as argument.
     """
-    if getattr(target, 'im_self', None) is not None:
+    if getattr(target, '__self__', None) is not None:
         # Turn a bound method into a BoundMethodWeakref instance.
         # Keep track of these instances for lookup by disconnect().
-        assert hasattr(target, 'im_func'), \
-            """safe_ref target {0!r} has im_self, but no im_func: \
+        assert hasattr(target, '__func__'), \
+            """safe_ref target {0!r} has __self__, but no __func__: \
             don't know how to create reference""".format(target)
         return get_bound_method_weakref(target=target,
                                         on_delete=on_delete)
-    if callable(on_delete):
+    if isinstance(on_delete, Callable):
         return weakref.ref(target, on_delete)
     else:
         return weakref.ref(target)
@@ -66,7 +68,7 @@ class BoundMethodWeakref(object):  # pragma: no cover
 
         weak reference to the target object
 
-    .. attribute:: weak_func
+    .. attribute:: weak_fun
 
         weak reference to the target function
 
@@ -112,10 +114,10 @@ class BoundMethodWeakref(object):  # pragma: no cover
         """Return a weak-reference-like instance for a bound method
 
         :param target: the instance-method target for the weak
-            reference, must have `im_self` and `im_func` attributes
+            reference, must have `__self__` and `__func__` attributes
             and be reconstructable via::
 
-                target.im_func.__get__(target.im_self)
+                target.__func__.__get__(target.__self__)
 
             which is true of built-in instance methods.
 
@@ -136,7 +138,7 @@ class BoundMethodWeakref(object):  # pragma: no cover
                 pass
             for function in methods:
                 try:
-                    if callable(function):
+                    if isinstance(function, Callable):
                         function(self)
                 except Exception as exc:
                     try:
@@ -147,10 +149,10 @@ class BoundMethodWeakref(object):  # pragma: no cover
 
         self.deletion_methods = [on_delete]
         self.key = self.calculate_key(target)
-        self.weak_self = weakref.ref(target.im_self, remove)
-        self.weak_func = weakref.ref(target.im_func, remove)
-        self.self_name = str(target.im_self)
-        self.func_name = str(target.im_func.__name__)
+        self.weak_self = weakref.ref(target.__self__, remove)
+        self.weak_fun = weakref.ref(target.__func__, remove)
+        self.self_name = str(target.__self__)
+        self.fun_name = str(target.__func__.__name__)
 
     def calculate_key(cls, target):
         """Calculate the reference key for this reference
@@ -158,7 +160,7 @@ class BoundMethodWeakref(object):  # pragma: no cover
         Currently this is a two-tuple of the `id()`'s of the
         target object and the target function respectively.
         """
-        return id(target.im_self), id(target.im_func)
+        return id(target.__self__), id(target.__func__)
     calculate_key = classmethod(calculate_key)
 
     def __str__(self):
@@ -166,14 +168,15 @@ class BoundMethodWeakref(object):  # pragma: no cover
         return '{0}( {1}.{2} )'.format(
             type(self).__name__,
             self.self_name,
-            self.func_name,
+            self.fun_name,
         )
 
     __repr__ = __str__
 
-    def __nonzero__(self):
+    def __bool__(self):
         """Whether we are still a valid reference"""
         return self() is not None
+    __nonzero__ = __bool__  # py2
 
     def __cmp__(self, other):
         """Compare with another reference"""
@@ -194,7 +197,7 @@ class BoundMethodWeakref(object):  # pragma: no cover
         """
         target = self.weak_self()
         if target is not None:
-            function = self.weak_func()
+            function = self.weak_fun()
             if function is not None:
                 return function.__get__(target)
 
@@ -224,10 +227,10 @@ class BoundNonDescriptorMethodWeakref(BoundMethodWeakref):  # pragma: no cover
         """Return a weak-reference-like instance for a bound method
 
         :param target: the instance-method target for the weak
-            reference, must have `im_self` and `im_func` attributes
+            reference, must have `__self__` and `__func__` attributes
             and be reconstructable via::
 
-                target.im_func.__get__(target.im_self)
+                target.__func__.__get__(target.__self__)
 
             which is true of built-in instance methods.
 
@@ -238,9 +241,9 @@ class BoundNonDescriptorMethodWeakref(BoundMethodWeakref):  # pragma: no cover
             which will be passed a pointer to this object.
 
         """
-        assert getattr(target.im_self, target.__name__) == target, \
+        assert getattr(target.__self__, target.__name__) == target, \
                "method %s isn't available as the attribute %s of %s" % (
-                    target, target.__name__, target.im_self)
+                    target, target.__name__, target.__self__)
         super(BoundNonDescriptorMethodWeakref, self).__init__(target,
                                                               on_delete)
 
@@ -258,7 +261,7 @@ class BoundNonDescriptorMethodWeakref(BoundMethodWeakref):  # pragma: no cover
         """
         target = self.weak_self()
         if target is not None:
-            function = self.weak_func()
+            function = self.weak_fun()
             if function is not None:
                 # Using curry() would be another option, but it erases the
                 # "signature" of the function. That is, after a function is

celery/utils/dispatch/signal.py

@@ -3,14 +3,16 @@
 from __future__ import absolute_import
 
 import weakref
+from collections import Callable
 from . import saferef
+from celery.five import range
 
 WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref)
 
 
 def _make_id(target):  # pragma: no cover
-    if hasattr(target, 'im_func'):
-        return (id(target.im_self), id(target.im_func))
+    if hasattr(target, '__func__'):
+        return (id(target.__self__), id(target.__func__))
     return id(target)
 
 
@@ -90,7 +92,7 @@ class Signal(object):  # pragma: no cover
 
             return _connect_signal
 
-        if args and callable(args[0]):
+        if args and isinstance(args[0], Callable):
             return _handle_options(*args[1:], **kwargs)(args[0])
         return _handle_options(*args, **kwargs)
 
@@ -117,7 +119,7 @@ class Signal(object):  # pragma: no cover
         else:
             lookup_key = (_make_id(receiver), _make_id(sender))
 
-        for index in xrange(len(self.receivers)):
+        for index in range(len(self.receivers)):
             (r_key, _) = self.receivers[index]
             if r_key == lookup_key:
                 del self.receivers[index]

celery/utils/functional.py

@@ -18,7 +18,7 @@ from kombu.utils import cached_property
 from kombu.utils.functional import promise, maybe_promise
 from kombu.utils.compat import OrderedDict
 
-from .compat import UserDict, UserList
+from celery.five import UserDict, UserList, items, keys, string_t
 
 KEYWORD_MARK = object()
 is_not_None = partial(operator.is_not, None)
@@ -46,7 +46,7 @@ class LRUCache(UserDict):
 
     def keys(self):
         # userdict.keys in py3k calls __getitem__
-        return self.data.keys()
+        return keys(self.data)
 
     def values(self):
         return list(self._iterate_values())
@@ -91,7 +91,7 @@ class LRUCache(UserDict):
 
 def is_list(l):
     """Returns true if object is list-like, but not a dict or string."""
-    return hasattr(l, '__iter__') and not isinstance(l, (dict, basestring))
+    return hasattr(l, '__iter__') and not isinstance(l, (dict, string_t))
 
 
 def maybe_list(l):
@@ -107,7 +107,7 @@ def memoize(maxsize=None, Cache=LRUCache):
 
         @wraps(fun)
         def _M(*args, **kwargs):
-            key = args + (KEYWORD_MARK, ) + tuple(sorted(kwargs.iteritems()))
+            key = args + (KEYWORD_MARK, ) + tuple(sorted(kwargs.items()))
             try:
                 with mutex:
                     value = cache[key]
@@ -263,6 +263,6 @@ class _regen(UserList, list):
         return list(self.__it)
 
 
-def dictfilter(d, **keys):
-    d = dict(d, **keys) if keys else d
-    return dict((k, v) for k, v in d.iteritems() if v is not None)
+def dictfilter(d, **filterkeys):
+    d = dict(d, **filterkeys) if filterkeys else d
+    return dict((k, v) for k, v in items(d) if v is not None)