Ask Solem 12 years ago
parent
commit
ceab8c087d

+ 5 - 5
celery/__init__.py

@@ -44,12 +44,12 @@ from .__compat__ import recreate_module
 
 
 old_module, new_module = recreate_module(__name__,  # pragma: no cover
 old_module, new_module = recreate_module(__name__,  # pragma: no cover
     by_module={
     by_module={
-        'celery.app':      ['Celery', 'bugreport', 'shared_task'],
+        'celery.app': ['Celery', 'bugreport', 'shared_task'],
         'celery.app.task': ['Task'],
         'celery.app.task': ['Task'],
-        'celery._state':   ['current_app', 'current_task'],
-        'celery.canvas':   ['chain', 'chord', 'chunks', 'group',
-                            'subtask', 'xmap', 'xstarmap'],
-        'celery.utils':    ['uuid'],
+        'celery._state': ['current_app', 'current_task'],
+        'celery.canvas': ['chain', 'chord', 'chunks', 'group',
+                          'subtask', 'xmap', 'xstarmap'],
+        'celery.utils': ['uuid'],
     },
     },
     direct={'task': 'celery.task'},
     direct={'task': 'celery.task'},
     __package__='celery', __file__=__file__,
     __package__='celery', __file__=__file__,

+ 2 - 1
celery/app/amqp.py

@@ -377,7 +377,8 @@ class AMQP(object):
     @property
     @property
     def producer_pool(self):
     def producer_pool(self):
         if self._producer_pool is None:
         if self._producer_pool is None:
-            self._producer_pool = ProducerPool(self.app.pool,
+            self._producer_pool = ProducerPool(
+                self.app.pool,
                 limit=self.app.pool.limit,
                 limit=self.app.pool.limit,
                 Producer=self.TaskProducer,
                 Producer=self.TaskProducer,
             )
             )

+ 2 - 1
celery/apps/beat.py

@@ -55,7 +55,8 @@ class Beat(configurated):
         self.max_interval = max_interval
         self.max_interval = max_interval
         self.socket_timeout = socket_timeout
         self.socket_timeout = socket_timeout
         self.no_color = no_color
         self.no_color = no_color
-        self.colored = app.log.colored(self.logfile,
+        self.colored = app.log.colored(
+            self.logfile,
             enabled=not no_color if no_color is not None else no_color,
             enabled=not no_color if no_color is not None else no_color,
         )
         )
         self.pidfile = pidfile
         self.pidfile = pidfile

+ 2 - 1
celery/apps/worker.py

@@ -139,7 +139,8 @@ class Worker(configurated):
             self.autoscale = [int(max_c), min_c and int(min_c) or 0]
             self.autoscale = [int(max_c), min_c and int(min_c) or 0]
         self._isatty = isatty(sys.stdout)
         self._isatty = isatty(sys.stdout)
 
 
-        self.colored = app.log.colored(self.logfile,
+        self.colored = app.log.colored(
+            self.logfile,
             enabled=not no_color if no_color is not None else no_color
             enabled=not no_color if no_color is not None else no_color
         )
         )
 
 

+ 3 - 5
celery/backends/mongodb.py

@@ -181,11 +181,9 @@ class MongoBackend(BaseDictBackend):
 
 
     def cleanup(self):
     def cleanup(self):
         """Delete expired metadata."""
         """Delete expired metadata."""
-        self.collection.remove({
-                'date_done': {
-                    '$lt': self.app.now() - self.expires,
-                 }
-        })
+        self.collection.remove(
+            {'date_done': {'$lt': self.app.now() - self.expires}},
+        )
 
 
     def __reduce__(self, args=(), kwargs={}):
     def __reduce__(self, args=(), kwargs={}):
         kwargs.update(
         kwargs.update(

+ 2 - 2
celery/backends/redis.py

@@ -53,8 +53,8 @@ class RedisBackend(KeyValueStoreBackend):
         conf = self.app.conf
         conf = self.app.conf
         if self.redis is None:
         if self.redis is None:
             raise ImproperlyConfigured(
             raise ImproperlyConfigured(
-                    'You need to install the redis library in order to use '
-                  + 'the Redis result store backend.')
+                'You need to install the redis library in order to use '
+                'the Redis result store backend.')
 
 
         # For compatibility with the old REDIS_* configuration keys.
         # For compatibility with the old REDIS_* configuration keys.
         def _get(key):
         def _get(key):

+ 8 - 8
celery/bin/base.py

@@ -249,14 +249,14 @@ class Command(object):
         return parser.parse_args(arguments)
         return parser.parse_args(arguments)
 
 
     def create_parser(self, prog_name, command=None):
     def create_parser(self, prog_name, command=None):
-        return self.prepare_parser(self.Parser(prog=prog_name,
-                           usage=self.usage(command),
-                           version=self.version,
-                           epilog=self.epilog,
-                           formatter=HelpFormatter(),
-                           description=self.description,
-                           option_list=(self.preload_options +
-                                        self.get_options())))
+        return self.prepare_parser(self.Parser(
+            prog=prog_name,
+            usage=self.usage(command),
+            version=self.version,
+            epilog=self.epilog,
+            formatter=HelpFormatter(),
+            description=self.description,
+            option_list=(self.preload_options + self.get_options())))
 
 
     def prepare_parser(self, parser):
     def prepare_parser(self, parser):
         docs = [self.parse_doc(doc) for doc in (self.doc, __doc__) if doc]
         docs = [self.parse_doc(doc) for doc in (self.doc, __doc__) if doc]

+ 3 - 3
celery/loaders/base.py

@@ -28,11 +28,11 @@ from celery.utils.functional import maybe_list
 
 
 BUILTIN_MODULES = frozenset()
 BUILTIN_MODULES = frozenset()
 
 
-ERROR_ENVVAR_NOT_SET = (
-"""The environment variable %r is not set,
+ERROR_ENVVAR_NOT_SET = """\
+The environment variable %r is not set,
 and as such the configuration could not be loaded.
 and as such the configuration could not be loaded.
 Please set this variable and make it point to
 Please set this variable and make it point to
-a configuration module.""")
+a configuration module."""
 
 
 CONFIG_INVALID_NAME = """
 CONFIG_INVALID_NAME = """
 Error: Module '%(module)s' doesn't exist, or it's not a valid \
 Error: Module '%(module)s' doesn't exist, or it's not a valid \

+ 8 - 6
celery/schedules.py

@@ -197,7 +197,9 @@ class crontab_parser(object):
         if len(toks) > 1:
         if len(toks) > 1:
             to = self._expand_number(toks[1])
             to = self._expand_number(toks[1])
             if to < fr:  # Wrap around max_ if necessary
             if to < fr:  # Wrap around max_ if necessary
-                return range(fr, self.min_ + self.max_) + range(self.min_, to + 1)
+                return range(fr,
+                             self.min_ + self.max_) + range(self.min_,
+                                                            to + 1)
             return range(fr, to + 1)
             return range(fr, to + 1)
         return [fr]
         return [fr]
 
 
@@ -396,11 +398,11 @@ class crontab(schedule):
                 datedata.moy = 0
                 datedata.moy = 0
         roll_over()
         roll_over()
 
 
-        while not (datetime(year=datedata.year,
-                            month=months_of_year[datedata.moy],
-                            day=days_of_month[datedata.dom]
-                           ).isoweekday() % 7
-                  ) in self.day_of_week:
+        while not datetime(
+                year=datedata.year,
+                month=months_of_year[datedata.moy],
+                day=days_of_month[datedata.dom]) \
+                    .isoweekday() % 7 in self.day_of_week:
             datedata.dom += 1
             datedata.dom += 1
             roll_over()
             roll_over()
 
 

+ 4 - 4
celery/task/__init__.py

@@ -40,10 +40,10 @@ class module(MagicModule):
 
 
 old_module, new_module = recreate_module(__name__,  # pragma: no cover
 old_module, new_module = recreate_module(__name__,  # pragma: no cover
     by_module={
     by_module={
-        'celery.task.base':   ['BaseTask', 'Task', 'PeriodicTask',
-                               'task', 'periodic_task'],
-        'celery.canvas':      ['group', 'chord', 'subtask'],
-        'celery.task.sets':   ['TaskSet'],
+        'celery.task.base': ['BaseTask', 'Task', 'PeriodicTask',
+                             'task', 'periodic_task'],
+        'celery.canvas': ['group', 'chord', 'subtask'],
+        'celery.task.sets': ['TaskSet'],
     },
     },
     base=module,
     base=module,
     __package__='celery.task',
     __package__='celery.task',

+ 2 - 2
celery/task/http.py

@@ -108,8 +108,8 @@ class MutableURL(object):
         scheme, netloc, path, params, query, fragment = self.parts
         scheme, netloc, path, params, query, fragment = self.parts
         query = urlencode(utf8dict(self.query.items()))
         query = urlencode(utf8dict(self.query.items()))
         components = [scheme + '://', netloc, path or '/',
         components = [scheme + '://', netloc, path or '/',
-                      ';%s' % params   if params   else '',
-                      '?%s' % query    if query    else '',
+                      ';%s' % params if params else '',
+                      '?%s' % query if query else '',
                       '#%s' % fragment if fragment else '']
                       '#%s' % fragment if fragment else '']
         return ''.join(filter(None, components))
         return ''.join(filter(None, components))
 
 

+ 8 - 8
celery/tests/backends/test_database.py

@@ -24,7 +24,7 @@ from celery.tests.utils import (
 try:
 try:
     import sqlalchemy  # noqa
     import sqlalchemy  # noqa
 except ImportError:
 except ImportError:
-    DatabaseBackend = Task = TaskSet = None
+    DatabaseBackend = Task = TaskSet = None  # noqa
 else:
 else:
     from celery.backends.database import DatabaseBackend
     from celery.backends.database import DatabaseBackend
     from celery.backends.database.models import Task, TaskSet
     from celery.backends.database.models import Task, TaskSet
@@ -87,7 +87,7 @@ class test_DatabaseBackend(Case):
             'task_id': 'xxx-does-not-exist-at-all',
             'task_id': 'xxx-does-not-exist-at-all',
             'result': None,
             'result': None,
             'traceback': None,
             'traceback': None,
-            }, tb.get_task_meta('xxx-does-not-exist-at-all'))
+        }, tb.get_task_meta('xxx-does-not-exist-at-all'))
 
 
     def test_mark_as_done(self):
     def test_mark_as_done(self):
         tb = DatabaseBackend()
         tb = DatabaseBackend()
@@ -133,9 +133,9 @@ class test_DatabaseBackend(Case):
             import traceback
             import traceback
             trace = '\n'.join(traceback.format_stack())
             trace = '\n'.join(traceback.format_stack())
             tb.mark_as_retry(tid, exception, traceback=trace)
             tb.mark_as_retry(tid, exception, traceback=trace)
-        self.assertEqual(tb.get_status(tid), states.RETRY)
-        self.assertIsInstance(tb.get_result(tid), KeyError)
-        self.assertEqual(tb.get_traceback(tid), trace)
+            self.assertEqual(tb.get_status(tid), states.RETRY)
+            self.assertIsInstance(tb.get_result(tid), KeyError)
+            self.assertEqual(tb.get_traceback(tid), trace)
 
 
     def test_mark_as_failure(self):
     def test_mark_as_failure(self):
         tb = DatabaseBackend()
         tb = DatabaseBackend()
@@ -147,9 +147,9 @@ class test_DatabaseBackend(Case):
             import traceback
             import traceback
             trace = '\n'.join(traceback.format_stack())
             trace = '\n'.join(traceback.format_stack())
             tb.mark_as_failure(tid3, exception, traceback=trace)
             tb.mark_as_failure(tid3, exception, traceback=trace)
-        self.assertEqual(tb.get_status(tid3), states.FAILURE)
-        self.assertIsInstance(tb.get_result(tid3), KeyError)
-        self.assertEqual(tb.get_traceback(tid3), trace)
+            self.assertEqual(tb.get_status(tid3), states.FAILURE)
+            self.assertIsInstance(tb.get_result(tid3), KeyError)
+            self.assertEqual(tb.get_traceback(tid3), trace)
 
 
     def test_forget(self):
     def test_forget(self):
         tb = DatabaseBackend(backend='memory://')
         tb = DatabaseBackend(backend='memory://')

+ 4 - 3
celery/tests/concurrency/test_concurrency.py

@@ -29,9 +29,10 @@ class test_BasePool(Case):
                      callback=gen_callback('callback'),
                      callback=gen_callback('callback'),
                      accept_callback=gen_callback('accept_callback'))
                      accept_callback=gen_callback('accept_callback'))
 
 
-        self.assertDictContainsSubset({
-                              'target': (1, (8, 16)),
-                              'callback': (2, (42, ))}, scratch)
+        self.assertDictContainsSubset(
+            {'target': (1, (8, 16)), 'callback': (2, (42, ))},
+            scratch,
+        )
         pa1 = scratch['accept_callback']
         pa1 = scratch['accept_callback']
         self.assertEqual(0, pa1[0])
         self.assertEqual(0, pa1[0])
         self.assertEqual(pa1[1][0], os.getpid())
         self.assertEqual(pa1[1][0], os.getpid())

+ 4 - 4
celery/tests/slow/test_buckets.py

@@ -31,10 +31,10 @@ class MockJob(object):
 
 
     def __eq__(self, other):
     def __eq__(self, other):
         if isinstance(other, self.__class__):
         if isinstance(other, self.__class__):
-            return bool(self.id == other.id \
-                    and self.name == other.name \
-                    and self.args == other.args \
-                    and self.kwargs == other.kwargs)
+            return bool(self.id == other.id
+                        and self.name == other.name
+                        and self.args == other.args
+                        and self.kwargs == other.kwargs)
         else:
         else:
             return self == other
             return self == other
 
 

+ 4 - 1
celery/tests/tasks/test_tasks.py

@@ -707,7 +707,10 @@ class test_crontab_parser(Case):
         self.assertEqual(crontab_parser(8).parse('*/2'), set([0, 2, 4, 6]))
         self.assertEqual(crontab_parser(8).parse('*/2'), set([0, 2, 4, 6]))
         self.assertEqual(crontab_parser().parse('2-9/5'), set([2, 7]))
         self.assertEqual(crontab_parser().parse('2-9/5'), set([2, 7]))
         self.assertEqual(crontab_parser().parse('2-10/5'), set([2, 7]))
         self.assertEqual(crontab_parser().parse('2-10/5'), set([2, 7]))
-        self.assertEqual(crontab_parser(min_=1).parse('55-5/3'), set([55, 58, 1, 4])) 
+        self.assertEqual(
+            crontab_parser(min_=1).parse('55-5/3'),
+            set([55, 58, 1, 4]),
+        )
         self.assertEqual(crontab_parser().parse('2-11/5,3'), set([2, 3, 7]))
         self.assertEqual(crontab_parser().parse('2-11/5,3'), set([2, 3, 7]))
         self.assertEqual(crontab_parser().parse('2-4/3,*/5,0-21/4'),
         self.assertEqual(crontab_parser().parse('2-4/3,*/5,0-21/4'),
                 set([0, 2, 4, 5, 8, 10, 12, 15, 16,
                 set([0, 2, 4, 5, 8, 10, 12, 15, 16,

+ 12 - 8
celery/tests/utilities/test_info.py

@@ -18,14 +18,18 @@ RANDTEXT_RES = """\
     lazy dog\
     lazy dog\
 """
 """
 
 
-QUEUES = {'queue1': {
-            'exchange': 'exchange1',
-            'exchange_type': 'type1',
-            'routing_key': 'bind1'},
-         'queue2': {
-            'exchange': 'exchange2',
-            'exchange_type': 'type2',
-            'routing_key': 'bind2'}}
+QUEUES = {
+    'queue1': {
+        'exchange': 'exchange1',
+        'exchange_type': 'type1',
+        'routing_key': 'bind1',
+    },
+    'queue2': {
+        'exchange': 'exchange2',
+        'exchange_type': 'type2',
+        'routing_key': 'bind2',
+    },
+}
 
 
 
 
 QUEUE_FORMAT1 = """. queue1:      exchange:exchange1(type1) binding:bind1"""
 QUEUE_FORMAT1 = """. queue1:      exchange:exchange1(type1) binding:bind1"""

+ 10 - 7
celery/tests/utils.py

@@ -2,11 +2,11 @@ from __future__ import absolute_import
 from __future__ import with_statement
 from __future__ import with_statement
 
 
 try:
 try:
-    import unittest
+    import unittest  # noqa
     unittest.skip
     unittest.skip
     from unittest.util import safe_repr, unorderable_list_difference
     from unittest.util import safe_repr, unorderable_list_difference
 except AttributeError:
 except AttributeError:
-    import unittest2 as unittest
+    import unittest2 as unittest  # noqa
     from unittest2.util import safe_repr, unorderable_list_difference  # noqa
     from unittest2.util import safe_repr, unorderable_list_difference  # noqa
 
 
 import importlib
 import importlib
@@ -110,7 +110,7 @@ class _AssertWarnsContext(_AssertRaisesBaseContext):
             if first_matching is None:
             if first_matching is None:
                 first_matching = w
                 first_matching = w
             if (self.expected_regex is not None and
             if (self.expected_regex is not None and
-                not self.expected_regex.search(str(w))):
+                    not self.expected_regex.search(str(w))):
                 continue
                 continue
             # store warning for later retrieval
             # store warning for later retrieval
             self.warning = w
             self.warning = w
@@ -165,6 +165,7 @@ class Case(unittest.TestCase):
         self.fail(self._formatMessage(msg, standard_msg))
         self.fail(self._formatMessage(msg, standard_msg))
 
 
     def assertItemsEqual(self, expected_seq, actual_seq, msg=None):
     def assertItemsEqual(self, expected_seq, actual_seq, msg=None):
+        missing = unexpected = None
         try:
         try:
             expected = sorted(expected_seq)
             expected = sorted(expected_seq)
             actual = sorted(actual_seq)
             actual = sorted(actual_seq)
@@ -179,11 +180,13 @@ class Case(unittest.TestCase):
 
 
         errors = []
         errors = []
         if missing:
         if missing:
-            errors.append('Expected, but missing:\n    %s' % (
-                           safe_repr(missing)))
+            errors.append(
+                'Expected, but missing:\n    %s' % (safe_repr(missing), ),
+            )
         if unexpected:
         if unexpected:
-            errors.append('Unexpected, but present:\n    %s' % (
-                           safe_repr(unexpected)))
+            errors.append(
+                'Unexpected, but present:\n    %s' % (safe_repr(unexpected), ),
+            )
         if errors:
         if errors:
             standardMsg = '\n'.join(errors)
             standardMsg = '\n'.join(errors)
             self.fail(self._formatMessage(msg, standardMsg))
             self.fail(self._formatMessage(msg, standardMsg))

+ 4 - 2
celery/tests/worker/test_control.py

@@ -266,8 +266,10 @@ class test_ControlPanel(Case):
         app = current_app
         app = current_app
         app.conf.CELERY_DISABLE_RATE_LIMITS = True
         app.conf.CELERY_DISABLE_RATE_LIMITS = True
         try:
         try:
-            e = self.panel.handle('rate_limit', arguments=dict(
-                 task_name=mytask.name, rate_limit='100/m'))
+            e = self.panel.handle(
+                'rate_limit',
+                arguments={'task_name': mytask.name,
+                           'rate_limit': '100/m'})
             self.assertIn('rate limits disabled', e.get('error'))
             self.assertIn('rate limits disabled', e.get('error'))
         finally:
         finally:
             app.conf.CELERY_DISABLE_RATE_LIMITS = False
             app.conf.CELERY_DISABLE_RATE_LIMITS = False

+ 3 - 3
celery/utils/__init__.py

@@ -129,7 +129,7 @@ def fun_takes_kwargs(fun, kwlist=[]):
     """
     """
     argspec = getattr(fun, 'argspec', getargspec(fun))
     argspec = getattr(fun, 'argspec', getargspec(fun))
     args, _varargs, keywords, _defaults = argspec
     args, _varargs, keywords, _defaults = argspec
-    if keywords != None:
+    if keywords is not None:
         return kwlist
         return kwlist
     return filter(partial(operator.contains, args), kwlist)
     return filter(partial(operator.contains, args), kwlist)
 
 
@@ -189,8 +189,8 @@ def maybe_reraise():
 
 
 
 
 def strtobool(term, table={'false': False, 'no': False, '0': False,
 def strtobool(term, table={'false': False, 'no': False, '0': False,
-                             'true':  True, 'yes': True,  '1': True,
-                             'on':    True, 'off': False}):
+                           'true': True, 'yes': True, '1': True,
+                           'on': True, 'off': False}):
     if isinstance(term, basestring):
     if isinstance(term, basestring):
         try:
         try:
             return table[term.lower()]
             return table[term.lower()]

+ 2 - 2
celery/utils/dispatch/saferef.py

@@ -239,8 +239,8 @@ class BoundNonDescriptorMethodWeakref(BoundMethodWeakref):  # pragma: no cover
 
 
         """
         """
         assert getattr(target.im_self, target.__name__) == target, \
         assert getattr(target.im_self, target.__name__) == target, \
-               "method %s isn't available as the attribute %s of %s" % (
-                    target, target.__name__, target.im_self)
+            "method %s isn't available as the attribute %s of %s" % (
+            target, target.__name__, target.im_self)
         super(BoundNonDescriptorMethodWeakref, self).__init__(target,
         super(BoundNonDescriptorMethodWeakref, self).__init__(target,
                                                               on_delete)
                                                               on_delete)
 
 

+ 1 - 1
celery/utils/mail.py

@@ -184,7 +184,7 @@ py-celery at %%(hostname)s.
     def should_send(self, context, exc):
     def should_send(self, context, exc):
         """Returns true or false depending on if a task error mail
         """Returns true or false depending on if a task error mail
         should be sent for this type of error."""
         should be sent for this type of error."""
-        allow_classes = tuple(map(symbol_by_name,  self.error_whitelist))
+        allow_classes = tuple(map(symbol_by_name, self.error_whitelist))
         return not self.error_whitelist or isinstance(exc, allow_classes)
         return not self.error_whitelist or isinstance(exc, allow_classes)
 
 
     def format_subject(self, context):
     def format_subject(self, context):

+ 1 - 1
celery/worker/consumer.py

@@ -828,7 +828,7 @@ class Consumer(object):
             q = queues[queue]
             q = queues[queue]
         else:
         else:
             exchange = queue if exchange is None else exchange
             exchange = queue if exchange is None else exchange
-            exchange_type = ('direct' if   exchange_type is None
+            exchange_type = ('direct' if exchange_type is None
                                       else exchange_type)
                                       else exchange_type)
             q = queues.select_add(queue,
             q = queues.select_add(queue,
                                   exchange=exchange,
                                   exchange=exchange,

+ 1 - 1
celery/worker/state.py

@@ -12,6 +12,7 @@
 from __future__ import absolute_import
 from __future__ import absolute_import
 
 
 import os
 import os
+import sys
 import platform
 import platform
 import shelve
 import shelve
 
 
@@ -104,7 +105,6 @@ if C_BENCH:  # pragma: no cover
 
 
         return __reserved(request)
         return __reserved(request)
 
 
-    import sys
     def task_ready(request):  # noqa
     def task_ready(request):  # noqa
         global all_count
         global all_count
         global bench_start
         global bench_start

+ 1 - 1
pavement.py

@@ -59,7 +59,7 @@ def verifyconfigref(options):
 def flake8(options):
 def flake8(options):
     noerror = getattr(options, 'noerror', False)
     noerror = getattr(options, 'noerror', False)
     complexity = getattr(options, 'complexity', 22)
     complexity = getattr(options, 'complexity', 22)
-    sh("""flake8 --ignore=E126,E127,E128 celery | perl -mstrict -mwarnings -nle'
+    sh("""flake8 --ignore=W602,E126,E127,E128 celery | perl -mstrict -mwarnings -nle'
         my $ignore = m/too complex \((\d+)\)/ && $1 le %s;
         my $ignore = m/too complex \((\d+)\)/ && $1 le %s;
         if (! $ignore) { print STDERR; our $FOUND_FLAKE = 1 }
         if (! $ignore) { print STDERR; our $FOUND_FLAKE = 1 }
     }{exit $FOUND_FLAKE;
     }{exit $FOUND_FLAKE;