Merge branch '3.0'

Conflicts:
	celery/loaders/base.py
	celery/loaders/default.py
	celery/utils/timer2.py

celery/bin/base.py

@@ -20,7 +20,7 @@ Preload Options
 
 .. cmdoption:: --config
 
-    name of the configuration module (default: `celeryconfig`)
+    Name of the configuration module
 
 .. _daemon-options:
 
@@ -136,7 +136,7 @@ class Command(object):
         Option('-A', '--app', default=None),
         Option('-b', '--broker', default=None),
         Option('--loader', default=None),
-        Option('--config', default='celeryconfig', dest='config_module'),
+        Option('--config', default=None),
     )
 
     #: Enable if the application should support config from the cmdline.
@@ -290,9 +290,9 @@ class Command(object):
         broker = preload_options.get('broker', None)
         if broker:
             os.environ['CELERY_BROKER_URL'] = broker
-        config_module = preload_options.get('config_module')
-        if config_module:
-            os.environ['CELERY_CONFIG_MODULE'] = config_module
+        config = preload_options.get('config')
+        if config:
+            os.environ['CELERY_CONFIG_MODULE'] = config
         if self.respects_app_option:
             if app and self.respects_app_option:
                 self.app = self.find_app(app)

celery/datastructures.py

@@ -352,9 +352,16 @@ class DictAttribute(object):
     `obj[k] -> obj.k`
 
     """
+    obj = None
 
     def __init__(self, obj):
-        self.obj = obj
+        object.__setattr__(self, 'obj', obj)
+
+    def __getattr__(self, key):
+        return getattr(self.obj, key)
+
+    def __setattr__(self, key, value):
+        return setattr(self.obj, key, value)
 
     def get(self, key, default=None):
         try:
@@ -382,14 +389,15 @@ class DictAttribute(object):
         return hasattr(self.obj, key)
 
     def _iterate_keys(self):
-        return iter(vars(self.obj))
+        return iter(dir(self.obj))
     iterkeys = _iterate_keys
 
     def __iter__(self):
         return self._iterate_keys()
 
     def _iterate_items(self):
-        return vars(self.obj).iteritems()
+        for key in self._iterate_keys():
+            yield getattr(self.obj, key)
     iteritems = _iterate_items
 
     if sys.version_info[0] == 3:  # pragma: no cover

celery/loaders/base.py

@@ -13,6 +13,7 @@ import imp
 import importlib
 import os
 import re
+import sys
 
 from datetime import datetime
 from itertools import imap
@@ -22,7 +23,9 @@ from kombu.utils.encoding import safe_str
 
 from celery.datastructures import DictAttribute
 from celery.exceptions import ImproperlyConfigured
-from celery.utils.imports import import_from_cwd, symbol_by_name
+from celery.utils.imports import (
+    import_from_cwd, symbol_by_name, NotAPackage, find_module,
+)
 from celery.utils.functional import maybe_list
 
 BUILTIN_MODULES = frozenset()
@@ -34,6 +37,14 @@ Please set this variable and make it point to
 a configuration module.""")
 
 _RACE_PROTECTION = False
+CONFIG_INVALID_NAME = """
+Error: Module '{module}' doesn't exist, or it's not a valid \
+Python module name.
+"""
+
+CONFIG_WITH_SUFFIX = CONFIG_INVALID_NAME + """
+Did you mean '{suggest}'?
+"""
 
 
 class BaseLoader(object):
@@ -150,6 +161,24 @@ class BaseLoader(object):
         self._conf = obj
         return True
 
+    def _import_config_module(self, name):
+        try:
+            self.find_module(name)
+        except NotAPackage:
+            if name.endswith('.py'):
+                raise NotAPackage, NotAPackage(
+                        CONFIG_WITH_SUFFIX.format(
+                            module=name,
+                            suggest=name[:-3])), sys.exc_info()[2]
+            raise NotAPackage, NotAPackage(
+                    CONFIG_INVALID_NAME.format(
+                        module=name)), sys.exc_info()[2]
+        else:
+            return self.import_from_cwd(name)
+
+    def find_module(self, module):
+        return find_module(module)
+
     def cmdline_config_parser(self, args, namespace='celery',
                 re_type=re.compile(r'\((\w+)\)'),
                 extra_types={'json': anyjson.loads},
@@ -210,6 +239,13 @@ class BaseLoader(object):
         mailer.send(message, fail_silently=fail_silently)
 
     def read_configuration(self):
+        try:
+            custom_config = os.environ['CELERY_CONFIG_MODULE']
+        except KeyError:
+            pass
+        else:
+            usercfg = self._import_config_module(custom_config)
+            return DictAttribute(usercfg)
         return {}
 
     def autodiscover_tasks(self, packages, related_name='tasks'):

celery/loaders/default.py

@@ -9,13 +9,11 @@
 from __future__ import absolute_import
 
 import os
-import sys
 import warnings
 
-from celery.datastructures import AttributeDict
+from celery.datastructures import DictAttribute
 from celery.exceptions import NotConfigured
 from celery.utils import strtobool
-from celery.utils.imports import NotAPackage, find_module
 
 from .base import BaseLoader
 
@@ -24,24 +22,12 @@ DEFAULT_CONFIG_MODULE = 'celeryconfig'
 #: Warns if configuration file is missing if :envvar:`C_WNOCONF` is set.
 C_WNOCONF = strtobool(os.environ.get('C_WNOCONF', False))
 
-CONFIG_INVALID_NAME = """
-Error: Module '{module}' doesn't exist, or it's not a valid \
-Python module name.
-"""
-
-CONFIG_WITH_SUFFIX = CONFIG_INVALID_NAME + """
-Did you mean '{suggest}'?
-"""
-
 
 class Loader(BaseLoader):
     """The loader used by the default app."""
 
     def setup_settings(self, settingsdict):
-        return AttributeDict(settingsdict)
-
-    def find_module(self, module):
-        return find_module(module)
+        return DictAttribute(settingsdict)
 
     def read_configuration(self):
         """Read configuration from :file:`celeryconfig.py` and configure
@@ -49,16 +35,7 @@ class Loader(BaseLoader):
         configname = os.environ.get('CELERY_CONFIG_MODULE',
                                      DEFAULT_CONFIG_MODULE)
         try:
-            self.find_module(configname)
-        except NotAPackage:
-            if configname.endswith('.py'):
-                raise NotAPackage, NotAPackage(
-                        CONFIG_WITH_SUFFIX.format(
-                            module=configname,
-                            suggest=configname[:-3])), sys.exc_info()[2]
-            raise NotAPackage, NotAPackage(
-                    CONFIG_INVALID_NAME.format(
-                        module=configname)), sys.exc_info()[2]
+            usercfg = self._import_config_module(configname)
         except ImportError:
             # billiard sets this if forked using execv
             if C_WNOCONF and not os.environ.get('FORKED_BY_MULTIPROCESSING'):
@@ -67,12 +44,5 @@ class Loader(BaseLoader):
                     'is available to Python.'.format(module=configname)))
             return self.setup_settings({})
         else:
-            celeryconfig = self.import_from_cwd(configname)
-            usercfg = dict((key, getattr(celeryconfig, key))
-                            for key in dir(celeryconfig)
-                                if self.wanted_module_item(key))
             self.configured = True
             return self.setup_settings(usercfg)
-
-    def wanted_module_item(self, item):
-        return not item.startswith('_')

celery/tests/app/test_loaders.py

@@ -153,22 +153,14 @@ class test_LoaderBase(Case):
 
 class test_DefaultLoader(Case):
 
-    def test_wanted_module_item(self):
-        l = default.Loader()
-        self.assertTrue(l.wanted_module_item('FOO'))
-        self.assertTrue(l.wanted_module_item('Foo'))
-        self.assertFalse(l.wanted_module_item('_FOO'))
-        self.assertFalse(l.wanted_module_item('__FOO'))
-        self.assertTrue(l.wanted_module_item('foo'))
-
-    @patch('celery.loaders.default.find_module')
+    @patch('celery.loaders.base.find_module')
     def test_read_configuration_not_a_package(self, find_module):
         find_module.side_effect = NotAPackage()
         l = default.Loader()
         with self.assertRaises(NotAPackage):
             l.read_configuration()
 
-    @patch('celery.loaders.default.find_module')
+    @patch('celery.loaders.base.find_module')
     def test_read_configuration_py_in_name(self, find_module):
         prev = os.environ['CELERY_CONFIG_MODULE']
         os.environ['CELERY_CONFIG_MODULE'] = 'celeryconfig.py'
@@ -180,7 +172,7 @@ class test_DefaultLoader(Case):
         finally:
             os.environ['CELERY_CONFIG_MODULE'] = prev
 
-    @patch('celery.loaders.default.find_module')
+    @patch('celery.loaders.base.find_module')
     def test_read_configuration_importerror(self, find_module):
         default.C_WNOCONF = True
         find_module.side_effect = ImportError()
@@ -238,7 +230,7 @@ class test_DefaultLoader(Case):
 
         with warnings.catch_warnings(record=True):
             l = _Loader()
-            self.assertDictEqual(l.conf, {})
+            self.assertFalse(l.configured)
             context_executed[0] = True
         self.assertTrue(context_executed[0])
 

celery/utils/timer2.py

@@ -17,9 +17,10 @@ import threading
 from datetime import datetime, timedelta
 from functools import wraps
 from itertools import count, imap
-from time import time, sleep, mktime
+from time import time, sleep
 
 from celery.utils.compat import THREAD_TIMEOUT_MAX
+from celery.utils.timeutils import timedelta_seconds, timezone
 from kombu.log import get_logger
 
 VERSION = (1, 0, 0)
@@ -31,6 +32,7 @@ __docformat__ = 'restructuredtext'
 
 DEFAULT_MAX_INTERVAL = 2
 TIMER_DEBUG = os.environ.get('TIMER_DEBUG')
+EPOCH = datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc)
 
 logger = get_logger('timer2')
 
@@ -71,7 +73,9 @@ class Entry(object):
 
 def to_timestamp(d):
     if isinstance(d, datetime):
-        return mktime(d.timetuple())
+        if d.tzinfo is None:
+            d = d.replace(tzinfo=timezone.utc)
+        return timedelta_seconds(d - EPOCH)
     return d