--config now works for all loaders

celery/bin/base.py

@@ -20,7 +20,7 @@ Preload Options
 
 .. cmdoption:: --config
 
-    name of the configuration module (default: `celeryconfig`)
+    Name of the configuration module
 
 .. _daemon-options:
 
@@ -133,7 +133,7 @@ class Command(object):
         Option('-A', '--app', default=None),
         Option('-b', '--broker', default=None),
         Option('--loader', default=None),
-        Option('--config', default='celeryconfig', dest='config_module'),
+        Option('--config', default=None),
     )
 
     #: Enable if the application should support config from the cmdline.
@@ -287,9 +287,9 @@ class Command(object):
         broker = preload_options.get('broker', None)
         if broker:
             os.environ['CELERY_BROKER_URL'] = broker
-        config_module = preload_options.get('config_module')
-        if config_module:
-            os.environ['CELERY_CONFIG_MODULE'] = config_module
+        config = preload_options.get('config')
+        if config:
+            os.environ['CELERY_CONFIG_MODULE'] = config
         if self.respects_app_option:
             if app and self.respects_app_option:
                 self.app = self.find_app(app)

celery/datastructures.py

@@ -232,9 +232,16 @@ class DictAttribute(object):
     `obj[k] -> obj.k`
 
     """
+    obj = None
 
     def __init__(self, obj):
-        self.obj = obj
+        object.__setattr__(self, 'obj', obj)
+
+    def __getattr__(self, key):
+        return getattr(self.obj, key)
+
+    def __setattr__(self, key, value):
+        return setattr(self.obj, key, value)
 
     def get(self, key, default=None):
         try:
@@ -262,14 +269,15 @@ class DictAttribute(object):
         return hasattr(self.obj, key)
 
     def _iterate_keys(self):
-        return iter(vars(self.obj))
+        return iter(dir(self.obj))
     iterkeys = _iterate_keys
 
     def __iter__(self):
         return self._iterate_keys()
 
     def _iterate_items(self):
-        return vars(self.obj).iteritems()
+        for key in self._iterate_keys():
+            yield getattr(self.obj, key)
     iteritems = _iterate_items
 
     if sys.version_info[0] == 3:  # pragma: no cover

celery/loaders/base.py

@@ -12,15 +12,18 @@ import anyjson
 import importlib
 import os
 import re
+import sys
 
 from datetime import datetime
 
 from kombu.utils import cached_property
 from kombu.utils.encoding import safe_str
 
-from celery.datastructures import DictAttribute
+from celery.datastructures import AttributeDict, DictAttribute
 from celery.exceptions import ImproperlyConfigured
-from celery.utils.imports import import_from_cwd, symbol_by_name
+from celery.utils.imports import (
+    import_from_cwd, symbol_by_name, NotAPackage, find_module,
+)
 from celery.utils.functional import maybe_list
 
 BUILTIN_MODULES = frozenset()
@@ -31,6 +34,15 @@ and as such the configuration could not be loaded.
 Please set this variable and make it point to
 a configuration module.""")
 
+CONFIG_INVALID_NAME = """
+Error: Module '%(module)s' doesn't exist, or it's not a valid \
+Python module name.
+"""
+
+CONFIG_WITH_SUFFIX = CONFIG_INVALID_NAME + """
+Did you mean '%(suggest)s'?
+"""
+
 
 class BaseLoader(object):
     """The base class for loaders.
@@ -145,6 +157,24 @@ class BaseLoader(object):
         self._conf = obj
         return True
 
+    def _import_config_module(self, name):
+        try:
+            self.find_module(name)
+        except NotAPackage:
+            if name.endswith('.py'):
+                raise NotAPackage, NotAPackage(
+                        CONFIG_WITH_SUFFIX % {
+                            'module': name,
+                            'suggest': name[:-3]}), sys.exc_info()[2]
+            raise NotAPackage, NotAPackage(
+                    CONFIG_INVALID_NAME % {
+                        'module': name}), sys.exc_info()[2]
+        else:
+            return self.import_from_cwd(name)
+
+    def find_module(self, module):
+        return find_module(module)
+
     def cmdline_config_parser(self, args, namespace='celery',
                 re_type=re.compile(r'\((\w+)\)'),
                 extra_types={'json': anyjson.loads},
@@ -205,6 +235,13 @@ class BaseLoader(object):
         mailer.send(message, fail_silently=fail_silently)
 
     def read_configuration(self):
+        try:
+            custom_config = os.environ['CELERY_CONFIG_MODULE']
+        except KeyError:
+            pass
+        else:
+            usercfg = self._import_config_module(custom_config)
+            return DictAttribute(usercfg)
         return {}
 
     @property

celery/loaders/default.py

@@ -9,13 +9,11 @@
 from __future__ import absolute_import
 
 import os
-import sys
 import warnings
 
-from celery.datastructures import AttributeDict
+from celery.datastructures import DictAttribute
 from celery.exceptions import NotConfigured
 from celery.utils import strtobool
-from celery.utils.imports import NotAPackage, find_module
 
 from .base import BaseLoader
 
@@ -24,24 +22,12 @@ DEFAULT_CONFIG_MODULE = 'celeryconfig'
 #: Warns if configuration file is missing if :envvar:`C_WNOCONF` is set.
 C_WNOCONF = strtobool(os.environ.get('C_WNOCONF', False))
 
-CONFIG_INVALID_NAME = """
-Error: Module '%(module)s' doesn't exist, or it's not a valid \
-Python module name.
-"""
-
-CONFIG_WITH_SUFFIX = CONFIG_INVALID_NAME + """
-Did you mean '%(suggest)s'?
-"""
-
 
 class Loader(BaseLoader):
     """The loader used by the default app."""
 
     def setup_settings(self, settingsdict):
-        return AttributeDict(settingsdict)
-
-    def find_module(self, module):
-        return find_module(module)
+        return DictAttribute(settingsdict)
 
     def read_configuration(self):
         """Read configuration from :file:`celeryconfig.py` and configure
@@ -49,16 +35,7 @@ class Loader(BaseLoader):
         configname = os.environ.get('CELERY_CONFIG_MODULE',
                                      DEFAULT_CONFIG_MODULE)
         try:
-            self.find_module(configname)
-        except NotAPackage:
-            if configname.endswith('.py'):
-                raise NotAPackage, NotAPackage(
-                        CONFIG_WITH_SUFFIX % {
-                            'module': configname,
-                            'suggest': configname[:-3]}), sys.exc_info()[2]
-            raise NotAPackage, NotAPackage(
-                    CONFIG_INVALID_NAME % {
-                        'module': configname}), sys.exc_info()[2]
+            usercfg = self._import_config_module(configname)
         except ImportError:
             # billiard sets this if forked using execv
             if C_WNOCONF and not os.environ.get('FORKED_BY_MULTIPROCESSING'):
@@ -67,12 +44,5 @@ class Loader(BaseLoader):
                     'is available to Python.' % (configname, )))
             return self.setup_settings({})
         else:
-            celeryconfig = self.import_from_cwd(configname)
-            usercfg = dict((key, getattr(celeryconfig, key))
-                            for key in dir(celeryconfig)
-                                if self.wanted_module_item(key))
             self.configured = True
             return self.setup_settings(usercfg)
-
-    def wanted_module_item(self, item):
-        return not item.startswith('_')

celery/tests/app/test_loaders.py

@@ -154,22 +154,14 @@ class test_LoaderBase(Case):
 
 class test_DefaultLoader(Case):
 
-    def test_wanted_module_item(self):
-        l = default.Loader()
-        self.assertTrue(l.wanted_module_item('FOO'))
-        self.assertTrue(l.wanted_module_item('Foo'))
-        self.assertFalse(l.wanted_module_item('_FOO'))
-        self.assertFalse(l.wanted_module_item('__FOO'))
-        self.assertTrue(l.wanted_module_item('foo'))
-
-    @patch('celery.loaders.default.find_module')
+    @patch('celery.loaders.base.find_module')
     def test_read_configuration_not_a_package(self, find_module):
         find_module.side_effect = NotAPackage()
         l = default.Loader()
         with self.assertRaises(NotAPackage):
             l.read_configuration()
 
-    @patch('celery.loaders.default.find_module')
+    @patch('celery.loaders.base.find_module')
     def test_read_configuration_py_in_name(self, find_module):
         prev = os.environ['CELERY_CONFIG_MODULE']
         os.environ['CELERY_CONFIG_MODULE'] = 'celeryconfig.py'
@@ -181,7 +173,7 @@ class test_DefaultLoader(Case):
         finally:
             os.environ['CELERY_CONFIG_MODULE'] = prev
 
-    @patch('celery.loaders.default.find_module')
+    @patch('celery.loaders.base.find_module')
     def test_read_configuration_importerror(self, find_module):
         default.C_WNOCONF = True
         find_module.side_effect = ImportError()
@@ -239,7 +231,7 @@ class test_DefaultLoader(Case):
 
         with catch_warnings(record=True):
             l = _Loader()
-            self.assertDictEqual(l.conf, {})
+            self.assertFalse(l.configured)
             context_executed[0] = True
         self.assertTrue(context_executed[0])