Configuration is now referencing the original dict, not copying it.

Also small fixes found after running django-celery test-suite

celery/app/base.py

@@ -2,6 +2,7 @@ import os
 import sys
 
 from datetime import timedelta
+from itertools import chain
 
 from celery import routes
 from celery.app.defaults import DEFAULTS
@@ -10,6 +11,55 @@ from celery.utils import noop, isatty
 from celery.utils.functional import wraps
 
 
+class MultiDictView(AttributeDict):
+    """View for one more more dicts.
+
+    * When getting a key, the dicts are searched in order.
+    * When setting a key, the key is added to the first dict.
+
+    """
+    dicts = None
+
+    def __init__(self, *dicts):
+        self.__dict__["dicts"] = dicts
+
+    def __getitem__(self, key):
+        for d in self.__dict__["dicts"]:
+            try:
+                return d[key]
+            except KeyError:
+                pass
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        self.__dict__["dicts"][0][key] = value
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def setdefault(self, key, default):
+        try:
+            return self[key]
+        except KeyError:
+            self[key] = default
+            return default
+
+    def __contains__(self, key):
+        for d in self.__dict__["dicts"]:
+            if key in d:
+                return True
+        return False
+
+    def __repr__(self):
+        return repr(dict(iter(self)))
+
+    def __iter__(self):
+        return chain(*[d.iteritems() for d in self.__dict__["dicts"]])
+
+
 class BaseApp(object):
     _amqp = None
     _backend = None
@@ -30,14 +80,13 @@ class BaseApp(object):
 
     def merge(self, a, b):
         """Like ``dict(a, **b)`` except it will keep values from ``a``
-        if the value in ``b`` is :const:`None`""".
+        if the value in ``b`` is :const:`None`."""
         b = dict(b)
         for key, value in a.items():
             if b.get(key) is None:
                 b[key] = value
         return b
 
-
     def AsyncResult(self, task_id, backend=None):
         from celery.result import BaseAsyncResult
         return BaseAsyncResult(task_id, app=self,
@@ -47,7 +96,6 @@ class BaseApp(object):
         from celery.result import TaskSetResult
         return TaskSetResult(taskset_id, results, app=self)
 
-
     def send_task(self, name, args=None, kwargs=None, countdown=None,
             eta=None, task_id=None, publisher=None, connection=None,
             connect_timeout=None, result_cls=None, expires=None,
@@ -117,12 +165,13 @@ class BaseApp(object):
 
     def pre_config_merge(self, c):
         if not c.get("CELERY_RESULT_BACKEND"):
-            c["CELERY_RESULT_BACKEND"] = c.get("CELERY_BACKEND")
+            rbackend = c.get("CELERY_BACKEND")
+            if rbackend:
+                c["CELERY_RESULT_BACKEND"] = backend
         if not c.get("BROKER_BACKEND"):
-            c["BROKER_BACKEND"] = c.get("BROKER_TRANSPORT")  or \
-                                    c.get("CARROT_BACKEND")
-        c.setdefault("CELERY_SEND_TASK_ERROR_EMAILS",
-                     c.get("SEND_CELERY_TASK_ERROR_EMAILS"))
+            cbackend = c.get("BROKER_TRANSPORT") or c.get("CARROT_BACKEND")
+            if cbackend:
+                c["BROKER_BACKEND"] = cbackend
         return c
 
     def post_config_merge(self, c):
@@ -185,7 +234,7 @@ class BaseApp(object):
         if self._conf is None:
             config = self.pre_config_merge(self.loader.conf)
             self._conf = self.post_config_merge(
-                            AttributeDict(DEFAULTS, **config))
+                            MultiDictView(config, DEFAULTS))
         return self._conf
 
     @property

celery/app/defaults.py

@@ -11,6 +11,7 @@ DEFAULT_TASK_LOG_FMT = " ".join("""
 
 
 DEFAULTS = {
+    "BROKER_BACKEND": None,
     "BROKER_CONNECTION_TIMEOUT": 4,
     "BROKER_CONNECTION_RETRY": True,
     "BROKER_CONNECTION_MAX_RETRIES": 100,
@@ -25,6 +26,8 @@ DEFAULTS = {
     "CELERY_ALWAYS_EAGER": False,
     "CELERY_EAGER_PROPAGATES_EXCEPTIONS": False,
     "CELERY_TASK_RESULT_EXPIRES": timedelta(days=1),
+    "CELERY_TASK_ERROR_WHITELIST": (),
+    "CELERY_IMPORTS": (),
     "CELERY_SEND_EVENTS": False,
     "CELERY_IGNORE_RESULT": False,
     "CELERY_STORE_ERRORS_EVEN_IF_IGNORED": False,
@@ -34,13 +37,13 @@ DEFAULTS = {
     "CELERYD_TASK_TIME_LIMIT": None,
     "CELERYD_TASK_SOFT_TIME_LIMIT": None,
     "CELERYD_MAX_TASKS_PER_CHILD": None,
-    "CELERY_ROUTES": None,
     "CELERY_CREATE_MISSING_QUEUES": True,
     "CELERY_DEFAULT_ROUTING_KEY": "celery",
     "CELERY_DEFAULT_QUEUE": "celery",
     "CELERY_DEFAULT_EXCHANGE": "celery",
     "CELERY_DEFAULT_EXCHANGE_TYPE": "direct",
     "CELERY_DEFAULT_DELIVERY_MODE": 2, # persistent
+    "CELERY_SEND_TASK_ERROR_EMAILS": False,
     "CELERY_ACKS_LATE": False,
     "CELERY_CACHE_BACKEND": None,
     "CELERY_CACHE_BACKEND_OPTIONS": {},

celery/events/snapshot.py

@@ -1,5 +1,6 @@
 from celery.utils import timer2
 
+from celery.app import app_or_default
 from celery.datastructures import TokenBucket
 from celery.events import EventReceiver
 from celery.events.state import State
@@ -20,7 +21,8 @@ class Polaroid(object):
         self.state = state
         self.freq = freq
         self.cleanup_freq = cleanup_freq
-        self.logger = logger or app.log.get_default_logger(name="celery.cam")
+        self.logger = logger or \
+                self.app.log.get_default_logger(name="celery.cam")
         self.maxrate = maxrate and TokenBucket(rate(maxrate))
 
     def install(self):

celery/loaders/base.py

@@ -47,7 +47,7 @@ class BaseLoader(object):
         return import_module(module)
 
     def import_default_modules(self):
-        imports = getattr(self.conf, "CELERY_IMPORTS", None) or []
+        imports = self.conf.get("CELERY_IMPORTS") or []
         imports = set(list(imports) + BUILTIN_MODULES)
         return map(self.import_task_module, imports)
 

celery/loaders/default.py

@@ -11,8 +11,6 @@ DEFAULT_CONFIG_MODULE = "celeryconfig"
 DEFAULT_SETTINGS = {
     "DEBUG": False,
     "ADMINS": (),
-    "DATABASE_ENGINE": "sqlite3",
-    "DATABASE_NAME": "celery.sqlite",
     "INSTALLED_APPS": ("celery", ),
     "CELERY_IMPORTS": (),
     "CELERY_TASK_ERROR_WHITELIST": (),

celery/task/base.py

@@ -427,7 +427,7 @@ class BaseTask(object):
         router = self.app.amqp.Router(queues)
 
         if self.app.conf.CELERY_ALWAYS_EAGER:
-            return apply(self, args, kwargs, task_id=task_id)
+            return self.apply(args, kwargs, task_id=task_id)
 
         options = dict(extract_exec_options(self), **options)
         options = router.route(options, self.name, args, kwargs)