Browse Source

Adds method Celery.add_defaults, which is like Celery.conf.update except a copy will not be made and supporting lazy intialization (if passing a callable instead of a dict)

Ask Solem 12 years ago
parent
commit
ae2d8618b6
2 changed files with 34 additions and 1 deletions
  1. 17 1
      celery/app/base.py
  2. 17 0
      docs/reference/celery.rst

+ 17 - 1
celery/app/base.py

@@ -73,6 +73,9 @@ class Celery(object):
         self.registry_cls = symbol_by_name(self.registry_cls)
         self.accept_magic_kwargs = accept_magic_kwargs
 
+        self.configured = False
+        self._pending_defaults = deque()
+
         self.finalized = False
         self._finalize_mutex = Lock()
         self._pending = deque()
@@ -158,11 +161,18 @@ class Celery(object):
 
                 pending = self._pending
                 while pending:
-                    maybe_evaluate(pending.pop())
+                    maybe_evaluate(pending.popleft())
 
                 for task in self._tasks.itervalues():
                     task.bind(self)
 
+    def add_defaults(self, fun):
+        if not callable(fun):
+            d, fun = fun, lambda: d
+        if self.configured:
+            return self.conf.add_defaults(fun())
+        self._pending_defaults.append(fun)
+
     def config_from_object(self, obj, silent=False):
         del(self.conf)
         return self.loader.config_from_object(obj, silent=silent)
@@ -297,8 +307,14 @@ class Celery(object):
         return backend(app=self, url=url)
 
     def _get_config(self):
+        self.configured = True
         s = Settings({}, [self.prepare_config(self.loader.conf),
                              deepcopy(DEFAULTS)])
+
+        # load lazy config dict initializers.
+        pending = self._pending_defaults
+        while pending:
+            s.add_defaults(pending.popleft()())
         if self._preconf:
             for key, value in self._preconf.iteritems():
                 setattr(s, key, value)

+ 17 - 0
docs/reference/celery.rst

@@ -114,6 +114,23 @@ Application
             >>> os.environ["CELERY_CONFIG_MODULE"] = "myapp.celeryconfig"
             >>> celery.config_from_envvar("CELERY_CONFIG_MODULE")
 
+    .. method:: Celery.add_defaults(d)
+
+        Add default configuration from dict ``d``.
+
+        If the argument is a callable function then it will be regarded
+        as a promise, and it won't be loaded until the configuration is
+        actually needed.
+
+        This method can be compared to::
+
+            >>> celery.conf.update(d)
+
+        with a difference that 1) no copy will be made and 2) the dict will
+        not be transferred when the worker spawns child processes, so
+        it's important that the same configuration happens at import time
+        when pickle restores the object on the other side.
+
     .. method:: Celery.start(argv=None)
 
         Run :program:`celery` using `argv`.