Fixes pickling of configuration modules when using execv/fork emulation. Closes #1126

celery/app/base.py

@@ -18,6 +18,7 @@ from copy import deepcopy
 from functools import wraps
 from operator import attrgetter
 
+from billiard import forking as _forking
 from billiard.util import register_after_fork
 from kombu.clocks import LamportClock
 from kombu.utils import cached_property
@@ -446,7 +447,7 @@ class Celery(object):
         when unpickling."""
         return {
             'main': self.main,
-            'changes': self.conf.changes,
+            'changes': self.conf._prepare_pickleable_changes(),
             'loader': self.loader_cls,
             'backend': self.backend_cls,
             'amqp': self.amqp_cls,
@@ -459,9 +460,10 @@ class Celery(object):
 
     def __reduce_args__(self):
         """Deprecated method, please use :meth:`__reduce_keys__` instead."""
-        return (self.main, self.conf.changes, self.loader_cls,
-                self.backend_cls, self.amqp_cls, self.events_cls,
-                self.log_cls, self.control_cls, self.accept_magic_kwargs)
+        return (self.main, self.conf._prepare_pickleable_changes(),
+                self.loader_cls, self.backend_cls, self.amqp_cls,
+                self.events_cls, self.log_cls, self.control_cls,
+                self.accept_magic_kwargs)
 
     @cached_property
     def Worker(self):

celery/app/utils.py

@@ -10,10 +10,13 @@ from __future__ import absolute_import
 
 import os
 import platform as _platform
+import types
+
+from billiard import forking as _forking
 
-from celery import datastructures
 from celery import platforms
 from celery.five import items
+from celery.datastructures import ConfigurationView, DictAttribute
 from celery.utils.text import pretty
 from celery.utils.imports import qualname
 
@@ -31,7 +34,7 @@ settings -> transport:{transport} results:{results}
 """
 
 
-class Settings(datastructures.ConfigurationView):
+class Settings(ConfigurationView):
     """Celery settings object."""
 
     @property
@@ -63,6 +66,25 @@ class Settings(datastructures.ConfigurationView):
         # the last stash is the default settings, so just skip that
         return Settings({}, self._order[:-1])
 
+    def _prepare_pickleable_changes(self):
+        # attempt to include keys from configuration modules,
+        # to work with multiprocessing execv/fork emulation.
+        # This is necessary when multiprocessing execv/fork emulation
+        # is enabled.  There may be a better way to do this, but attempts
+        # at forcing the subprocess to import the modules did not work out,
+        # because of some sys.path problem.  More at Issue #1126.
+        if _forking._forking_is_enabled:
+            return self.changes
+        R = {}
+        for d in reversed(self._order[:-1]):
+            if isinstance(d, DictAttribute):
+                d = object.__getattribute__(d, 'obj')
+                if isinstance(d, types.ModuleType):
+                    d = dict((k, v) for k, v in items(vars(d))
+                             if not k.startswith('__') and k.isupper())
+            R.update(d)
+        return R
+
     def find_option(self, name, namespace='celery'):
         """Search for option by name.