Merge branch '3.0'

Conflicts:
	Changelog
	README.rst
	celery/__init__.py
	celery/worker/__init__.py
	docs/includes/introduction.txt

Changelog

@@ -20,6 +20,32 @@ If you're looking for versions prior to 3.x you should see :ref:`history`.
 - `Task.apply_async` now supports timeout and soft_timeout arguments (Issue #802)
 - `App.control.Inspect.conf` can be used for inspecting worker configuration
 
+.. _version-3.0.5:
+
+3.0.5
+=====
+:release-date: 2012-08-01 04:00 P.M BST
+
+- Now depends on kombu 2.3.1 + billiard 2.7.3.11
+
+- Fixed a bug with the -B option (``cannot pickle thread.lock objects``)
+  (Issue #894 + Issue #892, + django-celery #154).
+
+- The :control:`restart_pool` control command now requires the
+  :setting:`CELERYD_POOL_RESTARTS` setting to be enabled
+
+    This change was necessary as the multiprocessing event that the restart
+    command depends on is responsible for creating many semaphores/file
+    descriptors, resulting in problems in some environments.
+
+- ``chain.apply`` now passes args to the first task (Issue #889).
+
+- Documented previously secret options to the Django-Celery monitor
+  in the monitoring userguide (Issue #396).
+
+- Old changelog are now organized in separate documents for each series,
+  see :ref:`history`.
+
 .. _version-3.0.4:
 
 3.0.4

README.rst

@@ -249,7 +249,7 @@ The following bundles are available:
 .. _`django-celery-with-redis`:
     http://pypi.python.org/pypi/django-celery-with-redis/
 .. _`django-celery-with-mongodb`:
-    http://pypi.python.org/pypi/django-celery-with-mongodb/
+    http://pypi.python.org/pypi/django-celery-with-mongdb/
 
 .. _celery-installing-from-source:
 

celery/__init__.py

@@ -7,8 +7,6 @@
 
 from __future__ import absolute_import
 
-from future_builtins import map
-
 SERIES = 'DEVEL'
 VERSION = (3, 1, 0, 'a1')
 __version__ = '.'.join(map(str, VERSION[0:3])) + ''.join(VERSION[3:])
@@ -16,8 +14,30 @@ __author__ = 'Ask Solem'
 __contact__ = 'ask@celeryproject.org'
 __homepage__ = 'http://celeryproject.org'
 __docformat__ = 'restructuredtext'
+__all__ = [
+    'celery', 'bugreport', 'shared_task', 'task',
+    'current_app', 'current_task',
+    'chain', 'chord', 'chunks', 'group', 'subtask',
+    'xmap', 'xstarmap', 'uuid', 'version', '__version__',
+]
 VERSION_BANNER = '{0} ({1})'.format(__version__, SERIES)
 
+# This is for static analyzers
+Celery = object
+bugreport = lambda *a, **kw: None
+shared_task = lambda *a, **kw: None
+Task = object
+current_app = object()
+current_task = object()
+chain = lambda *a, **kw: None
+chord = lambda *a, **kw: None
+chunks = lambda *a, **kw: None
+group = lambda *a, **kw: None
+subtask = lambda *a, **kw: None
+xmap = lambda *a, **kw: None
+xstarmap = lambda *a, **kw: None
+uuid = lambda: None
+
 # -eof meta-
 
 # Lazy loading

celery/app/defaults.py

@@ -169,6 +169,7 @@ NAMESPACES = {
         'MAX_TASKS_PER_CHILD': Option(type='int'),
         'POOL': Option(DEFAULT_POOL),
         'POOL_PUTLOCKS': Option(True, type='bool'),
+        'POOL_RESTARTS': Option(False, type='bool'),
         'PREFETCH_MULTIPLIER': Option(4, type='int'),
         'STATE_DB': Option(),
         'TASK_LOG_FORMAT': Option(DEFAULT_TASK_LOG_FMT),

celery/beat.py

@@ -387,6 +387,10 @@ class Service(object):
         self._is_shutdown = Event()
         self._is_stopped = Event()
 
+    def __reduce__(self):
+        return self.__class__, (self.max_interval, self.schedule_filename,
+                                self.scheduler_cls, self.app)
+
     def start(self, embedded_process=False):
         info('Celerybeat: Starting...')
         debug('Celerybeat: Ticking with max interval->%s',

celery/task/__init__.py

@@ -3,7 +3,10 @@
     celery.task
     ~~~~~~~~~~~
 
-    This is the old task module, it should not be used anymore.
+    This is the old task module, it should not be used anymore,
+    import from the main 'celery' module instead.
+    If you're looking for the decorator implementation then that's in
+    ``celery.app.base.Celery.task``.
 
 """
 from __future__ import absolute_import
@@ -12,6 +15,23 @@ from celery._state import current_app, current_task as current
 from celery.__compat__ import MagicModule, recreate_module
 from celery.local import Proxy
 
+__all__ = [
+    'BaseTask', 'Task', 'PeriodicTask',
+    'task', 'periodic_task',
+    'group', 'chord', 'subtask', 'TaskSet',
+]
+
+# This is for static analyzers
+BaseTask = object
+Task = object
+PeriodicTask = object
+task = lambda *a, **kw: None
+periodic_task = lambda *a, **kw: None
+group = lambda *a, **kw: None
+chord = lambda *a, **kw: None
+subtask = lambda *a, **kw: None
+TaskSet = object
+
 
 class module(MagicModule):
 

celery/worker/__init__.py

@@ -100,6 +100,7 @@ class WorkController(configurated):
     task_soft_time_limit = from_config()
     max_tasks_per_child = from_config()
     pool_putlocks = from_config()
+    pool_restarts = from_config()
     force_execv = from_config()
     prefetch_multiplier = from_config()
     state_db = from_config()

celery/worker/components.py

@@ -43,7 +43,8 @@ class Pool(bootsteps.StartStopComponent):
     name = 'worker.pool'
     requires = ('queues', )
 
-    def __init__(self, w, autoscale=None, no_execv=False, **kwargs):
+    def __init__(self, w, autoscale=None, autoreload=None,
+            no_execv=False, **kwargs):
         if isinstance(autoscale, basestring):
             max_c, _, min_c = autoscale.partition(',')
             autoscale = [int(max_c), min_c and int(min_c) or 0]
@@ -54,6 +55,7 @@ class Pool(bootsteps.StartStopComponent):
         w.no_execv = no_execv
         if w.autoscale:
             w.max_concurrency, w.min_concurrency = w.autoscale
+        self.autoreload_enabled = autoreload
 
     def on_poll_init(self, pool, hub):
         apply_after = hub.timer.apply_after
@@ -112,6 +114,7 @@ class Pool(bootsteps.StartStopComponent):
             w._quick_acquire = w.semaphore.acquire
             w._quick_release = w.semaphore.release
             max_restarts = 100
+        allow_restart = self.autoreload_enabled or w.pool_restarts
         pool = w.pool = self.instantiate(w.pool_cls, w.min_concurrency,
                             initargs=(w.app, w.hostname),
                             maxtasksperchild=w.max_tasks_per_child,
@@ -121,6 +124,7 @@ class Pool(bootsteps.StartStopComponent):
                             lost_worker_timeout=w.worker_lost_wait,
                             threads=threaded,
                             max_restarts=max_restarts,
+                            allow_restart=allow_restart,
                             semaphore=semaphore)
         if w.hub:
             w.hub.on_init.append(partial(self.on_poll_init, pool))

docs/configuration.rst

@@ -1429,6 +1429,16 @@ the built-in aliases: ``processes``, ``eventlet``, ``gevent``.
 
 Default is ``processes``.
 
+.. setting:: CELERYD_POOL_RESTARTS
+
+CELERYD_POOL_RESTARTS
+~~~~~~~~~~~~~~~~~~~~~
+
+If enabled the worker pool can be restarted using the
+:control:`pool_restart` remote control command.
+
+Disabled by default.
+
 .. setting:: CELERYD_AUTOSCALER
 
 CELERYD_AUTOSCALER

docs/userguide/workers.rst

@@ -588,6 +588,8 @@ Pool Restart Command
 
 .. versionadded:: 2.5
 
+Requires the :setting:`CELERYD_POOL_RESTARTS` setting to be enabled.
+
 The remote control command :control:`pool_restart` sends restart requests to
 the workers child processes.  It is particularly useful for forcing
 the worker to import new modules, or for reloading already imported

requirements/default.txt

@@ -1,3 +1,3 @@
-billiard>=2.7.3.10
+billiard>=2.7.3.11
 python-dateutil>=1.5,<2.0
-kombu>=2.3.0,<3.0
+kombu>=2.3.1,<3.0

setup.cfg

@@ -16,7 +16,7 @@ upload-dir = docs/.build/html
 [bdist_rpm]
 requires = uuid
            importlib
-           billiard>=2.7.3.10
+           billiard>=2.7.3.11
            python-dateutil >= 1.5
-           kombu >= 2.3.0
+           kombu >= 2.3.1
            ordereddict

setup.py

@@ -59,6 +59,9 @@ classes = """
     Programming Language :: Python :: 2
     Programming Language :: Python :: 2.6
     Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.2
+    Programming Language :: Python :: 3.3
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
     Programming Language :: Python :: Implementation :: Jython