Doc Improvements

celery/app/base.py

@@ -350,7 +350,6 @@ class Celery(object):
 
     @cached_property
     def Worker(self):
-        """Create new :class:`~celery.apps.worker.Worker` instance."""
         return self.subclass_with_self("celery.apps.worker:Worker")
 
     @cached_property
@@ -359,7 +358,6 @@ class Celery(object):
 
     @cached_property
     def Beat(self, **kwargs):
-        """Create new :class:`~celery.apps.beat.Beat` instance."""
         return self.subclass_with_self("celery.apps.beat:Beat")
 
     @cached_property
@@ -368,7 +366,6 @@ class Celery(object):
 
     @cached_property
     def Task(self):
-        """Default Task base class for this application."""
         return self.create_task_cls()
 
     @cached_property
@@ -401,48 +398,34 @@ class Celery(object):
 
     @cached_property
     def amqp(self):
-        """Sending/receiving messages.  See :class:`~celery.app.amqp.AMQP`."""
         return instantiate(self.amqp_cls, app=self)
 
     @cached_property
     def backend(self):
-        """Storing/retrieving task state.  See
-        :class:`~celery.backend.base.BaseBackend`."""
         return self._get_backend()
 
     @cached_property
     def conf(self):
-        """Current configuration (dict and attribute access)."""
         return self._get_config()
 
     @cached_property
     def control(self):
-        """Controlling worker nodes.  See
-        :class:`~celery.app.control.Control`."""
         return instantiate(self.control_cls, app=self)
 
     @cached_property
     def events(self):
-        """Sending/receiving events.  See :class:`~celery.events.Events`. """
         return instantiate(self.events_cls, app=self)
 
     @cached_property
     def loader(self):
-        """Current loader."""
         return get_loader_cls(self.loader_cls)(app=self)
 
     @cached_property
     def log(self):
-        """Logging utilities.  See :class:`~celery.app.log.Logging`."""
         return instantiate(self.log_cls, app=self)
 
     @cached_property
     def tasks(self):
-        """Registry of available tasks.
-
-        Accessing this attribute will also finalize the app.
-
-        """
         self.finalize()
         return self._tasks
 App = Celery  # compat

celery/events/__init__.py

@@ -78,7 +78,7 @@ class EventDispatcher(object):
     def __init__(self, connection=None, hostname=None, enabled=True,
             channel=None, buffer_while_offline=True, app=None,
             serializer=None):
-        self.app = app_or_default(app)
+        self.app = app_or_default(app or self.app)
         self.connection = connection
         self.channel = channel
         self.hostname = hostname or socket.gethostname()

celery/local.py

@@ -198,6 +198,13 @@ class PromiseProxy(Proxy):
         except AttributeError:
             return self.__evaluate__()
 
+    def __evaluated__(self):
+        try:
+            object.__getattribute__(self, "__thing")
+        except AttributeError:
+            return False
+        return True
+
     def __maybe_evaluate__(self):
         return self._get_current_object()
 

celery/worker/__init__.py

@@ -327,8 +327,8 @@ class WorkController(configurated):
         # ensure all task modules are imported in case an execv happens.
         task_modules = set(task.__class__.__module__
                             for task in self.app.tasks.itervalues())
-        self.app.conf.CELERY_INCLUDE = (
-            set(self.app.conf.CELERY_INCLUDE) + set(task_modules),
+        self.app.conf.CELERY_INCLUDE = tuple(
+            set(self.app.conf.CELERY_INCLUDE) | task_modules,
         )
 
         # Initialize boot steps

docs/configuration.rst

@@ -70,6 +70,18 @@ to be used.
 
 If not set then the systems default local time zone is used.
 
+.. warning::
+
+    Celery requires the :mod:`pytz` library to be installed,
+    when using custom time zones (other than UTC).  You can
+    install it using :program:`pip` or :program:`easy_install`::
+
+        $ pip install pytz
+
+    Pytz is a library that defines the timzones of the world,
+    it changes quite frequently so it is not included in the Python Standard
+    Library.
+
 .. _conf-tasks:
 
 Task settings

docs/getting-started/next-steps.rst

@@ -145,399 +145,3 @@ These options are described in more detailed in the :ref:`Workers Guide <guide-w
 
 *Canvas*: Designing Workflows
 =============================
-
-A :func:`~celery.subtask` wraps the signature of a single task invocation:
-arguments, keyword arguments and execution options.
-
-A subtask for the ``add`` task can be created like this::
-
-    >>> from celery import subtask
-    >>> subtask(add.name, args=(4, 4))
-
-or you can create one from the task itself::
-
-    >>> from proj.tasks import add
-    >>> add.subtask(args=(4, 4))
-
-It takes the same arguments as the :meth:`~@Task.apply_async` method::
-
-    >>> add.apply_async(args, kwargs, **options)
-    >>> add.subtask(args, kwargs, **options)
-
-    >>> add.apply_async((2, 2), countdown=1)
-    >>> add.subtask((2, 2), countdown=1)
-
-And like there is a :meth:`~@Task.delay` shortcut for `apply_async`
-there is an :meth:`~@Task.s` shortcut for subtask::
-
-    >>> add.s(*args, **kwargs)
-
-    >>> add.s(2, 2)
-    proj.tasks.add(2, 2)
-
-    >>> add.s(2, 2) == add.subtask((2, 2))
-    True
-
-You can't define options with :meth:`~@Task.s`, but a chaining
-``set`` call takes care of that::
-
-    >>> add.s(2, 2).set(countdown=1)
-    proj.tasks.add(2, 2)
-
-Partials
---------
-
-A subtask can be applied too::
-
-    >>> add.s(2, 2).delay()
-    >>> add.s(2, 2).apply_async(countdown=1)
-
-Specifying additional args, kwargs or options to ``apply_async``/``delay``
-creates partials:
-
-- Any arguments added will be prepended to the args in the signature::
-
-    >>> partial = add.s(2)          # incomplete signature
-    >>> partial.delay(4)            # 2 + 4
-    >>> partial.apply_async((4, ))  # same
-
-- Any keyword arguments added will be merged with the kwargs in the signature,
-  with the new keyword arguments taking precedence::
-
-    >>> s = add.s(2, 2)
-    >>> s.delay(debug=True)                    # -> add(2, 2, debug=True)
-    >>> s.apply_async(kwargs={"debug": True})  # same
-
-- Any options added will be merged with the options in the signature,
-  with the new options taking precedence::
-
-    >>> s = add.subtask((2, 2), countdown=10)
-    >>> s.apply_async(countdown=1)  # countdown is now 1
-
-You can also clone subtasks to augment these::
-
-    >>> s = add.s(2)
-    proj.tasks.add(2)
-
-    >>> s.clone(args=(4, ), kwargs={"debug": True})
-    proj.tasks.add(2, 4, debug=True)
-
-Partials are meant to be used with callbacks, any tasks linked or chord
-callbacks will be applied with the result of the parent task.
-Sometimes you want to specify a callback that does not take
-additional arguments, and in that case you can set the subtask
-to be immutable::
-
-    >>> add.apply_async((2, 2), link=reset_buffers.subtask(immutable=True))
-
-The ``.si()`` shortcut can also be used to create immutable subtasks::
-
-    >>> add.apply_async((2, 2), link=reset_buffers.si())
-
-Only the execution options can be set when a subtask is immutable,
-and it's not possible to apply the subtask with partial args/kwargs.
-
-.. note::
-
-    In this tutorial we use the prefix operator `~` to subtasks.
-    You probably shouldn't use it in your production code, but it's a handy shortcut
-    when testing with the Python shell::
-
-        >>> ~subtask
-
-        >>> # is the same as
-        >>> subtask.delay().get()
-
-Groups
-------
-
-A group can be used to execute several tasks in parallel.
-
-The :class:`~celery.group` function takes a list of subtasks::
-
-    >>> from celery import group
-    >>> from proj.tasks import add
-
-    >>> group(add.s(2, 2), add.s(4, 4))
-    (proj.tasks.add(2, 2), proj.tasks.add(4, 4))
-
-If you **call** the group, the tasks will be applied
-one after one in the current process, and a :class:`~@TaskSetResult`
-instance is returned which can be used to keep track of the results,
-or tell how many tasks are ready and so on::
-
-    >>> g = group(add.s(2, 2), add.s(4, 4))
-    >>> res = g()
-    >>> res.get()
-    [4, 8]
-
-However, if you call ``apply_async`` on the group it will
-send a special grouping task, so that the action of applying
-the tasks happens in a worker instead of the current process::
-
-    >>> res = g.apply_async()
-    >>> res.get()
-    [4, 8]
-
-Group also supports iterators::
-
-    >>> group(add.s(i, i) for i in xrange(100))()
-
-
-A group is a subclass instance, so it can be used in combination
-with other subtasks.
-
-Map & Starmap
--------------
-
-:class:`~celery.map` and :class:`~celery.starmap` are built-in tasks
-that calls the task for every element in a sequence.
-
-They differ from group in that
-
-- only one task message is sent
-
-- the operation is sequential.
-
-For example using ``map``:
-
-.. code-block:: python
-
-    >>> from proj.tasks import add
-
-    >>> ~xsum.map([range(10), range(100)])
-    [45, 4950]
-
-is the same as having a task doing:
-
-.. code-block:: python
-
-    @celery.task()
-    def temp():
-        return [xsum(range(10)), xsum(range(100))]
-
-and using ``starmap``::
-
-    >>> ~add.starmap(zip(range(10), range(10)))
-    [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
-
-is the same as having a task doing:
-
-.. code-block:: python
-
-    @celery.task()
-    def temp():
-        return [add(i, i) for i in range(10)]
-
-Both ``map`` and ``starmap`` are subtasks, so they can be used as
-other subtasks and combined in groups etc., for example
-to apply the starmap after 10 seconds::
-
-    >>> add.starmap(zip(range(10), range(10))).apply_async(countdown=10)
-
-.. _chunking-ov:
-
-Chunking
---------
-
--- Chunking lets you divide a iterable of work into pieces,
-   so that if you have one million objects, you can create
-   10 tasks with hundred thousand objects each.
-
-Some may worry that chunking your tasks results in a degradation
-of parallelism, but this is rarely true for a busy cluster
-and in practice since you are avoiding the overhead  of messaging
-it may considerably increase performance.
-
-To create a chunks subtask you can use :meth:`@Task.chunks`:
-
-.. code-block:: python
-
-    >>> add.chunks(zip(range(100), range(100)), 10)
-
-As with :class:`~celery.group` the act of **calling**
-the chunks will apply the tasks in the current process:
-
-.. code-block:: python
-
-    >>> from proj.tasks import add
-
-    >>> res = add.chunks(zip(range(100), range(100)), 10)()
-    >>> res.get()
-    [[0, 2, 4, 6, 8, 10, 12, 14, 16, 18],
-     [20, 22, 24, 26, 28, 30, 32, 34, 36, 38],
-     [40, 42, 44, 46, 48, 50, 52, 54, 56, 58],
-     [60, 62, 64, 66, 68, 70, 72, 74, 76, 78],
-     [80, 82, 84, 86, 88, 90, 92, 94, 96, 98],
-     [100, 102, 104, 106, 108, 110, 112, 114, 116, 118],
-     [120, 122, 124, 126, 128, 130, 132, 134, 136, 138],
-     [140, 142, 144, 146, 148, 150, 152, 154, 156, 158],
-     [160, 162, 164, 166, 168, 170, 172, 174, 176, 178],
-     [180, 182, 184, 186, 188, 190, 192, 194, 196, 198]]
-
-while calling ``.apply_async`` will create a dedicated
-task so that the individual tasks are applied in a worker
-instead::
-
-    >>> add.chunks(zip(range(100), range(100), 10)).apply_async()
-
-You can also convert chunks to a group::
-
-    >>> group = add.chunks(zip(range(100), range(100), 10)).group()
-
-and with the group skew the countdown of each task by increments
-of one::
-
-    >>> group.skew(start=1, stop=10)()
-
-which means that the first task will have a countdown of 1, the second
-a countdown of 2 and so on.
-
-
-Chaining tasks
---------------
-
-Tasks can be linked together, which in practice means adding
-a callback task::
-
-    >>> res = add.apply_async((2, 2), link=mul.s(16))
-    >>> res.get()
-    4
-
-The linked task will be applied with the result of its parent
-task as the first argument, which in the above case will result
-in ``mul(4, 16)`` since the result is 4.
-
-The results will keep track of what subtasks a task applies,
-and this can be accessed from the result instance::
-
-    >>> res.children
-    [<AsyncResult: 8c350acf-519d-4553-8a53-4ad3a5c5aeb4>]
-
-    >>> res.children[0].get()
-    64
-
-The result instance also has a :meth:`~@AsyncResult.collect` method
-that treats the result as a graph, enabling you to iterate over
-the results::
-
-    >>> list(res.collect())
-    [(<AsyncResult: 7b720856-dc5f-4415-9134-5c89def5664e>, 4),
-     (<AsyncResult: 8c350acf-519d-4553-8a53-4ad3a5c5aeb4>, 64)]
-
-By default :meth:`~@AsyncResult.collect` will raise an
-:exc:`~@IncompleteStream` exception if the graph is not fully
-formed (one of the tasks has not completed yet),
-but you can get an intermediate representation of the graph
-too::
-
-    >>> for result, value in res.collect(intermediate=True)):
-    ....
-
-You can link together as many tasks as you like,
-and subtasks can be linked too::
-
-    >>> s = add.s(2, 2)
-    >>> s.link(mul.s(4))
-    >>> s.link(log_result.s())
-
-You can also add *error callbacks* using the ``link_error`` argument::
-
-    >>> add.apply_async((2, 2), link_error=log_error.s())
-
-    >>> add.subtask((2, 2), link_error=log_error.s())
-
-Since exceptions can only be serialized when pickle is used
-the error callbacks take the id of the parent task as argument instead:
-
-.. code-block:: python
-
-    from proj.celery import celery
-
-    @celery.task()
-    def log_error(task_id):
-        result = celery.AsyncResult(task_id)
-        result.get(propagate=False)  # make sure result written.
-        with open("/var/errors/%s" % (task_id, )) as fh:
-            fh.write("--\n\n%s %s %s" % (
-                task_id, result.result, result.traceback))
-
-To make it even easier to link tasks together there is
-a special subtask called :class:`~celery.chain` that lets
-you chain tasks together:
-
-.. code-block:: python
-
-    >>> from celery import chain
-    >>> from proj.tasks import add, mul
-
-    # (4 + 4) * 8 * 10
-    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))
-    proj.tasks.add(4, 4) | proj.tasks.mul(8)
-
-
-Calling the chain will apply the tasks in the current process
-and return the result of the last task in the chain::
-
-    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))
-    >>> res.get()
-    640
-
-And calling ``apply_async`` will create a dedicated
-task so that the act of applying the chain happens
-in a worker::
-
-    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))
-    >>> res.get()
-    640
-
-It also sets ``parent`` attributes so that you can
-work your way up the chain to get intermediate results::
-
-    >>> res.parent.get()
-    64
-
-    >>> res.parent.parent.get()
-    8
-
-    >>> res.parent.parent
-    <AsyncResult: eeaad925-6778-4ad1-88c8-b2a63d017933>
-
-
-Chains can also be made using the ``|`` (pipe) operator::
-
-    >>> (add.s(2, 2) | mul.s(8) | mul.s(10)).apply_async()
-
-Graphs
-~~~~~~
-
-In addition you can work with the result graph as a
-:class:`~celery.datastructures.DependencyGraph`:
-
-.. code-block:: python
-
-    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))()
-
-    >>> res.parent.parent.graph
-    285fa253-fcf8-42ef-8b95-0078897e83e6(1)
-        463afec2-5ed4-4036-b22d-ba067ec64f52(0)
-    872c3995-6fa0-46ca-98c2-5a19155afcf0(2)
-        285fa253-fcf8-42ef-8b95-0078897e83e6(1)
-            463afec2-5ed4-4036-b22d-ba067ec64f52(0)
-
-You can even convert these graphs to *dot* format::
-
-    >>> with open("graph.dot", "w") as fh:
-    ...     res.parent.parent.graph.to_dot(fh)
-
-
-and create images::
-
-    $ dot -Tpng graph.dot -o graph.png
-
-.. image:: ../images/graph.png
-
-.. _chords-ov:
-
-Chords
-------

docs/reference/celery.rst

@@ -72,6 +72,8 @@ Application
 
         Task registry.
 
+        Accessing this attribute will also finalize the app.
+
     .. attribute:: Celery.pool
 
         Broker connection pool: :class:`~@pool`.

docs/userguide/application.rst

@@ -4,13 +4,108 @@
  Application
 =============
 
+.. contents::
+    :local:
+    :depth: 1
+
 The Celery library must instantiated before use, and this instance
 is called the application.
 
-Multiple Celery applications with different configuration
-and components can co-exist in the same process space,
-but there is always an app exposed as the 'current app' in each
-thread.
+The application is thread-safe so that multiple Celery applications
+with different configuration, components and tasks can co-exist in the
+same process space.
+
+Let's create one now:
+
+.. code-block:: python
+
+    >>> from celery import Celery
+    >>> celery = Celery()
+    >>> celery
+    <Celery __main__:0x100469fd0>
+
+The last line shows the textual representation of the application,
+which includes the name of the celery class (``Celery``), the name of the
+current main module (``__main__``), and the memory address of the object
+(``0x100469fd0``).
+
+Main Name
+=========
+
+Only one of these is important, and that is the main module name,
+let's look at why that is.
+
+When you send a task message in Celery, that message will not contain
+any source code, but only the name of the task you want to execute.
+This works similarly to how host names works on the internet: every worker
+maintains a mapping of task names to their actual functions, called the *task
+registry*.
+
+Whenever you define a task, that task will also be added to the local registry:
+
+.. code-block:: python
+
+    >>> @celery.task()
+    ... def add(x, y):
+    ...     return x + y
+
+    >>> add
+    <@task: __main__.add>
+
+    >>> add.name
+    __main__.add
+
+    >>> celery.tasks["__main__.add"]
+    <@task: __main__.add>
+
+and there we see that ``__main__`` again; whenever Celery is not able
+to detect what module the function belongs to, it uses the main module
+name to generate the beginning of the task name.
+
+This is only a problem in a limited set of use cases:
+
+    #. If the module that the task is defined in is run as a program.
+    #. If the application is created in the Python shell (REPL).
+
+For example here, where the tasks module is also used to start a worker:
+
+:file:`tasks.py`:
+
+.. code-block:: python
+
+    from celery import Celery
+    celery = Celery()
+
+    @celery.task()
+    def add(x, y): return x + y
+
+    if __name__ == '__main__':
+        celery.worker_main()
+
+When this module is executed the tasks will be named starting with "``__main__``",
+but when it the module is imported by another process, say to call a task,
+the tasks will be named starting with "``tasks``" (the real name of the module)::
+
+    >>> from tasks import add
+    >>> add.name
+    tasks.add
+
+You can specify another name for the main module:
+
+.. code-block:: python
+
+    >>> celery = Celery("tasks")
+    >>> celery.main
+    "tasks"
+
+    >>> @celery.task()
+    ... def add(x, y):
+    ...     return x + y
+
+    >>> add.name
+    tasks.add
+
+.. seealso:: :ref:`task-names`
 
 Configuration
 =============
@@ -19,21 +114,186 @@ There are lots of different options you can set that will change how
 Celery work.  These options can be set on the app instance directly,
 or you can use a dedicated configuration module.
 
-
-The current configuration is always available as :attr:`@Celery.conf`::
+The configuration is available as :attr:`@Celery.conf`::
 
     >>> celery.conf.CELERY_TIMEZONE
     "Europe/London"
 
-The configuration actually consists of multiple dictionaries
+where you can set configuration values directly::
+
+    >>> celery.conf.CELERY_ENABLE_UTC = True
+
+or you can update several keys at once by using the ``update`` method::
+
+    >>> celery.conf.update(
+    ...     CELERY_ENABLE_UTC=True,
+    ...     CELERY_TIMEZONE="Europe/London",
+    ...)
+
+The configuration object consists of multiple dictionaries
 that are consulted in order:
 
     #. Changes made at runtime.
-    #. Any configuration module (``loader.conf``).
+    #. The configuration module (if any)
     #. The default configuration (:mod:`celery.app.defaults`).
 
-When the app is serialized
-only the configuration changes will be transferred.
+
+.. seealso::
+
+    Go to the :ref:`Configuration reference <configuration>` for a complete
+    listing of all the available settings, and their default values.
+
+
+
+``config_from_object``
+----------------------
+
+.. sidebar:: Timezones & Pytz.
+
+    Setting a time zone other than UTC requires the :mod:`pytz` library
+    to be installed, see the :setting:`CELERY_TIMEZONE` setting for more
+    information.
+
+
+The :meth:`@Celery.config_from_object` method loads configuration
+from a configuration object.
+
+This can be a configuration module, or any object with configuration attributes.
+
+Note that any configuration that was previous set will be reset when
+:meth:`~@Celery.config_from_object` is called.  If you want to set additional
+configuration you should do so after.
+
+Example 1: Using the name of a module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: python
+
+    from celery import Celery
+
+    celery = Celery()
+    celery.config_from_object("celeryconfig")
+
+
+The ``celeryconfig`` module may then look like this:
+
+:file:`celeryconfig.py`:
+
+.. code-block:: python
+
+    CELERY_ENABLE_UTC = True
+    CELERY_TIMEZONE = "Europe/London"
+
+Example 2: Using a configuration module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: python
+
+    from celery import Celery
+
+    celery = Celery()
+    import celeryconfig
+    celery.config_from_object(celeryconfig)
+
+Example 3:  Using a configuration class/object
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: python
+
+    from celery import Celery
+
+    celery = Celery()
+
+    class Config:
+        CELERY_ENABLE_UTC = True
+        CELERY_TIMEZONE = "Europe/London"
+
+    celery.config_from_object(Config)
+
+``config_from_envvar``
+----------------------
+
+The :meth:`@Celery.config_from_envvar` takes the configuration module name
+from an environment variable
+
+For example -- to load configuration from a module specified in the
+environment variable named :envvar:`CELERY_CONFIG_MODULE`:
+
+.. code-block:: python
+
+    import os
+    from celery import Celery
+
+    #: Set default configuration module name
+    os.environ.setdefault("CELERY_CONFIG_MODULE", "celeryconfig")
+
+    celery = Celery()
+    celery.config_from_envvar("CELERY_CONFIG_MODULE")
+
+You can then specify the configuration module to use via the environment::
+
+    $ CELERY_CONFIG_MODULE="celeryconfig.prod" celery worker -l info
+
+Laziness
+========
+
+The application instance is lazy, meaning that it will not be evaluated
+until something is actually needed.
+
+Creating a :class:`@Celery` instance will only do the following:
+
+    #. Create a logical clock instance, used for events.
+    #. Create the task registry.
+    #. Set itself as the current app (but not if the ``set_as_current``
+       argument was disabled)
+    #. Call the :meth:`@Celery.on_init` callback (does nothing by default).
+
+The :meth:`~@Celery.task` decorator does not actually create the
+tasks at the point when it's called, instead it will defer the creation
+of the task to happen either when the task is used, or after the
+application has been *finalized*,
+
+This example shows how the task is not created until
+we use the task, or access an attribute (in this case :meth:`repr`):
+
+.. code-block:: python
+
+    >>> @celery.task()
+    >>> def add(x, y):
+    ...    return x + y
+
+    >>> type(add)
+    <class 'celery.local.PromiseProxy'>
+
+    >>> add.__evaluated__()
+    False
+
+    >>> add        # <-- causes repr(add) to happen
+    <@task: __main__.add>
+
+    >>> add.__evaluated__()
+    True
+
+*Finalization* of the appq happens either explicitly by calling
+:meth:`@Celery.finalize` -- or implicitly by accessing the :attr:`~@Celery.tasks`
+attribute.
+
+Finalizing the object will:
+
+    #. Copy tasks that must be shared between apps
+
+        Tasks are shared by default, but if the
+        ``shared`` argument to the task decorator is disabled,
+        then the task will be private to the app it's bound to.
+
+    #. Evaluate all pending task decorators.
+
+    #. Make sure all tasks are bound to the current app.
+
+        Tasks are bound to apps so that it can read default
+        values from the configuration.
+
+.. _default-app:
 
 .. topic:: The "default app".
 
@@ -62,6 +322,53 @@ only the configuration changes will be transferred.
     module-based API.
 
 
+Breaking the chain
+==================
+
+While it's possible to depend on the current app
+being set, the best practice is to always pass the app instance
+around to anything that needs it.
+
+We call this the "app chain", since it creates a chain
+of instances depending on the app being passed.
+
+The following example is considered bad practice:
+
+.. code-block:: python
+
+    from celery import current_app
+
+    class Scheduler(object):
+
+        def run(self):
+            app = current_app
+
+Instead it should take the ``app`` as an argument:
+
+.. code-block:: python
+
+    class Scheduler(object):
+
+        def __init__(self, app):
+            self.app = app
+
+Internally Celery uses the :func:`celery.app.app_or_default` function
+so that everything also works in the module-based compatibility API
+
+.. code-block:: python
+
+    from celery.app import app_or_default
+
+    class Scheduler(object):
+        def __init__(self, app=None):
+            self.app = app_or_default(app)
+
+In development you can set the :envvar:`CELERY_TRACE_APP`
+environment variable to raise an exception if the app
+chain breaks::
+
+    $ CELERY_TRACE_APP=1 celery worker -l info
+
 
 .. topic:: Evolving the API
 
@@ -100,7 +407,7 @@ only the configuration changes will be transferred.
     Later, it was decided that passing arbitrary call-ables
     was an anti-pattern, since it makes it very hard to use
     serializers other than pickle, and the feature was removed
-    in 2.0, replaced by task decorators::
+    in 2.0, replaced by task decorators:
 
     .. code-block:: python
 
@@ -109,3 +416,69 @@ only the configuration changes will be transferred.
         @task(send_error_emails=True)
         def hello(x):
             return "hello %s" % to
+
+Abstract Tasks
+==============
+
+All tasks created using the :meth:`~@Celery.task` decorator
+will inherit from the applications base :attr:`~@Celery.Task` class.
+
+You can specify a different base class with the ``base`` argument:
+
+.. code-block:: python
+
+    @celery.task(base=OtherTask):
+    def add(x, y):
+        return x + y
+
+To create a custom task class you should inherit from the neutral base
+class: :class:`celery.Task`.
+
+.. code-block:: python
+
+    from celery import Task
+
+    class DebugTask(Task):
+        abstract = True
+
+        def __call__(self, *args, **kwargs):
+            print("TASK STARTING: %s[%s]" % (self.name, self.request.id))
+            return self.run(*args, **kwargs)
+
+
+The neutral base class is special because it's not bound to any specific app
+yet.  Concrete subclasses of this class will be bound, so you should
+always mark generic base classes as ``abstract``
+
+Once a task is bound to an app it will read configuration to set default values
+and so on.
+
+It's also possible to change the default base class for an application
+by changing its :meth:`@Celery.Task` attribute:
+
+.. code-block:: python
+
+    >>> from celery import Celery, Task
+
+    >>> celery = Celery()
+
+    >>> class MyBaseTask(Task):
+    ...    abstract = True
+    ...    send_error_emails = True
+
+    >>> celery.Task = MyBaseTask
+    >>> celery.Task
+    <unbound MyBaseTask>
+
+    >>> @x.task()
+    ... def add(x, y):
+    ...     return x + y
+
+    >>> add
+    <@task: __main__.add>
+
+    >>> add.__class__.mro()
+    [<class add of <Celery __main__:0x1012b4410>>,
+     <unbound MyBaseTask>,
+     <unbound Task>,
+     <type 'object'>]

docs/userguide/canvas.rst

@@ -1,11 +1,12 @@
 .. _guide-canvas:
 
-============================
- Canvas: Building Workflows
-============================
+=============================
+ Canvas: Designing Workflows
+=============================
 
 .. contents::
     :local:
+    :depth: 1
 
 .. _canvas-subtasks:
 
@@ -14,32 +15,109 @@ Subtasks
 
 .. versionadded:: 2.0
 
-The :class:`~celery.subtask` type is used to wrap the arguments and
-execution options for a single task invocation:
+A :func:`~celery.subtask` wraps the signature of a single task invocation:
+arguments, keyword arguments and execution options.
 
-.. code-block:: python
+A subtask for the ``add`` task can be created like this::
 
-    from celery import subtask
+    >>> from celery import subtask
+    >>> subtask(add.name, args=(4, 4))
 
-    subtask(task_name_or_cls, args, kwargs, options)
+or you can create one from the task itself::
 
-For convenience every task also has a shortcut to create subtasks:
+    >>> from proj.tasks import add
+    >>> add.subtask(args=(4, 4))
 
-.. code-block:: python
+It supports the "Calling API" which means it takes the same arguments
+as the :meth:`~@Task.apply_async` method::
+
+    >>> add.apply_async(args, kwargs, **options)
+    >>> add.subtask(args, kwargs, **options)
+
+    >>> add.apply_async((2, 2), countdown=1)
+    >>> add.subtask((2, 2), countdown=1)
+
+And like there is a :meth:`~@Task.delay` shortcut for `apply_async`
+there is an :meth:`~@Task.s` shortcut for subtask::
+
+    >>> add.s(*args, **kwargs)
+
+    >>> add.s(2, 2)
+    proj.tasks.add(2, 2)
+
+    >>> add.s(2, 2) == add.subtask((2, 2))
+    True
+
+You can't define options with :meth:`~@Task.s`, but a chaining
+``set`` call takes care of that::
+
+    >>> add.s(2, 2).set(countdown=1)
+    proj.tasks.add(2, 2)
+
+Partials
+--------
+
+A subtask can be applied too::
+
+    >>> add.s(2, 2).delay()
+    >>> add.s(2, 2).apply_async(countdown=1)
+
+Specifying additional args, kwargs or options to ``apply_async``/``delay``
+creates partials:
+
+- Any arguments added will be prepended to the args in the signature::
+
+    >>> partial = add.s(2)          # incomplete signature
+    >>> partial.delay(4)            # 2 + 4
+    >>> partial.apply_async((4, ))  # same
+
+- Any keyword arguments added will be merged with the kwargs in the signature,
+  with the new keyword arguments taking precedence::
+
+    >>> s = add.s(2, 2)
+    >>> s.delay(debug=True)                    # -> add(2, 2, debug=True)
+    >>> s.apply_async(kwargs={"debug": True})  # same
 
-    task.subtask(args, kwargs, options)
+- Any options added will be merged with the options in the signature,
+  with the new options taking precedence::
 
-:class:`~celery.subtask` is actually a :class:`dict` subclass,
-which means it can be serialized with JSON or other encodings that doesn't
-support complex Python objects.
+    >>> s = add.subtask((2, 2), countdown=10)
+    >>> s.apply_async(countdown=1)  # countdown is now 1
 
-Also it can be regarded as a type, as the following usage works::
+You can also clone subtasks to augment these::
 
-    >>> s = subtask("tasks.add", args=(2, 2), kwargs={})
+    >>> s = add.s(2)
+    proj.tasks.add(2)
 
-    >>> subtask(dict(s))  # coerce dict into subtask
+    >>> s.clone(args=(4, ), kwargs={"debug": True})
+    proj.tasks.add(2, 4, debug=True)
+
+Partials are meant to be used with callbacks, any tasks linked or chord
+callbacks will be applied with the result of the parent task.
+Sometimes you want to specify a callback that does not take
+additional arguments, and in that case you can set the subtask
+to be immutable::
+
+    >>> add.apply_async((2, 2), link=reset_buffers.subtask(immutable=True))
+
+The ``.si()`` shortcut can also be used to create immutable subtasks::
+
+    >>> add.apply_async((2, 2), link=reset_buffers.si())
+
+Only the execution options can be set when a subtask is immutable,
+and it's not possible to apply the subtask with partial args/kwargs.
+
+.. note::
+
+    In this tutorial we use the prefix operator `~` to subtasks.
+    You probably shouldn't use it in your production code, but it's a handy shortcut
+    when testing with the Python shell::
+
+        >>> ~subtask
+
+        >>> # is the same as
+        >>> subtask.delay().get()
 
-This makes it excellent as a means to pass callbacks around to tasks.
 
 .. _canvas-callbacks:
 
@@ -54,8 +132,7 @@ to ``apply_async``:
 The callback will only be applied if the task exited successfully,
 and it will be applied with the return value of the parent task as argument.
 
-
-The best thing is that any arguments you add to `subtask`,
+As we mentioned earlier, any arguments you add to `subtask`,
 will be prepended to the arguments specified by the subtask itself!
 
 If you have the subtask::
@@ -76,16 +153,197 @@ arguments::
 As expected this will first launch one task calculating :math:`2 + 2`, then
 another task calculating :math:`4 + 8`.
 
+.. _canvas-chains:
+
+Chaining
+========
+
+Tasks can be linked together, which in practice means adding
+a callback task::
+
+    >>> res = add.apply_async((2, 2), link=mul.s(16))
+    >>> res.get()
+    4
+
+The linked task will be applied with the result of its parent
+task as the first argument, which in the above case will result
+in ``mul(4, 16)`` since the result is 4.
+
+The results will keep track of what subtasks a task applies,
+and this can be accessed from the result instance::
+
+    >>> res.children
+    [<AsyncResult: 8c350acf-519d-4553-8a53-4ad3a5c5aeb4>]
+
+    >>> res.children[0].get()
+    64
+
+The result instance also has a :meth:`~@AsyncResult.collect` method
+that treats the result as a graph, enabling you to iterate over
+the results::
+
+    >>> list(res.collect())
+    [(<AsyncResult: 7b720856-dc5f-4415-9134-5c89def5664e>, 4),
+     (<AsyncResult: 8c350acf-519d-4553-8a53-4ad3a5c5aeb4>, 64)]
+
+By default :meth:`~@AsyncResult.collect` will raise an
+:exc:`~@IncompleteStream` exception if the graph is not fully
+formed (one of the tasks has not completed yet),
+but you can get an intermediate representation of the graph
+too::
+
+    >>> for result, value in res.collect(intermediate=True)):
+    ....
+
+You can link together as many tasks as you like,
+and subtasks can be linked too::
+
+    >>> s = add.s(2, 2)
+    >>> s.link(mul.s(4))
+    >>> s.link(log_result.s())
+
+You can also add *error callbacks* using the ``link_error`` argument::
+
+    >>> add.apply_async((2, 2), link_error=log_error.s())
+
+    >>> add.subtask((2, 2), link_error=log_error.s())
+
+Since exceptions can only be serialized when pickle is used
+the error callbacks take the id of the parent task as argument instead:
+
+.. code-block:: python
+
+    from proj.celery import celery
+
+    @celery.task()
+    def log_error(task_id):
+        result = celery.AsyncResult(task_id)
+        result.get(propagate=False)  # make sure result written.
+        with open("/var/errors/%s" % (task_id, )) as fh:
+            fh.write("--\n\n%s %s %s" % (
+                task_id, result.result, result.traceback))
+
+To make it even easier to link tasks together there is
+a special subtask called :class:`~celery.chain` that lets
+you chain tasks together:
+
+.. code-block:: python
+
+    >>> from celery import chain
+    >>> from proj.tasks import add, mul
+
+    # (4 + 4) * 8 * 10
+    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))
+    proj.tasks.add(4, 4) | proj.tasks.mul(8)
+
+
+Calling the chain will apply the tasks in the current process
+and return the result of the last task in the chain::
+
+    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))
+    >>> res.get()
+    640
+
+And calling ``apply_async`` will create a dedicated
+task so that the act of applying the chain happens
+in a worker::
+
+    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))
+    >>> res.get()
+    640
+
+It also sets ``parent`` attributes so that you can
+work your way up the chain to get intermediate results::
+
+    >>> res.parent.get()
+    64
+
+    >>> res.parent.parent.get()
+    8
+
+    >>> res.parent.parent
+    <AsyncResult: eeaad925-6778-4ad1-88c8-b2a63d017933>
+
+
+Chains can also be made using the ``|`` (pipe) operator::
+
+    >>> (add.s(2, 2) | mul.s(8) | mul.s(10)).apply_async()
+
+Graphs
+------
+
+In addition you can work with the result graph as a
+:class:`~celery.datastructures.DependencyGraph`:
+
+.. code-block:: python
+
+    >>> res = chain(add.s(4, 4), mul.s(8), mul.s(10))()
+
+    >>> res.parent.parent.graph
+    285fa253-fcf8-42ef-8b95-0078897e83e6(1)
+        463afec2-5ed4-4036-b22d-ba067ec64f52(0)
+    872c3995-6fa0-46ca-98c2-5a19155afcf0(2)
+        285fa253-fcf8-42ef-8b95-0078897e83e6(1)
+            463afec2-5ed4-4036-b22d-ba067ec64f52(0)
+
+You can even convert these graphs to *dot* format::
+
+    >>> with open("graph.dot", "w") as fh:
+    ...     res.parent.parent.graph.to_dot(fh)
+
+
+and create images::
+
+    $ dot -Tpng graph.dot -o graph.png
+
+.. image:: ../images/graph.png
+
 .. _canvas-group:
 
 Groups
 ======
 
-The :class:`~celery.group` enables easy invocation of several
-tasks at once, and is then able to join the results in the same order as the
-tasks were invoked.
+A group can be used to execute several tasks in parallel.
+
+The :class:`~celery.group` function takes a list of subtasks::
+
+    >>> from celery import group
+    >>> from proj.tasks import add
+
+    >>> group(add.s(2, 2), add.s(4, 4))
+    (proj.tasks.add(2, 2), proj.tasks.add(4, 4))
+
+If you **call** the group, the tasks will be applied
+one after one in the current process, and a :class:`~@TaskSetResult`
+instance is returned which can be used to keep track of the results,
+or tell how many tasks are ready and so on::
+
+    >>> g = group(add.s(2, 2), add.s(4, 4))
+    >>> res = g()
+    >>> res.get()
+    [4, 8]
+
+However, if you call ``apply_async`` on the group it will
+send a special grouping task, so that the action of applying
+the tasks happens in a worker instead of the current process::
+
+    >>> res = g.apply_async()
+    >>> res.get()
+    [4, 8]
+
+Group also supports iterators::
+
+    >>> group(add.s(i, i) for i in xrange(100))()
+
+A group is a subclass instance, so it can be used in combination
+with other subtasks.
+
+Group Results
+-------------
 
-``group`` takes a list of :class:`~celery.subtask`'s::
+The group task returns a special result too,
+this result works just like normal task results, except
+that it works on the group as a whole::
 
     >>> from celery import group
     >>> from tasks import add
@@ -107,19 +365,7 @@ tasks were invoked.
     >>> result.join()
     [4, 8, 16, 32, 64]
 
-The first argument can alternatively be an iterator, like::
-
-    >>> group(add.subtask((i, i)) for i in range(100))
-
-.. _canvas-group-results:
-
-Group Results
--------------
-
-When a  :class:`~celery.group` is applied it returns a
-:class:`~celery.result.GroupResult` object.
-
-:class:`~celery.result.GroupResult` takes a list of
+The :class:`~celery.result.GroupResult` takes a list of
 :class:`~celery.result.AsyncResult` instances and operates on them as if it was a
 single task.
 
@@ -267,3 +513,111 @@ implemented in other backends (suggestions welcome!).
         def after_return(self, *args, **kwargs):
             do_something()
             super(MyTask, self).after_return(*args, **kwargs)
+
+Map & Starmap
+=============
+
+:class:`~celery.map` and :class:`~celery.starmap` are built-in tasks
+that calls the task for every element in a sequence.
+
+They differ from group in that
+
+- only one task message is sent
+
+- the operation is sequential.
+
+For example using ``map``:
+
+.. code-block:: python
+
+    >>> from proj.tasks import add
+
+    >>> ~xsum.map([range(10), range(100)])
+    [45, 4950]
+
+is the same as having a task doing:
+
+.. code-block:: python
+
+    @celery.task()
+    def temp():
+        return [xsum(range(10)), xsum(range(100))]
+
+and using ``starmap``::
+
+    >>> ~add.starmap(zip(range(10), range(10)))
+    [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
+
+is the same as having a task doing:
+
+.. code-block:: python
+
+    @celery.task()
+    def temp():
+        return [add(i, i) for i in range(10)]
+
+Both ``map`` and ``starmap`` are subtasks, so they can be used as
+other subtasks and combined in groups etc., for example
+to apply the starmap after 10 seconds::
+
+    >>> add.starmap(zip(range(10), range(10))).apply_async(countdown=10)
+
+.. _chunking-ov:
+
+Chunking
+========
+
+-- Chunking lets you divide a iterable of work into pieces,
+   so that if you have one million objects, you can create
+   10 tasks with hundred thousand objects each.
+
+Some may worry that chunking your tasks results in a degradation
+of parallelism, but this is rarely true for a busy cluster
+and in practice since you are avoiding the overhead  of messaging
+it may considerably increase performance.
+
+To create a chunks subtask you can use :meth:`@Task.chunks`:
+
+.. code-block:: python
+
+    >>> add.chunks(zip(range(100), range(100)), 10)
+
+As with :class:`~celery.group` the act of **calling**
+the chunks will apply the tasks in the current process:
+
+.. code-block:: python
+
+    >>> from proj.tasks import add
+
+    >>> res = add.chunks(zip(range(100), range(100)), 10)()
+    >>> res.get()
+    [[0, 2, 4, 6, 8, 10, 12, 14, 16, 18],
+     [20, 22, 24, 26, 28, 30, 32, 34, 36, 38],
+     [40, 42, 44, 46, 48, 50, 52, 54, 56, 58],
+     [60, 62, 64, 66, 68, 70, 72, 74, 76, 78],
+     [80, 82, 84, 86, 88, 90, 92, 94, 96, 98],
+     [100, 102, 104, 106, 108, 110, 112, 114, 116, 118],
+     [120, 122, 124, 126, 128, 130, 132, 134, 136, 138],
+     [140, 142, 144, 146, 148, 150, 152, 154, 156, 158],
+     [160, 162, 164, 166, 168, 170, 172, 174, 176, 178],
+     [180, 182, 184, 186, 188, 190, 192, 194, 196, 198]]
+
+while calling ``.apply_async`` will create a dedicated
+task so that the individual tasks are applied in a worker
+instead::
+
+    >>> add.chunks(zip(range(100), range(100), 10)).apply_async()
+
+You can also convert chunks to a group::
+
+    >>> group = add.chunks(zip(range(100), range(100), 10)).group()
+
+and with the group skew the countdown of each task by increments
+of one::
+
+    >>> group.skew(start=1, stop=10)()
+
+which means that the first task will have a countdown of 1, the second
+a countdown of 2 and so on.
+
+

docs/userguide/index.rst

@@ -8,14 +8,14 @@
 :Date: |today|
 
 .. toctree::
-    :maxdepth: 2
+    :maxdepth: 1
 
     application
     tasks
     calling
+    canvas
     workers
     periodic-tasks
-    canvas
     remote-tasks
     routing
     monitoring

docs/userguide/tasks.rst

@@ -4,29 +4,62 @@
  Tasks
 =======
 
+Tasks are the building blocks of Celery applications.
+
+A task can be created out of any callable and defines what happens
+when the worker receives a particular message.
+
+Every task has unique name which is referenced in the message,
+so that the worker can find the right task to execute.
+
+It's not a requirement, but it's a good idea to keep your tasks
+*idempotent*.  Idempotence means that a task can be applied multiple
+times without changing the result.
+
+This is important because the task message will not disappear
+until the message has been *acknowledged*. A worker can reserve
+many messages in advance and even if the worker is killed -- caused by a power failure
+or otherwise -- the message will be redelivered to another worker.
+
+But the worker cannot know if your tasks are idempotent, so the default
+behavior is to acknowledge the message in advance just before it's executed,
+this way a task that has been started will not be executed again.
+
+If your task is idempotent you can set the :attr:`acks_late` option
+to have the worker acknowledge the message *after* that task has been
+executed instead.  This way the task will be redelivered to another
+worker, even if the task has already started executing before.
+See also the FAQ entry :ref:`faq-acks_late-vs-retry`.
+
+--
+
+In this chapter you will learn all about defining tasks,
+and this is the **table of contents**:
+
 .. contents::
     :local:
     :depth: 1
 
+
 .. _task-basics:
 
 Basics
 ======
 
-A task is a class that encapsulates a function and its execution options.
-Given a function ``create_user`` taking two arguments: `username` and
-`password`, you can easily create a task from any function by using
-the task decorator:
+You can easily create a task from any callable by using
+the :meth:`~@Celery.task` decorator:
 
 .. code-block:: python
 
-    from django.contrib.auth import User
+    from .models import User
 
     @celery.task()
     def create_user(username, password):
         User.objects.create(username=username, password=password)
 
-Task options can be specified as arguments to the decorator:
+
+There are also many :ref:`options <task-options>` that can be set for the task,
+these can be specified as arguments to the decorator:
 
 .. code-block:: python
 
@@ -34,6 +67,8 @@ Task options can be specified as arguments to the decorator:
     def create_user(username, password):
         User.objects.create(username=username, password=password)
 
+
+
 .. sidebar:: How do I import the task decorator?
 
     The task decorator is available on your :class:`@Celery` instance,
@@ -63,6 +98,93 @@ Task options can be specified as arguments to the decorator:
         def add(x, y):
             return x + y
 
+.. _task-names:
+
+Names
+=====
+
+Every task must have a unique name, and a new name
+will be generated out of the function name if a custom name is not provided.
+
+For example:
+
+.. code-block:: python
+
+    >>> @celery.task(name="sum-of-two-numbers")
+    >>> def add(x, y):
+    ...     return x + y
+
+    >>> add.name
+    'sum-of-two-numbers'
+
+A best practice is to use the module name as a namespace,
+this way names won't collide if there's already a task with that name
+defined in another module.
+
+.. code-block:: python
+
+    >>> @celery.task(name="tasks.add")
+    >>> def add(x, y):
+    ...     return x + y
+
+You can tell the name of the task by investigating its name attribute::
+
+    >>> add.name
+    'tasks.add'
+
+Which is exactly the name that would have been generated anyway,
+if the module name is "tasks.py":
+
+:file:`tasks.py`:
+
+.. code-block:: python
+
+    @celery.task()
+    def add(x, y):
+        return x + y
+
+    >>> from tasks import add
+    >>> add.name
+    'tasks.add'
+
+.. _task-naming-relative-imports:
+
+Automatic naming and relative imports
+-------------------------------------
+
+Relative imports and automatic name generation does not go well together,
+so if you're using relative imports you should set the name explicitly.
+
+For example if the client imports the module "myapp.tasks" as ".tasks", and
+the worker imports the module as "myapp.tasks", the generated names won't match
+and an :exc:`~@NotRegistered` error will be raised by the worker.
+
+This is also the case if using Django and using `project.myapp`::
+
+    INSTALLED_APPS = ("project.myapp", )
+
+The worker will have the tasks registered as "project.myapp.tasks.*",
+while this is what happens in the client if the module is imported as
+"myapp.tasks":
+
+.. code-block:: python
+
+    >>> from myapp.tasks import add
+    >>> add.name
+    'myapp.tasks.add'
+
+For this reason you should never use "project.app", but rather
+add the project directory to the Python path::
+
+    import os
+    import sys
+    sys.path.append(os.getcwd())
+
+    INSTALLED_APPS = ("myapp", )
+
+This makes more sense from the reusable app perspective anyway.
+
+
 .. _task-request-info:
 
 Context
@@ -144,18 +266,21 @@ out/-err will be redirected to the workers logs by default (see
 
 .. _task-retry:
 
-Retrying a task if something fails
-==================================
+Retrying
+========
+
+:meth:`~@Task.retry` can be used to re-execute the task,
+for example in the event of recoverable errors.
 
-:meth:`~@Task.retry` can be used to re-execute the task in the event
-of temporary failure, or for any other reason.
+When you call ``retry`` it will send a new message, using the same
+task-id, and it will take care to make sure the message is delivered
+to the same queue as the originating task.
 
-A new message will the be sent using the same task-id, and sent
-to the same queue as the current task. This also means that you
-can track retries using the task's result instance (if a result
-backend is enabled).
+When a task is retried this is also recorded as a task state,
+so that you can track the progress of the task using the result
+instance (see :ref:`task-states`).
 
-An example:
+Here's an example using ``retry``:
 
 .. code-block:: python
 
@@ -168,11 +293,8 @@ An example:
             raise send_twitter_status.retry(exc=exc)
 
 Here we used the `exc` argument to pass the current exception to
-:meth:`~@Task.retry`. At each step of the retry this exception
-is available as the tombstone (result) of the task. When
-:attr:`~@Task.max_retries` has been exceeded this is the
-exception raised.  However, if an `exc` argument is not provided the
-:exc:`~@RetryTaskError` exception is raised instead.
+:meth:`~@Task.retry`.  Both the exception and the traceback will
+be available in the task state (if a result backend is enabled).
 
 .. note::
 
@@ -211,8 +333,16 @@ override this default.
 
 .. _task-options:
 
-Task options
-============
+List of Options
+===============
+
+The task decorator can take a number of options that change the way
+the task behaves, for example you can set the rate limit for a task
+using the :attr:`rate_limit` option.
+
+Any keyword argument passed to the task decorator will actually be set
+as an attribute of the resulting task class, and this is a list
+of the built-in attributes.
 
 General
 -------
@@ -359,97 +489,10 @@ General
 
     The API reference for :class:`~@Task`.
 
-.. _task-names:
-
-Task names
-==========
-
-The task type is identified by the *task name*.
-
-If not provided a name will be automatically generated using the module
-and class name.
-
-For example:
-
-.. code-block:: python
-
-    >>> @celery.task(name="sum-of-two-numbers")
-    >>> def add(x, y):
-    ...     return x + y
-
-    >>> add.name
-    'sum-of-two-numbers'
-
-A best practice is to use the module name as a prefix to classify the
-tasks using namespaces.  This way the name won't collide with the name from
-another module:
-
-.. code-block:: python
-
-    >>> @celery.task(name="tasks.add")
-    >>> def add(x, y):
-    ...     return x + y
-
-
-You can tell the name of the task by investigating its name attribute::
-
-    >>> add.name
-    'tasks.add'
-
-
-Which is exactly the name automatically generated for this
-task if the module name is "tasks.py":
-
-.. code-block:: python
-
-    >>> @celery.task()
-    >>> def add(x, y):
-    ...     return x + y
-
-    >>> add.name
-    'tasks.add'
-
-.. _task-naming-relative-imports:
-
-Automatic naming and relative imports
--------------------------------------
-
-Relative imports and automatic name generation does not go well together,
-so if you're using relative imports you should set the name explicitly.
-
-For example if the client imports the module "myapp.tasks" as ".tasks", and
-the worker imports the module as "myapp.tasks", the generated names won't match
-and an :exc:`~@NotRegistered` error will be raised by the worker.
-
-This is also the case if using Django and using `project.myapp`::
-
-    INSTALLED_APPS = ("project.myapp", )
-
-The worker will have the tasks registered as "project.myapp.tasks.*",
-while this is what happens in the client if the module is imported as
-"myapp.tasks":
-
-.. code-block:: python
-
-    >>> from myapp.tasks import add
-    >>> add.name
-    'myapp.tasks.add'
-
-For this reason you should never use "project.app", but rather
-add the project directory to the Python path::
-
-    import os
-    import sys
-    sys.path.append(os.getcwd())
-
-    INSTALLED_APPS = ("myapp", )
-
-This makes more sense from the reusable app perspective anyway.
-
 .. _task-states:
 
-Task States
-===========
+States
+======
 
 Celery can keep track of the tasks current state.  The state also contains the
 result of a successful task, or the exception and traceback information of a
@@ -697,8 +740,8 @@ you have to pass them as regular args:
 
 .. _task-custom-classes:
 
-Creating custom task classes
-============================
+Custom task classes
+===================
 
 All tasks inherit from the :class:`@Task` class.
 The :meth:`~@Task.run` method becomes the task body.

docs/userguide/workers.rst

@@ -21,7 +21,7 @@ Starting the worker
 
 You can start the worker in the foreground by executing the command::
 
-    $ celery worker --loglevel=INFO --app=app
+    $ celery worker --app=app -l info
 
 For a full list of available command line options see
 :mod:`~celery.bin.celeryd`, or simply do::
@@ -120,18 +120,133 @@ argument and defaults to the number of CPUs available on the machine.
     to find the numbers that works best for you, as this varies based on
     application, work load, task run times and other factors.
 
+.. _worker-remote-control:
+
+Remote control
+==============
+
+.. versionadded:: 2.0
+
+.. sidebar:: The ``celery`` command
+
+    The :program:`celery` program is used to execute remote control
+    commands from the command line.  It supports all of the commands
+    listed below.  See :ref:`monitoring-celeryctl` for more information.
+
+pool support: *processes, eventlet, gevent*, blocking:*threads/solo* (see note)
+broker support: *amqp, redis, mongodb*
+
+Workers have the ability to be remote controlled using a high-priority
+broadcast message queue.  The commands can be directed to all, or a specific
+list of workers.
+
+Commands can also have replies.  The client can then wait for and collect
+those replies.  Since there's no central authority to know how many
+workers are available in the cluster, there is also no way to estimate
+how many workers may send a reply, so the client has a configurable
+timeout — the deadline in seconds for replies to arrive in.  This timeout
+defaults to one second.  If the worker doesn't reply within the deadline
+it doesn't necessarily mean the worker didn't reply, or worse is dead, but
+may simply be caused by network latency or the worker being slow at processing
+commands, so adjust the timeout accordingly.
+
+In addition to timeouts, the client can specify the maximum number
+of replies to wait for.  If a destination is specified, this limit is set
+to the number of destination hosts.
+
+.. note::
+
+    The solo and threads pool supports remote control commands,
+    but any task executing will block any waiting control command,
+    so it is of limited use if the worker is very busy.  In that
+    case you must increase the timeout waitin for replies in the client.
+
+.. _worker-broadcast-fun:
+
+The :meth:`~@control.broadcast` function.
+----------------------------------------------------
+
+This is the client function used to send commands to the workers.
+Some remote control commands also have higher-level interfaces using
+:meth:`~@control.broadcast` in the background, like
+:meth:`~@control.rate_limit` and :meth:`~@control.ping`.
+
+Sending the :control:`rate_limit` command and keyword arguments::
+
+    >>> from celery.task.control import broadcast
+    >>> celery.control.broadcast("rate_limit",
+    ...                          arguments={"task_name": "myapp.mytask",
+    ...                                     "rate_limit": "200/m"})
+
+This will send the command asynchronously, without waiting for a reply.
+To request a reply you have to use the `reply` argument::
+
+    >>> celery.control.broadcast("rate_limit", {
+    ...     "task_name": "myapp.mytask", "rate_limit": "200/m"}, reply=True)
+    [{'worker1.example.com': 'New rate limit set successfully'},
+     {'worker2.example.com': 'New rate limit set successfully'},
+     {'worker3.example.com': 'New rate limit set successfully'}]
+
+Using the `destination` argument you can specify a list of workers
+to receive the command::
+
+    >>> celery.control.broadcast("rate_limit", {
+    ...     "task_name": "myapp.mytask",
+    ...     "rate_limit": "200/m"}, reply=True,
+    ...                             destination=["worker1.example.com"])
+    [{'worker1.example.com': 'New rate limit set successfully'}]
+
+
+Of course, using the higher-level interface to set rate limits is much
+more convenient, but there are commands that can only be requested
+using :meth:`~@control.broadcast`.
+
+.. control:: revoke
+
+Revoking tasks
+==============
+pool support: all
+broker support: *amqp, redis, mongodb*
+
+All worker nodes keeps a memory of revoked task ids, either in-memory or
+persistent on disk (see :ref:`worker-persistent-revokes`).
+
+When a worker receives a revoke request it will skip executing
+the task, but it won't terminate an already executing task unless
+the `terminate` option is set.
+
+If `terminate` is set the worker child process processing the task
+will be terminated.  The default signal sent is `TERM`, but you can
+specify this using the `signal` argument.  Signal can be the uppercase name
+of any signal defined in the :mod:`signal` module in the Python Standard
+Library.
+
+Terminating a task also revokes it.
+
+**Example**
+
+::
+
+    >>> celery.control.revoke("d9078da5-9915-40a0-bfa1-392c7bde42ed")
+
+    >>> celery.control.revoke("d9078da5-9915-40a0-bfa1-392c7bde42ed",
+    ...                       terminate=True)
+
+    >>> celery.control.revoke("d9078da5-9915-40a0-bfa1-392c7bde42ed",
+    ...                       terminate=True, signal="SIGKILL")
+
 .. _worker-persistent-revokes:
 
 Persistent revokes
-==================
+------------------
 
 Revoking tasks works by sending a broadcast message to all the workers,
 the workers then keep a list of revoked tasks in memory.
 
 If you want tasks to remain revoked after worker restart you need to
 specify a file for these to be stored in, either by using the `--statedb`
-argument to :mod:`~celery.bin.celeryd` or the :setting:`CELERYD_STATE_DB`
-setting.  See :setting:`CELERYD_STATE_DB` for more information.
+argument to :program:`celery worker` or the :setting:`CELERYD_STATE_DB`
+setting.
 
 Note that remote control commands must be working for revokes to work.
 Remote control commands are only supported by the amqp, redis and mongodb
@@ -204,6 +319,32 @@ two minutes::
 
 Only tasks that starts executing after the time limit change will be affected.
 
+.. _worker-rate-limits:
+
+Rate Limits
+===========
+
+.. control:: rate_limit
+
+Changing rate-limits at runtime
+-------------------------------
+
+Example changing the rate limit for the `myapp.mytask` task to accept
+200 tasks a minute on all servers::
+
+    >>> celery.control.rate_limit("myapp.mytask", "200/m")
+
+Example changing the rate limit on a single host by specifying the
+destination host name::
+
+    >>> celery.control.rate_limit("myapp.mytask", "200/m",
+    ...            destination=["worker1.example.com"])
+
+.. warning::
+
+    This won't affect workers with the
+    :setting:`CELERY_DISABLE_RATE_LIMITS` setting enabled.
+
 .. _worker-maxtasksperchild:
 
 Max tasks per child setting
@@ -302,200 +443,16 @@ environment variable::
 
     $ env CELERYD_FSNOTIFY=stat celery worker -l info --autoreload
 
-.. _worker-remote-control:
-
-Remote control
-==============
-
-.. versionadded:: 2.0
-
-pool support: *processes, eventlet, gevent*, blocking:*threads/solo* (see note)
-broker support: *amqp, redis, mongodb*
-
-Workers have the ability to be remote controlled using a high-priority
-broadcast message queue.  The commands can be directed to all, or a specific
-list of workers.
-
-Commands can also have replies.  The client can then wait for and collect
-those replies.  Since there's no central authority to know how many
-workers are available in the cluster, there is also no way to estimate
-how many workers may send a reply, so the client has a configurable
-timeout — the deadline in seconds for replies to arrive in.  This timeout
-defaults to one second.  If the worker doesn't reply within the deadline
-it doesn't necessarily mean the worker didn't reply, or worse is dead, but
-may simply be caused by network latency or the worker being slow at processing
-commands, so adjust the timeout accordingly.
-
-In addition to timeouts, the client can specify the maximum number
-of replies to wait for.  If a destination is specified, this limit is set
-to the number of destination hosts.
-
-.. seealso::
-
-    The :program:`celery` program is used to execute remote control
-    commands from the command line.  It supports all of the commands
-    listed below.  See :ref:`monitoring-celeryctl` for more information.
-
-.. note::
-
-    The solo and threads pool supports remote control commands,
-    but any task executing will block any waiting control command,
-    so it is of limited use if the worker is very busy.  In that
-    case you must increase the timeout waitin for replies in the client.
-
-.. _worker-broadcast-fun:
-
-The :meth:`~@control.broadcast` function.
-----------------------------------------------------
-
-This is the client function used to send commands to the workers.
-Some remote control commands also have higher-level interfaces using
-:meth:`~@control.broadcast` in the background, like
-:meth:`~@control.rate_limit` and :meth:`~@control.ping`.
-
-Sending the :control:`rate_limit` command and keyword arguments::
-
-    >>> from celery.task.control import broadcast
-    >>> celery.control.broadcast("rate_limit",
-    ...                          arguments={"task_name": "myapp.mytask",
-    ...                                     "rate_limit": "200/m"})
-
-This will send the command asynchronously, without waiting for a reply.
-To request a reply you have to use the `reply` argument::
-
-    >>> celery.control.broadcast("rate_limit", {
-    ...     "task_name": "myapp.mytask", "rate_limit": "200/m"}, reply=True)
-    [{'worker1.example.com': 'New rate limit set successfully'},
-     {'worker2.example.com': 'New rate limit set successfully'},
-     {'worker3.example.com': 'New rate limit set successfully'}]
-
-Using the `destination` argument you can specify a list of workers
-to receive the command::
-
-    >>> celery.control.broadcast("rate_limit", {
-    ...     "task_name": "myapp.mytask",
-    ...     "rate_limit": "200/m"}, reply=True,
-    ...                             destination=["worker1.example.com"])
-    [{'worker1.example.com': 'New rate limit set successfully'}]
-
-
-Of course, using the higher-level interface to set rate limits is much
-more convenient, but there are commands that can only be requested
-using :meth:`~@control.broadcast`.
-
-.. _worker-rate-limits:
-
-.. control:: rate_limit
-
-Rate limits
------------
-
-Example changing the rate limit for the `myapp.mytask` task to accept
-200 tasks a minute on all servers::
-
-    >>> celery.control.rate_limit("myapp.mytask", "200/m")
-
-Example changing the rate limit on a single host by specifying the
-destination host name::
-
-    >>> celery.control.rate_limit("myapp.mytask", "200/m",
-    ...            destination=["worker1.example.com"])
-
-.. warning::
-
-    This won't affect workers with the
-    :setting:`CELERY_DISABLE_RATE_LIMITS` setting on. To re-enable rate limits
-    then you have to restart the worker.
-
-.. control:: revoke
-
-Revoking tasks
---------------
-
-All worker nodes keeps a memory of revoked task ids, either in-memory or
-persistent on disk (see :ref:`worker-persistent-revokes`).
-
-When a worker receives a revoke request it will skip executing
-the task, but it won't terminate an already executing task unless
-the `terminate` option is set.
-
-If `terminate` is set the worker child process processing the task
-will be terminated.  The default signal sent is `TERM`, but you can
-specify this using the `signal` argument.  Signal can be the uppercase name
-of any signal defined in the :mod:`signal` module in the Python Standard
-Library.
-
-Terminating a task also revokes it.
-
-**Example**
-
-::
-
-    >>> celery.control.revoke("d9078da5-9915-40a0-bfa1-392c7bde42ed")
-
-    >>> celery.control.revoke("d9078da5-9915-40a0-bfa1-392c7bde42ed",
-    ...                       terminate=True)
-
-    >>> celery.control.revoke("d9078da5-9915-40a0-bfa1-392c7bde42ed",
-    ...                       terminate=True, signal="SIGKILL")
-
-.. control:: shutdown
-
-Remote shutdown
----------------
-
-This command will gracefully shut down the worker remotely::
-
-    >>> celery.control.broadcast("shutdown") # shutdown all workers
-    >>> celery.control.broadcast("shutdown, destination="worker1.example.com")
-
-.. control:: ping
-
-Ping
-----
-
-This command requests a ping from alive workers.
-The workers reply with the string 'pong', and that's just about it.
-It will use the default one second timeout for replies unless you specify
-a custom timeout::
-
-    >>> celery.control.ping(timeout=0.5)
-    [{'worker1.example.com': 'pong'},
-     {'worker2.example.com': 'pong'},
-     {'worker3.example.com': 'pong'}]
-
-:meth:`~@control.ping` also supports the `destination` argument,
-so you can specify which workers to ping::
-
-    >>> ping(['worker2.example.com', 'worker3.example.com'])
-    [{'worker2.example.com': 'pong'},
-     {'worker3.example.com': 'pong'}]
-
-.. _worker-enable-events:
-
-.. control:: enable_events
-.. control:: disable_events
-
-Enable/disable events
----------------------
-
-You can enable/disable events by using the `enable_events`,
-`disable_events` commands.  This is useful to temporarily monitor
-a worker using :program:`celery events`/:program:`celerymon`.
-
-.. code-block:: python
-
-    >>> celery.control.enable_events()
-    >>> celery.control.disable_events()
-
 .. _worker-autoreload:
 
-Autoreloading
--------------
+.. control:: pool_restart
+
+Pool Restart Command
+--------------------
 
 .. versionadded:: 2.5
 
-The remote control command ``pool_restart`` sends restart requests to
+The remote control command :control:`pool_restart` sends restart requests to
 the workers child processes.  It is particularly useful for forcing
 the worker to import new modules, or for reloading already imported
 modules.  This command does not interrupt executing tasks.
@@ -545,34 +502,6 @@ your own custom reloader by passing the ``reloader`` argument.
     - http://www.indelible.org/ink/python-reloading/
     - http://docs.python.org/library/functions.html#reload
 
-.. _worker-custom-control-commands:
-
-Writing your own remote control commands
-----------------------------------------
-
-Remote control commands are registered in the control panel and
-they take a single argument: the current
-:class:`~celery.worker.control.ControlDispatch` instance.
-From there you have access to the active
-:class:`~celery.worker.consumer.Consumer` if needed.
-
-Here's an example control command that restarts the broker connection:
-
-.. code-block:: python
-
-    from celery.worker.control import Panel
-
-    @Panel.register
-    def reset_connection(panel):
-        panel.logger.critical("Connection reset by remote control.")
-        panel.consumer.reset_connection()
-        return {"ok": "connection reset"}
-
-
-These can be added to task modules, or you can keep them in their own module
-then import them using the :setting:`CELERY_IMPORTS` setting::
-
-    CELERY_IMPORTS = ("myapp.worker.control", )
 
 .. _worker-inspect:
 
@@ -582,6 +511,9 @@ Inspecting workers
 :class:`@control.inspect` lets you inspect running workers.  It
 uses remote control commands under the hood.
 
+You can also use the ``celery`` command to inspect workers,
+and it supports the same commands as the :class:`@Celery.control` interface.
+
 .. code-block:: python
 
     # Inspect all nodes.
@@ -603,12 +535,7 @@ You can get a list of tasks registered in the worker using the
 :meth:`~@control.inspect.registered`::
 
     >>> i.registered()
-    [{'worker1.example.com': ['celery.delete_expired_task_meta',
-                              'celery.execute_remote',
-                              'celery.map_async',
-                              'celery.ping',
-                              'celery.task.http.HttpDispatchTask',
-                              'tasks.add',
+    [{'worker1.example.com': ['tasks.add',
                               'tasks.sleeptask']}]
 
 .. _worker-inspect-active-tasks:
@@ -649,7 +576,9 @@ You can get a list of tasks waiting to be scheduled by using
             "args": "[2]",
             "kwargs": "{}"}}]}]
 
-Note that these are tasks with an eta/countdown argument, not periodic tasks.
+.. note::
+
+    These are tasks with an eta/countdown argument, not periodic tasks.
 
 .. _worker-inspect-reserved:
 
@@ -668,3 +597,79 @@ You can get a list of these using
           "id": "32666e9b-809c-41fa-8e93-5ae0c80afbbf",
           "args": "(8,)",
           "kwargs": "{}"}]}]
+
+
+Additional Commands
+===================
+
+.. control:: shutdown
+
+Remote shutdown
+---------------
+
+This command will gracefully shut down the worker remotely::
+
+    >>> celery.control.broadcast("shutdown") # shutdown all workers
+    >>> celery.control.broadcast("shutdown, destination="worker1.example.com")
+
+.. control:: ping
+
+Ping
+----
+
+This command requests a ping from alive workers.
+The workers reply with the string 'pong', and that's just about it.
+It will use the default one second timeout for replies unless you specify
+a custom timeout::
+
+    >>> celery.control.ping(timeout=0.5)
+    [{'worker1.example.com': 'pong'},
+     {'worker2.example.com': 'pong'},
+     {'worker3.example.com': 'pong'}]
+
+:meth:`~@control.ping` also supports the `destination` argument,
+so you can specify which workers to ping::
+
+    >>> ping(['worker2.example.com', 'worker3.example.com'])
+    [{'worker2.example.com': 'pong'},
+     {'worker3.example.com': 'pong'}]
+
+.. _worker-enable-events:
+
+.. control:: enable_events
+.. control:: disable_events
+
+Enable/disable events
+---------------------
+
+You can enable/disable events by using the `enable_events`,
+`disable_events` commands.  This is useful to temporarily monitor
+a worker using :program:`celery events`/:program:`celerymon`.
+
+.. code-block:: python
+
+    >>> celery.control.enable_events()
+    >>> celery.control.disable_events()
+
+.. _worker-custom-control-commands:
+
+Writing your own remote control commands
+========================================
+
+Remote control commands are registered in the control panel and
+they take a single argument: the current
+:class:`~celery.worker.control.ControlDispatch` instance.
+From there you have access to the active
+:class:`~celery.worker.consumer.Consumer` if needed.
+
+Here's an example control command that restarts the broker connection:
+
+.. code-block:: python
+
+    from celery.worker.control import Panel
+
+    @Panel.register
+    def reset_connection(panel):
+        panel.logger.critical("Connection reset by remote control.")
+        panel.consumer.reset_connection()
+        return {"ok": "connection reset"}