Merge branch '2.4'

Conflicts:
	AUTHORS
	Changelog
	README.rst
	celery/__init__.py
	celery/platforms.py
	celery/utils/timeutils.py
	contrib/generic-init.d/celerybeat
	contrib/generic-init.d/celeryd
	contrib/generic-init.d/celeryevcam
	docs/includes/introduction.txt
	setup.py

AUTHORS

@@ -57,6 +57,7 @@ Joshua Ginsberg <jag@flowtheory.net>
 Juan Ignacio Catalano <catalanojuan@gmail.com>
 Juarez Bochi <jbochi@gmail.com>
 Jude Nagurney <jude@pwan.org>
+Julien Poissonnier <julien@caffeine.lu>
 Kevin Tran <hekevintran@gmail.com>
 Kornelijus Survila <kornholijo@gmail.com>
 Leo Dirac <leo@banyanbranch.com>

Changelog

@@ -184,6 +184,23 @@ News
 * Moved some common threading functionality to new module
   :mod:`celery.utils.threads`
 
+.. _version-2.4.5:
+
+2.4.5
+=====
+:release-date: 2011-12-02 05:00 P.M GMT
+:by: Ask Solem
+
+* Periodic task interval schedules were accidentally rounded down,
+  resulting in some periodic tasks being executed early.
+
+* Logging of humanized times in the celerybeat log is now more detailed.
+
+* New :ref:`brokers` section in the Getting Started part of the Documentation
+
+    This replaces the old :ref:`tut-otherqueues` tutorial, and adds
+    documentation for MongoDB, Beanstalk and CouchDB.
+
 .. _version-2.4.4:
 
 2.4.4

README.rst

@@ -4,7 +4,7 @@
 
 .. image:: http://cloud.github.com/downloads/ask/celery/celery_128.png
 
-:Version: 2.5.0a1
+:Version: 2.5.0b1
 :Web: http://celeryproject.org/
 :Download: http://pypi.python.org/pypi/celery/
 :Source: http://github.com/ask/celery/
@@ -111,7 +111,6 @@ Features
     +-----------------+----------------------------------------------------+
     | Fault-tolerant  | Excellent configurable error recovery when using   |
     |                 | `RabbitMQ`, ensures your tasks are never lost.     |
-    |                 | scenarios, and your tasks will never be lost.      |
     +-----------------+----------------------------------------------------+
     | Distributed     | Runs on one or more machines. Supports             |
     |                 | broker `clustering`_ and `HA`_ when used in        |
@@ -215,11 +214,45 @@ or from source.
 
 To install using `pip`,::
 
-    $ pip install Celery
+    $ pip install -U Celery
 
 To install using `easy_install`,::
 
-    $ easy_install Celery
+    $ easy_install -U Celery
+
+Bundles
+-------
+
+Celery also defines a group of bundles that can be used
+to install Celery and the dependencies for a given feature.
+
+The following bundles are available:
+
+:`celery-with-redis`_:
+    for using Redis as a broker.
+
+:`celery-with-mongodb`_:
+    for using MongoDB as a broker.
+
+:`django-celery-with-redis`_:
+    for Django, and using Redis as a broker.
+
+:`django-celery-with-mongodb`_:
+    for Django, and using MongoDB as a broker.
+
+:`bundle-celery`_:
+    convenience bundle installing *Celery* and related packages.
+
+.. _`celery-with-redis`:
+    http://pypi.python.org/pypi/celery-with-redis/
+.. _`celery-with-mongodb`:
+    http://pypi.python.org/pypi/celery-with-mongdb/
+.. _`django-celery-with-redis`:
+    http://pypi.python.org/pypi/django-celery-with-redis/
+.. _`django-celery-with-mongodb`:
+    http://pypi.python.org/pypi/django-celery-with-mongdb/
+.. _`bundle-celery`:
+    http://pypi.python.org/pypi/bundle-celery/
 
 .. _celery-installing-from-source:
 

celery/contrib/bundles.py

@@ -0,0 +1,47 @@
+from __future__ import absolute_import
+
+from celery import VERSION
+from bundle.extensions import Dist
+
+
+defaults = {"author": "Celery Project",
+            "author_email": "bundles@celeryproject.org",
+            "url": "http://celeryproject.org",
+            "license": "BSD"}
+celery = Dist("celery", VERSION, **defaults)
+django_celery = Dist("django-celery", VERSION, **defaults)
+flask_celery = Dist("Flask-Celery", VERSION, **defaults)
+
+bundles = [
+    celery.Bundle("celery-with-redis",
+        "Bundle installing the dependencies for Celery and Redis",
+        requires=["redis>=2.4.4"]),
+    celery.Bundle("celery-with-mongodb",
+        "Bundle installing the dependencies for Celery and MongoDB",
+        requires=["pymongo"]),
+    celery.Bundle("celery-with-couchdb",
+        "Bundle installing the dependencies for Celery and CouchDB",
+        requires=["couchdb"]),
+    celery.Bundle("celery-with-beanstalk",
+        "Bundle installing the dependencies for Celery and Beanstalk",
+        requires=["beanstalkc"]),
+
+    django_celery.Bundle("django-celery-with-redis",
+        "Bundle installing the dependencies for Django-Celery and Redis",
+        requires=["redis>=2.4.4"]),
+    django_celery.Bundle("django-celery-with-mongodb",
+        "Bundle installing the dependencies for Django-Celery and MongoDB",
+        requires=["pymongo"]),
+    django_celery.Bundle("django-celery-with-couchdb",
+        "Bundle installing the dependencies for Django-Celery and CouchDB",
+        requires=["couchdb"]),
+    django_celery.Bundle("django-celery-with-beanstalk",
+        "Bundle installing the dependencies for Django-Celery and Beanstalk",
+        requires=["beanstalkc"]),
+
+    celery.Bundle("bundle-celery",
+        "Bundle that installs Celery related modules",
+        requires=[django_celery, flask_celery,
+                  "django", "setproctitle", "celerymon",
+                  "cyme", "kombu-sqlalchemy", "django-kombu"]),
+]

celery/schedules.py

@@ -18,7 +18,7 @@ from datetime import datetime, timedelta
 from dateutil.relativedelta import relativedelta
 
 from .utils import is_iterable
-from .utils.timeutils import (timedelta_seconds, weekday,
+from .utils.timeutils import (timedelta_seconds, weekday, maybe_timedelta,
                               remaining, humanize_seconds)
 
 
@@ -30,7 +30,7 @@ class schedule(object):
     relative = False
 
     def __init__(self, run_every=None, relative=False):
-        self.run_every = run_every
+        self.run_every = maybe_timedelta(run_every)
         self.relative = relative
 
     def remaining_estimate(self, last_run_at):
@@ -62,18 +62,25 @@ class schedule(object):
         rem_delta = self.remaining_estimate(last_run_at)
         rem = timedelta_seconds(rem_delta)
         if rem == 0:
-            return True, timedelta_seconds(self.run_every)
+            return True, self.seconds
         return False, rem
 
     def __repr__(self):
-        return "<freq: %s>" % (
-                    humanize_seconds(timedelta_seconds(self.run_every)), )
+        return "<freq: %s>" % self.human_seconds
 
     def __eq__(self, other):
         if isinstance(other, schedule):
             return self.run_every == other.run_every
         return self.run_every == other
 
+    @property
+    def seconds(self):
+        return timedelta_seconds(self.run_every)
+
+    @property
+    def human_seconds(self):
+        return humanize_seconds(self.seconds)
+
 
 class crontab_parser(object):
     """Parser for crontab expressions. Any expression of the form 'groups'

celery/tests/test_utils/test_utils_timeutils.py

@@ -33,12 +33,12 @@ class test_timeutils(unittest.TestCase):
         self.assertEqual(timeutils.timedelta_seconds(delta), 0)
 
     def test_humanize_seconds(self):
-        t = ((4 * 60 * 60 * 24, "4 days"),
-             (1 * 60 * 60 * 24, "1 day"),
-             (4 * 60 * 60, "4 hours"),
-             (1 * 60 * 60, "1 hour"),
-             (4 * 60, "4 minutes"),
-             (1 * 60, "1 minute"),
+        t = ((4 * 60 * 60 * 24, "4.00 days"),
+             (1 * 60 * 60 * 24, "1.00 day"),
+             (4 * 60 * 60, "4.00 hours"),
+             (1 * 60 * 60, "1.00 hour"),
+             (4 * 60, "4.00 minutes"),
+             (1 * 60, "1.00 minute"),
              (4, "4.00 seconds"),
              (1, "1.00 second"),
              (4.3567631221, "4.36 seconds"),

celery/utils/timeutils.py

@@ -11,8 +11,6 @@
 """
 from __future__ import absolute_import
 
-import math
-
 from kombu.utils import cached_property
 
 from datetime import datetime, timedelta
@@ -35,10 +33,10 @@ RATE_MODIFIER_MAP = {"s": lambda n: n,
 
 HAVE_TIMEDELTA_TOTAL_SECONDS = hasattr(timedelta, "total_seconds")
 
-TIME_UNITS = (("day", 60 * 60 * 24, lambda n: int(math.ceil(n))),
-              ("hour", 60 * 60, lambda n: int(math.ceil(n))),
-              ("minute", 60, lambda n: int(math.ceil(n))),
-              ("second", 1, lambda n: "%.2f" % n))
+TIME_UNITS = (("day", 60 * 60 * 24.0, lambda n: "%.2f" % n),
+              ("hour", 60 * 60.0, lambda n: "%.2f" % n),
+              ("minute", 60.0, lambda n: "%.2f" % n),
+              ("second", 1.0, lambda n: "%.2f" % n))
 
 
 class UnknownTimezone(Exception):
@@ -134,14 +132,14 @@ def delta_resolution(dt, delta):
     return dt
 
 
-def remaining(start, ends_in, now=None, relative=True):
+def remaining(start, ends_in, now=None, relative=False):
     """Calculate the remaining time for a start date and a timedelta.
 
     e.g. "how many seconds left for 30 seconds after start?"
 
     :param start: Start :class:`~datetime.datetime`.
     :param ends_in: The end delta as a :class:`~datetime.timedelta`.
-    :keyword relative: If set to :const:`False`, the end time will be
+    :keyword relative: If enabled the end time will be
         calculated using :func:`delta_resolution` (i.e. rounded to the
         resolution of `ends_in`).
     :keyword now: Function returning the current time and date,
@@ -151,7 +149,7 @@ def remaining(start, ends_in, now=None, relative=True):
     now = now or datetime.utcnow()
 
     end_date = start + ends_in
-    if not relative:
+    if relative:
         end_date = delta_resolution(end_date, ends_in)
     return end_date - now
 
@@ -187,6 +185,7 @@ def weekday(name):
 def humanize_seconds(secs, prefix=""):
     """Show seconds in human form, e.g. 60 is "1 minute", 7200 is "2
     hours"."""
+    secs = float(secs)
     for unit, divider, formatter in TIME_UNITS:
         if secs >= divider:
             w = secs / divider

contrib/generic-init.d/celerybeat

@@ -65,7 +65,7 @@ if [ -n "$CELERYBEAT_USER" ]; then
 fi
 if [ -n "$CELERYBEAT_GROUP" ]; then
     DAEMON_OPTS="$DAEMON_OPTS --gid $CELERYBEAT_GROUP"
-    chown ":$CELERYBEAT_GROUP" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR
+    chgrp "$CELERYBEAT_GROUP" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR
 fi
 
 CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR}

contrib/generic-init.d/celeryd

@@ -65,7 +65,7 @@ if [ -n "$CELERYD_USER" ]; then
 fi
 if [ -n "$CELERYD_GROUP" ]; then
     DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP"
-    chown ":$CELERYD_GROUP" $CELERYD_LOG_DIR $CELERYD_PID_DIR
+    chgrp "$CELERYD_GROUP" $CELERYD_LOG_DIR $CELERYD_PID_DIR
 fi
 
 if [ -n "$CELERYD_CHDIR" ]; then

contrib/generic-init.d/celeryevcam

@@ -137,7 +137,7 @@ if [ -n "$CELERYEV_USER" ]; then
 fi
 if [ -n "$CELERYEV_GROUP" ]; then
     DAEMON_OPTS="$DAEMON_OPTS --gid $CELERYEV_GROUP"
-    chown "$CELERYEV_GROUP" $CELERYBEAT_LOG_DIR $CELERYEV_PID_DIR
+    chgrp "$CELERYEV_GROUP" $CELERYBEAT_LOG_DIR $CELERYEV_PID_DIR
 fi
 
 CELERYEV_CHDIR=${CELERYEV_CHDIR:-$CELERYD_CHDIR}

contrib/release/doc4allmods

@@ -7,7 +7,7 @@ SKIP_FILES="celery.backends.pyredis.rst
             celery.bin.celeryd_detach.rst
             celery.concurrency.processes._win.rst
             celery.contrib.rst
-            celery.contrib.batches.rst
+            celery.contrib.bundles.rst
             celery.worker.control.rst
             celery.worker.control.builtins.rst
             celery.worker.control.registry.rst

docs/configuration.rst

@@ -610,7 +610,7 @@ CELERY_DEFAULT_DELIVERY_MODE
 Can be `transient` or `persistent`.  The default is to send
 persistent messages.
 
-.. _conf-broker-connection:
+.. _conf-broker-settings:
 
 Broker Settings
 ---------------

docs/contributing.rst

@@ -553,15 +553,15 @@ is following the conventions.
 
 * Lines should not exceed 78 columns.
 
-  You can enforce this in :program:`vim` by setting the ``textwidth`` option::
+  You can enforce this in :program:`vim` by setting the ``textwidth`` option:
 
   .. code-block:: vim
 
         set textwidth=78
 
-    If adhering to this limit makes the code less readable, you have one more
-    character to go on, which means 78 is a soft limit, and 79 is the hard
-    limit :)
+  If adhering to this limit makes the code less readable, you have one more
+  character to go on, which means 78 is a soft limit, and 79 is the hard
+  limit :)
 
 * Import order
 
@@ -682,3 +682,20 @@ following:
     for series 2.4.
 
 * Also add the previous version under the "versions" tab.
+
+
+Updating bundles
+----------------
+
+First you need to make sure the bundle entrypoints have been installed,
+but either running `develop`, or `install`::
+
+    $ python setup.py develop
+
+Then make sure that you have your PyPI credentials stored in
+:file:`~/.pypirc`, and execute the command::
+
+    $ python setup.py upload_bundles
+
+If you broke something and need to update new versions of the bundles,
+then you can use ``upload_bundles_fix``.

docs/getting-started/brokers/beanstalk.rst

@@ -0,0 +1,57 @@
+.. _broker-beanstalk:
+
+=================
+ Using Beanstalk
+=================
+
+.. _broker-beanstalk-installation:
+
+Installation
+============
+
+For the Beanstalk support you have to install additional dependencies.
+You can install both Celery and these dependencies in one go using
+either the `celery-with-beanstalk`_, or the `django-celery-with-beanstalk`
+bundles::
+
+    $ pip install -U celery-with-beanstalk
+
+.. _`celery-with-beanstalk`:
+    http://pypi.python.org/pypi/celery-with-beanstalk
+.. _`django-celery-with-beanstalk`:
+    http://pypi.python.org/pypi/django-celery-with-beanstalk
+
+.. _broker-beanstalk-configuration:
+
+Configuration
+=============
+
+Configuration is easy, set the transport, and configure the location of
+your CouchDB database::
+
+    BROKER_URL = "beanstalk://localhost:11300"
+
+Where the URL is in the format of::
+
+    beanstalk://hostname:port
+
+The host name will default to ``localhost`` and the port to 11300,
+and so they are optional.
+
+.. _beanstalk-results-configuration:
+
+Results
+-------
+
+Using Beanstalk to store task state and results is currently **not supported**.
+
+.. _broker-beanstalk-limitations:
+
+Limitations
+===========
+
+The Beanstalk message transport does not currently support:
+
+    * Remote control commands (celeryctl, broadcast)
+    * Authentication
+

docs/getting-started/brokers/couchdb.rst

@@ -0,0 +1,55 @@
+.. _broker-couchdb:
+
+===============
+ Using CouchDB
+===============
+
+.. _broker-couchdb-installation:
+
+Installation
+============
+
+For the CouchDB support you have to install additional dependencies.
+You can install both Celery and these dependencies in one go using
+either the `celery-with-couchdb`_, or the `django-celery-with-couchdb` bundles::
+
+    $ pip install -U celery-with-couchdb
+
+.. _`celery-with-couchdb`:
+    http://pypi.python.org/pypi/celery-with-couchdb
+.. _`django-celery-with-couchdb`:
+    http://pypi.python.org/pypi/django-celery-with-couchdb
+
+.. _broker-couchdb-configuration:
+
+Configuration
+=============
+
+Configuration is easy, set the transport, and configure the location of
+your CouchDB database::
+
+    BROKER_URL = "couchdb://localhost:5984/database_name"
+
+Where the URL is in the format of::
+
+    couchdb://userid:password@hostname:port/database_name
+
+The host name will default to ``localhost`` and the port to 5984,
+and so they are optional.  userid and password are also optional,
+but needed if your CouchDB server requires authentication.
+
+.. _couchdb-results-configuration:
+
+Results
+-------
+
+Storing task state and results in CouchDB is currently **not supported**.
+
+.. _broker-couchdb-limitations:
+
+Limitations
+===========
+
+The Beanstalk message transport does not currently support:
+
+    * Remote control commands (celeryctl, broadcast)

docs/getting-started/brokers/django.rst

@@ -0,0 +1,58 @@
+.. _broker-django:
+
+===========================
+ Using the Django Database
+===========================
+
+.. _broker-django-installation:
+
+Installation
+============
+
+For the Django database transport support you have to install the
+`django-kombu` library::
+
+    $ pip install -U django-kombu
+
+.. _broker-django-configuration:
+
+Configuration
+=============
+
+The database transport uses the Django `DATABASE_*` settings for database
+configuration values.
+
+#. Set your broker transport::
+
+    BROKER_URL = "django://"
+
+#. Add :mod:`djkombu` to `INSTALLED_APPS`::
+
+    INSTALLED_APPS = ("djkombu", )
+
+#. Verify your database settings::
+
+    DATABASE_ENGINE = "mysql"
+    DATABASE_NAME = "mydb"
+    DATABASE_USER = "myuser"
+    DATABASE_PASSWORD = "secret"
+
+  The above is just an example, if you haven't configured your database before
+  you should read the Django database settings reference:
+  http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine
+
+#. Sync your database schema::
+
+    $ python manage.py syncdb
+
+.. _broker-django-limitations:
+
+Limitations
+===========
+
+The Django database transport does not currently support:
+
+    * Remote control commands (celeryev, broadcast)
+    * Events, including the Django Admin monitor.
+    * Using more than a few workers (can lead to messages being executed
+      multiple times).

docs/getting-started/brokers/index.rst

@@ -0,0 +1,21 @@
+.. _brokers:
+
+=====================
+ Brokers
+=====================
+
+:Release: |version|
+:Date: |today|
+
+Celery supports several message transport alternatives.
+
+.. toctree::
+    :maxdepth: 1
+
+    rabbitmq
+    redis
+    sqlalchemy
+    django
+    mongodb
+    couchdb
+    beanstalk

docs/getting-started/brokers/mongodb.rst

@@ -0,0 +1,56 @@
+.. _broker-mongodb:
+
+===============
+ Using MongoDB
+===============
+
+.. _broker-mongodb-installation:
+
+Installation
+============
+
+For the MongoDB support you have to install additional dependencies.
+You can install both Celery and these dependencies in one go using
+either the `celery-with-mongodb`_, or the `django-celery-with-mongodb` bundles::
+
+    $ pip install -U celery-with-mongodb
+
+.. _`celery-with-mongodb`:
+    http://pypi.python.org/pypi/celery-with-mongodb
+.. _`django-celery-with-mongodb`:
+    http://pypi.python.org/pypi/django-celery-with-mongodb
+
+.. _broker-mongodb-configuration:
+
+Configuration
+=============
+
+Configuration is easy, set the transport, and configure the location of
+your MongoDB database::
+
+    BROKER_URL = "mongodb://localhost:27017/database_name"
+
+Where the URL is in the format of::
+
+    mongodb://userid:password@hostname:port/database_name
+
+The host name will default to ``localhost`` and the port to 27017,
+and so they are optional.  userid and password are also optional,
+but needed if your MongoDB server requires authentication.
+
+.. _mongodb-results-configuration:
+
+Results
+-------
+
+If you also want to store the state and return values of tasks in MongoDB,
+you should see :ref:`conf-mongodb-result-backend`.
+
+.. _broker-mongodb-limitations:
+
+Limitations
+===========
+
+The mongodb message transport currently does not support:
+
+    * Remote control commands (celeryctl, broadcast)

docs/getting-started/broker-installation.rst → docs/getting-started/brokers/rabbitmq.rst

@@ -1,16 +1,29 @@
-.. _broker-installation:
+.. _broker-rabbitmq:
 
-=====================
- Broker Installation
-=====================
+================
+ Using RabbitMQ
+================
 
 .. contents::
     :local:
 
+Installation & Configuration
+============================
+
+RabbitMQ is the default broker so it does not require any additional
+dependencies or initial configuration, other than the URL location of
+the broker instance you want to use::
+
+    >>> BROKER_URL = "amqp://guest:guest@localhost:5672//"
+
+For a description of broker URLs and a full list of the
+various broker configuration options available to Celery,
+see :ref:`conf-broker-settings`.
+
 .. _installing-rabbitmq:
 
-Installing RabbitMQ
-===================
+Installing the RabbitMQ Server
+==============================
 
 See `Installing RabbitMQ`_ over at RabbitMQ's website. For Mac OS X
 see `Installing RabbitMQ on OS X`_.
@@ -28,7 +41,7 @@ see `Installing RabbitMQ on OS X`_.
 .. _rabbitmq-configuration:
 
 Setting up RabbitMQ
-===================
+-------------------
 
 To use celery we need to create a RabbitMQ user, a virtual host and
 allow that user access to that virtual host::
@@ -48,7 +61,7 @@ See the RabbitMQ `Admin Guide`_ for more information about `access control`_.
 .. _rabbitmq-osx-installation:
 
 Installing RabbitMQ on OS X
-===========================
+---------------------------
 
 The easiest way to install RabbitMQ on Snow Leopard is using `Homebrew`_; the new
 and shiny package management system for OS X.
@@ -89,7 +102,7 @@ Finally, we can install rabbitmq using :program:`brew`::
 .. _rabbitmq-osx-system-hostname:
 
 Configuring the system host name
---------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If you're using a DHCP server that is giving you a random host name, you need
 to permanently configure the host name. This is because RabbitMQ uses the host name
@@ -126,7 +139,7 @@ then RabbitMQ will try to use `rabbit@23`, which is an illegal host name.
 .. _rabbitmq-osx-start-stop:
 
 Starting/Stopping the RabbitMQ server
--------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To start the server::
 
@@ -143,4 +156,3 @@ Never use :program:`kill` to stop the RabbitMQ server, but rather use the
     $ sudo rabbitmqctl stop
 
 When the server is running, you can continue reading `Setting up RabbitMQ`_.
-

docs/getting-started/brokers/redis.rst

@@ -0,0 +1,52 @@
+.. _broker-redis:
+
+=============
+ Using Redis
+=============
+
+.. _broker-redis-installation:
+
+Installation
+============
+
+For the Redis support you have to install additional dependencies.
+You can install both Celery and these dependencies in one go using
+ehter the `celery-with-redis`_, or the `django-celery-with-redis` bundles::
+
+    $ pip install -U celery-with-redis
+
+.. _`celery-with-redis`:
+    http://pypi.python.org/pypi/celery-with-redis
+.. _`django-celery-with-redis`:
+    http://pypi.python.org/pypi/django-celery-with-redis
+
+.. _broker-redis-configuration:
+
+Configuration
+=============
+
+Configuration is easy, set the transport, and configure the location of
+your Redis database::
+
+    BROKER_URL = "redis://localhost:6379/0"
+
+
+Where the URL is in the format of::
+
+    redis://userid:password@hostname:port/db_number
+
+.. _redis-results-configuration:
+
+Results
+-------
+
+If you also want to store the state and return values of tasks in Redis,
+you should configure these settings::
+
+    CELERY_RESULT_BACKEND = "redis"
+    CELERY_REDIS_HOST = "localhost"
+    CELERY_REDIS_PORT = 6379
+    CELERY_REDIS_DB = 0
+
+For a complete list of options supported by the Redis result backend see
+:ref:`conf-redis-result-backend`

docs/getting-started/brokers/sqlalchemy.rst

@@ -0,0 +1,74 @@
+.. _broker-sqlalchemy:
+
+==================
+ Using SQLAlchemy
+==================
+
+.. _broker-sqlalchemy-installation:
+
+Installation
+============
+
+For the SQLAlchemy transport you have to install the
+`kombu-sqlalchemy` library::
+
+    $ pip install -U kombu-sqlalchemy
+
+.. _broker-sqlalchemy-configuration:
+
+Configuration
+=============
+
+This transport uses only the :setting:`BROKER_HOST` setting, which have to be
+an SQLAlchemy database URI.
+
+#. Set your broker transport::
+
+    BROKER_TRANSPORT = "sqlalchemy"
+
+#. Configure the database URI::
+
+    BROKER_HOST = "sqlite:///celerydb.sqlite"
+
+Please see `SQLAlchemy: Supported Databases`_ for a table of supported databases.
+Some other `SQLAlchemy Connection String`_, examples:
+
+.. code-block:: python
+
+    # sqlite (filename)
+    BROKER_HOST = "sqlite:///celerydb.sqlite"
+
+    # mysql
+    BROKER_HOST = "mysql://scott:tiger@localhost/foo"
+
+    # postgresql
+    BROKER_HOST = "postgresql://scott:tiger@localhost/mydatabase"
+
+    # oracle
+    BROKER_HOST = "oracle://scott:tiger@127.0.0.1:1521/sidname"
+
+.. _`SQLAlchemy: Supported Databases`:
+    http://www.sqlalchemy.org/docs/core/engines.html#supported-databases
+
+.. _`SQLAlchemy Connection String`:
+    http://www.sqlalchemy.org/docs/core/engines.html#database-urls
+
+.. _sqlalchemy-results-configuration:
+
+Results
+-------
+
+To store results in the database as well, you should configure the result
+backend.  See :ref:`conf-database-result-backend`.
+
+.. _broker-sqlalchemy-limitations:
+
+Limitations
+===========
+
+The SQLAlchemy database transport does not currently support:
+
+    * Remote control commands (celeryev, broadcast)
+    * Events, including the Django Admin monitor.
+    * Using more than a few workers (can lead to messages being executed
+      multiple times).

docs/getting-started/first-steps-with-celery.rst

@@ -18,35 +18,32 @@ messages.
 
 There are several choices available, including:
 
-* `RabbitMQ`_
+* :ref:`broker-rabbitmq`
 
-Feature-complete, safe and durable. If not losing tasks
+`RabbitMQ`_ is feature-complete, safe and durable. If not losing tasks
 is important to you, then this is your best option.
 
-See :ref:`broker-installation` for more about installing and configuring
-RabbitMQ.
+* :ref:`broker-redis`
 
-* `Redis`_
-
-Also feature-complete, but power failures or abrubt termination
+`Redis`_ is also feature-complete, but power failures or abrupt termination
 may result in data loss.
 
-See :ref:`otherqueues-redis` for configuration details.
-
-* Databases
+* :ref:`broker-sqlalchemy`
+* :ref:`broker-django`
 
 Using a database as a message queue is not recommended, but can be sufficient
-for very small installations.  Celery can use the SQLAlchemy and Django ORMS.
-See :ref:`otherqueues-sqlalchemy` or :ref:`otherqueues-django`.
+for very small installations.  Celery can use the SQLAlchemy and Django ORM.
 
 * and more.
 
 In addition to the above, there are several other transport implementations
-to choose from, including CouchDB, Beanstalk, MongoDB, and SQS.  See the Kombu
-documentation for more information.
+to choose from, including :ref:`broker-couchdb`, :ref:`broker-beanstalk`,
+:ref:`broker-mongodb`, and SQS.  There is a `Transport Comparison`_
+in the Kombu documentation.
 
 .. _`RabbitMQ`: http://www.rabbitmq.com/
 .. _`Redis`: http://redis.io/
+.. _`Transport Comparison`: http://kombu.rtfd.org/transport-comparison
 
 .. _celerytut-simple-tasks:
 

docs/getting-started/index.rst

@@ -9,6 +9,6 @@
     :maxdepth: 2
 
     introduction
-    broker-installation
+    brokers/index
     first-steps-with-celery
     resources

docs/includes/introduction.txt

@@ -106,7 +106,6 @@ Features
     +-----------------+----------------------------------------------------+
     | Fault-tolerant  | Excellent configurable error recovery when using   |
     |                 | `RabbitMQ`, ensures your tasks are never lost.     |
-    |                 | scenarios, and your tasks will never be lost.      |
     +-----------------+----------------------------------------------------+
     | Distributed     | Runs on one or more machines. Supports             |
     |                 | broker `clustering`_ and `HA`_ when used in        |
@@ -210,11 +209,45 @@ or from source.
 
 To install using `pip`,::
 
-    $ pip install Celery
+    $ pip install -U Celery
 
 To install using `easy_install`,::
 
-    $ easy_install Celery
+    $ easy_install -U Celery
+
+Bundles
+-------
+
+Celery also defines a group of bundles that can be used
+to install Celery and the dependencies for a given feature.
+
+The following bundles are available:
+
+:`celery-with-redis`_:
+    for using Redis as a broker.
+
+:`celery-with-mongodb`_:
+    for using MongoDB as a broker.
+
+:`django-celery-with-redis`_:
+    for Django, and using Redis as a broker.
+
+:`django-celery-with-mongodb`_:
+    for Django, and using MongoDB as a broker.
+
+:`bundle-celery`_:
+    convenience bundle installing *Celery* and related packages.
+
+.. _`celery-with-redis`:
+    http://pypi.python.org/pypi/celery-with-redis/
+.. _`celery-with-mongodb`:
+    http://pypi.python.org/pypi/celery-with-mongdb/
+.. _`django-celery-with-redis`:
+    http://pypi.python.org/pypi/django-celery-with-redis/
+.. _`django-celery-with-mongodb`:
+    http://pypi.python.org/pypi/django-celery-with-mongdb/
+.. _`bundle-celery`:
+    http://pypi.python.org/pypi/bundle-celery/
 
 .. _celery-installing-from-source:
 

docs/tutorials/otherqueues.rst

@@ -4,148 +4,23 @@
  Using Celery with Redis/Database as the messaging queue.
 ==========================================================
 
-.. contents::
-    :local:
-
 .. _otherqueues-redis:
 
 Redis
 =====
 
-For the Redis support you have to install the Python redis client::
-
-    $ pip install -U redis
-
-.. _otherqueues-redis-conf:
-
-Configuration
--------------
-
-Configuration is easy, set the transport, and configure the location of
-your Redis database::
-
-    BROKER_URL = "redis://localhost:6379/0"
-
-
-Where the URL is in the format of::
-
-    redis://userid:password@hostname:port/db_number
-
-
-Results
-~~~~~~~
-
-You probably also want to store results in Redis::
-
-    CELERY_RESULT_BACKEND = "redis"
-    CELERY_REDIS_HOST = "localhost"
-    CELERY_REDIS_PORT = 6379
-    CELERY_REDIS_DB = 0
-
-For a complete list of options supported by the Redis result backend see
-:ref:`conf-redis-result-backend`
-
-If you don't intend to consume results you should disable them::
-
-    CELERY_IGNORE_RESULT = True
+This section has been moved to :ref:`broker-redis`.
 
 .. _otherqueues-sqlalchemy:
 
 SQLAlchemy
 ==========
 
-.. _otherqueues-sqlalchemy-conf:
-
-For the SQLAlchemy transport you have to install the
-`kombu-sqlalchemy` library::
-
-    $ pip install -U kombu-sqlalchemy
-
-Configuration
--------------
-
-This transport uses only the :setting:`BROKER_HOST` setting, which have to be
-an SQLAlchemy database URI.
-
-#. Set your broker transport::
-
-    BROKER_TRANSPORT = "sqlalchemy"
-
-#. Configure the database URI::
-
-    BROKER_HOST = "sqlite:///celerydb.sqlite"
-
-Please see `SQLAlchemy: Supported Databases`_ for a table of supported databases.
-Some other `SQLAlchemy Connection String`_, examples:
-
-.. code-block:: python
-
-    # sqlite (filename)
-    BROKER_HOST = "sqlite:///celerydb.sqlite"
-
-    # mysql
-    BROKER_HOST = "mysql://scott:tiger@localhost/foo"
-
-    # postgresql
-    BROKER_HOST = "postgresql://scott:tiger@localhost/mydatabase"
-
-    # oracle
-    BROKER_HOST = "oracle://scott:tiger@127.0.0.1:1521/sidname"
-
-.. _`SQLAlchemy: Supported Databases`:
-    http://www.sqlalchemy.org/docs/core/engines.html#supported-databases
-
-.. _`SQLAlchemy Connection String`:
-    http://www.sqlalchemy.org/docs/core/engines.html#database-urls
-
-Results
-~~~~~~~
-
-To store results in the database as well, you should configure the result
-backend.  See :ref:`conf-database-result-backend`.
-
-If you don't intend to consume results you should disable them::
-
-    CELERY_IGNORE_RESULT = True
+This section has been moved to :ref:`broker-sqlalchemy`.
 
 .. _otherqueues-django:
 
 Django Database
 ===============
 
-.. _otherqueues-django-conf:
-
-For the Django database transport support you have to install the
-`django-kombu` library::
-
-    $ pip install -U django-kombu
-
-Configuration
--------------
-
-The database backend uses the Django `DATABASE_*` settings for database
-configuration values.
-
-#. Set your broker transport::
-
-    BROKER_TRANSPORT = "django"
-
-#. Add :mod:`djkombu` to `INSTALLED_APPS`::
-
-    INSTALLED_APPS = ("djkombu", )
-
-
-#. Verify you database settings::
-
-    DATABASE_ENGINE = "mysql"
-    DATABASE_NAME = "mydb"
-    DATABASE_USER = "myuser"
-    DATABASE_PASSWORD = "secret"
-
-  The above is just an example, if you haven't configured your database before
-  you should read the Django database settings reference:
-  http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine
-
-#. Sync your database schema.
-
-    $ python manage.py syncdb
+This section has been moved to :ref:`broker-django`.

docs/userguide/tasks.rst

@@ -74,7 +74,7 @@ attributes:
                 task.  Used by e.g. :meth:`~celery.task.base.BaseTask.retry`
                 to resend the task to the same destination queue.
 
-  **NOTE** As some messaging backends doesn't have advanced routing
+  **NOTE** As some messaging backends don't have advanced routing
   capabilities, you can't trust the availability of keys in this mapping.
 
 
@@ -556,7 +556,7 @@ limitations.
   increase the polling intervals of operations such as `result.wait()`, and
   `tasksetresult.join()`
 
-* Some databases uses a default transaction isolation level that
+* Some databases use a default transaction isolation level that
   is not suitable for polling tables for changes.
 
   In MySQL the default transaction isolation level is `REPEATABLE-READ`, which
@@ -576,7 +576,7 @@ PENDING
 ~~~~~~~
 
 Task is waiting for execution or unknown.
-Any task id that is not know is implied to be in the pending state.
+Any task id that is not known is implied to be in the pending state.
 
 .. state:: STARTED
 
@@ -644,7 +644,7 @@ you could have a look at :mod:`abortable tasks <~celery.contrib.abortable>`
 which defines its own custom :state:`ABORTED` state.
 
 Use :meth:`Task.update_state <celery.task.base.BaseTask.update_state>` to
-update a tasks state::
+update a task's state::
 
     @task
     def upload_files(filenames):
@@ -666,7 +666,7 @@ Creating pickleable exceptions
 A little known Python fact is that exceptions must behave a certain
 way to support being pickled.
 
-Tasks that raises exceptions that are not pickleable will not work
+Tasks that raise exceptions that are not pickleable will not work
 properly when Pickle is used as the serializer.
 
 To make sure that your exceptions are pickleable the exception
@@ -722,7 +722,7 @@ Creating custom task classes
 ============================
 
 All tasks inherit from the :class:`celery.task.Task` class.
-The tasks body is its :meth:`run` method.
+The task's body is its :meth:`run` method.
 
 The following code,
 

requirements/docs.txt

@@ -1,4 +1,3 @@
 Sphinx
 sphinxcontrib-issuetracker>=0.9
-Sphinx-PyPI-upload
 SQLAlchemy

requirements/pkgutils.txt

@@ -1,3 +1,5 @@
 paver
 flake8
 tox
+Sphinx-PyPI-upload
+bundle>=1.1.0

setup.py

@@ -5,16 +5,6 @@ import sys
 import codecs
 import platform
 
-extra = {}
-tests_require = ["nose", "nose-cover3", "sqlalchemy", "mock"]
-is_py3k  = sys.version_info >= (3, 0)
-if is_py3k:
-    extra.update(use_2to3=True)
-elif sys.version_info < (2, 7):
-    tests_require.append("unittest2")
-elif sys.version_info <= (2, 5):
-    tests_require.append("simplejson")
-
 if sys.version_info < (2, 5):
     raise Exception("Celery requires Python 2.5 or higher.")
 
@@ -28,7 +18,48 @@ except ImportError:
     from setuptools import setup, find_packages           # noqa
     from setuptools.command.test import test              # noqa
 
-# -- Parse meta
+NAME = "celery"
+entrypoints = {}
+extra = {}
+
+# -*- Classifiers -*-
+
+classes = """
+    Development Status :: 5 - Production/Stable
+    License :: OSI Approved :: BSD License
+    Topic :: System :: Distributed Computing
+    Topic :: Software Development :: Object Brokering
+    Intended Audience :: Developers
+    Intended Audience :: Information Technology
+    Intended Audience :: Science/Research
+    Intended Audience :: Financial and Insurance Industry
+    Intended Audience :: Healthcare Industry
+    Environment :: No Input/Output (Daemon)
+    Environment :: Console
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.5
+    Programming Language :: Python :: 2.6
+    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.2
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: Implementation :: PyPy
+    Programming Language :: Python :: Implementation :: Jython
+    Operating System :: OS Independent
+    Operating System :: POSIX
+    Operating System :: Microsoft :: Windows
+    Operating System :: MacOS :: MacOS X
+"""
+classifiers = [s.strip() for s in classes.split('\n') if s]
+
+# -*- Python 3 -*-
+is_py3k  = sys.version_info >= (3, 0)
+if is_py3k:
+    extra.update(use_2to3=True)
+
+# -*- Distribution Meta -*-
+
 import re
 re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)')
 re_vers = re.compile(r'VERSION\s*=\s*\((.*?)\)')
@@ -64,8 +95,8 @@ try:
                 meta.update(handler(m))
 finally:
     meta_fh.close()
-# --
 
+# -*- Custom Commands -*-
 
 class quicktest(test):
     extra_env = dict(SKIP_RLIMITS=1, QUICKTEST=1)
@@ -75,6 +106,8 @@ class quicktest(test):
             os.environ[env_name] = str(env_value)
         test.run(self, *args, **kwargs)
 
+# -*- Installation Dependencies -*-
+
 install_requires = []
 try:
     import importlib  # noqa
@@ -101,13 +134,24 @@ if is_jython:
     install_requires.append("threadpool")
     install_requires.append("simplejson")
 
+# -*- Tests Requires -*-
+
+tests_require = ["nose", "nose-cover3", "sqlalchemy", "mock"]
+if sys.version_info < (2, 7):
+    tests_require.append("unittest2")
+elif sys.version_info <= (2, 5):
+    tests_require.append("simplejson")
+
+# -*- Long Description -*-
+
 if os.path.exists("README.rst"):
     long_description = codecs.open("README.rst", "r", "utf-8").read()
 else:
     long_description = "See http://pypi.python.org/pypi/celery"
 
+# -*- Entry Points -*- #
 
-console_scripts = [
+console_scripts = entrypoints["console_scripts"] = [
         'celerybeat = celery.bin.celerybeat:main',
         'camqadm = celery.bin.camqadm:main',
         'celeryev = celery.bin.celeryev:main',
@@ -119,6 +163,10 @@ if platform.system() == "Windows":
 else:
     console_scripts.append('celeryd = celery.bin.celeryd:main')
 
+# bundles: Only relevant for Celery developers.
+entrypoints["bundle.bundles"] = ["celery = celery.contrib.bundles:bundles"]
+
+# -*- %%% -*-
 
 setup(
     name="celery",
@@ -133,38 +181,9 @@ setup(
     zip_safe=False,
     install_requires=install_requires,
     tests_require=tests_require,
-    cmdclass={"test": test,
-              "quicktest": quicktest},
     test_suite="nose.collector",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "License :: OSI Approved :: BSD License",
-        "Topic :: System :: Distributed Computing",
-        "Topic :: Software Development :: Object Brokering",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Information Technology",
-        "Intended Audience :: Science/Research",
-        "Intended Audience :: Financial and Insurance Industry",
-        "Intended Audience :: Healthcare Industry",
-        "Environment :: No Input/Output (Daemon)",
-        "Environment :: Console",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.5",
-        "Programming Language :: Python :: 2.6",
-        "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.2",
-        "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Python :: Implementation :: PyPy",
-        "Programming Language :: Python :: Implementation :: Jython",
-        "Operating System :: OS Independent",
-        "Operating System :: POSIX",
-        "Operating System :: Microsoft :: Windows",
-        "Operating System :: MacOS :: MacOS X",
-    ],
-    entry_points={
-        'console_scripts': console_scripts,
-    },
+    cmdclass={"quicktest": quicktest},
+    classifiers=classifiers,
+    entry_points=entrypoints,
     long_description=long_description,
     **extra)