|
@@ -106,6 +106,7 @@ and the worker currently defines two blueprints: **Worker**, and **Consumer**
|
|
|
|
|
|
----------------------------------------------------------
|
|
|
|
|
|
+.. _extending-worker_blueprint:
|
|
|
|
|
|
Worker
|
|
|
======
|
|
@@ -118,21 +119,31 @@ to the Consumer blueprint.
|
|
|
The :class:`~celery.worker.WorkController` is the core worker implementation,
|
|
|
and contains several methods and attributes that you can use in your bootstep.
|
|
|
|
|
|
+.. _extending-worker_blueprint-attributes:
|
|
|
+
|
|
|
Attributes
|
|
|
----------
|
|
|
|
|
|
+.. _extending-worker-app:
|
|
|
+
|
|
|
.. attribute:: app
|
|
|
|
|
|
The current app instance.
|
|
|
|
|
|
+.. _extending-worker-hostname:
|
|
|
+
|
|
|
.. attribute:: hostname
|
|
|
|
|
|
The workers node name (e.g. `worker1@example.com`)
|
|
|
|
|
|
+.. _extending-worker-blueprint:
|
|
|
+
|
|
|
.. attribute:: blueprint
|
|
|
|
|
|
This is the worker :class:`~celery.bootsteps.Blueprint`.
|
|
|
|
|
|
+.. _extending-worker-hub:
|
|
|
+
|
|
|
.. attribute:: hub
|
|
|
|
|
|
Event loop object (:class:`~kombu.async.Hub`). You can use
|
|
@@ -148,6 +159,8 @@ Attributes
|
|
|
class WorkerStep(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.components:Hub',)
|
|
|
|
|
|
+.. _extending-worker-pool:
|
|
|
+
|
|
|
.. attribute:: pool
|
|
|
|
|
|
The current process/eventlet/gevent/thread pool.
|
|
@@ -160,6 +173,8 @@ Attributes
|
|
|
class WorkerStep(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.components:Pool',)
|
|
|
|
|
|
+.. _extending-worker-timer:
|
|
|
+
|
|
|
.. attribute:: timer
|
|
|
|
|
|
:class:`~kombu.async.timer.Timer` used to schedule functions.
|
|
@@ -171,6 +186,8 @@ Attributes
|
|
|
class WorkerStep(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.components:Timer',)
|
|
|
|
|
|
+.. _extending-worker-statedb:
|
|
|
+
|
|
|
.. attribute:: statedb
|
|
|
|
|
|
:class:`Database <celery.worker.state.Persistent>`` to persist state between
|
|
@@ -185,6 +202,8 @@ Attributes
|
|
|
class WorkerStep(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.components:Statedb',)
|
|
|
|
|
|
+.. _extending-worker-autoscaler:
|
|
|
+
|
|
|
.. attribute:: autoscaler
|
|
|
|
|
|
:class:`~celery.worker.autoscaler.Autoscaler` used to automatically grow
|
|
@@ -199,6 +218,8 @@ Attributes
|
|
|
class WorkerStep(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.autoscaler:Autoscaler',)
|
|
|
|
|
|
+.. _extending-worker-autoreloader:
|
|
|
+
|
|
|
.. attribute:: autoreloader
|
|
|
|
|
|
:class:`~celery.worker.autoreloder.Autoreloader` used to automatically
|
|
@@ -212,6 +233,9 @@ Attributes
|
|
|
class WorkerStep(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.autoreloader:Autoreloader',)
|
|
|
|
|
|
+Example worker bootstep
|
|
|
+-----------------------
|
|
|
+
|
|
|
An example Worker bootstep could be:
|
|
|
|
|
|
.. code-block:: python
|
|
@@ -243,7 +267,6 @@ An example Worker bootstep could be:
|
|
|
Every method is passed the current ``WorkController`` instance as the first
|
|
|
argument.
|
|
|
|
|
|
-
|
|
|
Another example could use the timer to wake up at regular intervals:
|
|
|
|
|
|
.. code-block:: python
|
|
@@ -276,6 +299,8 @@ Another example could use the timer to wake up at regular intervals:
|
|
|
if req.time_start and time() - req.time_start > self.timeout:
|
|
|
raise SystemExit()
|
|
|
|
|
|
+.. _extending-consumer_blueprint:
|
|
|
+
|
|
|
Consumer
|
|
|
========
|
|
|
|
|
@@ -289,25 +314,37 @@ be possible to restart your blueprint. An additional 'shutdown' method is
|
|
|
defined for consumer bootsteps, this method is called when the worker is
|
|
|
shutdown.
|
|
|
|
|
|
+.. _extending-consumer-attributes:
|
|
|
+
|
|
|
Attributes
|
|
|
----------
|
|
|
|
|
|
+.. _extending-consumer-app:
|
|
|
+
|
|
|
.. attribute:: app
|
|
|
|
|
|
The current app instance.
|
|
|
|
|
|
+.. _extending-consumer-controller:
|
|
|
+
|
|
|
.. attribute:: controller
|
|
|
|
|
|
The parent :class:`~@WorkController` object that created this consumer.
|
|
|
|
|
|
+.. _extending-consumer-hostname:
|
|
|
+
|
|
|
.. attribute:: hostname
|
|
|
|
|
|
The workers node name (e.g. `worker1@example.com`)
|
|
|
|
|
|
+.. _extending-consumer-blueprint:
|
|
|
+
|
|
|
.. attribute:: blueprint
|
|
|
|
|
|
This is the worker :class:`~celery.bootsteps.Blueprint`.
|
|
|
|
|
|
+.. _extending-consumer-hub:
|
|
|
+
|
|
|
.. attribute:: hub
|
|
|
|
|
|
Event loop object (:class:`~kombu.async.Hub`). You can use
|
|
@@ -323,6 +360,7 @@ Attributes
|
|
|
class WorkerStep(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker:Hub',)
|
|
|
|
|
|
+.. _extending-consumer-connection:
|
|
|
|
|
|
.. attribute:: connection
|
|
|
|
|
@@ -336,6 +374,8 @@ Attributes
|
|
|
class Step(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.consumer:Connection',)
|
|
|
|
|
|
+.. _extending-consumer-event_dispatcher:
|
|
|
+
|
|
|
.. attribute:: event_dispatcher
|
|
|
|
|
|
A :class:`@events.Dispatcher` object that can be used to send events.
|
|
@@ -347,6 +387,8 @@ Attributes
|
|
|
class Step(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.consumer:Events',)
|
|
|
|
|
|
+.. _extending-consumer-gossip:
|
|
|
+
|
|
|
.. attribute:: gossip
|
|
|
|
|
|
Worker to worker broadcast communication
|
|
@@ -406,15 +448,21 @@ Attributes
|
|
|
This does not necessarily mean the worker is actually offline, so use a time
|
|
|
out mechanism if the default heartbeat timeout is not sufficient.
|
|
|
|
|
|
+.. _extending-consumer-pool:
|
|
|
+
|
|
|
.. attribute:: pool
|
|
|
|
|
|
The current process/eventlet/gevent/thread pool.
|
|
|
See :class:`celery.concurrency.base.BasePool`.
|
|
|
|
|
|
+.. _extending-consumer-timer:
|
|
|
+
|
|
|
.. attribute:: timer
|
|
|
|
|
|
:class:`Timer <celery.utils.timer2.Schedule` used to schedule functions.
|
|
|
|
|
|
+.. _extending-consumer-heart:
|
|
|
+
|
|
|
.. attribute:: heart
|
|
|
|
|
|
Responsible for sending worker event heartbeats
|
|
@@ -427,6 +475,8 @@ Attributes
|
|
|
class Step(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.consumer:Heart',)
|
|
|
|
|
|
+.. _extending-consumer-task_consumer:
|
|
|
+
|
|
|
.. attribute:: task_consumer
|
|
|
|
|
|
The :class:`kombu.Consumer` object used to consume task messages.
|
|
@@ -438,6 +488,8 @@ Attributes
|
|
|
class Step(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.consumer:Heart',)
|
|
|
|
|
|
+.. _extending-consumer-strategies:
|
|
|
+
|
|
|
.. attribute:: strategies
|
|
|
|
|
|
Every registered task type has an entry in this mapping,
|
|
@@ -460,6 +512,7 @@ Attributes
|
|
|
class Step(bootsteps.StartStopStep):
|
|
|
requires = ('celery.worker.consumer:Heart',)
|
|
|
|
|
|
+.. _extending-consumer-task_buckets:
|
|
|
|
|
|
.. attribute:: task_buckets
|
|
|
|
|
@@ -475,6 +528,8 @@ Attributes
|
|
|
|
|
|
.. _`token bucket algorithm`: http://en.wikipedia.org/wiki/Token_bucket
|
|
|
|
|
|
+.. _extending_consumer-qos:
|
|
|
+
|
|
|
.. attribute:: qos
|
|
|
|
|
|
The :class:`~kombu.common.QoS` object can be used to change the
|