yangck
/
celery


			
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593
							<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">


<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    
    <title>Calling Tasks &mdash; Celery 2.6.0rc4 documentation</title>
    
    <link rel="stylesheet" href="../_static/celery.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/issuetracker.css" type="text/css" />
    
    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '../',
        VERSION:     '2.6.0rc4',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  true
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <link rel="top" title="Celery 2.6.0rc4 documentation" href="../index.html" />
    <link rel="up" title="User Guide" href="index.html" />
    <link rel="next" title="Workers Guide" href="workers.html" />
    <link rel="prev" title="Tasks" href="tasks.html" /> 
  </head>
  <body>
    <div class="related">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../genindex.html" title="General Index"
             accesskey="I">index</a></li>
        <li class="right" >
          <a href="../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="right" >
          <a href="workers.html" title="Workers Guide"
             accesskey="N">next</a> |</li>
        <li class="right" >
          <a href="tasks.html" title="Tasks"
             accesskey="P">previous</a> |</li>
        <li><a href="../index.html">Celery 2.6.0rc4 documentation</a> &raquo;</li>
          <li><a href="index.html" accesskey="U">User Guide</a> &raquo;</li> 
      </ul>
    </div>  

    <div class="document">
      <div class="documentwrapper">
        <div class="bodywrapper">
          <div class="body">
            
<div class="deck">

    
        <p class="developmentversion">
        This document is for Celery's development version, which can be
        significantly different from previous releases. Get old docs here:

        <a href="http://docs.celeryproject.org/en/latest/userguide/calling.html">2.5</a>.
        </p>
        

</div>
    <div class="section" id="calling-tasks">
<span id="guide-calling"></span><h1>Calling Tasks<a class="headerlink" href="#calling-tasks" title="Permalink to this headline">¶</a></h1>
<div class="contents local topic" id="contents">
<ul class="simple">
<li><a class="reference internal" href="#basics" id="id1">Basics</a></li>
<li><a class="reference internal" href="#linking-callbacks-errbacks" id="id2">Linking (callbacks/errbacks)</a></li>
<li><a class="reference internal" href="#eta-and-countdown" id="id3">ETA and countdown</a></li>
<li><a class="reference internal" href="#expiration" id="id4">Expiration</a></li>
<li><a class="reference internal" href="#message-sending-retry" id="id5">Message Sending Retry</a></li>
<li><a class="reference internal" href="#serializers" id="id6">Serializers</a></li>
<li><a class="reference internal" href="#compression" id="id7">Compression</a></li>
<li><a class="reference internal" href="#connections" id="id8">Connections</a></li>
<li><a class="reference internal" href="#routing-options" id="id9">Routing options</a></li>
</ul>
</div>
<div class="section" id="basics">
<span id="calling-basics"></span><h2><a class="toc-backref" href="#id1">Basics</a><a class="headerlink" href="#basics" title="Permalink to this headline">¶</a></h2>
<p>This document describes Celery&#8217;s uniform &#8220;Calling API&#8221;
used by task instances and the <a class="reference internal" href="canvas.html#guide-canvas"><em>canvas</em></a>.</p>
<p>The API defines a standard set of execution options, as well as three methods:</p>
<blockquote>
<div><ul>
<li><p class="first"><tt class="docutils literal"><span class="pre">apply_async(args[,</span> <span class="pre">kwargs[,</span> <span class="pre">...]])</span></tt></p>
<blockquote>
<div><p>Sends a task message.</p>
</div></blockquote>
</li>
<li><p class="first"><tt class="docutils literal"><span class="pre">delay(*args,</span> <span class="pre">**kwargs)</span></tt></p>
<blockquote>
<div><p>Shortcut to send a task message, but does not support execution
options.</p>
</div></blockquote>
</li>
<li><p class="first"><tt class="docutils literal"><span class="pre">apply()</span></tt></p>
<blockquote>
<div><p>Does not send a message but executes the task inline instead.</p>
</div></blockquote>
</li>
</ul>
</div></blockquote>
<div class="topic" id="calling-cheat">
<p class="topic-title first">Quick Cheat Sheet</p>
<ul>
<li><dl class="first docutils">
<dt><tt class="docutils literal"><span class="pre">T.delay(arg,</span> <span class="pre">kwarg=value)</span></tt></dt>
<dd><p class="first last">always a shortcut to <tt class="docutils literal"><span class="pre">.apply_async</span></tt>.</p>
</dd>
</dl>
</li>
<li><p class="first"><tt class="docutils literal"><span class="pre">T.apply_async((arg,</span> <span class="pre">),</span> <span class="pre">{&quot;kwarg&quot;:</span> <span class="pre">value})</span></tt></p>
</li>
<li><dl class="first docutils">
<dt><tt class="docutils literal"><span class="pre">T.apply_async(countdown=10)</span></tt></dt>
<dd><p class="first last">executes 10 seconds from now.</p>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><tt class="docutils literal"><span class="pre">T.apply_async(eta=now</span> <span class="pre">+</span> <span class="pre">timedelta(seconds=10))</span></tt></dt>
<dd><p class="first last">executes 10 seconds from now, specifed using <tt class="docutils literal"><span class="pre">eta</span></tt></p>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><tt class="docutils literal"><span class="pre">T.apply_async(countdown=60,</span> <span class="pre">expires=120)</span></tt></dt>
<dd><p class="first last">executes in one minute from now, but expires after 2 minutes.</p>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><tt class="docutils literal"><span class="pre">T.apply_async(expires=now</span> <span class="pre">+</span> <span class="pre">timedelta(days=2))</span></tt></dt>
<dd><p class="first last">expires in 2 days, set using <a class="reference external" href="http://docs.python.org/dev/library/datetime.html#datetime.datetime" title="(in Python v3.3)"><tt class="xref py py-class docutils literal"><span class="pre">datetime</span></tt></a>.</p>
</dd>
</dl>
</li>
</ul>
</div>
<div class="section" id="example">
<h3>Example<a class="headerlink" href="#example" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="../reference/celery.app.task.html#celery.app.task.Task.delay" title="celery.app.task.Task.delay"><tt class="xref py py-meth docutils literal"><span class="pre">delay()</span></tt></a> method is convenient as it looks like calling a regular
function:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">task</span><span class="o">.</span><span class="n">delay</span><span class="p">(</span><span class="n">arg1</span><span class="p">,</span> <span class="n">arg2</span><span class="p">,</span> <span class="n">kwarg1</span><span class="o">=</span><span class="s">&quot;x&quot;</span><span class="p">,</span> <span class="n">kwarg2</span><span class="o">=</span><span class="s">&quot;y&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>Using <a class="reference internal" href="../reference/celery.app.task.html#celery.app.task.Task.apply_async" title="celery.app.task.Task.apply_async"><tt class="xref py py-meth docutils literal"><span class="pre">apply_async()</span></tt></a> instead we have to write:</p>
<div class="highlight-python"><pre>.. code-block:: python</pre>
</div>
<blockquote>
<div>task.apply_async(args=[arg1, arg2], kwargs={&#8220;kwarg1&#8221;: &#8220;x&#8221;, &#8220;kwarg2&#8221;: &#8220;y&#8221;})</div></blockquote>
<div class="sidebar">
<p class="first sidebar-title">Tip</p>
<p class="last">If the task is not registered in the current process
you can use <a class="reference internal" href="../reference/celery.html#celery.Celery.send_task" title="celery.Celery.send_task"><tt class="xref py py-meth docutils literal"><span class="pre">send_task()</span></tt></a> to call the task by name instead.</p>
</div>
<p>So <cite>delay</cite> is clearly convenient, but it doesn&#8217;t give you as much control as using
<cite>apply_async</cite>.  With <cite>apply_async</cite> you can override the execution options
available as attributes on the <a class="reference internal" href="../reference/celery.app.task.html#celery.app.task.Task" title="celery.app.task.Task"><tt class="xref py py-class docutils literal"><span class="pre">Task</span></tt></a> class (see <a class="reference internal" href="tasks.html#task-options"><em>Task options</em></a>).
In addition you can set countdown/eta, task expiry, provide a custom broker
connection and more.</p>
<p>The rest of this document will go into the task execution
options in detail.  All examples use a task
called <cite>add</cite>, returning the sum of two arguments:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="nd">@celery.task</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">add</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
    <span class="k">return</span> <span class="n">x</span> <span class="o">+</span> <span class="n">y</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title first">There&#8217;s another way...</p>
<p>You will learn more about this later while reading about the <a class="reference internal" href="canvas.html#guide-canvas"><em>Canvas</em></a>, but <a class="reference internal" href="../reference/celery.html#celery.subtask" title="celery.subtask"><tt class="xref py py-class docutils literal"><span class="pre">subtask</span></tt></a>&#8216;s are objects used to pass around
the signature of a task invocation, (for example to send it over the
network), and they also support the Calling API:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">task</span><span class="o">.</span><span class="n">s</span><span class="p">(</span><span class="n">arg1</span><span class="p">,</span> <span class="n">arg2</span><span class="p">,</span> <span class="n">kwarg1</span><span class="o">=</span><span class="s">&quot;x&quot;</span><span class="p">,</span> <span class="n">kwargs2</span><span class="o">=</span><span class="s">&quot;y&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">apply_async</span><span class="p">()</span>
</pre></div>
</div>
</div>
</div>
</div>
<div class="section" id="linking-callbacks-errbacks">
<span id="calling-links"></span><h2><a class="toc-backref" href="#id2">Linking (callbacks/errbacks)</a><a class="headerlink" href="#linking-callbacks-errbacks" title="Permalink to this headline">¶</a></h2>
<p>Celery supports linking tasks together so that one task follows another.
The callback task will be applied with the result of the parent task
as a partial argument:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">link</span><span class="o">=</span><span class="n">add</span><span class="o">.</span><span class="n">s</span><span class="p">(</span><span class="mi">16</span><span class="p">))</span>
</pre></div>
</div>
<div class="sidebar">
<p class="first sidebar-title">What is <tt class="docutils literal"><span class="pre">s</span></tt>?</p>
<p>The <tt class="docutils literal"><span class="pre">add.s</span></tt> call used here is called a subtask, we talk
more about subtasks in the <a class="reference internal" href="canvas.html#guide-canvas"><em>canvas guide</em></a>,
where you can also learn about <a class="reference internal" href="../reference/celery.html#celery.chain" title="celery.chain"><tt class="xref py py-class docutils literal"><span class="pre">chain</span></tt></a>, which
is a simpler way to chain tasks together.</p>
<p class="last">In practice the <tt class="docutils literal"><span class="pre">link</span></tt> execution option is considered an internal
primitive, and you will probably not use it directly, but
rather use chains instead.</p>
</div>
<p>Here the result of the first task (4) will be sent to a new
task that adds 16 to the previous result, forming the expression
<img class="math" src="../_images/math/5bec3f9bd85f65932e771e2f003f923f49ed1a94.png" alt="(2 + 2) + 16 = 20"/></p>
<p>You can also cause a callback to be applied if task raises an exception
(<em>errback</em>), but this behaves differently from a regular callback
in that it will be passed the id of the parent task, not the result.
This is because it may not always be possible to serialize
the exception raised, and so this way the error callback requires
a result backend to be enabled, and the task must retrieve the result
of the task instead.</p>
<p>This is an example error callback:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="nd">@celery.task</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">error_handler</span><span class="p">(</span><span class="n">uuid</span><span class="p">):</span>
    <span class="n">result</span> <span class="o">=</span> <span class="n">AsyncResult</span><span class="p">(</span><span class="n">uuid</span><span class="p">)</span>
    <span class="n">exc</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">propagate</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
    <span class="k">print</span><span class="p">(</span><span class="s">&quot;Task </span><span class="si">%r</span><span class="s"> raised exception: </span><span class="si">%r</span><span class="se">\n</span><span class="si">%r</span><span class="s">&quot;</span> <span class="o">%</span> <span class="p">(</span>
          <span class="n">exc</span><span class="p">,</span> <span class="n">result</span><span class="o">.</span><span class="n">traceback</span><span class="p">))</span>
</pre></div>
</div>
<p>it can be added to the task using the <tt class="docutils literal"><span class="pre">link_error</span></tt> execution
option:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">link_error</span><span class="o">=</span><span class="n">error_handler</span><span class="o">.</span><span class="n">s</span><span class="p">())</span>
</pre></div>
</div>
<p>In addition, both the <tt class="docutils literal"><span class="pre">link</span></tt> and <tt class="docutils literal"><span class="pre">link_error</span></tt> options can be expressed
as a list:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">link</span><span class="o">=</span><span class="p">[</span><span class="n">add</span><span class="o">.</span><span class="n">s</span><span class="p">(</span><span class="mi">16</span><span class="p">),</span> <span class="n">other_task</span><span class="o">.</span><span class="n">s</span><span class="p">()])</span>
</pre></div>
</div>
<p>The callbacks/errbacks will then be called in order, and all
callbacks will be called with the return value of the parent task
as a partial argument.</p>
</div>
<div class="section" id="eta-and-countdown">
<span id="calling-eta"></span><h2><a class="toc-backref" href="#id3">ETA and countdown</a><a class="headerlink" href="#eta-and-countdown" title="Permalink to this headline">¶</a></h2>
<p>The ETA (estimated time of arrival) lets you set a specific date and time that
is the earliest time at which your task will be executed.  <cite>countdown</cite> is
a shortcut to set eta by seconds into the future.</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">result</span> <span class="o">=</span> <span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">countdown</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>    <span class="c"># this takes at least 3 seconds to return</span>
<span class="go">20</span>
</pre></div>
</div>
<p>The task is guaranteed to be executed at some time <em>after</em> the
specified date and time, but not necessarily at that exact time.
Possible reasons for broken deadlines may include many items waiting
in the queue, or heavy network latency.  To make sure your tasks
are executed in a timely manner you should monitor the queue for congestion. Use
Munin, or similar tools, to receive alerts, so appropriate action can be
taken to ease the workload.  See <a class="reference internal" href="monitoring.html#monitoring-munin"><em>Munin</em></a>.</p>
<p>While <cite>countdown</cite> is an integer, <cite>eta</cite> must be a <a class="reference external" href="http://docs.python.org/dev/library/datetime.html#datetime.datetime" title="(in Python v3.3)"><tt class="xref py py-class docutils literal"><span class="pre">datetime</span></tt></a>
object, specifying an exact date and time (including millisecond precision,
and timezone information):</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">tomorrow</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">+</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">eta</span><span class="o">=</span><span class="n">tomorrow</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="expiration">
<span id="calling-expiration"></span><h2><a class="toc-backref" href="#id4">Expiration</a><a class="headerlink" href="#expiration" title="Permalink to this headline">¶</a></h2>
<p>The <cite>expires</cite> argument defines an optional expiry time,
either as seconds after task publish, or a specific date and time using
<a class="reference external" href="http://docs.python.org/dev/library/datetime.html#datetime.datetime" title="(in Python v3.3)"><tt class="xref py py-class docutils literal"><span class="pre">datetime</span></tt></a>:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">&gt;&gt;&gt; </span><span class="c"># Task expires after one minute from now.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> <span class="n">expires</span><span class="o">=</span><span class="mi">60</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c"># Also supports datetime</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> <span class="n">kwargs</span><span class="p">,</span>
<span class="gp">... </span>                <span class="n">expires</span><span class="o">=</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">+</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
</pre></div>
</div>
<p>When a worker receives an expired task it will mark
the task as <a class="reference internal" href="tasks.html#std:state-REVOKED"><tt class="xref std std-state docutils literal"><span class="pre">REVOKED</span></tt></a> (<a class="reference internal" href="../reference/celery.exceptions.html#celery.exceptions.TaskRevokedError" title="celery.exceptions.TaskRevokedError"><tt class="xref py py-exc docutils literal"><span class="pre">TaskRevokedError</span></tt></a>).</p>
</div>
<div class="section" id="message-sending-retry">
<span id="calling-retry"></span><h2><a class="toc-backref" href="#id5">Message Sending Retry</a><a class="headerlink" href="#message-sending-retry" title="Permalink to this headline">¶</a></h2>
<p>Celery will automatically retry sending messages in the event of connection
failure, and retry behavior can be configured &#8211; like how often to retry, or a maximum
number of retries &#8211; or disabled all together.</p>
<p>To disable retry you can set the <tt class="docutils literal"><span class="pre">retry</span></tt> execution option to <tt class="xref py py-const docutils literal"><span class="pre">False</span></tt>:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">retry</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title first">Related Settings</p>
<table class="hlist"><tr><td><ul class="simple">
<li><a class="reference internal" href="../configuration.html#std:setting-CELERY_TASK_PUBLISH_RETRY"><tt class="xref std std-setting docutils literal"><span class="pre">CELERY_TASK_PUBLISH_RETRY</span></tt></a></li>
</ul>
</td><td><ul class="simple">
<li><a class="reference internal" href="../configuration.html#std:setting-CELERY_TASK_PUBLISH_RETRY_POLICY"><tt class="xref std std-setting docutils literal"><span class="pre">CELERY_TASK_PUBLISH_RETRY_POLICY</span></tt></a></li>
</ul>
</td></tr></table>
</div>
<div class="section" id="retry-policy">
<h3>Retry Policy<a class="headerlink" href="#retry-policy" title="Permalink to this headline">¶</a></h3>
<p>A retry policy is a mapping that controls how retries behave,
and can contain the following keys:</p>
<ul>
<li><p class="first"><cite>max_retries</cite></p>
<blockquote>
<div><p>Maximum number of retries before giving up, in this case the
exception that caused the retry to fail will be raised.</p>
<p>A value of 0 or <tt class="xref py py-const docutils literal"><span class="pre">None</span></tt> means it will retry forever.</p>
<p>The default is to retry 3 times.</p>
</div></blockquote>
</li>
<li><p class="first"><cite>interval_start</cite></p>
<blockquote>
<div><p>Defines the number of seconds (float or integer) to wait between
retries.  Default is 0, which means the first retry will be
instantaneous.</p>
</div></blockquote>
</li>
<li><p class="first"><cite>interval_step</cite></p>
<blockquote>
<div><p>On each consecutive retry this number will be added to the retry
delay (float or integer).  Default is 0.2.</p>
</div></blockquote>
</li>
<li><p class="first"><cite>interval_max</cite></p>
<blockquote>
<div><p>Maximum number of seconds (float or integer) to wait between
retries.  Default is 0.2.</p>
</div></blockquote>
</li>
</ul>
<p>For example, the default policy correlates to:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">retry</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">retry_policy</span><span class="o">=</span><span class="p">{</span>
    <span class="s">&quot;max_retries&quot;</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
    <span class="s">&quot;interval_start&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
    <span class="s">&quot;interval_step&quot;</span><span class="p">:</span> <span class="mf">0.2</span><span class="p">,</span>
    <span class="s">&quot;interval_max&quot;</span><span class="p">:</span> <span class="mf">0.2</span><span class="p">,</span>
<span class="p">})</span>
</pre></div>
</div>
<p>the maximum time spent retrying will be 0.4 seconds.  It is set relatively
short by default because a connection failure could lead to a retry pile effect
if the broker connection is down: e.g. many web server processes waiting
to retry blocking other incoming requests.</p>
</div>
</div>
<div class="section" id="serializers">
<span id="calling-serializers"></span><h2><a class="toc-backref" href="#id6">Serializers</a><a class="headerlink" href="#serializers" title="Permalink to this headline">¶</a></h2>
<div class="sidebar">
<p class="first sidebar-title">Security</p>
<p>The pickle module allows for execution of arbitrary functions,
please see the <a class="reference internal" href="security.html#guide-security"><em>security guide</em></a>.</p>
<p class="last">Celery also comes with a special serializer that uses
cryptography to sign your messages.</p>
</div>
<p>Data transferred between clients and workers needs to be serialized,
so every message in Celery has a <tt class="docutils literal"><span class="pre">content_type</span></tt> header that
describes the serialization method used to encode it.</p>
<p>The default serializer is <a class="reference external" href="http://docs.python.org/dev/library/pickle.html#pickle" title="(in Python v3.3)"><tt class="xref py py-mod docutils literal"><span class="pre">pickle</span></tt></a>, but you can
change this using the <a class="reference internal" href="../configuration.html#std:setting-CELERY_TASK_SERIALIZER"><tt class="xref std std-setting docutils literal"><span class="pre">CELERY_TASK_SERIALIZER</span></tt></a> setting,
or for each individual task, or even per message.</p>
<p>There&#8217;s built-in support for <a class="reference external" href="http://docs.python.org/dev/library/pickle.html#pickle" title="(in Python v3.3)"><tt class="xref py py-mod docutils literal"><span class="pre">pickle</span></tt></a>, <cite>JSON</cite>, <cite>YAML</cite>
and <cite>msgpack</cite>, and you can also add your own custom serializers by registering
them into the Kombu serializer registry (see <a class="reference external" href="http://packages.python.org/kombu/introduction.html#serialization-of-data">Kombu: Serialization of Data</a>).</p>
<p>Each option has its advantages and disadvantages.</p>
<dl class="docutils">
<dt>json &#8211; JSON is supported in many programming languages, is now</dt>
<dd><p class="first">a standard part of Python (since 2.6), and is fairly fast to decode
using the modern Python libraries such as <tt class="xref py py-mod docutils literal"><span class="pre">cjson</span></tt> or <tt class="xref py py-mod docutils literal"><span class="pre">simplejson</span></tt>.</p>
<p>The primary disadvantage to JSON is that it limits you to the following
data types: strings, Unicode, floats, boolean, dictionaries, and lists.
Decimals and dates are notably missing.</p>
<p>Also, binary data will be transferred using Base64 encoding, which will
cause the transferred data to be around 34% larger than an encoding which
supports native binary types.</p>
<p>However, if your data fits inside the above constraints and you need
cross-language support, the default setting of JSON is probably your
best choice.</p>
<p class="last">See <a class="reference external" href="http://json.org">http://json.org</a> for more information.</p>
</dd>
<dt>pickle &#8211; If you have no desire to support any language other than</dt>
<dd><p class="first">Python, then using the pickle encoding will gain you the support of
all built-in Python data types (except class instances), smaller
messages when sending binary files, and a slight speedup over JSON
processing.</p>
<p class="last">See <a class="reference external" href="http://docs.python.org/library/pickle.html">http://docs.python.org/library/pickle.html</a> for more information.</p>
</dd>
<dt>yaml &#8211; YAML has many of the same characteristics as json,</dt>
<dd><p class="first">except that it natively supports more data types (including dates,
recursive references, etc.)</p>
<p>However, the Python libraries for YAML are a good bit slower than the
libraries for JSON.</p>
<p>If you need a more expressive set of data types and need to maintain
cross-language compatibility, then YAML may be a better fit than the above.</p>
<p class="last">See <a class="reference external" href="http://yaml.org/">http://yaml.org/</a> for more information.</p>
</dd>
<dt>msgpack &#8211; msgpack is a binary serialization format that is closer to JSON</dt>
<dd><p class="first">in features.  It is very young however, and support should be considered
experimental at this point.</p>
<p class="last">See <a class="reference external" href="http://msgpack.org/">http://msgpack.org/</a> for more information.</p>
</dd>
</dl>
<p>The encoding used is available as a message header, so the worker knows how to
deserialize any task.  If you use a custom serializer, this serializer must
be available for the worker.</p>
<p>The following order is used to decide which serializer
to use when sending a task:</p>
<blockquote>
<div><ol class="arabic simple">
<li>The <cite>serializer</cite> execution option.</li>
<li>The <a class="reference internal" href="../reference/celery.app.task.html#celery.app.task.Task.serializer" title="celery.app.task.Task.serializer"><tt class="xref py py-attr docutils literal"><span class="pre">Task.serializer</span></tt></a> attribute</li>
<li>The <a class="reference internal" href="../configuration.html#std:setting-CELERY_TASK_SERIALIZER"><tt class="xref std std-setting docutils literal"><span class="pre">CELERY_TASK_SERIALIZER</span></tt></a> setting.</li>
</ol>
</div></blockquote>
<p>Example setting a custom serializer for a single task invocation:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> <span class="n">serializer</span><span class="o">=</span><span class="s">&quot;json&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="compression">
<span id="calling-compression"></span><h2><a class="toc-backref" href="#id7">Compression</a><a class="headerlink" href="#compression" title="Permalink to this headline">¶</a></h2>
<p>Celery can compress the messages using either <em>gzip</em>, or <em>bzip2</em>.
You can also create your own compression schemes and register
them in the <a class="reference external" href="http://kombu.readthedocs.org/en/latest/reference/kombu.compression.html#kombu.compression.register" title="(in Kombu v2.2)"><tt class="xref py py-func docutils literal"><span class="pre">kombu</span> <span class="pre">compression</span> <span class="pre">registry</span></tt></a>.</p>
<p>The following order is used to decide which compression scheme
to use when sending a task:</p>
<blockquote>
<div><ol class="arabic simple">
<li>The <cite>compression</cite> execution option.</li>
<li>The <tt class="xref py py-attr docutils literal"><span class="pre">Task.compression</span></tt> attribute.</li>
<li>The <a class="reference internal" href="../configuration.html#std:setting-CELERY_MESSAGE_COMPRESSION"><tt class="xref std std-setting docutils literal"><span class="pre">CELERY_MESSAGE_COMPRESSION</span></tt></a> attribute.</li>
</ol>
</div></blockquote>
<p>Example specifying the compression used when calling a task:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">compression</span><span class="o">=</span><span class="s">&quot;zlib&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="connections">
<span id="calling-connections"></span><h2><a class="toc-backref" href="#id8">Connections</a><a class="headerlink" href="#connections" title="Permalink to this headline">¶</a></h2>
<div class="sidebar">
<p class="first sidebar-title">Automatic Pool Support</p>
<p>Since version 2.3 there is support for automatic connection pools,
so you don&#8217;t have to manually handle connections and publishers
to reuse connections.</p>
<p>The connection pool is enabled by default since version 2.5.</p>
<p class="last">See the <a class="reference internal" href="../configuration.html#std:setting-BROKER_POOL_LIMIT"><tt class="xref std std-setting docutils literal"><span class="pre">BROKER_POOL_LIMIT</span></tt></a> setting for more information.</p>
</div>
<p>You can handle the connection manually by creating a
publisher:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">with</span> <span class="n">add</span><span class="o">.</span><span class="n">app</span><span class="o">.</span><span class="n">pool</span><span class="o">.</span><span class="n">acquire</span><span class="p">(</span><span class="n">block</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span> <span class="k">as</span> <span class="n">connection</span><span class="p">:</span>
    <span class="k">with</span> <span class="n">add</span><span class="o">.</span><span class="n">get_publisher</span><span class="p">(</span><span class="n">connection</span><span class="p">)</span> <span class="k">as</span> <span class="n">publisher</span><span class="p">:</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">args</span> <span class="ow">in</span> <span class="n">numbers</span><span class="p">:</span>
                <span class="n">res</span> <span class="o">=</span> <span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">publisher</span><span class="o">=</span><span class="n">publisher</span><span class="p">)</span>
                <span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
<span class="k">print</span><span class="p">([</span><span class="n">res</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="k">for</span> <span class="n">res</span> <span class="ow">in</span> <span class="n">results</span><span class="p">])</span>
</pre></div>
</div>
<p>Though this particular example is much better expressed as a group:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">celery</span> <span class="kn">import</span> <span class="n">group</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">numbers</span> <span class="o">=</span> <span class="p">[(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">),</span> <span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> <span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">16</span><span class="p">)]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">res</span> <span class="o">=</span> <span class="n">group</span><span class="p">(</span><span class="n">add</span><span class="o">.</span><span class="n">subtask</span><span class="p">(</span><span class="n">n</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">numbers</span><span class="p">)</span><span class="o">.</span><span class="n">apply_async</span><span class="p">()</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">res</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<span class="go">[4, 8, 16, 32]</span>
</pre></div>
</div>
</div>
<div class="section" id="routing-options">
<span id="calling-routing"></span><h2><a class="toc-backref" href="#id9">Routing options</a><a class="headerlink" href="#routing-options" title="Permalink to this headline">¶</a></h2>
<p>Celery can route tasks to different queues.</p>
<p>Simple routing (name &lt;-&gt; name) is accomplished using the <tt class="docutils literal"><span class="pre">queue</span></tt> option:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="n">add</span><span class="o">.</span><span class="n">apply_async</span><span class="p">(</span><span class="n">queue</span><span class="o">=</span><span class="s">&quot;priority.high&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>You can then assign workers to the <tt class="docutils literal"><span class="pre">priority.high</span></tt> queue by using
the workers <em class="xref std std-option">-Q</em> argument:</p>
<div class="highlight-python"><pre>$ celery worker -l info -Q celery,priority.high</pre>
</div>
<div class="admonition-see-also admonition seealso">
<p class="first admonition-title">See also</p>
<p>Hard-coding queue names in code is not recommended, the best practice
is to use configuration routers (<a class="reference internal" href="../configuration.html#std:setting-CELERY_ROUTES"><tt class="xref std std-setting docutils literal"><span class="pre">CELERY_ROUTES</span></tt></a>).</p>
<p class="last">To find out more about routing, please see <a class="reference internal" href="routing.html#guide-routing"><em>Routing Tasks</em></a>.</p>
</div>
<div class="section" id="advanced-options">
<h3>Advanced Options<a class="headerlink" href="#advanced-options" title="Permalink to this headline">¶</a></h3>
<p>These options are for advanced users who want to take use of
AMQP&#8217;s full routing capabilities. Interested parties may read the
<a class="reference internal" href="routing.html#guide-routing"><em>routing guide</em></a>.</p>
<ul>
<li><p class="first">exchange</p>
<blockquote>
<div><p>Name of exchange (or a <a class="reference external" href="http://kombu.readthedocs.org/en/latest/reference/kombu.entity.html#kombu.entity.Exchange" title="(in Kombu v2.2)"><tt class="xref py py-class docutils literal"><span class="pre">kombu.entity.Exchange</span></tt></a>) to
send the message to.</p>
</div></blockquote>
</li>
<li><p class="first">routing_key</p>
<blockquote>
<div><p>Routing key used to determine.</p>
</div></blockquote>
</li>
<li><p class="first">mandatory</p>
<blockquote>
<div><p>This sets the delivery to be mandatory.  An exception will be raised
if there are no running workers able to take on the task.</p>
<p>Not supported by <tt class="xref py py-mod docutils literal"><span class="pre">amqplib</span></tt>.</p>
</div></blockquote>
</li>
<li><p class="first">immediate</p>
<blockquote>
<div><p>Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.</p>
<p>Not supported by <tt class="xref py py-mod docutils literal"><span class="pre">amqplib</span></tt>.</p>
</div></blockquote>
</li>
<li><p class="first">priority</p>
<blockquote>
<div><p>A number between <cite>0</cite> and <cite>9</cite>, where <cite>0</cite> is the highest priority.</p>
<p>Supported by: redis, beanstalk</p>
</div></blockquote>
</li>
</ul>
</div>
</div>
</div>


          </div>
        </div>
      </div>
      <div class="sphinxsidebar">
        <div class="sphinxsidebarwrapper"><p class="logo"><a href="../index.html">
  <img class="logo" src="http://cloud.github.com/downloads/celery/celery/celery_128.png" alt="Logo"/>
</a></p>
  <h4>Previous topic</h4>
  <p class="topless"><a href="tasks.html"
                        title="previous chapter">Tasks</a></p>
  <h4>Next topic</h4>
  <p class="topless"><a href="workers.html"
                        title="next chapter">Workers Guide</a></p>
  <h3>This Page</h3>
  <ul class="this-page-menu">
    <li><a href="../_sources/userguide/calling.txt"
           rel="nofollow">Show Source</a></li>
  </ul>
<div id="searchbox" style="display: none">
  <h3>Quick search</h3>
    <form class="search" action="../search.html" method="get">
      <input type="text" name="q" />
      <input type="submit" value="Go" />
      <input type="hidden" name="check_keywords" value="yes" />
      <input type="hidden" name="area" value="default" />
    </form>
    <p class="searchtip" style="font-size: 90%">
    Enter search terms or a module, class or function name.
    </p>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
        </div>
      </div>
      <div class="clearer"></div>
    </div>
    <div class="related">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../genindex.html" title="General Index"
             >index</a></li>
        <li class="right" >
          <a href="../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="right" >
          <a href="workers.html" title="Workers Guide"
             >next</a> |</li>
        <li class="right" >
          <a href="tasks.html" title="Tasks"
             >previous</a> |</li>
        <li><a href="../index.html">Celery 2.6.0rc4 documentation</a> &raquo;</li>
          <li><a href="index.html" >User Guide</a> &raquo;</li> 
      </ul>
    </div>
    <div class="footer">
        &copy; Copyright 2009-2012, Ask Solem &amp; Contributors.
    </div>
  </body>
</html>