canvas.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892
  1. # -*- coding: utf-8 -*-
  2. """
  3. celery.canvas
  4. ~~~~~~~~~~~~~
  5. Composing task workflows.
  6. Documentation for some of these types are in :mod:`celery`.
  7. You should import these from :mod:`celery` and not this module.
  8. """
  9. from __future__ import absolute_import
  10. from collections import MutableSequence, deque
  11. from copy import deepcopy
  12. from functools import partial as _partial, reduce
  13. from operator import itemgetter
  14. from itertools import chain as _chain
  15. from kombu.utils import cached_property, fxrange, reprcall, uuid
  16. from celery._state import current_app, get_current_worker_task
  17. from celery.utils.functional import (
  18. maybe_list, is_list, regen,
  19. chunks as _chunks,
  20. )
  21. from celery.utils.text import truncate
  22. __all__ = ['Signature', 'chain', 'xmap', 'xstarmap', 'chunks',
  23. 'group', 'chord', 'signature', 'maybe_signature']
  24. class _getitem_property(object):
  25. """Attribute -> dict key descriptor.
  26. The target object must support ``__getitem__``,
  27. and optionally ``__setitem__``.
  28. Example:
  29. >>> from collections import defaultdict
  30. >>> class Me(dict):
  31. ... deep = defaultdict(dict)
  32. ...
  33. ... foo = _getitem_property('foo')
  34. ... deep_thing = _getitem_property('deep.thing')
  35. >>> me = Me()
  36. >>> me.foo
  37. None
  38. >>> me.foo = 10
  39. >>> me.foo
  40. 10
  41. >>> me['foo']
  42. 10
  43. >>> me.deep_thing = 42
  44. >>> me.deep_thing
  45. 42
  46. >>> me.deep
  47. defaultdict(<type 'dict'>, {'thing': 42})
  48. """
  49. def __init__(self, keypath):
  50. path, _, self.key = keypath.rpartition('.')
  51. self.path = path.split('.') if path else None
  52. def _path(self, obj):
  53. return (reduce(lambda d, k: d[k], [obj] + self.path) if self.path
  54. else obj)
  55. def __get__(self, obj, type=None):
  56. if obj is None:
  57. return type
  58. return self._path(obj).get(self.key)
  59. def __set__(self, obj, value):
  60. self._path(obj)[self.key] = value
  61. def maybe_unroll_group(g):
  62. """Unroll group with only one member."""
  63. # Issue #1656
  64. try:
  65. size = len(g.tasks)
  66. except TypeError:
  67. try:
  68. size = g.tasks.__length_hint__()
  69. except (AttributeError, TypeError):
  70. pass
  71. else:
  72. return list(g.tasks)[0] if size == 1 else g
  73. else:
  74. return g.tasks[0] if size == 1 else g
  75. def task_name_from(task):
  76. return getattr(task, 'name', task)
  77. class Signature(dict):
  78. """Class that wraps the arguments and execution options
  79. for a single task invocation.
  80. Used as the parts in a :class:`group` and other constructs,
  81. or to pass tasks around as callbacks while being compatible
  82. with serializers with a strict type subset.
  83. :param task: Either a task class/instance, or the name of a task.
  84. :keyword args: Positional arguments to apply.
  85. :keyword kwargs: Keyword arguments to apply.
  86. :keyword options: Additional options to :meth:`Task.apply_async`.
  87. Note that if the first argument is a :class:`dict`, the other
  88. arguments will be ignored and the values in the dict will be used
  89. instead.
  90. >>> s = signature('tasks.add', args=(2, 2))
  91. >>> signature(s)
  92. {'task': 'tasks.add', args=(2, 2), kwargs={}, options={}}
  93. """
  94. TYPES = {}
  95. _app = _type = None
  96. @classmethod
  97. def register_type(cls, subclass, name=None):
  98. cls.TYPES[name or subclass.__name__] = subclass
  99. return subclass
  100. @classmethod
  101. def from_dict(self, d, app=None):
  102. typ = d.get('subtask_type')
  103. if typ:
  104. return self.TYPES[typ].from_dict(d, app=app)
  105. return Signature(d, app=app)
  106. def __init__(self, task=None, args=None, kwargs=None, options=None,
  107. type=None, subtask_type=None, immutable=False,
  108. app=None, **ex):
  109. self._app = app
  110. init = dict.__init__
  111. if isinstance(task, dict):
  112. return init(self, task) # works like dict(d)
  113. # Also supports using task class/instance instead of string name.
  114. try:
  115. task_name = task.name
  116. except AttributeError:
  117. task_name = task
  118. else:
  119. self._type = task
  120. init(self,
  121. task=task_name, args=tuple(args or ()),
  122. kwargs=kwargs or {},
  123. options=dict(options or {}, **ex),
  124. subtask_type=subtask_type,
  125. immutable=immutable)
  126. def __call__(self, *partial_args, **partial_kwargs):
  127. args, kwargs, _ = self._merge(partial_args, partial_kwargs, None)
  128. return self.type(*args, **kwargs)
  129. def delay(self, *partial_args, **partial_kwargs):
  130. return self.apply_async(partial_args, partial_kwargs)
  131. def apply(self, args=(), kwargs={}, **options):
  132. """Apply this task locally."""
  133. # For callbacks: extra args are prepended to the stored args.
  134. args, kwargs, options = self._merge(args, kwargs, options)
  135. return self.type.apply(args, kwargs, **options)
  136. def _merge(self, args=(), kwargs={}, options={}):
  137. if self.immutable:
  138. return (self.args, self.kwargs,
  139. dict(self.options, **options) if options else self.options)
  140. return (tuple(args) + tuple(self.args) if args else self.args,
  141. dict(self.kwargs, **kwargs) if kwargs else self.kwargs,
  142. dict(self.options, **options) if options else self.options)
  143. def clone(self, args=(), kwargs={}, **opts):
  144. # need to deepcopy options so origins links etc. is not modified.
  145. if args or kwargs or opts:
  146. args, kwargs, opts = self._merge(args, kwargs, opts)
  147. else:
  148. args, kwargs, opts = self.args, self.kwargs, self.options
  149. s = Signature.from_dict({'task': self.task, 'args': tuple(args),
  150. 'kwargs': kwargs, 'options': deepcopy(opts),
  151. 'subtask_type': self.subtask_type,
  152. 'immutable': self.immutable}, app=self._app)
  153. s._type = self._type
  154. return s
  155. partial = clone
  156. def freeze(self, _id=None, group_id=None, chord=None, root_id=None):
  157. opts = self.options
  158. try:
  159. tid = opts['task_id']
  160. except KeyError:
  161. tid = opts['task_id'] = _id or uuid()
  162. root_id = opts.setdefault('root_id', root_id)
  163. if 'reply_to' not in opts:
  164. opts['reply_to'] = self.app.oid
  165. if group_id:
  166. opts['group_id'] = group_id
  167. if chord:
  168. opts['chord'] = chord
  169. return self.AsyncResult(tid)
  170. _freeze = freeze
  171. def replace(self, args=None, kwargs=None, options=None):
  172. s = self.clone()
  173. if args is not None:
  174. s.args = args
  175. if kwargs is not None:
  176. s.kwargs = kwargs
  177. if options is not None:
  178. s.options = options
  179. return s
  180. def set(self, immutable=None, **options):
  181. if immutable is not None:
  182. self.set_immutable(immutable)
  183. self.options.update(options)
  184. return self
  185. def set_immutable(self, immutable):
  186. self.immutable = immutable
  187. def apply_async(self, args=(), kwargs={}, route_name=None, **options):
  188. try:
  189. _apply = self._apply_async
  190. except IndexError: # no tasks for chain, etc to find type
  191. return
  192. # For callbacks: extra args are prepended to the stored args.
  193. if args or kwargs or options:
  194. args, kwargs, options = self._merge(args, kwargs, options)
  195. else:
  196. args, kwargs, options = self.args, self.kwargs, self.options
  197. return _apply(args, kwargs, **options)
  198. def append_to_list_option(self, key, value):
  199. items = self.options.setdefault(key, [])
  200. if not isinstance(items, MutableSequence):
  201. items = self.options[key] = [items]
  202. if value not in items:
  203. items.append(value)
  204. return value
  205. def link(self, callback):
  206. return self.append_to_list_option('link', callback)
  207. def link_error(self, errback):
  208. return self.append_to_list_option('link_error', errback)
  209. def flatten_links(self):
  210. return list(_chain.from_iterable(_chain(
  211. [[self]],
  212. (link.flatten_links()
  213. for link in maybe_list(self.options.get('link')) or [])
  214. )))
  215. def __or__(self, other):
  216. if isinstance(other, group):
  217. other = maybe_unroll_group(other)
  218. if not isinstance(self, chain) and isinstance(other, chain):
  219. return chain((self, ) + other.tasks, app=self._app)
  220. elif isinstance(other, chain):
  221. return chain(*self.tasks + other.tasks, app=self._app)
  222. elif isinstance(other, Signature):
  223. if isinstance(self, chain):
  224. return chain(*self.tasks + (other, ), app=self._app)
  225. return chain(self, other, app=self._app)
  226. return NotImplemented
  227. def __deepcopy__(self, memo):
  228. memo[id(self)] = self
  229. return dict(self)
  230. def __invert__(self):
  231. return self.apply_async().get()
  232. def __reduce__(self):
  233. # for serialization, the task type is lazily loaded,
  234. # and not stored in the dict itself.
  235. return signature, (dict(self), )
  236. def __json__(self):
  237. return dict(self)
  238. def reprcall(self, *args, **kwargs):
  239. args, kwargs, _ = self._merge(args, kwargs, {})
  240. return reprcall(self['task'], args, kwargs)
  241. def election(self):
  242. type = self.type
  243. app = type.app
  244. tid = self.options.get('task_id') or uuid()
  245. with app.producer_or_acquire(None) as P:
  246. props = type.backend.on_task_call(P, tid)
  247. app.control.election(tid, 'task', self.clone(task_id=tid, **props),
  248. connection=P.connection)
  249. return type.AsyncResult(tid)
  250. def __repr__(self):
  251. return self.reprcall()
  252. @property
  253. def name(self):
  254. # for duck typing compatibility with Task.name
  255. return self.task
  256. @cached_property
  257. def type(self):
  258. return self._type or self.app.tasks[self['task']]
  259. @cached_property
  260. def app(self):
  261. return self._app or current_app
  262. @cached_property
  263. def AsyncResult(self):
  264. try:
  265. return self.type.AsyncResult
  266. except KeyError: # task not registered
  267. return self.app.AsyncResult
  268. @cached_property
  269. def _apply_async(self):
  270. try:
  271. return self.type.apply_async
  272. except KeyError:
  273. return _partial(self.app.send_task, self['task'])
  274. id = _getitem_property('options.task_id')
  275. task = _getitem_property('task')
  276. args = _getitem_property('args')
  277. kwargs = _getitem_property('kwargs')
  278. options = _getitem_property('options')
  279. subtask_type = _getitem_property('subtask_type')
  280. immutable = _getitem_property('immutable')
  281. @Signature.register_type
  282. class chain(Signature):
  283. tasks = _getitem_property('kwargs.tasks')
  284. def __init__(self, *tasks, **options):
  285. tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0])
  286. else tasks)
  287. Signature.__init__(
  288. self, 'celery.chain', (), {'tasks': tasks}, **options
  289. )
  290. self.subtask_type = 'chain'
  291. def __call__(self, *args, **kwargs):
  292. if self.tasks:
  293. return self.apply_async(args, kwargs)
  294. def apply_async(self, args=(), kwargs={}, **options):
  295. # python is best at unpacking kwargs, so .run is here to do that.
  296. app = self.app
  297. if app.conf.CELERY_ALWAYS_EAGER:
  298. return self.apply(args, kwargs, **options)
  299. return self.run(args, kwargs, app=app, **(
  300. dict(self.options, **options) if options else self.options))
  301. def run(self, args=(), kwargs={}, group_id=None, chord=None,
  302. task_id=None, link=None, link_error=None,
  303. publisher=None, producer=None, root_id=None, app=None, **options):
  304. app = app or self.app
  305. args = tuple(args) + tuple(self.args)
  306. tasks, results = self.prepare_steps(
  307. args, self.tasks, root_id, link_error,
  308. )
  309. if not results:
  310. return
  311. result = results[-1]
  312. last_task = tasks[-1]
  313. if group_id:
  314. last_task.set(group_id=group_id)
  315. if chord:
  316. last_task.set(chord=chord)
  317. if task_id:
  318. last_task.set(task_id=task_id)
  319. result = last_task.type.AsyncResult(task_id)
  320. # make sure we can do a link() and link_error() on a chain object.
  321. if link:
  322. tasks[-1].set(link=link)
  323. tasks[0].apply_async(**options)
  324. return result
  325. def prepare_steps(self, args, tasks,
  326. root_id=None, link_error=None, app=None,
  327. from_dict=Signature.from_dict):
  328. app = app or self.app
  329. steps = deque(tasks)
  330. next_step = prev_task = prev_res = None
  331. tasks, results = [], []
  332. i = 0
  333. while steps:
  334. task = steps.popleft()
  335. if not isinstance(task, Signature):
  336. task = from_dict(task, app=app)
  337. if not i: # first task
  338. # first task gets partial args from chain
  339. task = task.clone(args)
  340. res = task.freeze(root_id=root_id)
  341. root_id = res.id if root_id is None else root_id
  342. else:
  343. task = task.clone()
  344. res = task.freeze(root_id=root_id)
  345. i += 1
  346. if isinstance(task, group):
  347. task = maybe_unroll_group(task)
  348. if isinstance(task, chain):
  349. # splice the chain
  350. steps.extendleft(reversed(task.tasks))
  351. continue
  352. elif isinstance(task, group) and steps and \
  353. not isinstance(steps[0], group):
  354. # automatically upgrade group(...) | s to chord(group, s)
  355. try:
  356. next_step = steps.popleft()
  357. # for chords we freeze by pretending it's a normal
  358. # signature instead of a group.
  359. res = Signature.freeze(next_step)
  360. task = chord(
  361. task, body=next_step,
  362. task_id=res.task_id, root_id=root_id,
  363. )
  364. except IndexError:
  365. pass # no callback, so keep as group.
  366. if prev_task:
  367. # link previous task to this task.
  368. prev_task.link(task)
  369. # set AsyncResult.parent
  370. if not res.parent:
  371. res.parent = prev_res
  372. if link_error:
  373. task.set(link_error=link_error)
  374. if not isinstance(prev_task, chord):
  375. results.append(res)
  376. tasks.append(task)
  377. prev_task, prev_res = task, res
  378. return tasks, results
  379. def apply(self, args=(), kwargs={}, **options):
  380. last, fargs = None, args
  381. for task in self.tasks:
  382. res = task.clone(fargs).apply(
  383. last and (last.get(), ), **dict(self.options, **options))
  384. res.parent, last, fargs = last, res, None
  385. return last
  386. @classmethod
  387. def from_dict(self, d, app=None):
  388. tasks = d['kwargs']['tasks']
  389. if tasks:
  390. if isinstance(tasks, tuple): # aaaargh
  391. tasks = d['kwargs']['tasks'] = list(tasks)
  392. # First task must be signature object to get app
  393. tasks[0] = maybe_signature(tasks[0], app=app)
  394. return chain(*tasks, app=app, **d['options'])
  395. @property
  396. def app(self):
  397. app = self._app
  398. if app is None:
  399. try:
  400. app = self.tasks[0]._app
  401. except (KeyError, IndexError):
  402. pass
  403. return app or current_app
  404. def __repr__(self):
  405. return ' | '.join(repr(t) for t in self.tasks)
  406. class _basemap(Signature):
  407. _task_name = None
  408. _unpack_args = itemgetter('task', 'it')
  409. def __init__(self, task, it, **options):
  410. Signature.__init__(
  411. self, self._task_name, (),
  412. {'task': task, 'it': regen(it)}, immutable=True, **options
  413. )
  414. def apply_async(self, args=(), kwargs={}, **opts):
  415. # need to evaluate generators
  416. task, it = self._unpack_args(self.kwargs)
  417. return self.type.apply_async(
  418. (), {'task': task, 'it': list(it)},
  419. route_name=task_name_from(self.kwargs.get('task')), **opts
  420. )
  421. @classmethod
  422. def from_dict(cls, d, app=None):
  423. return cls(*cls._unpack_args(d['kwargs']), app=app, **d['options'])
  424. @Signature.register_type
  425. class xmap(_basemap):
  426. _task_name = 'celery.map'
  427. def __repr__(self):
  428. task, it = self._unpack_args(self.kwargs)
  429. return '[{0}(x) for x in {1}]'.format(task.task,
  430. truncate(repr(it), 100))
  431. @Signature.register_type
  432. class xstarmap(_basemap):
  433. _task_name = 'celery.starmap'
  434. def __repr__(self):
  435. task, it = self._unpack_args(self.kwargs)
  436. return '[{0}(*x) for x in {1}]'.format(task.task,
  437. truncate(repr(it), 100))
  438. @Signature.register_type
  439. class chunks(Signature):
  440. _unpack_args = itemgetter('task', 'it', 'n')
  441. def __init__(self, task, it, n, **options):
  442. Signature.__init__(
  443. self, 'celery.chunks', (),
  444. {'task': task, 'it': regen(it), 'n': n},
  445. immutable=True, **options
  446. )
  447. @classmethod
  448. def from_dict(self, d, app=None):
  449. return chunks(*self._unpack_args(d['kwargs']), app=app, **d['options'])
  450. def apply_async(self, args=(), kwargs={}, **opts):
  451. return self.group().apply_async(
  452. args, kwargs,
  453. route_name=task_name_from(self.kwargs.get('task')), **opts
  454. )
  455. def __call__(self, **options):
  456. return self.apply_async(**options)
  457. def group(self):
  458. # need to evaluate generators
  459. task, it, n = self._unpack_args(self.kwargs)
  460. return group((xstarmap(task, part, app=self._app)
  461. for part in _chunks(iter(it), n)),
  462. app=self._app)
  463. @classmethod
  464. def apply_chunks(cls, task, it, n, app=None):
  465. return cls(task, it, n, app=app)()
  466. def _maybe_group(tasks):
  467. if isinstance(tasks, group):
  468. tasks = list(tasks.tasks)
  469. elif isinstance(tasks, Signature):
  470. tasks = [tasks]
  471. else:
  472. tasks = regen(tasks)
  473. return tasks
  474. @Signature.register_type
  475. class group(Signature):
  476. tasks = _getitem_property('kwargs.tasks')
  477. def __init__(self, *tasks, **options):
  478. if len(tasks) == 1:
  479. tasks = _maybe_group(tasks[0])
  480. Signature.__init__(
  481. self, 'celery.group', (), {'tasks': tasks}, **options
  482. )
  483. self.subtask_type = 'group'
  484. @classmethod
  485. def from_dict(self, d, app=None):
  486. return group(d['kwargs']['tasks'], app=app, **d['options'])
  487. def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict,
  488. Signature=Signature, from_dict=Signature.from_dict):
  489. for task in tasks:
  490. if isinstance(task, dict):
  491. if isinstance(task, Signature):
  492. # local sigs are always of type Signature, and we
  493. # clone them to make sure we do not modify the originals.
  494. task = task.clone()
  495. else:
  496. # serialized sigs must be converted to Signature.
  497. task = from_dict(task, app=app)
  498. if isinstance(task, group):
  499. # needs yield_from :(
  500. unroll = task._prepared(
  501. task.tasks, partial_args, group_id, root_id, app,
  502. )
  503. for taskN, resN in unroll:
  504. yield taskN, resN
  505. else:
  506. if partial_args and not task.immutable:
  507. task.args = tuple(partial_args) + tuple(task.args)
  508. yield task, task.freeze(group_id=group_id, root_id=root_id)
  509. def _apply_tasks(self, tasks, producer=None, app=None, **options):
  510. app = app or self.app
  511. with app.producer_or_acquire(producer) as producer:
  512. for sig, res in tasks:
  513. sig.apply_async(producer=producer, add_to_parent=False,
  514. **options)
  515. yield res
  516. def _freeze_gid(self, options):
  517. # remove task_id and use that as the group_id,
  518. # if we don't remove it then every task will have the same id...
  519. options = dict(self.options, **options)
  520. options['group_id'] = group_id = (
  521. options.pop('task_id', uuid()))
  522. return options, group_id, options.get('root_id')
  523. def apply_async(self, args=(), kwargs=None, add_to_parent=True,
  524. producer=None, **options):
  525. app = self.app
  526. if app.conf.CELERY_ALWAYS_EAGER:
  527. return self.apply(args, kwargs, **options)
  528. if not self.tasks:
  529. return self.freeze()
  530. options, group_id, root_id = self._freeze_gid(options)
  531. tasks = self._prepared(self.tasks, args, group_id, root_id, app)
  532. result = self.app.GroupResult(
  533. group_id, list(self._apply_tasks(tasks, producer, app, **options)),
  534. )
  535. parent_task = get_current_worker_task()
  536. if add_to_parent and parent_task:
  537. parent_task.add_trail(result)
  538. return result
  539. def apply(self, args=(), kwargs={}, **options):
  540. app = self.app
  541. if not self.tasks:
  542. return self.freeze() # empty group returns GroupResult
  543. options, group_id, root_id = self._freeze_gid(options)
  544. tasks = self._prepared(self.tasks, args, group_id, root_id, app)
  545. return app.GroupResult(group_id, [
  546. sig.apply(**options) for sig, _ in tasks
  547. ])
  548. def set_immutable(self, immutable):
  549. for task in self.tasks:
  550. task.set_immutable(immutable)
  551. def link(self, sig):
  552. # Simply link to first task
  553. sig = sig.clone().set(immutable=True)
  554. return self.tasks[0].link(sig)
  555. def link_error(self, sig):
  556. sig = sig.clone().set(immutable=True)
  557. return self.tasks[0].link_error(sig)
  558. def __call__(self, *partial_args, **options):
  559. return self.apply_async(partial_args, **options)
  560. def _freeze_unroll(self, new_tasks, group_id, chord, root_id):
  561. stack = deque(self.tasks)
  562. while stack:
  563. task = maybe_signature(stack.popleft(), app=self._app).clone()
  564. if isinstance(task, group):
  565. stack.extendleft(task.tasks)
  566. else:
  567. new_tasks.append(task)
  568. yield task.freeze(group_id=group_id,
  569. chord=chord, root_id=root_id)
  570. def freeze(self, _id=None, group_id=None, chord=None, root_id=None):
  571. opts = self.options
  572. try:
  573. gid = opts['task_id']
  574. except KeyError:
  575. gid = opts['task_id'] = uuid()
  576. if group_id:
  577. opts['group_id'] = group_id
  578. if chord:
  579. opts['chord'] = chord
  580. root_id = opts.setdefault('root_id', root_id)
  581. new_tasks = []
  582. # Need to unroll subgroups early so that chord gets the
  583. # right result instance for chord_unlock etc.
  584. results = list(self._freeze_unroll(
  585. new_tasks, group_id, chord, root_id,
  586. ))
  587. if isinstance(self.tasks, MutableSequence):
  588. self.tasks[:] = new_tasks
  589. else:
  590. self.tasks = new_tasks
  591. return self.app.GroupResult(gid, results)
  592. _freeze = freeze
  593. def skew(self, start=1.0, stop=None, step=1.0):
  594. it = fxrange(start, stop, step, repeatlast=True)
  595. for task in self.tasks:
  596. task.set(countdown=next(it))
  597. return self
  598. def __iter__(self):
  599. return iter(self.tasks)
  600. def __repr__(self):
  601. return repr(self.tasks)
  602. @property
  603. def app(self):
  604. app = self._app
  605. if app is None:
  606. try:
  607. app = self.tasks[0].app
  608. except (KeyError, IndexError):
  609. pass
  610. return app if app is not None else current_app
  611. @Signature.register_type
  612. class chord(Signature):
  613. def __init__(self, header, body=None, task='celery.chord',
  614. args=(), kwargs={}, **options):
  615. Signature.__init__(
  616. self, task, args,
  617. dict(kwargs, header=_maybe_group(header),
  618. body=maybe_signature(body, app=self._app)), **options
  619. )
  620. self.subtask_type = 'chord'
  621. def freeze(self, *args, **kwargs):
  622. return self.body.freeze(*args, **kwargs)
  623. @classmethod
  624. def from_dict(self, d, app=None):
  625. args, d['kwargs'] = self._unpack_args(**d['kwargs'])
  626. return self(*args, app=app, **d)
  627. @staticmethod
  628. def _unpack_args(header=None, body=None, **kwargs):
  629. # Python signatures are better at extracting keys from dicts
  630. # than manually popping things off.
  631. return (header, body), kwargs
  632. @cached_property
  633. def app(self):
  634. return self._get_app(self.body)
  635. def _get_app(self, body=None):
  636. app = self._app
  637. if app is None:
  638. app = self.tasks[0]._app
  639. if app is None and body is not None:
  640. app = body._app
  641. return app if app is not None else current_app
  642. def apply_async(self, args=(), kwargs={}, task_id=None,
  643. producer=None, publisher=None, connection=None,
  644. router=None, result_cls=None, **options):
  645. body = kwargs.get('body') or self.kwargs['body']
  646. kwargs = dict(self.kwargs, **kwargs)
  647. body = body.clone(**options)
  648. app = self._get_app(body)
  649. tasks = (self.tasks.clone() if isinstance(self.tasks, group)
  650. else group(self.tasks))
  651. if app.conf.CELERY_ALWAYS_EAGER:
  652. return self.apply((), kwargs,
  653. body=body, task_id=task_id, **options)
  654. return self.run(tasks, body, args, task_id=task_id, **options)
  655. def apply(self, args=(), kwargs={}, propagate=True, body=None, **options):
  656. body = self.body if body is None else body
  657. tasks = (self.tasks.clone() if isinstance(self.tasks, group)
  658. else group(self.tasks))
  659. return body.apply(
  660. args=(tasks.apply().get(propagate=propagate), ),
  661. )
  662. def _traverse_tasks(self, tasks, value=None):
  663. stack = deque(tasks)
  664. while stack:
  665. task = stack.popleft()
  666. if isinstance(task, group):
  667. stack.extend(task.tasks)
  668. else:
  669. yield task if value is None else value
  670. def __length_hint__(self):
  671. return sum(self._traverse_tasks(self.tasks, 1))
  672. def run(self, header, body, partial_args, app=None, interval=None,
  673. countdown=1, max_retries=None, propagate=None, eager=False,
  674. task_id=None, **options):
  675. app = app or self._get_app(body)
  676. propagate = (app.conf.CELERY_CHORD_PROPAGATES
  677. if propagate is None else propagate)
  678. group_id = uuid()
  679. root_id = body.options.get('root_id')
  680. if 'chord_size' not in body:
  681. body['chord_size'] = self.__length_hint__()
  682. options = dict(self.options, **options) if options else self.options
  683. if options:
  684. body.options.update(options)
  685. results = header.freeze(
  686. group_id=group_id, chord=body, root_id=root_id).results
  687. bodyres = body.freeze(task_id, root_id=root_id)
  688. parent = app.backend.apply_chord(
  689. header, partial_args, group_id, body,
  690. interval=interval, countdown=countdown,
  691. options=options, max_retries=max_retries,
  692. propagate=propagate, result=results)
  693. bodyres.parent = parent
  694. return bodyres
  695. def __call__(self, body=None, **options):
  696. return self.apply_async((), {'body': body} if body else {}, **options)
  697. def clone(self, *args, **kwargs):
  698. s = Signature.clone(self, *args, **kwargs)
  699. # need to make copy of body
  700. try:
  701. s.kwargs['body'] = s.kwargs['body'].clone()
  702. except (AttributeError, KeyError):
  703. pass
  704. return s
  705. def link(self, callback):
  706. self.body.link(callback)
  707. return callback
  708. def link_error(self, errback):
  709. self.body.link_error(errback)
  710. return errback
  711. def set_immutable(self, immutable):
  712. # changes mutability of header only, not callback.
  713. for task in self.tasks:
  714. task.set_immutable(immutable)
  715. def __repr__(self):
  716. if self.body:
  717. return self.body.reprcall(self.tasks)
  718. return '<chord without body: {0.tasks!r}>'.format(self)
  719. tasks = _getitem_property('kwargs.header')
  720. body = _getitem_property('kwargs.body')
  721. def signature(varies, *args, **kwargs):
  722. if isinstance(varies, dict):
  723. if isinstance(varies, Signature):
  724. return varies.clone()
  725. return Signature.from_dict(varies)
  726. return Signature(varies, *args, **kwargs)
  727. subtask = signature # XXX compat
  728. def maybe_signature(d, app=None):
  729. if d is not None:
  730. if isinstance(d, dict):
  731. if not isinstance(d, Signature):
  732. d = signature(d)
  733. elif isinstance(d, list):
  734. return [maybe_signature(s, app=app) for s in d]
  735. if app is not None:
  736. d._app = app
  737. return d
  738. maybe_subtask = maybe_signature # XXX compat