canvas.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896
  1. # -*- coding: utf-8 -*-
  2. """
  3. celery.canvas
  4. ~~~~~~~~~~~~~
  5. Composing task workflows.
  6. Documentation for some of these types are in :mod:`celery`.
  7. You should import these from :mod:`celery` and not this module.
  8. """
  9. from __future__ import absolute_import
  10. from collections import MutableSequence, deque
  11. from copy import deepcopy
  12. from functools import partial as _partial, reduce
  13. from operator import itemgetter
  14. from itertools import chain as _chain
  15. from kombu.utils import cached_property, fxrange, reprcall, uuid
  16. from celery._state import current_app, get_current_worker_task
  17. from celery.utils.functional import (
  18. maybe_list, is_list, regen,
  19. chunks as _chunks,
  20. )
  21. from celery.utils.text import truncate
  22. __all__ = ['Signature', 'chain', 'xmap', 'xstarmap', 'chunks',
  23. 'group', 'chord', 'signature', 'maybe_signature']
  24. class _getitem_property(object):
  25. """Attribute -> dict key descriptor.
  26. The target object must support ``__getitem__``,
  27. and optionally ``__setitem__``.
  28. Example:
  29. >>> from collections import defaultdict
  30. >>> class Me(dict):
  31. ... deep = defaultdict(dict)
  32. ...
  33. ... foo = _getitem_property('foo')
  34. ... deep_thing = _getitem_property('deep.thing')
  35. >>> me = Me()
  36. >>> me.foo
  37. None
  38. >>> me.foo = 10
  39. >>> me.foo
  40. 10
  41. >>> me['foo']
  42. 10
  43. >>> me.deep_thing = 42
  44. >>> me.deep_thing
  45. 42
  46. >>> me.deep
  47. defaultdict(<type 'dict'>, {'thing': 42})
  48. """
  49. def __init__(self, keypath):
  50. path, _, self.key = keypath.rpartition('.')
  51. self.path = path.split('.') if path else None
  52. def _path(self, obj):
  53. return (reduce(lambda d, k: d[k], [obj] + self.path) if self.path
  54. else obj)
  55. def __get__(self, obj, type=None):
  56. if obj is None:
  57. return type
  58. return self._path(obj).get(self.key)
  59. def __set__(self, obj, value):
  60. self._path(obj)[self.key] = value
  61. def maybe_unroll_group(g):
  62. """Unroll group with only one member."""
  63. # Issue #1656
  64. try:
  65. size = len(g.tasks)
  66. except TypeError:
  67. try:
  68. size = g.tasks.__length_hint__()
  69. except (AttributeError, TypeError):
  70. pass
  71. else:
  72. return list(g.tasks)[0] if size == 1 else g
  73. else:
  74. return g.tasks[0] if size == 1 else g
  75. def task_name_from(task):
  76. return getattr(task, 'name', task)
  77. class Signature(dict):
  78. """Class that wraps the arguments and execution options
  79. for a single task invocation.
  80. Used as the parts in a :class:`group` and other constructs,
  81. or to pass tasks around as callbacks while being compatible
  82. with serializers with a strict type subset.
  83. :param task: Either a task class/instance, or the name of a task.
  84. :keyword args: Positional arguments to apply.
  85. :keyword kwargs: Keyword arguments to apply.
  86. :keyword options: Additional options to :meth:`Task.apply_async`.
  87. Note that if the first argument is a :class:`dict`, the other
  88. arguments will be ignored and the values in the dict will be used
  89. instead.
  90. >>> s = signature('tasks.add', args=(2, 2))
  91. >>> signature(s)
  92. {'task': 'tasks.add', args=(2, 2), kwargs={}, options={}}
  93. """
  94. TYPES = {}
  95. _app = _type = None
  96. @classmethod
  97. def register_type(cls, subclass, name=None):
  98. cls.TYPES[name or subclass.__name__] = subclass
  99. return subclass
  100. @classmethod
  101. def from_dict(self, d, app=None):
  102. typ = d.get('subtask_type')
  103. if typ:
  104. return self.TYPES[typ].from_dict(d, app=app)
  105. return Signature(d, app=app)
  106. def __init__(self, task=None, args=None, kwargs=None, options=None,
  107. type=None, subtask_type=None, immutable=False,
  108. app=None, **ex):
  109. self._app = app
  110. init = dict.__init__
  111. if isinstance(task, dict):
  112. return init(self, task) # works like dict(d)
  113. # Also supports using task class/instance instead of string name.
  114. try:
  115. task_name = task.name
  116. except AttributeError:
  117. task_name = task
  118. else:
  119. self._type = task
  120. init(self,
  121. task=task_name, args=tuple(args or ()),
  122. kwargs=kwargs or {},
  123. options=dict(options or {}, **ex),
  124. subtask_type=subtask_type,
  125. immutable=immutable)
  126. def __call__(self, *partial_args, **partial_kwargs):
  127. args, kwargs, _ = self._merge(partial_args, partial_kwargs, None)
  128. return self.type(*args, **kwargs)
  129. def delay(self, *partial_args, **partial_kwargs):
  130. return self.apply_async(partial_args, partial_kwargs)
  131. def apply(self, args=(), kwargs={}, **options):
  132. """Apply this task locally."""
  133. # For callbacks: extra args are prepended to the stored args.
  134. args, kwargs, options = self._merge(args, kwargs, options)
  135. return self.type.apply(args, kwargs, **options)
  136. def _merge(self, args=(), kwargs={}, options={}):
  137. if self.immutable:
  138. return (self.args, self.kwargs,
  139. dict(self.options, **options) if options else self.options)
  140. return (tuple(args) + tuple(self.args) if args else self.args,
  141. dict(self.kwargs, **kwargs) if kwargs else self.kwargs,
  142. dict(self.options, **options) if options else self.options)
  143. def clone(self, args=(), kwargs={}, **opts):
  144. # need to deepcopy options so origins links etc. is not modified.
  145. if args or kwargs or opts:
  146. args, kwargs, opts = self._merge(args, kwargs, opts)
  147. else:
  148. args, kwargs, opts = self.args, self.kwargs, self.options
  149. s = Signature.from_dict({'task': self.task, 'args': tuple(args),
  150. 'kwargs': kwargs, 'options': deepcopy(opts),
  151. 'subtask_type': self.subtask_type,
  152. 'immutable': self.immutable}, app=self._app)
  153. s._type = self._type
  154. return s
  155. partial = clone
  156. def freeze(self, _id=None, group_id=None, chord=None, root_id=None):
  157. opts = self.options
  158. try:
  159. tid = opts['task_id']
  160. except KeyError:
  161. tid = opts['task_id'] = _id or uuid()
  162. root_id = opts.setdefault('root_id', root_id)
  163. if 'reply_to' not in opts:
  164. opts['reply_to'] = self.app.oid
  165. if group_id:
  166. opts['group_id'] = group_id
  167. if chord:
  168. opts['chord'] = chord
  169. return self.AsyncResult(tid)
  170. _freeze = freeze
  171. def replace(self, args=None, kwargs=None, options=None):
  172. s = self.clone()
  173. if args is not None:
  174. s.args = args
  175. if kwargs is not None:
  176. s.kwargs = kwargs
  177. if options is not None:
  178. s.options = options
  179. return s
  180. def set(self, immutable=None, **options):
  181. if immutable is not None:
  182. self.set_immutable(immutable)
  183. self.options.update(options)
  184. return self
  185. def set_immutable(self, immutable):
  186. self.immutable = immutable
  187. def apply_async(self, args=(), kwargs={}, route_name=None, **options):
  188. try:
  189. _apply = self._apply_async
  190. except IndexError: # no tasks for chain, etc to find type
  191. return
  192. # For callbacks: extra args are prepended to the stored args.
  193. if args or kwargs or options:
  194. args, kwargs, options = self._merge(args, kwargs, options)
  195. else:
  196. args, kwargs, options = self.args, self.kwargs, self.options
  197. return _apply(args, kwargs, **options)
  198. def append_to_list_option(self, key, value):
  199. items = self.options.setdefault(key, [])
  200. if not isinstance(items, MutableSequence):
  201. items = self.options[key] = [items]
  202. if value not in items:
  203. items.append(value)
  204. return value
  205. def link(self, callback):
  206. return self.append_to_list_option('link', callback)
  207. def link_error(self, errback):
  208. return self.append_to_list_option('link_error', errback)
  209. def flatten_links(self):
  210. return list(_chain.from_iterable(_chain(
  211. [[self]],
  212. (link.flatten_links()
  213. for link in maybe_list(self.options.get('link')) or [])
  214. )))
  215. def __or__(self, other):
  216. if isinstance(other, group):
  217. other = maybe_unroll_group(other)
  218. if not isinstance(self, chain) and isinstance(other, chain):
  219. return chain((self, ) + other.tasks, app=self._app)
  220. elif isinstance(other, chain):
  221. return chain(*self.tasks + other.tasks, app=self._app)
  222. elif isinstance(other, Signature):
  223. if isinstance(self, chain):
  224. return chain(*self.tasks + (other, ), app=self._app)
  225. return chain(self, other, app=self._app)
  226. return NotImplemented
  227. def __deepcopy__(self, memo):
  228. memo[id(self)] = self
  229. return dict(self)
  230. def __invert__(self):
  231. return self.apply_async().get()
  232. def __reduce__(self):
  233. # for serialization, the task type is lazily loaded,
  234. # and not stored in the dict itself.
  235. return signature, (dict(self), )
  236. def __json__(self):
  237. return dict(self)
  238. def reprcall(self, *args, **kwargs):
  239. args, kwargs, _ = self._merge(args, kwargs, {})
  240. return reprcall(self['task'], args, kwargs)
  241. def election(self):
  242. type = self.type
  243. app = type.app
  244. tid = self.options.get('task_id') or uuid()
  245. with app.producer_or_acquire(None) as P:
  246. props = type.backend.on_task_call(P, tid)
  247. app.control.election(tid, 'task', self.clone(task_id=tid, **props),
  248. connection=P.connection)
  249. return type.AsyncResult(tid)
  250. def __repr__(self):
  251. return self.reprcall()
  252. @property
  253. def name(self):
  254. # for duck typing compatibility with Task.name
  255. return self.task
  256. @cached_property
  257. def type(self):
  258. return self._type or self.app.tasks[self['task']]
  259. @cached_property
  260. def app(self):
  261. return self._app or current_app
  262. @cached_property
  263. def AsyncResult(self):
  264. try:
  265. return self.type.AsyncResult
  266. except KeyError: # task not registered
  267. return self.app.AsyncResult
  268. @cached_property
  269. def _apply_async(self):
  270. try:
  271. return self.type.apply_async
  272. except KeyError:
  273. return _partial(self.app.send_task, self['task'])
  274. id = _getitem_property('options.task_id')
  275. task = _getitem_property('task')
  276. args = _getitem_property('args')
  277. kwargs = _getitem_property('kwargs')
  278. options = _getitem_property('options')
  279. subtask_type = _getitem_property('subtask_type')
  280. immutable = _getitem_property('immutable')
  281. @Signature.register_type
  282. class chain(Signature):
  283. tasks = _getitem_property('kwargs.tasks')
  284. def __init__(self, *tasks, **options):
  285. tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0])
  286. else tasks)
  287. Signature.__init__(
  288. self, 'celery.chain', (), {'tasks': tasks}, **options
  289. )
  290. self.subtask_type = 'chain'
  291. def __call__(self, *args, **kwargs):
  292. if self.tasks:
  293. return self.apply_async(args, kwargs)
  294. def apply_async(self, args=(), kwargs={}, **options):
  295. # python is best at unpacking kwargs, so .run is here to do that.
  296. app = self.app
  297. if app.conf.CELERY_ALWAYS_EAGER:
  298. return self.apply(args, kwargs, **options)
  299. return self.run(args, kwargs, app=app, **(
  300. dict(self.options, **options) if options else self.options))
  301. def run(self, args=(), kwargs={}, group_id=None, chord=None,
  302. task_id=None, link=None, link_error=None,
  303. publisher=None, producer=None, root_id=None, app=None, **options):
  304. app = app or self.app
  305. args = (tuple(args) + tuple(self.args)
  306. if args and not self.immutable else self.args)
  307. tasks, results = self.prepare_steps(
  308. args, self.tasks, root_id, link_error,
  309. )
  310. if not results:
  311. return
  312. result = results[-1]
  313. last_task = tasks[-1]
  314. if group_id:
  315. last_task.set(group_id=group_id)
  316. if chord:
  317. last_task.set(chord=chord)
  318. if task_id:
  319. last_task.set(task_id=task_id)
  320. result = last_task.type.AsyncResult(task_id)
  321. # make sure we can do a link() and link_error() on a chain object.
  322. if link:
  323. tasks[-1].set(link=link)
  324. tasks[0].apply_async(**options)
  325. return result
  326. def prepare_steps(self, args, tasks,
  327. root_id=None, link_error=None, app=None,
  328. from_dict=Signature.from_dict):
  329. app = app or self.app
  330. steps = deque(tasks)
  331. next_step = prev_task = prev_res = None
  332. tasks, results = [], []
  333. i = 0
  334. while steps:
  335. task = steps.popleft()
  336. if not isinstance(task, Signature):
  337. task = from_dict(task, app=app)
  338. if not i: # first task
  339. # first task gets partial args from chain
  340. task = task.clone(args)
  341. res = task.freeze(root_id=root_id)
  342. root_id = res.id if root_id is None else root_id
  343. else:
  344. task = task.clone()
  345. res = task.freeze(root_id=root_id)
  346. i += 1
  347. if isinstance(task, group):
  348. task = maybe_unroll_group(task)
  349. if isinstance(task, chain):
  350. # splice the chain
  351. steps.extendleft(reversed(task.tasks))
  352. continue
  353. elif isinstance(task, group) and steps and \
  354. not isinstance(steps[0], group):
  355. # automatically upgrade group(...) | s to chord(group, s)
  356. try:
  357. next_step = steps.popleft()
  358. # for chords we freeze by pretending it's a normal
  359. # signature instead of a group.
  360. res = Signature.freeze(next_step)
  361. task = chord(
  362. task, body=next_step,
  363. task_id=res.task_id, root_id=root_id,
  364. )
  365. except IndexError:
  366. pass # no callback, so keep as group.
  367. if prev_task:
  368. # link previous task to this task.
  369. prev_task.link(task)
  370. # set AsyncResult.parent
  371. if not res.parent:
  372. res.parent = prev_res
  373. if link_error:
  374. task.set(link_error=link_error)
  375. if not isinstance(prev_task, chord):
  376. results.append(res)
  377. tasks.append(task)
  378. prev_task, prev_res = task, res
  379. return tasks, results
  380. def apply(self, args=(), kwargs={}, **options):
  381. last, fargs = None, args
  382. for task in self.tasks:
  383. res = task.clone(fargs).apply(
  384. last and (last.get(), ), **dict(self.options, **options))
  385. res.parent, last, fargs = last, res, None
  386. return last
  387. @classmethod
  388. def from_dict(self, d, app=None):
  389. tasks = d['kwargs']['tasks']
  390. if tasks:
  391. if isinstance(tasks, tuple): # aaaargh
  392. tasks = d['kwargs']['tasks'] = list(tasks)
  393. # First task must be signature object to get app
  394. tasks[0] = maybe_signature(tasks[0], app=app)
  395. return chain(*tasks, app=app, **d['options'])
  396. @property
  397. def app(self):
  398. app = self._app
  399. if app is None:
  400. try:
  401. app = self.tasks[0]._app
  402. except (KeyError, IndexError):
  403. pass
  404. return app or current_app
  405. def __repr__(self):
  406. return ' | '.join(repr(t) for t in self.tasks)
  407. class _basemap(Signature):
  408. _task_name = None
  409. _unpack_args = itemgetter('task', 'it')
  410. def __init__(self, task, it, **options):
  411. Signature.__init__(
  412. self, self._task_name, (),
  413. {'task': task, 'it': regen(it)}, immutable=True, **options
  414. )
  415. def apply_async(self, args=(), kwargs={}, **opts):
  416. # need to evaluate generators
  417. task, it = self._unpack_args(self.kwargs)
  418. return self.type.apply_async(
  419. (), {'task': task, 'it': list(it)},
  420. route_name=task_name_from(self.kwargs.get('task')), **opts
  421. )
  422. @classmethod
  423. def from_dict(cls, d, app=None):
  424. return cls(*cls._unpack_args(d['kwargs']), app=app, **d['options'])
  425. @Signature.register_type
  426. class xmap(_basemap):
  427. _task_name = 'celery.map'
  428. def __repr__(self):
  429. task, it = self._unpack_args(self.kwargs)
  430. return '[{0}(x) for x in {1}]'.format(task.task,
  431. truncate(repr(it), 100))
  432. @Signature.register_type
  433. class xstarmap(_basemap):
  434. _task_name = 'celery.starmap'
  435. def __repr__(self):
  436. task, it = self._unpack_args(self.kwargs)
  437. return '[{0}(*x) for x in {1}]'.format(task.task,
  438. truncate(repr(it), 100))
  439. @Signature.register_type
  440. class chunks(Signature):
  441. _unpack_args = itemgetter('task', 'it', 'n')
  442. def __init__(self, task, it, n, **options):
  443. Signature.__init__(
  444. self, 'celery.chunks', (),
  445. {'task': task, 'it': regen(it), 'n': n},
  446. immutable=True, **options
  447. )
  448. @classmethod
  449. def from_dict(self, d, app=None):
  450. return chunks(*self._unpack_args(d['kwargs']), app=app, **d['options'])
  451. def apply_async(self, args=(), kwargs={}, **opts):
  452. return self.group().apply_async(
  453. args, kwargs,
  454. route_name=task_name_from(self.kwargs.get('task')), **opts
  455. )
  456. def __call__(self, **options):
  457. return self.apply_async(**options)
  458. def group(self):
  459. # need to evaluate generators
  460. task, it, n = self._unpack_args(self.kwargs)
  461. return group((xstarmap(task, part, app=self._app)
  462. for part in _chunks(iter(it), n)),
  463. app=self._app)
  464. @classmethod
  465. def apply_chunks(cls, task, it, n, app=None):
  466. return cls(task, it, n, app=app)()
  467. def _maybe_group(tasks):
  468. if isinstance(tasks, group):
  469. tasks = list(tasks.tasks)
  470. elif isinstance(tasks, Signature):
  471. tasks = [tasks]
  472. else:
  473. tasks = regen(tasks)
  474. return tasks
  475. @Signature.register_type
  476. class group(Signature):
  477. tasks = _getitem_property('kwargs.tasks')
  478. def __init__(self, *tasks, **options):
  479. if len(tasks) == 1:
  480. tasks = _maybe_group(tasks[0])
  481. Signature.__init__(
  482. self, 'celery.group', (), {'tasks': tasks}, **options
  483. )
  484. self.subtask_type = 'group'
  485. @classmethod
  486. def from_dict(self, d, app=None):
  487. return group(d['kwargs']['tasks'], app=app, **d['options'])
  488. def _prepared(self, tasks, partial_args, group_id, root_id, app, dict=dict,
  489. Signature=Signature, from_dict=Signature.from_dict):
  490. for task in tasks:
  491. if isinstance(task, dict):
  492. if isinstance(task, Signature):
  493. # local sigs are always of type Signature, and we
  494. # clone them to make sure we do not modify the originals.
  495. task = task.clone()
  496. else:
  497. # serialized sigs must be converted to Signature.
  498. task = from_dict(task, app=app)
  499. if isinstance(task, group):
  500. # needs yield_from :(
  501. unroll = task._prepared(
  502. task.tasks, partial_args, group_id, root_id, app,
  503. )
  504. for taskN, resN in unroll:
  505. yield taskN, resN
  506. else:
  507. if partial_args and not task.immutable:
  508. task.args = tuple(partial_args) + tuple(task.args)
  509. yield task, task.freeze(group_id=group_id, root_id=root_id)
  510. def _apply_tasks(self, tasks, producer=None, app=None,
  511. add_to_parent=None, **options):
  512. app = app or self.app
  513. with app.producer_or_acquire(producer) as producer:
  514. for sig, res in tasks:
  515. sig.apply_async(producer=producer, add_to_parent=False,
  516. **options)
  517. yield res
  518. def _freeze_gid(self, options):
  519. # remove task_id and use that as the group_id,
  520. # if we don't remove it then every task will have the same id...
  521. options = dict(self.options, **options)
  522. options['group_id'] = group_id = (
  523. options.pop('task_id', uuid()))
  524. return options, group_id, options.get('root_id')
  525. def apply_async(self, args=(), kwargs=None, add_to_parent=True,
  526. producer=None, **options):
  527. app = self.app
  528. if app.conf.CELERY_ALWAYS_EAGER:
  529. return self.apply(args, kwargs, **options)
  530. if not self.tasks:
  531. return self.freeze()
  532. options, group_id, root_id = self._freeze_gid(options)
  533. tasks = self._prepared(self.tasks, args, group_id, root_id, app)
  534. result = self.app.GroupResult(
  535. group_id, list(self._apply_tasks(tasks, producer, app, **options)),
  536. )
  537. parent_task = get_current_worker_task()
  538. if add_to_parent and parent_task:
  539. parent_task.add_trail(result)
  540. return result
  541. def apply(self, args=(), kwargs={}, **options):
  542. app = self.app
  543. if not self.tasks:
  544. return self.freeze() # empty group returns GroupResult
  545. options, group_id, root_id = self._freeze_gid(options)
  546. tasks = self._prepared(self.tasks, args, group_id, root_id, app)
  547. return app.GroupResult(group_id, [
  548. sig.apply(**options) for sig, _ in tasks
  549. ])
  550. def set_immutable(self, immutable):
  551. for task in self.tasks:
  552. task.set_immutable(immutable)
  553. def link(self, sig):
  554. # Simply link to first task
  555. sig = sig.clone().set(immutable=True)
  556. return self.tasks[0].link(sig)
  557. def link_error(self, sig):
  558. sig = sig.clone().set(immutable=True)
  559. return self.tasks[0].link_error(sig)
  560. def __call__(self, *partial_args, **options):
  561. return self.apply_async(partial_args, **options)
  562. def _freeze_unroll(self, new_tasks, group_id, chord, root_id):
  563. stack = deque(self.tasks)
  564. while stack:
  565. task = maybe_signature(stack.popleft(), app=self._app).clone()
  566. if isinstance(task, group):
  567. stack.extendleft(task.tasks)
  568. else:
  569. new_tasks.append(task)
  570. yield task.freeze(group_id=group_id,
  571. chord=chord, root_id=root_id)
  572. def freeze(self, _id=None, group_id=None, chord=None, root_id=None):
  573. opts = self.options
  574. try:
  575. gid = opts['task_id']
  576. except KeyError:
  577. gid = opts['task_id'] = uuid()
  578. if group_id:
  579. opts['group_id'] = group_id
  580. if chord:
  581. opts['chord'] = chord
  582. root_id = opts.setdefault('root_id', root_id)
  583. new_tasks = []
  584. # Need to unroll subgroups early so that chord gets the
  585. # right result instance for chord_unlock etc.
  586. results = list(self._freeze_unroll(
  587. new_tasks, group_id, chord, root_id,
  588. ))
  589. if isinstance(self.tasks, MutableSequence):
  590. self.tasks[:] = new_tasks
  591. else:
  592. self.tasks = new_tasks
  593. return self.app.GroupResult(gid, results)
  594. _freeze = freeze
  595. def skew(self, start=1.0, stop=None, step=1.0):
  596. it = fxrange(start, stop, step, repeatlast=True)
  597. for task in self.tasks:
  598. task.set(countdown=next(it))
  599. return self
  600. def __iter__(self):
  601. return iter(self.tasks)
  602. def __repr__(self):
  603. return repr(self.tasks)
  604. @property
  605. def app(self):
  606. app = self._app
  607. if app is None:
  608. try:
  609. app = self.tasks[0].app
  610. except (KeyError, IndexError):
  611. pass
  612. return app if app is not None else current_app
  613. @Signature.register_type
  614. class chord(Signature):
  615. def __init__(self, header, body=None, task='celery.chord',
  616. args=(), kwargs={}, **options):
  617. Signature.__init__(
  618. self, task, args,
  619. dict(kwargs, header=_maybe_group(header),
  620. body=maybe_signature(body, app=self._app)), **options
  621. )
  622. self.subtask_type = 'chord'
  623. def freeze(self, *args, **kwargs):
  624. return self.body.freeze(*args, **kwargs)
  625. @classmethod
  626. def from_dict(self, d, app=None):
  627. args, d['kwargs'] = self._unpack_args(**d['kwargs'])
  628. return self(*args, app=app, **d)
  629. @staticmethod
  630. def _unpack_args(header=None, body=None, **kwargs):
  631. # Python signatures are better at extracting keys from dicts
  632. # than manually popping things off.
  633. return (header, body), kwargs
  634. @cached_property
  635. def app(self):
  636. return self._get_app(self.body)
  637. def _get_app(self, body=None):
  638. app = self._app
  639. if app is None:
  640. app = self.tasks[0]._app
  641. if app is None and body is not None:
  642. app = body._app
  643. return app if app is not None else current_app
  644. def apply_async(self, args=(), kwargs={}, task_id=None,
  645. producer=None, publisher=None, connection=None,
  646. router=None, result_cls=None, **options):
  647. args = (tuple(args) + tuple(self.args)
  648. if args and not self.immutable else self.args)
  649. body = kwargs.get('body') or self.kwargs['body']
  650. kwargs = dict(self.kwargs, **kwargs)
  651. body = body.clone(**options)
  652. app = self._get_app(body)
  653. tasks = (self.tasks.clone() if isinstance(self.tasks, group)
  654. else group(self.tasks))
  655. if app.conf.CELERY_ALWAYS_EAGER:
  656. return self.apply((), kwargs,
  657. body=body, task_id=task_id, **options)
  658. return self.run(tasks, body, args, task_id=task_id, **options)
  659. def apply(self, args=(), kwargs={}, propagate=True, body=None, **options):
  660. body = self.body if body is None else body
  661. tasks = (self.tasks.clone() if isinstance(self.tasks, group)
  662. else group(self.tasks))
  663. return body.apply(
  664. args=(tasks.apply().get(propagate=propagate), ),
  665. )
  666. def _traverse_tasks(self, tasks, value=None):
  667. stack = deque(tasks)
  668. while stack:
  669. task = stack.popleft()
  670. if isinstance(task, group):
  671. stack.extend(task.tasks)
  672. else:
  673. yield task if value is None else value
  674. def __length_hint__(self):
  675. return sum(self._traverse_tasks(self.tasks, 1))
  676. def run(self, header, body, partial_args, app=None, interval=None,
  677. countdown=1, max_retries=None, propagate=None, eager=False,
  678. task_id=None, **options):
  679. app = app or self._get_app(body)
  680. propagate = (app.conf.CELERY_CHORD_PROPAGATES
  681. if propagate is None else propagate)
  682. group_id = uuid()
  683. root_id = body.options.get('root_id')
  684. if 'chord_size' not in body:
  685. body['chord_size'] = self.__length_hint__()
  686. options = dict(self.options, **options) if options else self.options
  687. if options:
  688. body.options.update(options)
  689. results = header.freeze(
  690. group_id=group_id, chord=body, root_id=root_id).results
  691. bodyres = body.freeze(task_id, root_id=root_id)
  692. parent = app.backend.apply_chord(
  693. header, partial_args, group_id, body,
  694. interval=interval, countdown=countdown,
  695. options=options, max_retries=max_retries,
  696. propagate=propagate, result=results)
  697. bodyres.parent = parent
  698. return bodyres
  699. def __call__(self, body=None, **options):
  700. return self.apply_async((), {'body': body} if body else {}, **options)
  701. def clone(self, *args, **kwargs):
  702. s = Signature.clone(self, *args, **kwargs)
  703. # need to make copy of body
  704. try:
  705. s.kwargs['body'] = s.kwargs['body'].clone()
  706. except (AttributeError, KeyError):
  707. pass
  708. return s
  709. def link(self, callback):
  710. self.body.link(callback)
  711. return callback
  712. def link_error(self, errback):
  713. self.body.link_error(errback)
  714. return errback
  715. def set_immutable(self, immutable):
  716. # changes mutability of header only, not callback.
  717. for task in self.tasks:
  718. task.set_immutable(immutable)
  719. def __repr__(self):
  720. if self.body:
  721. return self.body.reprcall(self.tasks)
  722. return '<chord without body: {0.tasks!r}>'.format(self)
  723. tasks = _getitem_property('kwargs.header')
  724. body = _getitem_property('kwargs.body')
  725. def signature(varies, *args, **kwargs):
  726. if isinstance(varies, dict):
  727. if isinstance(varies, Signature):
  728. return varies.clone()
  729. return Signature.from_dict(varies)
  730. return Signature(varies, *args, **kwargs)
  731. subtask = signature # XXX compat
  732. def maybe_signature(d, app=None):
  733. if d is not None:
  734. if isinstance(d, dict):
  735. if not isinstance(d, Signature):
  736. d = signature(d)
  737. elif isinstance(d, list):
  738. return [maybe_signature(s, app=app) for s in d]
  739. if app is not None:
  740. d._app = app
  741. return d
  742. maybe_subtask = maybe_signature # XXX compat