result.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989
  1. # -*- coding: utf-8 -*-
  2. """Task results/state and results for groups of tasks."""
  3. from __future__ import absolute_import, unicode_literals
  4. import time
  5. from collections import OrderedDict, deque
  6. from contextlib import contextmanager
  7. from copy import copy
  8. from kombu.utils.objects import cached_property
  9. from vine import Thenable, barrier, promise
  10. from . import current_app
  11. from . import states
  12. from ._state import _set_task_join_will_block, task_join_will_block
  13. from .app import app_or_default
  14. from .exceptions import ImproperlyConfigured, IncompleteStream, TimeoutError
  15. from .five import (
  16. items, python_2_unicode_compatible, range, string_t, monotonic,
  17. )
  18. from .utils import deprecated
  19. from .utils.graph import DependencyGraph, GraphFormatter
  20. try:
  21. import tblib
  22. except ImportError:
  23. tblib = None
  24. __all__ = [
  25. 'ResultBase', 'AsyncResult', 'ResultSet',
  26. 'GroupResult', 'EagerResult', 'result_from_tuple',
  27. ]
  28. E_WOULDBLOCK = """\
  29. Never call result.get() within a task!
  30. See http://docs.celeryq.org/en/latest/userguide/tasks.html\
  31. #task-synchronous-subtasks
  32. """
  33. def assert_will_not_block():
  34. if task_join_will_block():
  35. raise RuntimeError(E_WOULDBLOCK)
  36. @contextmanager
  37. def allow_join_result():
  38. reset_value = task_join_will_block()
  39. _set_task_join_will_block(False)
  40. try:
  41. yield
  42. finally:
  43. _set_task_join_will_block(reset_value)
  44. class ResultBase(object):
  45. """Base class for results."""
  46. #: Parent result (if part of a chain)
  47. parent = None
  48. @Thenable.register
  49. @python_2_unicode_compatible
  50. class AsyncResult(ResultBase):
  51. """Query task state.
  52. Arguments:
  53. id (str): See :attr:`id`.
  54. backend (Backend): See :attr:`backend`.
  55. """
  56. app = None
  57. #: Error raised for timeouts.
  58. TimeoutError = TimeoutError
  59. #: The task's UUID.
  60. id = None
  61. #: The task result backend to use.
  62. backend = None
  63. def __init__(self, id, backend=None,
  64. task_name=None, # deprecated
  65. app=None, parent=None):
  66. if id is None:
  67. raise ValueError(
  68. 'AsyncResult requires valid id, not {0}'.format(type(id)))
  69. self.app = app_or_default(app or self.app)
  70. self.id = id
  71. self.backend = backend or self.app.backend
  72. self.parent = parent
  73. self.on_ready = promise(self._on_fulfilled, weak=True)
  74. self._cache = None
  75. def then(self, callback, on_error=None, weak=False):
  76. self.backend.add_pending_result(self, weak=weak)
  77. return self.on_ready.then(callback, on_error)
  78. def _on_fulfilled(self, result):
  79. self.backend.remove_pending_result(self)
  80. return result
  81. def as_tuple(self):
  82. parent = self.parent
  83. return (self.id, parent and parent.as_tuple()), None
  84. def forget(self):
  85. """Forget about (and possibly remove the result of) this task."""
  86. self._cache = None
  87. self.backend.forget(self.id)
  88. def revoke(self, connection=None, terminate=False, signal=None,
  89. wait=False, timeout=None):
  90. """Send revoke signal to all workers.
  91. Any worker receiving the task, or having reserved the
  92. task, *must* ignore it.
  93. Arguments:
  94. terminate (bool): Also terminate the process currently working
  95. on the task (if any).
  96. signal (str): Name of signal to send to process if terminate.
  97. Default is TERM.
  98. wait (bool): Wait for replies from workers.
  99. The ``timeout`` argument specifies the seconds to wait.
  100. Disabled by default.
  101. timeout (float): Time in seconds to wait for replies when
  102. ``wait`` is enabled.
  103. """
  104. self.app.control.revoke(self.id, connection=connection,
  105. terminate=terminate, signal=signal,
  106. reply=wait, timeout=timeout)
  107. def get(self, timeout=None, propagate=True, interval=0.5,
  108. no_ack=True, follow_parents=True, callback=None, on_message=None,
  109. on_interval=None, disable_sync_subtasks=True,
  110. EXCEPTION_STATES=states.EXCEPTION_STATES,
  111. PROPAGATE_STATES=states.PROPAGATE_STATES):
  112. """Wait until task is ready, and return its result.
  113. Warning:
  114. Waiting for tasks within a task may lead to deadlocks.
  115. Please read :ref:`task-synchronous-subtasks`.
  116. Arguments:
  117. timeout (float): How long to wait, in seconds, before the
  118. operation times out.
  119. propagate (bool): Re-raise exception if the task failed.
  120. interval (float): Time to wait (in seconds) before retrying to
  121. retrieve the result. Note that this does not have any effect
  122. when using the RPC/redis result store backends, as they don't
  123. use polling.
  124. no_ack (bool): Enable amqp no ack (automatically acknowledge
  125. message). If this is :const:`False` then the message will
  126. **not be acked**.
  127. follow_parents (bool): Re-raise any exception raised by
  128. parent tasks.
  129. disable_sync_subtasks (bool): Disable tasks to wait for sub tasks
  130. this is the default configuration. CAUTION do not enable this
  131. unless you must.
  132. Raises:
  133. celery.exceptions.TimeoutError: if `timeout` isn't
  134. :const:`None` and the result does not arrive within
  135. `timeout` seconds.
  136. Exception: If the remote call raised an exception then that
  137. exception will be re-raised in the caller process.
  138. """
  139. if disable_sync_subtasks:
  140. assert_will_not_block()
  141. _on_interval = promise()
  142. if follow_parents and propagate and self.parent:
  143. on_interval = promise(self._maybe_reraise_parent_error, weak=True)
  144. self._maybe_reraise_parent_error()
  145. if on_interval:
  146. _on_interval.then(on_interval)
  147. if self._cache:
  148. if propagate:
  149. self.maybe_throw(callback=callback)
  150. return self.result
  151. self.backend.add_pending_result(self)
  152. return self.backend.wait_for_pending(
  153. self, timeout=timeout,
  154. interval=interval,
  155. on_interval=_on_interval,
  156. no_ack=no_ack,
  157. propagate=propagate,
  158. callback=callback,
  159. on_message=on_message,
  160. )
  161. wait = get # deprecated alias to :meth:`get`.
  162. def _maybe_reraise_parent_error(self):
  163. for node in reversed(list(self._parents())):
  164. node.maybe_throw()
  165. def _parents(self):
  166. node = self.parent
  167. while node:
  168. yield node
  169. node = node.parent
  170. def collect(self, intermediate=False, **kwargs):
  171. """Collect results as they return.
  172. Iterator, like :meth:`get` will wait for the task to complete,
  173. but will also follow :class:`AsyncResult` and :class:`ResultSet`
  174. returned by the task, yielding ``(result, value)`` tuples for each
  175. result in the tree.
  176. An example would be having the following tasks:
  177. .. code-block:: python
  178. from celery import group
  179. from proj.celery import app
  180. @app.task(trail=True)
  181. def A(how_many):
  182. return group(B.s(i) for i in range(how_many))()
  183. @app.task(trail=True)
  184. def B(i):
  185. return pow2.delay(i)
  186. @app.task(trail=True)
  187. def pow2(i):
  188. return i ** 2
  189. .. code-block:: pycon
  190. >>> from celery.result import ResultBase
  191. >>> from proj.tasks import A
  192. >>> result = A.delay(10)
  193. >>> [v for v in result.collect()
  194. ... if not isinstance(v, (ResultBase, tuple))]
  195. [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
  196. Note:
  197. The ``Task.trail`` option must be enabled
  198. so that the list of children is stored in ``result.children``.
  199. This is the default but enabled explicitly for illustration.
  200. Yields:
  201. Tuple[AsyncResult, Any]: tuples containing the result instance
  202. of the child task, and the return value of that task.
  203. """
  204. for _, R in self.iterdeps(intermediate=intermediate):
  205. yield R, R.get(**kwargs)
  206. def get_leaf(self):
  207. value = None
  208. for _, R in self.iterdeps():
  209. value = R.get()
  210. return value
  211. def iterdeps(self, intermediate=False):
  212. stack = deque([(None, self)])
  213. while stack:
  214. parent, node = stack.popleft()
  215. yield parent, node
  216. if node.ready():
  217. stack.extend((node, child) for child in node.children or [])
  218. else:
  219. if not intermediate:
  220. raise IncompleteStream()
  221. def ready(self):
  222. """Return :const:`True` if the task has executed.
  223. If the task is still running, pending, or is waiting
  224. for retry then :const:`False` is returned.
  225. """
  226. return self.state in self.backend.READY_STATES
  227. def successful(self):
  228. """Return :const:`True` if the task executed successfully."""
  229. return self.state == states.SUCCESS
  230. def failed(self):
  231. """Return :const:`True` if the task failed."""
  232. return self.state == states.FAILURE
  233. def throw(self, *args, **kwargs):
  234. self.on_ready.throw(*args, **kwargs)
  235. def maybe_throw(self, propagate=True, callback=None):
  236. cache = self._get_task_meta() if self._cache is None else self._cache
  237. state, value, tb = (
  238. cache['status'], cache['result'], cache.get('traceback'))
  239. if state in states.PROPAGATE_STATES and propagate:
  240. self.throw(value, self._to_remote_traceback(tb))
  241. if callback is not None:
  242. callback(self.id, value)
  243. return value
  244. maybe_reraise = maybe_throw # XXX compat alias
  245. def _to_remote_traceback(self, tb):
  246. if tb and tblib is not None and self.app.conf.task_remote_tracebacks:
  247. return tblib.Traceback.from_string(tb).as_traceback()
  248. def build_graph(self, intermediate=False, formatter=None):
  249. graph = DependencyGraph(
  250. formatter=formatter or GraphFormatter(root=self.id, shape='oval'),
  251. )
  252. for parent, node in self.iterdeps(intermediate=intermediate):
  253. graph.add_arc(node)
  254. if parent:
  255. graph.add_edge(parent, node)
  256. return graph
  257. def __str__(self):
  258. """`str(self) -> self.id`."""
  259. return str(self.id)
  260. def __hash__(self):
  261. """`hash(self) -> hash(self.id)`."""
  262. return hash(self.id)
  263. def __repr__(self):
  264. return '<{0}: {1}>'.format(type(self).__name__, self.id)
  265. def __eq__(self, other):
  266. if isinstance(other, AsyncResult):
  267. return other.id == self.id
  268. elif isinstance(other, string_t):
  269. return other == self.id
  270. return NotImplemented
  271. def __ne__(self, other):
  272. res = self.__eq__(other)
  273. return True if res is NotImplemented else not res
  274. def __copy__(self):
  275. return self.__class__(
  276. self.id, self.backend, None, self.app, self.parent,
  277. )
  278. def __reduce__(self):
  279. return self.__class__, self.__reduce_args__()
  280. def __reduce_args__(self):
  281. return self.id, self.backend, None, None, self.parent
  282. @cached_property
  283. def graph(self):
  284. return self.build_graph()
  285. @property
  286. def supports_native_join(self):
  287. return self.backend.supports_native_join
  288. @property
  289. def children(self):
  290. return self._get_task_meta().get('children')
  291. def _maybe_set_cache(self, meta):
  292. if meta:
  293. state = meta['status']
  294. if state in states.READY_STATES:
  295. d = self._set_cache(self.backend.meta_from_decoded(meta))
  296. self.on_ready(self)
  297. return d
  298. return meta
  299. def _get_task_meta(self):
  300. if self._cache is None:
  301. return self._maybe_set_cache(self.backend.get_task_meta(self.id))
  302. return self._cache
  303. def _iter_meta(self):
  304. return iter([self._get_task_meta()])
  305. def _set_cache(self, d):
  306. children = d.get('children')
  307. if children:
  308. d['children'] = [
  309. result_from_tuple(child, self.app) for child in children
  310. ]
  311. self._cache = d
  312. return d
  313. @property
  314. def result(self):
  315. """Task return value.
  316. Note:
  317. When the task has been executed, this contains the return value.
  318. If the task raised an exception, this will be the exception
  319. instance.
  320. """
  321. return self._get_task_meta()['result']
  322. info = result
  323. @property
  324. def traceback(self):
  325. """Get the traceback of a failed task."""
  326. return self._get_task_meta().get('traceback')
  327. @property
  328. def state(self):
  329. """The tasks current state.
  330. Possible values includes:
  331. *PENDING*
  332. The task is waiting for execution.
  333. *STARTED*
  334. The task has been started.
  335. *RETRY*
  336. The task is to be retried, possibly because of failure.
  337. *FAILURE*
  338. The task raised an exception, or has exceeded the retry limit.
  339. The :attr:`result` attribute then contains the
  340. exception raised by the task.
  341. *SUCCESS*
  342. The task executed successfully. The :attr:`result` attribute
  343. then contains the tasks return value.
  344. """
  345. return self._get_task_meta()['status']
  346. status = state # XXX compat
  347. @property
  348. def task_id(self):
  349. """Compat. alias to :attr:`id`."""
  350. return self.id
  351. @task_id.setter # noqa
  352. def task_id(self, id):
  353. self.id = id
  354. @Thenable.register
  355. @python_2_unicode_compatible
  356. class ResultSet(ResultBase):
  357. """A collection of results.
  358. Arguments:
  359. results (Sequence[AsyncResult]): List of result instances.
  360. """
  361. _app = None
  362. #: List of results in in the set.
  363. results = None
  364. def __init__(self, results, app=None, ready_barrier=None, **kwargs):
  365. self._app = app
  366. self._cache = None
  367. self.results = results
  368. self.on_ready = promise(args=(self,))
  369. self._on_full = ready_barrier or barrier(results)
  370. if self._on_full:
  371. self._on_full.then(promise(self.on_ready, weak=True))
  372. def add(self, result):
  373. """Add :class:`AsyncResult` as a new member of the set.
  374. Does nothing if the result is already a member.
  375. """
  376. if result not in self.results:
  377. self.results.append(result)
  378. if self._on_full:
  379. self._on_full.add(result)
  380. def _on_ready(self):
  381. self.backend.remove_pending_result(self)
  382. if self.backend.is_async:
  383. self._cache = [r.get() for r in self.results]
  384. self.on_ready()
  385. def remove(self, result):
  386. """Remove result from the set; it must be a member.
  387. Raises:
  388. KeyError: if the result isn't a member.
  389. """
  390. if isinstance(result, string_t):
  391. result = self.app.AsyncResult(result)
  392. try:
  393. self.results.remove(result)
  394. except ValueError:
  395. raise KeyError(result)
  396. def discard(self, result):
  397. """Remove result from the set if it is a member.
  398. Does nothing if it's not a member.
  399. """
  400. try:
  401. self.remove(result)
  402. except KeyError:
  403. pass
  404. def update(self, results):
  405. """Extend from iterable of results."""
  406. self.results.extend(r for r in results if r not in self.results)
  407. def clear(self):
  408. """Remove all results from this set."""
  409. self.results[:] = [] # don't create new list.
  410. def successful(self):
  411. """Return true if all tasks successful.
  412. Returns:
  413. bool: true if all of the tasks finished
  414. successfully (i.e. didn't raise an exception).
  415. """
  416. return all(result.successful() for result in self.results)
  417. def failed(self):
  418. """Return true if any of the tasks failed.
  419. Returns:
  420. bool: true if one of the tasks failed.
  421. (i.e., raised an exception)
  422. """
  423. return any(result.failed() for result in self.results)
  424. def maybe_throw(self, callback=None, propagate=True):
  425. for result in self.results:
  426. result.maybe_throw(callback=callback, propagate=propagate)
  427. maybe_reraise = maybe_throw # XXX compat alias.
  428. def waiting(self):
  429. """Return true if any of the tasks are incomplete.
  430. Returns:
  431. bool: true if one of the tasks are still
  432. waiting for execution.
  433. """
  434. return any(not result.ready() for result in self.results)
  435. def ready(self):
  436. """Did all of the tasks complete? (either by success of failure).
  437. Returns:
  438. bool: true if all of the tasks have been executed.
  439. """
  440. return all(result.ready() for result in self.results)
  441. def completed_count(self):
  442. """Task completion count.
  443. Returns:
  444. int: the number of tasks completed.
  445. """
  446. return sum(int(result.successful()) for result in self.results)
  447. def forget(self):
  448. """Forget about (and possible remove the result of) all the tasks."""
  449. for result in self.results:
  450. result.forget()
  451. def revoke(self, connection=None, terminate=False, signal=None,
  452. wait=False, timeout=None):
  453. """Send revoke signal to all workers for all tasks in the set.
  454. Arguments:
  455. terminate (bool): Also terminate the process currently working
  456. on the task (if any).
  457. signal (str): Name of signal to send to process if terminate.
  458. Default is TERM.
  459. wait (bool): Wait for replies from worker.
  460. The ``timeout`` argument specifies the number of seconds
  461. to wait. Disabled by default.
  462. timeout (float): Time in seconds to wait for replies when
  463. the ``wait`` argument is enabled.
  464. """
  465. self.app.control.revoke([r.id for r in self.results],
  466. connection=connection, timeout=timeout,
  467. terminate=terminate, signal=signal, reply=wait)
  468. def __iter__(self):
  469. return iter(self.results)
  470. def __getitem__(self, index):
  471. """`res[i] -> res.results[i]`."""
  472. return self.results[index]
  473. @deprecated.Callable('4.0', '5.0')
  474. def iterate(self, timeout=None, propagate=True, interval=0.5):
  475. """Deprecated method, use :meth:`get` with a callback argument."""
  476. elapsed = 0.0
  477. results = OrderedDict((result.id, copy(result))
  478. for result in self.results)
  479. while results:
  480. removed = set()
  481. for task_id, result in items(results):
  482. if result.ready():
  483. yield result.get(timeout=timeout and timeout - elapsed,
  484. propagate=propagate)
  485. removed.add(task_id)
  486. else:
  487. if result.backend.subpolling_interval:
  488. time.sleep(result.backend.subpolling_interval)
  489. for task_id in removed:
  490. results.pop(task_id, None)
  491. time.sleep(interval)
  492. elapsed += interval
  493. if timeout and elapsed >= timeout:
  494. raise TimeoutError('The operation timed out')
  495. def get(self, timeout=None, propagate=True, interval=0.5,
  496. callback=None, no_ack=True, on_message=None):
  497. """See :meth:`join`.
  498. This is here for API compatibility with :class:`AsyncResult`,
  499. in addition it uses :meth:`join_native` if available for the
  500. current result backend.
  501. """
  502. if self._cache is not None:
  503. return self._cache
  504. return (self.join_native if self.supports_native_join else self.join)(
  505. timeout=timeout, propagate=propagate,
  506. interval=interval, callback=callback, no_ack=no_ack,
  507. on_message=on_message,
  508. )
  509. def join(self, timeout=None, propagate=True, interval=0.5,
  510. callback=None, no_ack=True, on_message=None, on_interval=None):
  511. """Gather the results of all tasks as a list in order.
  512. Note:
  513. This can be an expensive operation for result store
  514. backends that must resort to polling (e.g., database).
  515. You should consider using :meth:`join_native` if your backend
  516. supports it.
  517. Warning:
  518. Waiting for tasks within a task may lead to deadlocks.
  519. Please see :ref:`task-synchronous-subtasks`.
  520. Arguments:
  521. timeout (float): The number of seconds to wait for results
  522. before the operation times out.
  523. propagate (bool): If any of the tasks raises an exception,
  524. the exception will be re-raised when this flag is set.
  525. interval (float): Time to wait (in seconds) before retrying to
  526. retrieve a result from the set. Note that this does not have
  527. any effect when using the amqp result store backend,
  528. as it does not use polling.
  529. callback (Callable): Optional callback to be called for every
  530. result received. Must have signature ``(task_id, value)``
  531. No results will be returned by this function if a callback
  532. is specified. The order of results is also arbitrary when a
  533. callback is used. To get access to the result object for
  534. a particular id you'll have to generate an index first:
  535. ``index = {r.id: r for r in gres.results.values()}``
  536. Or you can create new result objects on the fly:
  537. ``result = app.AsyncResult(task_id)`` (both will
  538. take advantage of the backend cache anyway).
  539. no_ack (bool): Automatic message acknowledgment (Note that if this
  540. is set to :const:`False` then the messages
  541. *will not be acknowledged*).
  542. Raises:
  543. celery.exceptions.TimeoutError: if ``timeout`` isn't
  544. :const:`None` and the operation takes longer than ``timeout``
  545. seconds.
  546. """
  547. assert_will_not_block()
  548. time_start = monotonic()
  549. remaining = None
  550. if on_message is not None:
  551. raise ImproperlyConfigured(
  552. 'Backend does not support on_message callback')
  553. results = []
  554. for result in self.results:
  555. remaining = None
  556. if timeout:
  557. remaining = timeout - (monotonic() - time_start)
  558. if remaining <= 0.0:
  559. raise TimeoutError('join operation timed out')
  560. value = result.get(
  561. timeout=remaining, propagate=propagate,
  562. interval=interval, no_ack=no_ack, on_interval=on_interval,
  563. )
  564. if callback:
  565. callback(result.id, value)
  566. else:
  567. results.append(value)
  568. return results
  569. def then(self, callback, on_error=None, weak=False):
  570. return self.on_ready.then(callback, on_error)
  571. def iter_native(self, timeout=None, interval=0.5, no_ack=True,
  572. on_message=None, on_interval=None):
  573. """Backend optimized version of :meth:`iterate`.
  574. .. versionadded:: 2.2
  575. Note that this does not support collecting the results
  576. for different task types using different backends.
  577. This is currently only supported by the amqp, Redis and cache
  578. result backends.
  579. """
  580. return self.backend.iter_native(
  581. self,
  582. timeout=timeout, interval=interval, no_ack=no_ack,
  583. on_message=on_message, on_interval=on_interval,
  584. )
  585. def join_native(self, timeout=None, propagate=True,
  586. interval=0.5, callback=None, no_ack=True,
  587. on_message=None, on_interval=None):
  588. """Backend optimized version of :meth:`join`.
  589. .. versionadded:: 2.2
  590. Note that this does not support collecting the results
  591. for different task types using different backends.
  592. This is currently only supported by the amqp, Redis and cache
  593. result backends.
  594. """
  595. assert_will_not_block()
  596. order_index = None if callback else {
  597. result.id: i for i, result in enumerate(self.results)
  598. }
  599. acc = None if callback else [None for _ in range(len(self))]
  600. for task_id, meta in self.iter_native(timeout, interval, no_ack,
  601. on_message, on_interval):
  602. value = meta['result']
  603. if propagate and meta['status'] in states.PROPAGATE_STATES:
  604. raise value
  605. if callback:
  606. callback(task_id, value)
  607. else:
  608. acc[order_index[task_id]] = value
  609. return acc
  610. def _iter_meta(self):
  611. return (meta for _, meta in self.backend.get_many(
  612. {r.id for r in self.results}, max_iterations=1,
  613. ))
  614. def _failed_join_report(self):
  615. return (res for res in self.results
  616. if res.backend.is_cached(res.id) and
  617. res.state in states.PROPAGATE_STATES)
  618. def __len__(self):
  619. return len(self.results)
  620. def __eq__(self, other):
  621. if isinstance(other, ResultSet):
  622. return other.results == self.results
  623. return NotImplemented
  624. def __ne__(self, other):
  625. res = self.__eq__(other)
  626. return True if res is NotImplemented else not res
  627. def __repr__(self):
  628. return '<{0}: [{1}]>'.format(type(self).__name__,
  629. ', '.join(r.id for r in self.results))
  630. @property
  631. def supports_native_join(self):
  632. try:
  633. return self.results[0].supports_native_join
  634. except IndexError:
  635. pass
  636. @property
  637. def app(self):
  638. if self._app is None:
  639. self._app = (self.results[0].app if self.results else
  640. current_app._get_current_object())
  641. return self._app
  642. @app.setter
  643. def app(self, app): # noqa
  644. self._app = app
  645. @property
  646. def backend(self):
  647. return self.app.backend if self.app else self.results[0].backend
  648. @Thenable.register
  649. @python_2_unicode_compatible
  650. class GroupResult(ResultSet):
  651. """Like :class:`ResultSet`, but with an associated id.
  652. This type is returned by :class:`~celery.group`.
  653. It enables inspection of the tasks state and return values as
  654. a single entity.
  655. Arguments:
  656. id (str): The id of the group.
  657. results (Sequence[AsyncResult]): List of result instances.
  658. parent (ResultBase): Parent result of this group.
  659. """
  660. #: The UUID of the group.
  661. id = None
  662. #: List/iterator of results in the group
  663. results = None
  664. def __init__(self, id=None, results=None, parent=None, **kwargs):
  665. self.id = id
  666. self.parent = parent
  667. ResultSet.__init__(self, results, **kwargs)
  668. def save(self, backend=None):
  669. """Save group-result for later retrieval using :meth:`restore`.
  670. Example:
  671. >>> def save_and_restore(result):
  672. ... result.save()
  673. ... result = GroupResult.restore(result.id)
  674. """
  675. return (backend or self.app.backend).save_group(self.id, self)
  676. def delete(self, backend=None):
  677. """Remove this result if it was previously saved."""
  678. (backend or self.app.backend).delete_group(self.id)
  679. def __reduce__(self):
  680. return self.__class__, self.__reduce_args__()
  681. def __reduce_args__(self):
  682. return self.id, self.results
  683. def __bool__(self):
  684. return bool(self.id or self.results)
  685. __nonzero__ = __bool__ # Included for Py2 backwards compatibility
  686. def __eq__(self, other):
  687. if isinstance(other, GroupResult):
  688. return (
  689. other.id == self.id and
  690. other.results == self.results and
  691. other.parent == self.parent
  692. )
  693. return NotImplemented
  694. def __ne__(self, other):
  695. res = self.__eq__(other)
  696. return True if res is NotImplemented else not res
  697. def __repr__(self):
  698. return '<{0}: {1} [{2}]>'.format(type(self).__name__, self.id,
  699. ', '.join(r.id for r in self.results))
  700. def as_tuple(self):
  701. return (self.id, self.parent), [r.as_tuple() for r in self.results]
  702. @property
  703. def children(self):
  704. return self.results
  705. @classmethod
  706. def restore(cls, id, backend=None, app=None):
  707. """Restore previously saved group result."""
  708. app = app or cls.app
  709. backend = backend or (app.backend if app else current_app.backend)
  710. return backend.restore_group(id)
  711. @Thenable.register
  712. @python_2_unicode_compatible
  713. class EagerResult(AsyncResult):
  714. """Result that we know has already been executed."""
  715. def __init__(self, id, ret_value, state, traceback=None):
  716. # pylint: disable=super-init-not-called
  717. # XXX should really not be inheriting from AsyncResult
  718. self.id = id
  719. self._result = ret_value
  720. self._state = state
  721. self._traceback = traceback
  722. self.on_ready = promise()
  723. self.on_ready(self)
  724. def then(self, callback, on_error=None, weak=False):
  725. return self.on_ready.then(callback, on_error)
  726. def _get_task_meta(self):
  727. return self._cache
  728. def __reduce__(self):
  729. return self.__class__, self.__reduce_args__()
  730. def __reduce_args__(self):
  731. return (self.id, self._result, self._state, self._traceback)
  732. def __copy__(self):
  733. cls, args = self.__reduce__()
  734. return cls(*args)
  735. def ready(self):
  736. return True
  737. def get(self, timeout=None, propagate=True, **kwargs):
  738. if self.successful():
  739. return self.result
  740. elif self.state in states.PROPAGATE_STATES:
  741. if propagate:
  742. raise self.result
  743. return self.result
  744. wait = get # XXX Compat (remove 5.0)
  745. def forget(self):
  746. pass
  747. def revoke(self, *args, **kwargs):
  748. self._state = states.REVOKED
  749. def __repr__(self):
  750. return '<EagerResult: {0.id}>'.format(self)
  751. @property
  752. def _cache(self):
  753. return {
  754. 'task_id': self.id,
  755. 'result': self._result,
  756. 'status': self._state,
  757. 'traceback': self._traceback,
  758. }
  759. @property
  760. def result(self):
  761. """The tasks return value."""
  762. return self._result
  763. @property
  764. def state(self):
  765. """The tasks state."""
  766. return self._state
  767. status = state
  768. @property
  769. def traceback(self):
  770. """The traceback if the task failed."""
  771. return self._traceback
  772. @property
  773. def supports_native_join(self):
  774. return False
  775. def result_from_tuple(r, app=None):
  776. """Deserialize result from tuple."""
  777. # earlier backends may just pickle, so check if
  778. # result is already prepared.
  779. app = app_or_default(app)
  780. Result = app.AsyncResult
  781. if not isinstance(r, ResultBase):
  782. res, nodes = r
  783. id, parent = res if isinstance(res, (list, tuple)) else (res, None)
  784. if parent:
  785. parent = result_from_tuple(parent, app)
  786. if nodes:
  787. return app.GroupResult(
  788. id, [result_from_tuple(child, app) for child in nodes],
  789. parent=parent,
  790. )
  791. return Result(id, parent=parent)
  792. return r