test_result.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872
  1. from __future__ import absolute_import, unicode_literals
  2. import pytest
  3. import traceback
  4. from contextlib import contextmanager
  5. from case import Mock, call, patch, skip
  6. from celery import uuid
  7. from celery import states
  8. from celery.backends.base import SyncBackendMixin
  9. from celery.exceptions import (
  10. CPendingDeprecationWarning, ImproperlyConfigured,
  11. IncompleteStream, TimeoutError,
  12. )
  13. from celery.five import range
  14. from celery.result import (
  15. AsyncResult,
  16. EagerResult,
  17. ResultSet,
  18. result_from_tuple,
  19. assert_will_not_block,
  20. )
  21. from celery.utils.serialization import pickle
  22. PYTRACEBACK = """\
  23. Traceback (most recent call last):
  24. File "foo.py", line 2, in foofunc
  25. don't matter
  26. File "bar.py", line 3, in barfunc
  27. don't matter
  28. Doesn't matter: really!\
  29. """
  30. def mock_task(name, state, result, traceback=None):
  31. return dict(
  32. id=uuid(), name=name, state=state,
  33. result=result, traceback=traceback,
  34. )
  35. def save_result(app, task):
  36. traceback = task.get('traceback') or 'Some traceback'
  37. if task['state'] == states.SUCCESS:
  38. app.backend.mark_as_done(task['id'], task['result'])
  39. elif task['state'] == states.RETRY:
  40. app.backend.mark_as_retry(
  41. task['id'], task['result'], traceback=traceback,
  42. )
  43. else:
  44. app.backend.mark_as_failure(
  45. task['id'], task['result'], traceback=traceback,
  46. )
  47. def make_mock_group(app, size=10):
  48. tasks = [mock_task('ts%d' % i, states.SUCCESS, i) for i in range(size)]
  49. [save_result(app, task) for task in tasks]
  50. return [app.AsyncResult(task['id']) for task in tasks]
  51. class test_AsyncResult:
  52. def setup(self):
  53. self.app.conf.result_cache_max = 100
  54. self.app.conf.result_serializer = 'pickle'
  55. self.task1 = mock_task('task1', states.SUCCESS, 'the')
  56. self.task2 = mock_task('task2', states.SUCCESS, 'quick')
  57. self.task3 = mock_task('task3', states.FAILURE, KeyError('brown'))
  58. self.task4 = mock_task('task3', states.RETRY, KeyError('red'))
  59. self.task5 = mock_task(
  60. 'task3', states.FAILURE, KeyError('blue'), PYTRACEBACK,
  61. )
  62. for task in (self.task1, self.task2,
  63. self.task3, self.task4, self.task5):
  64. save_result(self.app, task)
  65. @self.app.task(shared=False)
  66. def mytask():
  67. pass
  68. self.mytask = mytask
  69. @patch('celery.result.task_join_will_block')
  70. def test_assert_will_not_block(self, task_join_will_block):
  71. task_join_will_block.return_value = True
  72. with pytest.raises(RuntimeError):
  73. assert_will_not_block()
  74. task_join_will_block.return_value = False
  75. assert_will_not_block()
  76. def test_without_id(self):
  77. with pytest.raises(ValueError):
  78. AsyncResult(None, app=self.app)
  79. def test_compat_properties(self):
  80. x = self.app.AsyncResult('1')
  81. assert x.task_id == x.id
  82. x.task_id = '2'
  83. assert x.id == '2'
  84. @pytest.mark.usefixtures('depends_on_current_app')
  85. def test_reduce_direct(self):
  86. x = AsyncResult('1', app=self.app)
  87. fun, args = x.__reduce__()
  88. assert fun(*args) == x
  89. def test_children(self):
  90. x = self.app.AsyncResult('1')
  91. children = [EagerResult(str(i), i, states.SUCCESS) for i in range(3)]
  92. x._cache = {'children': children, 'status': states.SUCCESS}
  93. x.backend = Mock()
  94. assert x.children
  95. assert len(x.children) == 3
  96. def test_propagates_for_parent(self):
  97. x = self.app.AsyncResult(uuid())
  98. x.backend = Mock(name='backend')
  99. x.backend.get_task_meta.return_value = {}
  100. x.backend.wait_for_pending.return_value = 84
  101. x.parent = EagerResult(uuid(), KeyError('foo'), states.FAILURE)
  102. with pytest.raises(KeyError):
  103. x.get(propagate=True)
  104. x.backend.wait_for_pending.assert_not_called()
  105. x.parent = EagerResult(uuid(), 42, states.SUCCESS)
  106. assert x.get(propagate=True) == 84
  107. x.backend.wait_for_pending.assert_called()
  108. def test_get_children(self):
  109. tid = uuid()
  110. x = self.app.AsyncResult(tid)
  111. child = [self.app.AsyncResult(uuid()).as_tuple()
  112. for i in range(10)]
  113. x._cache = {'children': child}
  114. assert x.children
  115. assert len(x.children) == 10
  116. x._cache = {'status': states.SUCCESS}
  117. x.backend._cache[tid] = {'result': None}
  118. assert x.children is None
  119. def test_build_graph_get_leaf_collect(self):
  120. x = self.app.AsyncResult('1')
  121. x.backend._cache['1'] = {'status': states.SUCCESS, 'result': None}
  122. c = [EagerResult(str(i), i, states.SUCCESS) for i in range(3)]
  123. x.iterdeps = Mock()
  124. x.iterdeps.return_value = (
  125. (None, x),
  126. (x, c[0]),
  127. (c[0], c[1]),
  128. (c[1], c[2])
  129. )
  130. x.backend.READY_STATES = states.READY_STATES
  131. assert x.graph
  132. assert x.get_leaf() is 2
  133. it = x.collect()
  134. assert list(it) == [
  135. (x, None),
  136. (c[0], 0),
  137. (c[1], 1),
  138. (c[2], 2),
  139. ]
  140. def test_iterdeps(self):
  141. x = self.app.AsyncResult('1')
  142. c = [EagerResult(str(i), i, states.SUCCESS) for i in range(3)]
  143. x._cache = {'status': states.SUCCESS, 'result': None, 'children': c}
  144. for child in c:
  145. child.backend = Mock()
  146. child.backend.get_children.return_value = []
  147. it = x.iterdeps()
  148. assert list(it) == [
  149. (None, x),
  150. (x, c[0]),
  151. (x, c[1]),
  152. (x, c[2]),
  153. ]
  154. x._cache = None
  155. x.ready = Mock()
  156. x.ready.return_value = False
  157. with pytest.raises(IncompleteStream):
  158. list(x.iterdeps())
  159. list(x.iterdeps(intermediate=True))
  160. def test_eq_not_implemented(self):
  161. assert self.app.AsyncResult('1') != object()
  162. @pytest.mark.usefixtures('depends_on_current_app')
  163. def test_reduce(self):
  164. a1 = self.app.AsyncResult('uuid')
  165. restored = pickle.loads(pickle.dumps(a1))
  166. assert restored.id == 'uuid'
  167. a2 = self.app.AsyncResult('uuid')
  168. assert pickle.loads(pickle.dumps(a2)).id == 'uuid'
  169. def test_maybe_set_cache_empty(self):
  170. self.app.AsyncResult('uuid')._maybe_set_cache(None)
  171. def test_set_cache__children(self):
  172. r1 = self.app.AsyncResult('id1')
  173. r2 = self.app.AsyncResult('id2')
  174. r1._set_cache({'children': [r2.as_tuple()]})
  175. assert r2 in r1.children
  176. def test_successful(self):
  177. ok_res = self.app.AsyncResult(self.task1['id'])
  178. nok_res = self.app.AsyncResult(self.task3['id'])
  179. nok_res2 = self.app.AsyncResult(self.task4['id'])
  180. assert ok_res.successful()
  181. assert not nok_res.successful()
  182. assert not nok_res2.successful()
  183. pending_res = self.app.AsyncResult(uuid())
  184. assert not pending_res.successful()
  185. def test_raising(self):
  186. notb = self.app.AsyncResult(self.task3['id'])
  187. withtb = self.app.AsyncResult(self.task5['id'])
  188. with pytest.raises(KeyError):
  189. notb.get()
  190. try:
  191. withtb.get()
  192. except KeyError:
  193. tb = traceback.format_exc()
  194. assert ' File "foo.py", line 2, in foofunc' not in tb
  195. assert ' File "bar.py", line 3, in barfunc' not in tb
  196. assert 'KeyError:' in tb
  197. assert "'blue'" in tb
  198. else:
  199. raise AssertionError('Did not raise KeyError.')
  200. @skip.unless_module('tblib')
  201. def test_raising_remote_tracebacks(self):
  202. withtb = self.app.AsyncResult(self.task5['id'])
  203. old, self.app.conf.remote_tracebacks = (
  204. self.app.conf.remote_tracebacks, True)
  205. try:
  206. withtb.get()
  207. except KeyError:
  208. tb = traceback.format_exc()
  209. assert ' File "foo.py", line 2, in foofunc' in tb
  210. assert ' File "bar.py", line 3, in barfunc' in tb
  211. assert 'KeyError:' in tb
  212. assert "'blue'" in tb
  213. else:
  214. raise AssertionError('Did not raise KeyError.')
  215. finally:
  216. self.app.conf.remote_tracebacks = old
  217. def test_str(self):
  218. ok_res = self.app.AsyncResult(self.task1['id'])
  219. ok2_res = self.app.AsyncResult(self.task2['id'])
  220. nok_res = self.app.AsyncResult(self.task3['id'])
  221. assert str(ok_res) == self.task1['id']
  222. assert str(ok2_res) == self.task2['id']
  223. assert str(nok_res) == self.task3['id']
  224. pending_id = uuid()
  225. pending_res = self.app.AsyncResult(pending_id)
  226. assert str(pending_res) == pending_id
  227. def test_repr(self):
  228. ok_res = self.app.AsyncResult(self.task1['id'])
  229. ok2_res = self.app.AsyncResult(self.task2['id'])
  230. nok_res = self.app.AsyncResult(self.task3['id'])
  231. assert repr(ok_res) == '<AsyncResult: %s>' % (self.task1['id'],)
  232. assert repr(ok2_res) == '<AsyncResult: %s>' % (self.task2['id'],)
  233. assert repr(nok_res) == '<AsyncResult: %s>' % (self.task3['id'],)
  234. pending_id = uuid()
  235. pending_res = self.app.AsyncResult(pending_id)
  236. assert repr(pending_res) == '<AsyncResult: %s>' % (pending_id,)
  237. def test_hash(self):
  238. assert (hash(self.app.AsyncResult('x0w991')) ==
  239. hash(self.app.AsyncResult('x0w991')))
  240. assert (hash(self.app.AsyncResult('x0w991')) !=
  241. hash(self.app.AsyncResult('x1w991')))
  242. def test_get_traceback(self):
  243. ok_res = self.app.AsyncResult(self.task1['id'])
  244. nok_res = self.app.AsyncResult(self.task3['id'])
  245. nok_res2 = self.app.AsyncResult(self.task4['id'])
  246. assert not ok_res.traceback
  247. assert nok_res.traceback
  248. assert nok_res2.traceback
  249. pending_res = self.app.AsyncResult(uuid())
  250. assert not pending_res.traceback
  251. def test_get__backend_gives_None(self):
  252. res = self.app.AsyncResult(self.task1['id'])
  253. res.backend.wait_for = Mock(name='wait_for')
  254. res.backend.wait_for.return_value = None
  255. assert res.get() is None
  256. def test_get(self):
  257. ok_res = self.app.AsyncResult(self.task1['id'])
  258. ok2_res = self.app.AsyncResult(self.task2['id'])
  259. nok_res = self.app.AsyncResult(self.task3['id'])
  260. nok2_res = self.app.AsyncResult(self.task4['id'])
  261. callback = Mock(name='callback')
  262. assert ok_res.get(callback=callback) == 'the'
  263. callback.assert_called_with(ok_res.id, 'the')
  264. assert ok2_res.get() == 'quick'
  265. with pytest.raises(KeyError):
  266. nok_res.get()
  267. assert nok_res.get(propagate=False)
  268. assert isinstance(nok2_res.result, KeyError)
  269. assert ok_res.info == 'the'
  270. def test_eq_ne(self):
  271. r1 = self.app.AsyncResult(self.task1['id'])
  272. r2 = self.app.AsyncResult(self.task1['id'])
  273. r3 = self.app.AsyncResult(self.task2['id'])
  274. assert r1 == r2
  275. assert r1 != r3
  276. assert r1 == r2.id
  277. assert r1 != r3.id
  278. @pytest.mark.usefixtures('depends_on_current_app')
  279. def test_reduce_restore(self):
  280. r1 = self.app.AsyncResult(self.task1['id'])
  281. fun, args = r1.__reduce__()
  282. assert fun(*args) == r1
  283. def test_get_timeout(self):
  284. res = self.app.AsyncResult(self.task4['id']) # has RETRY state
  285. with pytest.raises(TimeoutError):
  286. res.get(timeout=0.001)
  287. pending_res = self.app.AsyncResult(uuid())
  288. with patch('celery.result.time') as _time:
  289. with pytest.raises(TimeoutError):
  290. pending_res.get(timeout=0.001, interval=0.001)
  291. _time.sleep.assert_called_with(0.001)
  292. def test_get_timeout_longer(self):
  293. res = self.app.AsyncResult(self.task4['id']) # has RETRY state
  294. with patch('celery.result.time') as _time:
  295. with pytest.raises(TimeoutError):
  296. res.get(timeout=1, interval=1)
  297. _time.sleep.assert_called_with(1)
  298. def test_ready(self):
  299. oks = (self.app.AsyncResult(self.task1['id']),
  300. self.app.AsyncResult(self.task2['id']),
  301. self.app.AsyncResult(self.task3['id']))
  302. assert all(result.ready() for result in oks)
  303. assert not self.app.AsyncResult(self.task4['id']).ready()
  304. assert not self.app.AsyncResult(uuid()).ready()
  305. class test_ResultSet:
  306. def test_resultset_repr(self):
  307. assert repr(self.app.ResultSet(
  308. [self.app.AsyncResult(t) for t in ['1', '2', '3']]))
  309. def test_eq_other(self):
  310. assert self.app.ResultSet([
  311. self.app.AsyncResult(t) for t in [1, 3, 3]]) != 1
  312. rs1 = self.app.ResultSet([self.app.AsyncResult(1)])
  313. rs2 = self.app.ResultSet([self.app.AsyncResult(1)])
  314. assert rs1 == rs2
  315. def test_get(self):
  316. x = self.app.ResultSet([self.app.AsyncResult(t) for t in [1, 2, 3]])
  317. b = x.results[0].backend = Mock()
  318. b.supports_native_join = False
  319. x.join_native = Mock()
  320. x.join = Mock()
  321. x.get()
  322. x.join.assert_called()
  323. b.supports_native_join = True
  324. x.get()
  325. x.join_native.assert_called()
  326. def test_eq_ne(self):
  327. g1 = self.app.ResultSet([
  328. self.app.AsyncResult('id1'),
  329. self.app.AsyncResult('id2'),
  330. ])
  331. g2 = self.app.ResultSet([
  332. self.app.AsyncResult('id1'),
  333. self.app.AsyncResult('id2'),
  334. ])
  335. g3 = self.app.ResultSet([
  336. self.app.AsyncResult('id3'),
  337. self.app.AsyncResult('id1'),
  338. ])
  339. assert g1 == g2
  340. assert g1 != g3
  341. assert g1 != object()
  342. def test_takes_app_from_first_task(self):
  343. x = ResultSet([self.app.AsyncResult('id1')])
  344. assert x.app is x.results[0].app
  345. x.app = self.app
  346. assert x.app is self.app
  347. def test_get_empty(self):
  348. x = self.app.ResultSet([])
  349. assert x.supports_native_join is None
  350. x.join = Mock(name='join')
  351. x.get()
  352. x.join.assert_called()
  353. def test_add(self):
  354. x = self.app.ResultSet([self.app.AsyncResult(1)])
  355. x.add(self.app.AsyncResult(2))
  356. assert len(x) == 2
  357. x.add(self.app.AsyncResult(2))
  358. assert len(x) == 2
  359. @contextmanager
  360. def dummy_copy(self):
  361. with patch('celery.result.copy') as copy:
  362. def passt(arg):
  363. return arg
  364. copy.side_effect = passt
  365. yield
  366. def test_iterate_respects_subpolling_interval(self):
  367. r1 = self.app.AsyncResult(uuid())
  368. r2 = self.app.AsyncResult(uuid())
  369. backend = r1.backend = r2.backend = Mock()
  370. backend.subpolling_interval = 10
  371. ready = r1.ready = r2.ready = Mock()
  372. def se(*args, **kwargs):
  373. ready.side_effect = KeyError()
  374. return False
  375. ready.return_value = False
  376. ready.side_effect = se
  377. x = self.app.ResultSet([r1, r2])
  378. with self.dummy_copy():
  379. with patch('celery.result.time') as _time:
  380. with pytest.warns(CPendingDeprecationWarning):
  381. with pytest.raises(KeyError):
  382. list(x.iterate())
  383. _time.sleep.assert_called_with(10)
  384. backend.subpolling_interval = 0
  385. with patch('celery.result.time') as _time:
  386. with pytest.warns(CPendingDeprecationWarning):
  387. with pytest.raises(KeyError):
  388. ready.return_value = False
  389. ready.side_effect = se
  390. list(x.iterate())
  391. _time.sleep.assert_not_called()
  392. def test_times_out(self):
  393. r1 = self.app.AsyncResult(uuid)
  394. r1.ready = Mock()
  395. r1.ready.return_value = False
  396. x = self.app.ResultSet([r1])
  397. with self.dummy_copy():
  398. with patch('celery.result.time'):
  399. with pytest.warns(CPendingDeprecationWarning):
  400. with pytest.raises(TimeoutError):
  401. list(x.iterate(timeout=1))
  402. def test_add_discard(self):
  403. x = self.app.ResultSet([])
  404. x.add(self.app.AsyncResult('1'))
  405. assert self.app.AsyncResult('1') in x.results
  406. x.discard(self.app.AsyncResult('1'))
  407. x.discard(self.app.AsyncResult('1'))
  408. x.discard('1')
  409. assert self.app.AsyncResult('1') not in x.results
  410. x.update([self.app.AsyncResult('2')])
  411. def test_clear(self):
  412. x = self.app.ResultSet([])
  413. r = x.results
  414. x.clear()
  415. assert x.results is r
  416. class MockAsyncResultFailure(AsyncResult):
  417. @property
  418. def result(self):
  419. return KeyError('baz')
  420. @property
  421. def state(self):
  422. return states.FAILURE
  423. def get(self, propagate=True, **kwargs):
  424. if propagate:
  425. raise self.result
  426. return self.result
  427. class MockAsyncResultSuccess(AsyncResult):
  428. forgotten = False
  429. def forget(self):
  430. self.forgotten = True
  431. @property
  432. def result(self):
  433. return 42
  434. @property
  435. def state(self):
  436. return states.SUCCESS
  437. def get(self, **kwargs):
  438. return self.result
  439. class SimpleBackend(SyncBackendMixin):
  440. ids = []
  441. def __init__(self, ids=[]):
  442. self.ids = ids
  443. def _ensure_not_eager(self):
  444. pass
  445. def get_many(self, *args, **kwargs):
  446. return ((id, {'result': i, 'status': states.SUCCESS})
  447. for i, id in enumerate(self.ids))
  448. class test_GroupResult:
  449. def setup(self):
  450. self.size = 10
  451. self.ts = self.app.GroupResult(
  452. uuid(), make_mock_group(self.app, self.size),
  453. )
  454. @pytest.mark.usefixtures('depends_on_current_app')
  455. def test_is_pickleable(self):
  456. ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
  457. assert pickle.loads(pickle.dumps(ts)) == ts
  458. ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
  459. assert pickle.loads(pickle.dumps(ts2)) == ts2
  460. @pytest.mark.usefixtures('depends_on_current_app')
  461. def test_reduce(self):
  462. ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
  463. fun, args = ts.__reduce__()
  464. ts2 = fun(*args)
  465. assert ts2.id == ts.id
  466. assert ts == ts2
  467. def test_eq_ne(self):
  468. ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
  469. ts2 = self.app.GroupResult(ts.id, ts.results)
  470. ts3 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
  471. ts4 = self.app.GroupResult(ts.id, [self.app.AsyncResult(uuid())])
  472. assert ts == ts2
  473. assert ts != ts3
  474. assert ts != ts4
  475. assert ts != object()
  476. def test_len(self):
  477. assert len(self.ts) == self.size
  478. def test_eq_other(self):
  479. assert self.ts != 1
  480. @pytest.mark.usefixtures('depends_on_current_app')
  481. def test_pickleable(self):
  482. assert pickle.loads(pickle.dumps(self.ts))
  483. def test_iterate_raises(self):
  484. ar = MockAsyncResultFailure(uuid(), app=self.app)
  485. ts = self.app.GroupResult(uuid(), [ar])
  486. with pytest.warns(CPendingDeprecationWarning):
  487. it = ts.iterate()
  488. with pytest.raises(KeyError):
  489. next(it)
  490. def test_forget(self):
  491. subs = [MockAsyncResultSuccess(uuid(), app=self.app),
  492. MockAsyncResultSuccess(uuid(), app=self.app)]
  493. ts = self.app.GroupResult(uuid(), subs)
  494. ts.forget()
  495. for sub in subs:
  496. assert sub.forgotten
  497. def test_getitem(self):
  498. subs = [MockAsyncResultSuccess(uuid(), app=self.app),
  499. MockAsyncResultSuccess(uuid(), app=self.app)]
  500. ts = self.app.GroupResult(uuid(), subs)
  501. assert ts[0] is subs[0]
  502. def test_save_restore(self):
  503. subs = [MockAsyncResultSuccess(uuid(), app=self.app),
  504. MockAsyncResultSuccess(uuid(), app=self.app)]
  505. ts = self.app.GroupResult(uuid(), subs)
  506. ts.save()
  507. with pytest.raises(AttributeError):
  508. ts.save(backend=object())
  509. assert self.app.GroupResult.restore(ts.id).results == ts.results
  510. ts.delete()
  511. assert self.app.GroupResult.restore(ts.id) is None
  512. with pytest.raises(AttributeError):
  513. self.app.GroupResult.restore(ts.id, backend=object())
  514. def test_join_native(self):
  515. backend = SimpleBackend()
  516. results = [self.app.AsyncResult(uuid(), backend=backend)
  517. for i in range(10)]
  518. ts = self.app.GroupResult(uuid(), results)
  519. ts.app.backend = backend
  520. backend.ids = [result.id for result in results]
  521. res = ts.join_native()
  522. assert res == list(range(10))
  523. callback = Mock(name='callback')
  524. assert not ts.join_native(callback=callback)
  525. callback.assert_has_calls([
  526. call(r.id, i) for i, r in enumerate(ts.results)
  527. ])
  528. def test_join_native_raises(self):
  529. ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
  530. ts.iter_native = Mock()
  531. ts.iter_native.return_value = iter([
  532. (uuid(), {'status': states.FAILURE, 'result': KeyError()})
  533. ])
  534. with pytest.raises(KeyError):
  535. ts.join_native(propagate=True)
  536. def test_failed_join_report(self):
  537. res = Mock()
  538. ts = self.app.GroupResult(uuid(), [res])
  539. res.state = states.FAILURE
  540. res.backend.is_cached.return_value = True
  541. assert next(ts._failed_join_report()) is res
  542. res.backend.is_cached.return_value = False
  543. with pytest.raises(StopIteration):
  544. next(ts._failed_join_report())
  545. def test_repr(self):
  546. assert repr(
  547. self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())]))
  548. def test_children_is_results(self):
  549. ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
  550. assert ts.children is ts.results
  551. def test_iter_native(self):
  552. backend = SimpleBackend()
  553. results = [self.app.AsyncResult(uuid(), backend=backend)
  554. for i in range(10)]
  555. ts = self.app.GroupResult(uuid(), results)
  556. ts.app.backend = backend
  557. backend.ids = [result.id for result in results]
  558. assert len(list(ts.iter_native())) == 10
  559. def test_iterate_yields(self):
  560. ar = MockAsyncResultSuccess(uuid(), app=self.app)
  561. ar2 = MockAsyncResultSuccess(uuid(), app=self.app)
  562. ts = self.app.GroupResult(uuid(), [ar, ar2])
  563. with pytest.warns(CPendingDeprecationWarning):
  564. it = ts.iterate()
  565. assert next(it) == 42
  566. assert next(it) == 42
  567. def test_iterate_eager(self):
  568. ar1 = EagerResult(uuid(), 42, states.SUCCESS)
  569. ar2 = EagerResult(uuid(), 42, states.SUCCESS)
  570. ts = self.app.GroupResult(uuid(), [ar1, ar2])
  571. with pytest.warns(CPendingDeprecationWarning):
  572. it = ts.iterate()
  573. assert next(it) == 42
  574. assert next(it) == 42
  575. def test_join_timeout(self):
  576. ar = MockAsyncResultSuccess(uuid(), app=self.app)
  577. ar2 = MockAsyncResultSuccess(uuid(), app=self.app)
  578. ar3 = self.app.AsyncResult(uuid())
  579. ts = self.app.GroupResult(uuid(), [ar, ar2, ar3])
  580. with pytest.raises(TimeoutError):
  581. ts.join(timeout=0.0000001)
  582. ar4 = self.app.AsyncResult(uuid())
  583. ar4.get = Mock()
  584. ts2 = self.app.GroupResult(uuid(), [ar4])
  585. assert ts2.join(timeout=0.1)
  586. callback = Mock(name='callback')
  587. assert not ts2.join(timeout=0.1, callback=callback)
  588. callback.assert_called_with(ar4.id, ar4.get())
  589. def test_iter_native_when_empty_group(self):
  590. ts = self.app.GroupResult(uuid(), [])
  591. assert list(ts.iter_native()) == []
  592. def test_iterate_simple(self):
  593. with pytest.warns(CPendingDeprecationWarning):
  594. it = self.ts.iterate()
  595. results = sorted(list(it))
  596. assert results == list(range(self.size))
  597. def test___iter__(self):
  598. assert list(iter(self.ts)) == self.ts.results
  599. def test_join(self):
  600. joined = self.ts.join()
  601. assert joined == list(range(self.size))
  602. def test_successful(self):
  603. assert self.ts.successful()
  604. def test_failed(self):
  605. assert not self.ts.failed()
  606. def test_maybe_throw(self):
  607. self.ts.results = [Mock(name='r1')]
  608. self.ts.maybe_throw()
  609. self.ts.results[0].maybe_throw.assert_called_with(
  610. callback=None, propagate=True,
  611. )
  612. def test_join__on_message(self):
  613. with pytest.raises(ImproperlyConfigured):
  614. self.ts.join(on_message=Mock())
  615. def test_waiting(self):
  616. assert not self.ts.waiting()
  617. def test_ready(self):
  618. assert self.ts.ready()
  619. def test_completed_count(self):
  620. assert self.ts.completed_count() == len(self.ts)
  621. class test_pending_AsyncResult:
  622. def test_result(self, app):
  623. res = app.AsyncResult(uuid())
  624. assert res.result is None
  625. class test_failed_AsyncResult:
  626. def setup(self):
  627. self.size = 11
  628. self.app.conf.result_serializer = 'pickle'
  629. results = make_mock_group(self.app, 10)
  630. failed = mock_task('ts11', states.FAILURE, KeyError('Baz'))
  631. save_result(self.app, failed)
  632. failed_res = self.app.AsyncResult(failed['id'])
  633. self.ts = self.app.GroupResult(uuid(), results + [failed_res])
  634. def test_completed_count(self):
  635. assert self.ts.completed_count() == len(self.ts) - 1
  636. def test_iterate_simple(self):
  637. with pytest.warns(CPendingDeprecationWarning):
  638. it = self.ts.iterate()
  639. def consume():
  640. return list(it)
  641. with pytest.raises(KeyError):
  642. consume()
  643. def test_join(self):
  644. with pytest.raises(KeyError):
  645. self.ts.join()
  646. def test_successful(self):
  647. assert not self.ts.successful()
  648. def test_failed(self):
  649. assert self.ts.failed()
  650. class test_pending_Group:
  651. def setup(self):
  652. self.ts = self.app.GroupResult(
  653. uuid(), [self.app.AsyncResult(uuid()),
  654. self.app.AsyncResult(uuid())])
  655. def test_completed_count(self):
  656. assert self.ts.completed_count() == 0
  657. def test_ready(self):
  658. assert not self.ts.ready()
  659. def test_waiting(self):
  660. assert self.ts.waiting()
  661. def test_join(self):
  662. with pytest.raises(TimeoutError):
  663. self.ts.join(timeout=0.001)
  664. def test_join_longer(self):
  665. with pytest.raises(TimeoutError):
  666. self.ts.join(timeout=1)
  667. class test_EagerResult:
  668. def setup(self):
  669. @self.app.task(shared=False)
  670. def raising(x, y):
  671. raise KeyError(x, y)
  672. self.raising = raising
  673. def test_wait_raises(self):
  674. res = self.raising.apply(args=[3, 3])
  675. with pytest.raises(KeyError):
  676. res.wait()
  677. assert res.wait(propagate=False)
  678. def test_wait(self):
  679. res = EagerResult('x', 'x', states.RETRY)
  680. res.wait()
  681. assert res.state == states.RETRY
  682. assert res.status == states.RETRY
  683. def test_forget(self):
  684. res = EagerResult('x', 'x', states.RETRY)
  685. res.forget()
  686. def test_revoke(self):
  687. res = self.raising.apply(args=[3, 3])
  688. assert not res.revoke()
  689. class test_tuples:
  690. def test_AsyncResult(self):
  691. x = self.app.AsyncResult(uuid())
  692. assert x, result_from_tuple(x.as_tuple() == self.app)
  693. assert x, result_from_tuple(x == self.app)
  694. def test_with_parent(self):
  695. x = self.app.AsyncResult(uuid())
  696. x.parent = self.app.AsyncResult(uuid())
  697. y = result_from_tuple(x.as_tuple(), self.app)
  698. assert y == x
  699. assert y.parent == x.parent
  700. assert isinstance(y.parent, AsyncResult)
  701. def test_compat(self):
  702. uid = uuid()
  703. x = result_from_tuple([uid, []], app=self.app)
  704. assert x.id == uid
  705. def test_GroupResult(self):
  706. x = self.app.GroupResult(
  707. uuid(), [self.app.AsyncResult(uuid()) for _ in range(10)],
  708. )
  709. assert x, result_from_tuple(x.as_tuple() == self.app)
  710. assert x, result_from_tuple(x == self.app)