Renamed celery.worker.pool -> celery.concurrency.processes

billiard pool previously named celery.concurrency.processes is now
celery.concurrency.processes.pool

Changelog

@@ -177,7 +177,7 @@ News
     =====================================  =====================================
     **Module name**                        **celery equivalent**
     =====================================  =====================================
-    ``billiard.pool``                      ``celery.concurrency.processes``
+    ``billiard.pool``                      ``celery.concurrency.processes.pool``
     ``billiard.serialization``             ``celery.serialization``
     ``billiard.utils.functional``          ``celery.utils.functional``
     =====================================  =====================================
@@ -756,7 +756,7 @@ Fixes
 
     .. code-block:: python
 
-        CELERYD_POOL = "celery.worker.pool.TaskPool"
+        CELERYD_POOL = "celery.concurrency.processes.TaskPool"
         CELERYD_MEDIATOR = "celery.worker.controllers.Mediator"
         CELERYD_ETA_SCHEDULER = "celery.worker.controllers.ScheduleController"
         CELERYD_LISTENER = "celery.worker.listener.CarrotListener"

celery/worker/pool.py → celery/concurrency/processes/__init__.py

@@ -5,10 +5,11 @@ Process Pools.
 """
 
 from celery import log
-from celery.concurrency.processes import Pool, RUN
 from celery.datastructures import ExceptionInfo
 from celery.utils.functional import curry
 
+from celery.concurrency.processes.pool import Pool, RUN
+
 
 class TaskPool(object):
     """Process Pool for processing tasks in parallel.

celery/concurrency/processes.py → celery/concurrency/processes/pool.py

@@ -24,6 +24,8 @@ import signal
 from multiprocessing import Process, cpu_count, TimeoutError
 from multiprocessing.util import Finalize, debug
 
+from celery.exceptions import SoftTimeLimitExceeded, TimeLimitExceeded
+
 #
 # Constants representing the state of a pool
 #
@@ -48,13 +50,6 @@ def mapstar(args):
 # Code run by worker processes
 #
 
-class TimeLimitExceeded(Exception):
-    """The time limit has been exceeded and the job has been terminated."""
-
-class SoftTimeLimitExceeded(Exception):
-    """The soft time limit has been exceeded. This exception is raised
-    to give the job a chance to clean up."""
-
 def soft_timeout_sighandler(signum, frame):
     raise SoftTimeLimitExceeded()
 

celery/conf.py

@@ -45,7 +45,7 @@ _DEFAULTS = {
     "CELERY_BROKER_CONNECTION_MAX_RETRIES": 100,
     "CELERY_ACKS_LATE": False,
     "CELERYD_POOL_PUTLOCKS": True,
-    "CELERYD_POOL": "celery.worker.pool.TaskPool",
+    "CELERYD_POOL": "celery.concurrency.processes.TaskPool",
     "CELERYD_MEDIATOR": "celery.worker.controllers.Mediator",
     "CELERYD_ETA_SCHEDULER": "celery.worker.controllers.ScheduleController",
     "CELERYD_LISTENER": "celery.worker.listener.CarrotListener",

celery/exceptions.py

@@ -3,7 +3,6 @@
 Common Exceptions
 
 """
-from celery.concurrency.processes import SoftTimeLimitExceeded as _STLE
 
 UNREGISTERED_FMT = """
 Task of kind %s is not registered, please make sure it's imported.
@@ -14,7 +13,11 @@ class RouteNotFound(KeyError):
     """Task routed to a queue not in the routing table (CELERY_QUEUES)."""
 
 
-class SoftTimeLimitExceeded(_STLE):
+class TimeLimitExceeded(Exception):
+    """The time limit has been exceeded and the job has been terminated."""
+
+
+class SoftTimeLimitExceeded(Exception):
     """The soft time limit has been exceeded. This exception is raised
     to give the task a chance to clean up."""
     pass

celery/task/base.py

@@ -485,7 +485,7 @@ class Task(object):
         """The method the worker calls to execute the task.
 
         :param wrapper: A :class:`celery.worker.job.TaskWrapper`.
-        :param pool: A :class:`celery.worker.pool.TaskPool` object.
+        :param pool: A task pool.
         :param loglevel: Current loglevel.
         :param logfile: Name of the currently used logfile.
 

celery/tests/test_pool.py

@@ -1,10 +1,11 @@
-import unittest2 as unittest
+import sys
+import time
 import logging
 import itertools
-import time
-from celery.worker.pool import TaskPool
+import unittest2 as unittest
+
+from celery.concurrency.processes import TaskPool
 from celery.datastructures import ExceptionInfo
-import sys
 
 
 def do_something(i):

celery/tests/test_worker_job.py

@@ -13,7 +13,7 @@ from celery.task.base import Task
 from celery.utils import gen_unique_id
 from celery.result import AsyncResult
 from celery.worker.job import WorkerTaskTrace, TaskWrapper
-from celery.worker.pool import TaskPool
+from celery.concurrency.processes import TaskPool
 from celery.backends import default_backend
 from celery.exceptions import RetryTaskError, NotRegistered
 from celery.decorators import task as task_dec

celery/utils/__init__.py

@@ -207,23 +207,23 @@ def get_cls_by_name(name, aliases={}):
 
     Example::
 
-        celery.worker.pool.TaskPool
-                           ^- class name
+        celery.concurrency.processes.TaskPool
+                                    ^- class name
 
     If ``aliases`` is provided, a dict containing short name/long name
     mappings, the name is looked up in the aliases first.
 
     Examples:
 
-        >>> get_cls_by_name("celery.worker.pool.TaskPool")
-        <class 'celery.worker.pool.TaskPool'>
+        >>> get_cls_by_name("celery.concurrency.processes.TaskPool")
+        <class 'celery.concurrency.processes.TaskPool'>
 
         >>> get_cls_by_name("default", {
-        ...     "default": "celery.worker.pool.TaskPool"})
-        <class 'celery.worker.pool.TaskPool'>
+        ...     "default": "celery.concurrency.processes.TaskPool"})
+        <class 'celery.concurrency.processes.TaskPool'>
 
         # Does not try to look up non-string names.
-        >>> from celery.worker.pool import TaskPool
+        >>> from celery.concurrency.processes import TaskPool
         >>> get_cls_by_name(TaskPool) is TaskPool
         True
 

docs/configuration.rst

@@ -588,7 +588,7 @@ Custom Component Classes (advanced)
 * CELERYD_POOL
 
     Name of the task pool class used by the worker.
-    Default is ``"celery.worker.pool.TaskPool"``.
+    Default is ``"celery.concurrency.processes.TaskPool"``.
 
 * CELERYD_LISTENER
 

docs/internals/reference/celery.concurrency.processes.pool.rst

@@ -0,0 +1,9 @@
+===================================================================
+ extended multiprocessing.pool - celery.concurrency.processes.pool
+===================================================================
+
+.. currentmodule:: celery.concurrency.processes.pool
+
+.. automodule:: celery.concurrency.processes.pool
+    :members:
+    :undoc-members:

docs/internals/reference/celery.worker.pool.rst

@@ -1,9 +0,0 @@
-================================
- Task Pool - celery.worker.pool
-================================
-
-.. currentmodule:: celery.worker.pool
-
-.. automodule:: celery.worker.pool
-    :members:
-    :undoc-members:

docs/internals/reference/index.rst

@@ -14,12 +14,14 @@
     celery.worker.controllers
     celery.worker.buckets
     celery.worker.scheduler
-    celery.worker.pool
     celery.worker.heartbeat
     celery.worker.control
     celery.worker.control.builtins
     celery.worker.control.registry
     celery.worker.revoke
+    celery.concurrency.processes
+    celery.concurrency.processes.pool
+    celery.concurrency.threads
     celery.beat
     celery.backends
     celery.backends.base
@@ -29,8 +31,6 @@
     celery.backends.pyredis
     celery.backends.tyrant
     celery.execute.trace
-    celery.concurrency.processes
-    celery.concurrency.threads
     celery.serialization
     celery.datastructures
     celery.routes

docs/reference/celery.conf.rst

@@ -235,7 +235,7 @@ Configuration - celery.conf
 .. data:: CELERYD_POOL
 
     Name of the task pool class used by the worker.
-    Default is ``"celery.worker.pool.TaskPool"``.
+    Default is ``"celery.concurrency.processes.TaskPool"``.
 
 .. data:: CELERYD_LISTENER