Got a 3x performance gain by setting the prefetch count to 4 times the concurrency, (from an average task round-trip of 0.1s to 0.03s!). So added a new setting: CELERYD_PREFETCH_MULTIPLIER which is set to 4 by default.

celery/conf.py

@@ -28,6 +28,7 @@ _DEFAULTS = {
     "CELERY_BROKER_CONNECTION_RETRY": True,
     "CELERY_BROKER_CONNECTION_MAX_RETRIES": 100,
     "CELERYD_CONCURRENCY": 0, # defaults to cpu count
+    "CELERYD_PREFETCH_MULTIPLIER": 4,
     "CELERYD_LOG_FORMAT": DEFAULT_P_LOG_FMT,
     "CELERYD_LOG_LEVEL": "WARN",
     "CELERYD_LOG_FILE": None, # stderr
@@ -94,6 +95,7 @@ CELERYD_LOG_LEVEL = _get("CELERYD_LOG_LEVEL",
                         compat=["CELERYD_DAEMON_LOG_LEVEL"])
 CELERYD_LOG_LEVEL = LOG_LEVELS[CELERYD_LOG_LEVEL.upper()]
 CELERYD_CONCURRENCY = _get("CELERYD_CONCURRENCY")
+CELERYD_PREFETCH_MULTIPLIER = _get("CELERYD_PREFETCH_MULTIPLIER")
 
 # <--- Message routing                             <-   --   --- - ----- -- #
 QUEUES = _get("CELERY_QUEUES")

celery/worker/__init__.py

@@ -134,11 +134,12 @@ class WorkController(object):
         self.clockservice = self.embed_clockservice and ClockServiceThread(
                                 logger=self.logger,
                                 max_interval=1) or None
+        prefetch_count = concurrency * conf.CELERYD_PREFETCH_MULTIPLIER
         self.listener = CarrotListener(self.ready_queue,
                                        self.eta_schedule,
                                        logger=self.logger,
                                        send_events=send_events,
-                                       initial_prefetch_count=concurrency)
+                                       initial_prefetch_count=prefetch_count)
 
         # The order is important here;
         #   the first in the list is the first to start,

docs/configuration.rst

@@ -48,6 +48,16 @@ Concurrency settings
     Defaults to the number of CPUs/cores available.
 
 
+* CELERYD_PREFETCH_MULTIPLIER
+    How many messages to prefetch at a time multiplied by the number of
+    concurrent processes. The default is 4 (four messages for each
+    process). The default setting seems pretty good here, but if you have
+    very long running tasks waiting in the queue and you have to start the
+    workers, make note that the first worker to start will receive four times the
+    number of messages initially, which might not be fairly balanced among the
+    workers.
+
+
 Task result backend settings
 ============================