Default log format is now includes %I to avoid race conditions (one logfile per child process)

celery/bin/multi.py

@@ -16,13 +16,13 @@ Examples
     # this.  The abbreviation %n will be expanded to the current
     # node name.
     $ celery multi start Leslie -E --pidfile=/var/run/celery/%n.pid
-                                    --logfile=/var/log/celery/%n.log
+                                   --logfile=/var/log/celery/%n%I.log
 
 
     # You need to add the same arguments when you restart,
     # as these are not persisted anywhere.
     $ celery multi restart Leslie -E --pidfile=/var/run/celery/%n.pid
-                                     --logfile=/var/run/celery/%n.log
+                                     --logfile=/var/run/celery/%n%I.log
 
     # To stop the node, you need to specify the same pidfile.
     $ celery multi stop Leslie --pidfile=/var/run/celery/%n.pid
@@ -252,7 +252,7 @@ class MultiTool(object):
 
     def with_detacher_default_options(self, p):
         _setdefaultopt(p.options, ['--pidfile', '-p'], '%n.pid')
-        _setdefaultopt(p.options, ['--logfile', '-f'], '%n.log')
+        _setdefaultopt(p.options, ['--logfile', '-f'], '%n%I.log')
         p.options.setdefault(
             '--cmd',
             '-m {0}'.format(celery_exe('worker', '--detach')),

docs/tutorials/daemonizing.rst

@@ -56,7 +56,7 @@ must also export them (e.g. ``export DISPLAY=":0"``)
 
         $ celery multi start worker1 \
             --pidfile="$HOME/run/celery/%n.pid" \
-            --logfile="$HOME/log/celery/%n.log"
+            --logfile="$HOME/log/celery/%n%I.log"
 
         $ celery multi restart worker1 --pidfile="$HOME/run/celery/%n.pid"
 
@@ -97,7 +97,7 @@ This is an example configuration for a Python project.
     CELERYD_OPTS="--time-limit=300 --concurrency=8"
 
     # %n will be replaced with the first part of the nodename.
-    CELERYD_LOG_FILE="/var/log/celery/%n.log"
+    CELERYD_LOG_FILE="/var/log/celery/%n%I.log"
     CELERYD_PID_FILE="/var/run/celery/%n.pid"
 
     # Workers should run as an unprivileged user.
@@ -156,7 +156,9 @@ Available options
     Full path to the PID file. Default is /var/run/celery/%n.pid
 
 * CELERYD_LOG_FILE
-    Full path to the worker log file. Default is /var/log/celery/%n.log
+    Full path to the worker log file. Default is /var/log/celery/%n%I.log
+    **Note**: Using `%I` is important when using the prefork pool as having
+    multiple processes share the same log file will lead to race conditions.
 
 * CELERYD_LOG_LEVEL
     Worker log level. Default is INFO.
@@ -311,8 +313,10 @@ This is an example configuration for a Python project:
     # Extra command-line arguments to the worker
     CELERYD_OPTS="--time-limit=300 --concurrency=8"
 
-    # %n will be replaced with the first part of the nodename.
-    CELERYD_LOG_FILE="/var/log/celery/%n.log"
+    # - %n will be replaced with the first part of the nodename.
+    # - %I will be replaced with the current child process index
+    #   and is important when using the prefork pool to avoid race conditions.
+    CELERYD_LOG_FILE="/var/log/celery/%n%I.log"
     CELERYD_PID_FILE="/var/run/celery/%n.pid"
 
 .. _generic-systemd-celeryd-django-example:
@@ -339,8 +343,9 @@ This is an example configuration for those using `django-celery`:
     # Extra command-line arguments to the worker
     CELERYD_OPTS="--time-limit=300 --concurrency=8"
 
-    # %n will be replaced with the first part of the nodename.
-    CELERYD_LOG_FILE="/var/log/celery/%n.log"
+    # - %n will be replaced with the first part of the nodename.
+    # - %I will be replaced with the current child process index
+    CELERYD_LOG_FILE="/var/log/celery/%n%I.log"
     CELERYD_PID_FILE="/var/run/celery/%n.pid"
 
 To add an environment variable such as DJANGO_SETTINGS_MODULE use the

extra/centos/celeryd

@@ -29,7 +29,7 @@
 # Setting `prog` here allows you to symlink this init script, making it easy
 # to run multiple processes on the system.
 
-# If we're invoked via SysV-style runlevel scripts we need to follow the 
+# If we're invoked via SysV-style runlevel scripts we need to follow the
 # link from rcX.d before working out the script name.
 if [[ `dirname $0` == /etc/rc*.d ]]; then
     target="$(readlink $0)"
@@ -48,7 +48,7 @@ source /etc/init.d/functions
 SLEEP_SECONDS=5
 
 DEFAULT_PID_FILE="/var/run/celery/$prog-%n.pid"
-DEFAULT_LOG_FILE="/var/log/celery/$prog-%n.log"
+DEFAULT_LOG_FILE="/var/log/celery/$prog-%n%I.log"
 DEFAULT_LOG_LEVEL="INFO"
 DEFAULT_NODES="celery"
 DEFAULT_CELERYD="-m celery.bin.celeryd_detach"

extra/centos/celeryd.sysconfig

@@ -16,8 +16,10 @@
 # Create log/pid dirs, if they don't already exist
 #CELERY_CREATE_DIRS=1
 
-# %n will be replaced with the nodename
-#CELERYD_LOG_FILE="/path/to/my_application/log/%n.log"
+# - %n will be replaced with the first part of the nodename.
+# - %I will be replaced with the current child process index
+#   and is important when using the prefork pool to avoid race conditions.
+#CELERYD_LOG_FILE="/path/to/my_application/log/%n%I.log"
 #CELERYD_PID_FILE="/var/run/celery/%n.pid"
 
 # Workers run as an unprivileged user

extra/generic-init.d/celeryd

@@ -48,7 +48,7 @@ SCRIPT_NAME="$(basename "$SCRIPT_FILE")"
 
 DEFAULT_USER="celery"
 DEFAULT_PID_FILE="/var/run/celery/%n.pid"
-DEFAULT_LOG_FILE="/var/log/celery/%n.log"
+DEFAULT_LOG_FILE="/var/log/celery/%n%I.log"
 DEFAULT_LOG_LEVEL="INFO"
 DEFAULT_NODES="celery"
 DEFAULT_CELERYD="-m celery worker --detach"

extra/systemd/celery.conf

@@ -6,7 +6,7 @@ CELERYD_NODES="worker"
 CELERYD_OPTS=""
 CELERY_BIN="/usr/bin/python2 -m celery"
 CELERYD_PID_FILE="/var/run/celery/%n.pid"
-CELERYD_LOG_FILE="/var/log/celery/%n.log"
+CELERYD_LOG_FILE="/var/log/celery/%n%I.log"
 CELERYD_LOG_LEVEL="INFO"
 
 d /run/celery 0755 user users -