Add new option that forces using billiard process name in logs; fixes #1330

celery/app/base.py

@@ -31,6 +31,7 @@ from celery.loaders import get_loader_cls
 from celery.local import PromiseProxy, maybe_evaluate
 from celery.utils.functional import first
 from celery.utils.imports import instantiate, symbol_by_name
+from celery.utils.log import ensure_process_aware_logger
 from celery.utils.objects import mro_lookup
 
 from .annotations import prepare as prepare_annotations
@@ -256,6 +257,12 @@ class Celery(object):
         self.conf.update(self.loader.cmdline_config_parser(argv, namespace))
 
     def autodiscover_tasks(self, packages, related_name='tasks'):
+        if self.conf.CELERY_FORCE_BILLIARD_LOGGING:
+            # we'll use billiard's processName instead of
+            # multiprocessing's one in all the loggers
+            # created after this call
+            ensure_process_aware_logger()
+
         self.loader.autodiscover_tasks(packages, related_name)
 
     def send_task(self, name, args=None, kwargs=None, countdown=None,

celery/app/defaults.py

@@ -94,6 +94,7 @@ NAMESPACES = {
         'ACKS_LATE': Option(False, type='bool'),
         'ALWAYS_EAGER': Option(False, type='bool'),
         'ANNOTATIONS': Option(type='any'),
+        'FORCE_BILLIARD_LOGGING': Option(True, type='bool'),
         'BROADCAST_QUEUE': Option('celeryctl'),
         'BROADCAST_EXCHANGE': Option('celeryctl'),
         'BROADCAST_EXCHANGE_TYPE': Option('fanout'),

docs/configuration.rst

@@ -1510,6 +1510,23 @@ Can be one of :const:`DEBUG`, :const:`INFO`, :const:`WARNING`,
 
 Default is :const:`WARNING`.
 
+.. setting:: CELERY_FORCE_BILLIARD_LOGGING
+
+CELERY_FORCE_BILLIARD_LOGGING
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 3.1
+
+Celery uses :mod:`multiprocessing`'s fork called `billiard` as a pool
+implementation. Python assumes we use :mod:`multiprocessing` when trying
+to log `processName` though. By default this option forces Celery to modify
+the logger class as early as possible in order to provide correct process
+name in log messages. If you are going to use :mod:`multiprocessing` along
+with Celery, you can disable this behavior by setting
+`CELERY_FORCE_BILLIARD_LOGGING = False`.
+
+Default is :const:`True`.
+
 .. _conf-security:
 
 Security