Merge branch 'nott/logging_processName'

celery/app/base.py

@@ -31,6 +31,7 @@ from celery.loaders import get_loader_cls
 from celery.local import PromiseProxy, maybe_evaluate
 from celery.utils.functional import first
 from celery.utils.imports import instantiate, symbol_by_name
+from celery.utils.log import ensure_process_aware_logger
 from celery.utils.objects import mro_lookup
 
 from .annotations import prepare as prepare_annotations
@@ -260,6 +261,12 @@ class Celery(object):
         self.conf.update(self.loader.cmdline_config_parser(argv, namespace))
 
     def autodiscover_tasks(self, packages, related_name='tasks'):
+        if self.conf.CELERY_FORCE_BILLIARD_LOGGING:
+            # we'll use billiard's processName instead of
+            # multiprocessing's one in all the loggers
+            # created after this call
+            ensure_process_aware_logger()
+
         self.loader.autodiscover_tasks(packages, related_name)
 
     def send_task(self, name, args=None, kwargs=None, countdown=None,

celery/app/defaults.py

@@ -94,6 +94,7 @@ NAMESPACES = {
         'ACKS_LATE': Option(False, type='bool'),
         'ALWAYS_EAGER': Option(False, type='bool'),
         'ANNOTATIONS': Option(type='any'),
+        'FORCE_BILLIARD_LOGGING': Option(True, type='bool'),
         'BROADCAST_QUEUE': Option('celeryctl'),
         'BROADCAST_EXCHANGE': Option('celeryctl'),
         'BROADCAST_EXCHANGE_TYPE': Option('fanout'),

celery/app/log.py

@@ -32,8 +32,6 @@ from celery.utils.log import (
 )
 from celery.utils.term import colored
 
-PY3 = sys.version_info[0] == 3
-
 MP_LOG = os.environ.get('MP_LOG', False)
 
 
@@ -95,8 +93,7 @@ class Logging(object):
         format = format or self.format
         colorize = self.supports_color(colorize, logfile)
         reset_multiprocessing_logger()
-        if not PY3:
-            ensure_process_aware_logger()
+        ensure_process_aware_logger()
         receivers = signals.setup_logging.send(
             sender=None, loglevel=loglevel, logfile=logfile,
             format=format, colorize=colorize,

celery/utils/log.py

@@ -112,12 +112,6 @@ class ColorFormatter(logging.Formatter):
                     type(msg), exc)
                 record.exc_info = True
 
-        if not PY3 and 'processName' not in record.__dict__:
-            # Very ugly, but have to make sure processName is supported
-            # by foreign logger instances.
-            # (processName is always supported by Python 2.7)
-            process_name = current_process and current_process()._name or ''
-            record.__dict__['processName'] = process_name
         return safe_str(logging.Formatter.format(self, record))
 
 

docs/configuration.rst

@@ -1507,6 +1507,23 @@ Can be one of :const:`DEBUG`, :const:`INFO`, :const:`WARNING`,
 
 Default is :const:`WARNING`.
 
+.. setting:: CELERY_FORCE_BILLIARD_LOGGING
+
+CELERY_FORCE_BILLIARD_LOGGING
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 3.1
+
+Celery uses :mod:`multiprocessing`'s fork called `billiard` as a pool
+implementation. Python assumes we use :mod:`multiprocessing` when trying
+to log `processName` though. By default this option forces Celery to modify
+the logger class as early as possible in order to provide correct process
+name in log messages. If you are going to use :mod:`multiprocessing` along
+with Celery, you can disable this behavior by setting
+`CELERY_FORCE_BILLIARD_LOGGING = False`.
+
+Default is :const:`True`.
+
 .. _conf-security:
 
 Security