#321 alternative solution to log rotate, use WatchedFileHandlers

to watch the logfiles, and use system or custom solutions for
logrotate.

celery/log.py

@@ -14,6 +14,7 @@ from celery import signals
 from celery import current_app
 from celery.utils import LOG_LEVELS, isatty
 from celery.utils.compat import LoggerAdapter
+from celery.utils.compat import WatchedFileHandler
 from celery.utils.encoding import safe_str
 from celery.utils.patch import ensure_process_aware_logger
 from celery.utils.term import colored
@@ -23,7 +24,7 @@ class ColorFormatter(logging.Formatter):
     #: Loglevel -> Color mapping.
     COLORS = colored().names
     colors = {"DEBUG": COLORS["blue"], "WARNING": COLORS["yellow"],
-              "ERROR": COLORS["red"],  "CRITICAL": COLORS["magenta"]}
+              "ERROR": COLORS["red"], "CRITICAL": COLORS["magenta"]}
 
     def __init__(self, msg, use_color=True):
         logging.Formatter.__init__(self, msg)
@@ -132,7 +133,7 @@ class Logging(object):
             logfile = sys.__stderr__
         if hasattr(logfile, "write"):
             return logging.StreamHandler(logfile)
-        return logging.FileHandler(logfile)
+        return WatchedFileHandler(logfile)
 
     def get_default_logger(self, loglevel=None, name="celery"):
         """Get default logger instance.

celery/utils/compat.py

@@ -481,3 +481,67 @@ try:
     chain_from_iterable = getattr(chain, "from_iterable")
 except AttributeError:
     chain_from_iterable = _compat_chain_from_iterable
+
+
+############## logging.handlers.WatchedFileHandler ##########################
+import os
+from stat import ST_DEV, ST_INO
+import platform as _platform
+
+if _platform.system() == "Windows":
+    #since windows doesn't go with WatchedFileHandler use FileHandler instead
+    WatchedFileHandler = logging.FileHandler
+else:
+    try:
+        from logging.handlers import WatchedFileHandler
+    except ImportError:
+        class WatchedFileHandler(logging.FileHandler):
+            """
+            A handler for logging to a file, which watches the file
+            to see if it has changed while in use. This can happen because of
+            usage of programs such as newsyslog and logrotate which perform
+            log file rotation. This handler, intended for use under Unix,
+            watches the file to see if it has changed since the last emit.
+            (A file has changed if its device or inode have changed.)
+            If it has changed, the old file stream is closed, and the file
+            opened to get a new stream.
+        
+            This handler is not appropriate for use under Windows, because
+            under Windows open files cannot be moved or renamed - logging
+            opens the files with exclusive locks - and so there is no need
+            for such a handler. Furthermore, ST_INO is not supported under
+            Windows; stat always returns zero for this value.
+        
+            This handler is based on a suggestion and patch by Chad J.
+            Schroeder.
+            """
+            def __init__(self, filename, mode='a', encoding=None, delay=0):
+                logging.FileHandler.__init__(self, filename, mode, encoding, delay)
+                if not os.path.exists(self.baseFilename):
+                    self.dev, self.ino = -1, -1
+                else:
+                    stat = os.stat(self.baseFilename)
+                    self.dev, self.ino = stat[ST_DEV], stat[ST_INO]
+
+            def emit(self, record):
+                """
+                Emit a record.
+        
+                First check if the underlying file has changed, and if it
+                has, close the old stream and reopen the file to get the
+                current stream.
+                """
+                if not os.path.exists(self.baseFilename):
+                    stat = None
+                    changed = 1
+                else:
+                    stat = os.stat(self.baseFilename)
+                    changed = (stat[ST_DEV] != self.dev) or (stat[ST_INO] != self.ino)
+                if changed and self.stream is not None:
+                    self.stream.flush()
+                    self.stream.close()
+                    self.stream = self._open()
+                    if stat is None:
+                        stat = os.stat(self.baseFilename)
+                    self.dev, self.ino = stat[ST_DEV], stat[ST_INO]
+                logging.FileHandler.emit(self, record)