Fix celery beat --detach in PyPy

While running celery beat under PyPy, file descriptor pointing to
/dev/urandom is closed while daemonizing. This makes shelve, and
in turn beat's scheduler, unable to access it, hence the startup
fails with OSError 9. This is fixed by /dev/urandom's fd to
keep lsit passed to close_open_fds.

CONTRIBUTORS.txt

@@ -193,3 +193,4 @@ Justin Patrin, 2015/08/06
 Juan Rossi, 2015/08/10
 Piotr Maślanka, 2015/08/24
 Gerald Manipon, 2015/10/19
+Krzysztof Bujniewicz, 2015/10/21

Changelog

@@ -31,6 +31,10 @@ new in Celery 3.1.
 - **Beat**: PyPy shelve may raise ``KeyError`` when setting keys
   (Issue #2862).
 
+- **Programs**: :program:`celery beat --deatched` now working on PyPy.
+
+    Fix contributed by Krzysztof Bujniewicz.
+
 - **Results**: Redis result backend now ensures all pipelines are cleaned up.
 
     Contributed by Justin Patrin.
@@ -57,6 +61,34 @@ new in Celery 3.1.
 
     Fix contributed by Sukrit Khera.
 
+- **Results**: RPC/amqp backends did not deserialize exceptions properly
+  (Issue #2691).
+
+    Fix contributed by Sukrit Khera.
+
+- **Programs**: Fixed problem with :program:`celery amqp`'s
+  ``basic_publish`` (Issue #2013).
+
+- **Worker**: Embedded beat now properly sets app for thread/process
+  (Issue #2594).
+
+- **Documentation**: Many improvements and typos fixed.
+
+    Contributions by:
+
+        Carlos Garcia-Dubus
+        D. Yu
+        jerry
+        Jocelyn Delalande
+        Josh Kupershmidt
+        Juan Rossi
+        kanemra
+        Paul Pearce
+        Pavel Savchenko
+        Sean Wang
+        Seungha Kim
+        Zhaorong Ma
+
 .. _version-3.1.18:
 
 3.1.18

celery/platforms.py

@@ -43,7 +43,8 @@ __all__ = ['EX_OK', 'EX_FAILURE', 'EX_UNAVAILABLE', 'EX_USAGE', 'SYSTEM',
            'close_open_fds', 'DaemonContext', 'detached', 'parse_uid',
            'parse_gid', 'setgroups', 'initgroups', 'setgid', 'setuid',
            'maybe_drop_privileges', 'signals', 'set_process_title',
-           'set_mp_process_title', 'get_errno_name', 'ignore_errno']
+           'set_mp_process_title', 'get_errno_name', 'ignore_errno',
+           'fd_by_path']
 
 # exitcodes
 EX_OK = getattr(os, 'EX_OK', 0)
@@ -264,6 +265,43 @@ def _create_pidlock(pidfile):
     pidlock.acquire()
     return pidlock
 
+def fd_by_path(paths):
+    """
+    Return a list of fds.
+
+    This method returns list of fds corresponding to
+    file paths passed in paths variable.
+
+    :keyword paths: List of file paths go get fd for.
+
+    :returns: :list:.
+
+    **Example**:
+
+    .. code-block:: python
+
+        keep = fd_by_path(['/dev/urandom',
+                           '/my/precious/'])
+    """
+    stats = set()
+    for path in paths:
+        try:
+            fd = os.open(path, os.O_RDONLY)
+        except OSError:
+            continue
+        try:
+            stats.add(os.fstat(fd)[1:3])
+        finally:
+            os.close(fd)
+
+    def fd_in_stats(fd):
+        try:
+            return os.fstat(fd)[1:3] in stats
+        except OSError:
+            return False
+
+    return [fd for fd in range(get_fdmax(2048)) if fd_in_stats(fd)]
+
 
 if hasattr(os, 'closerange'):
 
@@ -321,7 +359,10 @@ class DaemonContext(object):
                 self.after_chdir()
 
             if not self.fake:
-                close_open_fds(self.stdfds)
+                # We need to keep /dev/urandom from closing because
+                # shelve needs it, and Beat needs shelve to start.
+                keep = list(self.stdfds) + fd_by_path(['/dev/urandom'])
+                close_open_fds(keep)
                 for fd in self.stdfds:
                     self.redirect_to_null(maybe_fileno(fd))
 

celery/tests/utils/test_platforms.py

@@ -4,6 +4,7 @@ import errno
 import os
 import sys
 import signal
+import tempfile
 
 from celery import _find_option_with_arg
 from celery import platforms
@@ -27,6 +28,7 @@ from celery.platforms import (
     setgroups,
     _setgroups_hack,
     close_open_fds,
+    fd_by_path,
 )
 
 try:
@@ -54,6 +56,14 @@ class test_find_option_with_arg(Case):
             'bar'
         )
 
+class test_fd_by_path(Case):
+
+    def test_finds(self):
+        test_file = tempfile.NamedTemporaryFile()
+        keep = fd_by_path([test_file.name])
+        self.assertEqual(keep, [test_file.file.fileno()])
+        test_file.close()
+
 
 class test_close_open_fds(Case):