Adds --detach|-D option to celery worker command. Closes #800

celery/bin/celery.py

@@ -248,6 +248,10 @@ class worker(Delegate):
     """
     Command = 'celery.bin.celeryd:WorkerCommand'
 
+    def run_from_argv(self, prog_name, argv):
+        self.target.maybe_detach(argv)
+        super(worker, self).run_from_argv(prog_name, argv)
+
 
 @command
 class events(Delegate):

celery/bin/celeryd.py

@@ -130,12 +130,17 @@ class WorkerCommand(Command):
     enable_config_from_cmdline = True
     supports_args = False
 
-    def execute_from_commandline(self, argv=None, dopts=['-D', '--detach']):
+    def execute_from_commandline(self, argv=None):
+        self.maybe_detach(argv)
+        return super(WorkerCommand, self).execute_from_commandline(argv)
+
+    def maybe_detach(self, argv, dopts=['-D', '--detach']):
         argv = list(sys.argv) if argv is None else argv
         if any(arg in argv for arg in dopts):
             argv = [arg for arg in argv if arg not in dopts]
-            return _detach.detached_celeryd().execute_from_commandline(argv)
-        return super(WorkerCommand, self).execute_from_commandline(argv)
+            # never returns
+            _detach.detached_celeryd().execute_from_commandline(argv)
+            raise SystemExit(0)
 
     def run(self, hostname=None, pool_cls=None, loglevel=None,
             app=None, **kwargs):

celery/bin/celeryd_multi.py

@@ -153,7 +153,8 @@ class MultiTool(object):
         self.commands = {'start': self.start,
                          'show': self.show,
                          'stop': self.stop,
-                         'stop_verify': self.stop_verify,
+                         'stopwait': self.stopwait,
+                         'stop_verify': self.stopwait,
                          'restart': self.restart,
                          'kill': self.kill,
                          'names': self.names,
@@ -225,7 +226,7 @@ class MultiTool(object):
     def with_detacher_default_options(self, p):
         p.options.setdefault('--pidfile', 'celeryd@%n.pid')
         p.options.setdefault('--logfile', 'celeryd@%n.log')
-        p.options.setdefault('--cmd', '-m celery.bin.celeryd_detach')
+        p.options.setdefault('--cmd', '-m celery worker --detach')
 
     def signal_node(self, nodename, pid, sig):
         try:
@@ -345,11 +346,12 @@ class MultiTool(object):
         self._stop_nodes(p, cmd, retry=2, callback=on_node_shutdown)
         self.retval = int(any(retvals))
 
-    def stop_verify(self, argv, cmd):
+    def stopwait(self, argv, cmd):
         self.splash()
         p = NamespacedOptionParser(argv)
         self.with_detacher_default_options(p)
         return self._stop_nodes(p, cmd, retry=2)
+    stop_verify = stopwait  # compat
 
     def expand(self, argv, cmd=None):
         template = argv[0]

celery/platforms.py

@@ -272,7 +272,9 @@ class DaemonContext(object):
             os.chdir(self.workdir)
             os.umask(self.umask)
 
-            os.closerange(1, get_fdmax(default=2048))
+            for fd in reversed(range(get_fdmax(default=2048))):
+                with ignore_EBADF():
+                    os.close(fd)
             os.open(DAEMON_REDIRECT_TO, os.O_RDWR)
             os.dup2(0, 1)
             os.dup2(0, 2)

celery/tests/bin/test_celeryd_multi.py

@@ -417,9 +417,9 @@ class test_MultiTool(Case):
         self.assertTrue(self.t.verbose)
         self.assertTrue(self.t.no_color)
 
-    def test_stop_verify(self):
+    def test_stopwait(self):
         self.t._stop_nodes = Mock()
-        self.t.stop_verify(['foo', 'bar', 'baz'], 'celeryd')
+        self.t.stopwait(['foo', 'bar', 'baz'], 'celeryd')
         self.assertEqual(self.t._stop_nodes.call_args[1]['retry'], 2)
 
     @patch('celery.bin.celeryd_multi.MultiTool')

extra/centos/celeryd.init

@@ -35,8 +35,8 @@ prog="$(basename $0)"
 # on RHEL systems.
 [ -f "/etc/sysconfig/$prog" ] && . /etc/sysconfig/$prog
 
-CELERYD=${CELERYD:-"-m celery.bin.celeryd_detach"}
-CELERYD_MULTI=${CELERYD_MULTI:-"/usr/bin/celeryd-multi"}
+CELERYD=${CELERYD:-"-m celery worker --detach"}
+CELERY=${CELERY:-"/usr/bin/celery"}
 CELERYD_PID_FILE=${CELERYD_PID_FILE:-"/var/run/celeryd/$prog.pid"}
 CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-"/var/log/celeryd/$prog.log"}
 CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-"INFO"}
@@ -70,7 +70,7 @@ start_workers () {
     fi
 
     echo -n $"Starting $prog: "
-    $CELERYD_MULTI start $prog \
+    $CELERY multi start $prog \
                                --pidfile=$CELERYD_PID_FILE \
                                --logfile=$CELERYD_LOG_FILE \
                                --loglevel=$CELERYD_LOG_LEVEL \
@@ -99,7 +99,7 @@ stop_workers () {
     fi
 
     # First, try to nicely shut it down.
-    $CELERYD_MULTI stop $prog --pidfile=$CELERYD_PID_FILE --quiet
+    $CELERY stop $prog --pidfile=$CELERYD_PID_FILE --quiet
     RETVAL=$?
 
     # SLeep a few seconds. (this was part of the original script; we can't
@@ -109,7 +109,7 @@ stop_workers () {
 
     # If we haven't ended, explicitly kill it!
     if [ -f $CELERYD_PID_FILE ] && [ -e /proc/$(cat $CELERYD_PID_FILE) ]; then
-        $CELERYD_MULTI stop $prog -KILL --pidfile=$CELERYD_PID_FILE --quiet
+        $CELERY stop $prog -KILL --pidfile=$CELERYD_PID_FILE --quiet
     fi
 
     if [ "$RETVAL" == "0" ]; then

extra/generic-init.d/celerybeat

@@ -24,7 +24,7 @@
 DEFAULT_PID_FILE="/var/run/celerybeat.pid"
 DEFAULT_LOG_FILE="/var/log/celerybeat.log"
 DEFAULT_LOG_LEVEL="INFO"
-DEFAULT_CELERYBEAT="celerybeat"
+DEFAULT_CELERYBEAT="celery beat"
 
 # /etc/init.d/ssh: start and stop the celery task worker daemon.
 

extra/generic-init.d/celeryd

@@ -24,7 +24,7 @@ DEFAULT_PID_FILE="/var/run/celeryd@%n.pid"
 DEFAULT_LOG_FILE="/var/log/celeryd@%n.log"
 DEFAULT_LOG_LEVEL="INFO"
 DEFAULT_NODES="celery"
-DEFAULT_CELERYD="-m celery.bin.celeryd_detach"
+DEFAULT_CELERYD="-m celery worker --detach"
 
 # /etc/init.d/celeryd: start and stop the celery task worker daemon.
 
@@ -38,9 +38,9 @@ fi
 CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}}
 CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}}
 CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}}
-CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"}
+CELERYD_MULTI=${CELERYD_MULTI:-"celery multi"}
 CELERYD=${CELERYD:-$DEFAULT_CELERYD}
-CELERYCTL=${CELERYCTL:="celeryctl"}
+CELERYCTL=${CELERYCTL:="celery"}
 CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES}
 
 export CELERY_LOADER
@@ -97,7 +97,7 @@ export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
 
 
 stop_workers () {
-    $CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE"
+    $CELERYD_MULTI stopwait $CELERYD_NODES --pidfile="$CELERYD_PID_FILE"
 }