'celeryd-multi detach' now actually launches and daemonizes multiple workers

celery/bin/celeryd_multi.py

@@ -1,11 +1,14 @@
-import sys
+import os
+import shlex
 import socket
+import sys
 
 from celery.utils.compat import defaultdict
 
 EXAMPLES = """
 Some examples:
 
+    # The "start" command just prints the commands needed to start the workers.
     # Advanced example with 10 workers:
     #   * Three of the workers processes the images and video queue
     #   * Two of the workers processes the data queue with loglevel DEBUG
@@ -13,20 +16,25 @@ Some examples:
     $ celeryd-multi start 10 -l INFO -Q:1-3 images,video -Q:4,5:data
         -Q default -L:4,5 DEBUG
 
+    # To actually start the workers into the background you need to
+    # use the "detach" command:
+    $ celeryd-multi detach 10 -l INFO -Q:1-3 images,video -Q:4,5:data
+        -Q default -L:4,5 DEBUG
+
     # get commands to start 10 workers, with 3 processes each
-    $ celeryd-multi start 3 -c 3
+    $ celeryd-multi detach 3 -c 3
     celeryd -n celeryd1.myhost -c 3
     celeryd -n celeryd2.myhost -c 3
     celeryd- n celeryd3.myhost -c 3
 
     # start 3 named workers
-    $ celeryd-multi start image video data -c 3
+    $ celeryd-multi detach image video data -c 3
     celeryd -n image.myhost -c 3
     celeryd -n video.myhost -c 3
     celeryd -n data.myhost -c 3
 
     # specify custom hostname
-    $ celeryd-multi start 2 -n worker.example.com -c 3
+    $ celeryd-multi detach 2 -n worker.example.com -c 3
     celeryd -n celeryd1.worker.example.com -c 3
     celeryd -n celeryd2.worker.example.com -c 3
 
@@ -34,20 +42,20 @@ Some examples:
     # but you can also modify the options for ranges of or single workers
 
     # 3 workers: Two with 3 processes, and one with 10 processes.
-    $ celeryd-multi start 3 -c 3 -c:1 10
+    $ celeryd-multi detach 3 -c 3 -c:1 10
     celeryd -n celeryd1.myhost -c 10
     celeryd -n celeryd2.myhost -c 3
     celeryd -n celeryd3.myhost -c 3
 
     # can also specify options for named workers
-    $ celeryd-multi start image video data -c 3 -c:image 10
+    $ celeryd-multi detach image video data -c 3 -c:image 10
     celeryd -n image.myhost -c 10
     celeryd -n video.myhost -c 3
     celeryd -n data.myhost -c 3
 
     # ranges and lists of workers in options is also allowed:
     # (-c:1-3 can also be written as -c:1,2,3)
-    $ celeryd-multi start 5 -c 3  -c:1-3 10
+    $ celeryd-multi detach 5 -c 3  -c:1-3 10
     celeryd -n celeryd1.myhost -c 10
     celeryd -n celeryd2.myhost -c 10
     celeryd -n celeryd3.myhost -c 10
@@ -55,7 +63,7 @@ Some examples:
     celeryd -n celeryd5.myhost -c 3
 
     # lists also works with named workers
-    $ celeryd-multi start foo bar baz xuzzy -c 3 -c:foo,bar,baz 10
+    $ celeryd-multi detach foo bar baz xuzzy -c 3 -c:foo,bar,baz 10
     celeryd -n foo.myhost -c 10
     celeryd -n bar.myhost -c 10
     celeryd -n baz.myhost -c 10
@@ -185,10 +193,19 @@ def say(m):
     sys.stderr.write("%s\n" % (m, ))
 
 
+def detach_worker(argv):
+    path = sys.executable
+    argv = [sys.executable, "-m", "celery.bin.celeryd_detach"] + argv
+
+    if os.fork() == 0:
+        os.execv(path, argv)
+
+
 class MultiTool(object):
 
     def __init__(self):
         self.commands = {"start": self.start,
+                         "detach": self.detach,
                          "names": self.names,
                          "expand": self.expand,
                          "get": self.get,
@@ -224,6 +241,13 @@ class MultiTool(object):
         print("\n".join(worker
                         for _, worker, _ in multi_args(p, cmd)))
 
+    def detach(self, argv, cmd):
+        p = NamespacedOptionParser(argv)
+        p.options.setdefault("--pidfile", "celeryd@%n.pid")
+        p.options.setdefault("--logfile", "celeryd@%n.log")
+        for argv in [argv for _, argv, _ in multi_args(p, cmd)]:
+            detach_worker(shlex.split(argv))
+
     def expand(self, argv, cmd=None):
         template = argv[0]
         p = NamespacedOptionParser(argv[1:])