[multi] Now supports FQDN nodenames (name@hostname), new expansion variable %N added

celery/bin/celeryd_multi.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
 
-.. program:: celeryd-multi
+.. program:: celery multi
 
 Examples
 ========
@@ -9,83 +9,86 @@ Examples
 .. code-block:: bash
 
     # Single worker with explicit name and events enabled.
-    $ celeryd-multi start Leslie -E
+    $ celery multi start Leslie -E
 
     # Pidfiles and logfiles are stored in the current directory
     # by default.  Use --pidfile and --logfile argument to change
-    # this.  The abbreviation %n will be expanded to the current
+    # this.  The abbreviation %N will be expanded to the current
     # node name.
-    $ celeryd-multi start Leslie -E --pidfile=/var/run/celery/%n.pid
-                                    --logfile=/var/log/celery/%n.log
+    $ celery multi start Leslie -E --pidfile=/var/run/celery/%N.pid
+                                    --logfile=/var/log/celery/%N.log
 
 
     # You need to add the same arguments when you restart,
     # as these are not persisted anywhere.
-    $ celeryd-multi restart Leslie -E --pidfile=/var/run/celery/%n.pid
-                                      --logfile=/var/run/celery/%n.log
+    $ celery multi restart Leslie -E --pidfile=/var/run/celery/%N.pid
+                                     --logfile=/var/run/celery/%N.log
 
     # To stop the node, you need to specify the same pidfile.
-    $ celeryd-multi stop Leslie --pidfile=/var/run/celery/%n.pid
+    $ celery multi stop Leslie --pidfile=/var/run/celery/%N.pid
 
     # 3 workers, with 3 processes each
-    $ celeryd-multi start 3 -c 3
-    celeryd -n celeryd1.myhost -c 3
-    celeryd -n celeryd2.myhost -c 3
-    celeryd- n celeryd3.myhost -c 3
+    $ celery multi start 3 -c 3
+    celery worker -n celery1@myhost -c 3
+    celery worker -n celery2@myhost -c 3
+    celery worker -n celery3@myhost -c 3
 
     # start 3 named workers
-    $ celeryd-multi start image video data -c 3
-    celeryd -n image.myhost -c 3
-    celeryd -n video.myhost -c 3
-    celeryd -n data.myhost -c 3
+    $ celery multi start image video data -c 3
+    celery worker -n image@myhost -c 3
+    celery worker -n video@myhost -c 3
+    celery worker -n data@myhost -c 3
 
     # specify custom hostname
-    $ celeryd-multi start 2 -n worker.example.com -c 3
-    celeryd -n celeryd1.worker.example.com -c 3
-    celeryd -n celeryd2.worker.example.com -c 3
+    $ celery multi start 2 --hostname=worker.example.com -c 3
+    celery worker -n celery1@worker.example.com -c 3
+    celery worker -n celery2@worker.example.com -c 3
+
+    # specify fully qualified nodenames
+    $ celery multi start foo@worker.example.com bar@worker.example.com -c 3
 
     # Advanced example starting 10 workers in the background:
     #   * Three of the workers processes the images and video queue
     #   * Two of the workers processes the data queue with loglevel DEBUG
     #   * the rest processes the default' queue.
-    $ celeryd-multi start 10 -l INFO -Q:1-3 images,video -Q:4,5 data
+    $ celery multi start 10 -l INFO -Q:1-3 images,video -Q:4,5 data
         -Q default -L:4,5 DEBUG
 
     # You can show the commands necessary to start the workers with
     # the 'show' command:
-    $ celeryd-multi show 10 -l INFO -Q:1-3 images,video -Q:4,5 data
+    $ celery multi show 10 -l INFO -Q:1-3 images,video -Q:4,5 data
         -Q default -L:4,5 DEBUG
 
-    # Additional options are added to each celeryd',
+    # Additional options are added to each celery worker' comamnd,
     # but you can also modify the options for ranges of, or specific workers
 
     # 3 workers: Two with 3 processes, and one with 10 processes.
-    $ celeryd-multi start 3 -c 3 -c:1 10
-    celeryd -n celeryd1.myhost -c 10
-    celeryd -n celeryd2.myhost -c 3
-    celeryd -n celeryd3.myhost -c 3
+    $ celery multi start 3 -c 3 -c:1 10
+    celery worker -n celery1@myhost -c 10
+    celery worker -n celery2@myhost -c 3
+    celery worker -n celery3@myhost -c 3
 
     # can also specify options for named workers
-    $ celeryd-multi start image video data -c 3 -c:image 10
-    celeryd -n image.myhost -c 10
-    celeryd -n video.myhost -c 3
-    celeryd -n data.myhost -c 3
+    $ celery multi start image video data -c 3 -c:image 10
+    celery worker -n image@myhost -c 10
+    celery worker -n video@myhost -c 3
+    celery worker -n data@myhost -c 3
 
     # ranges and lists of workers in options is also allowed:
     # (-c:1-3 can also be written as -c:1,2,3)
-    $ celeryd-multi start 5 -c 3  -c:1-3 10
-    celeryd -n celeryd1.myhost -c 10
-    celeryd -n celeryd2.myhost -c 10
-    celeryd -n celeryd3.myhost -c 10
-    celeryd -n celeryd4.myhost -c 3
-    celeryd -n celeryd5.myhost -c 3
+    $ celery multi start 5 -c 3  -c:1-3 10
+    celery worker -n celery1@myhost -c 10
+    celery worker -n celery2@myhost -c 10
+    celery worker -n celery3@myhost -c 10
+    celery worker -n celery4@myhost -c 3
+    celery worker -n celery5@myhost -c 3
 
     # lists also works with named workers
-    $ celeryd-multi start foo bar baz xuzzy -c 3 -c:foo,bar,baz 10
-    celeryd -n foo.myhost -c 10
-    celeryd -n bar.myhost -c 10
-    celeryd -n baz.myhost -c 10
-    celeryd -n xuzzy.myhost -c 3
+    $ celery multi start foo bar baz xuzzy -c 3 -c:foo,bar,baz 10
+    celery worker -n foo@myhost -c 10
+    celery worker -n bar@myhost -c 10
+    celery worker -n baz@myhost -c 10
+    celery worker -n xuzzy@myhost -c 3
 
 """
 from __future__ import absolute_import, print_function
@@ -107,7 +110,7 @@ from kombu.utils.encoding import from_utf8
 from celery import VERSION_BANNER
 from celery.five import items
 from celery.platforms import Pidfile, IS_WINDOWS
-from celery.utils import term
+from celery.utils import term, nodesplit
 from celery.utils.text import pluralize
 
 SIGNAMES = set(sig for sig in dir(signal)
@@ -115,13 +118,13 @@ SIGNAMES = set(sig for sig in dir(signal)
 SIGMAP = dict((getattr(signal, name), name) for name in SIGNAMES)
 
 USAGE = """\
-usage: {prog_name} start <node1 node2 nodeN|range> [celeryd options]
+usage: {prog_name} start <node1 node2 nodeN|range> [worker options]
        {prog_name} stop <n1 n2 nN|range> [-SIG (default: -TERM)]
-       {prog_name} restart <n1 n2 nN|range> [-SIG] [celeryd options]
+       {prog_name} restart <n1 n2 nN|range> [-SIG] [worker options]
        {prog_name} kill <n1 n2 nN|range>
 
-       {prog_name} show <n1 n2 nN|range> [celeryd options]
-       {prog_name} get hostname <n1 n2 nN|range> [-qv] [celeryd options]
+       {prog_name} show <n1 n2 nN|range> [worker options]
+       {prog_name} get hostname <n1 n2 nN|range> [-qv] [worker options]
        {prog_name} names <n1 n2 nN|range>
        {prog_name} expand template <n1 n2 nN|range>
        {prog_name} help
@@ -150,7 +153,7 @@ class MultiTool(object):
         self.quiet = quiet
         self.verbose = verbose
         self.no_color = no_color
-        self.prog_name = 'celeryd-multi'
+        self.prog_name = 'celery multi'
         self.commands = {'start': self.start,
                          'show': self.show,
                          'stop': self.stop,
@@ -163,7 +166,7 @@ class MultiTool(object):
                          'get': self.get,
                          'help': self.help}
 
-    def execute_from_commandline(self, argv, cmd='celeryd'):
+    def execute_from_commandline(self, argv, cmd='celery worker'):
         argv = list(argv)   # don't modify callers argv.
 
         # Reserve the --nosplash|--quiet|-q/--verbose options.
@@ -225,8 +228,8 @@ class MultiTool(object):
         self.retcode = int(any(retcodes))
 
     def with_detacher_default_options(self, p):
-        p.options.setdefault('--pidfile', 'celeryd@%n.pid')
-        p.options.setdefault('--logfile', 'celeryd@%n.log')
+        p.options.setdefault('--pidfile', '%N.pid')
+        p.options.setdefault('--logfile', '%N.log')
         p.options.setdefault('--cmd', '-m celery worker --detach')
 
     def signal_node(self, nodename, pid, sig):
@@ -293,7 +296,7 @@ class MultiTool(object):
             self.note('')
 
     def getpids(self, p, cmd, callback=None):
-        pidfile_template = p.options.setdefault('--pidfile', 'celeryd@%n.pid')
+        pidfile_template = p.options.setdefault('--pidfile', '%N.pid')
 
         nodes = []
         for nodename, argv, expander in multi_args(p, cmd):
@@ -370,7 +373,7 @@ class MultiTool(object):
     def splash(self):
         if not self.nosplash:
             c = self.colored
-            self.note(c.cyan('celeryd-multi v{0}'.format(VERSION_BANNER)))
+            self.note(c.cyan('celery multi v{0}'.format(VERSION_BANNER)))
 
     def waitexec(self, argv, path=sys.executable):
         args = ' '.join([path] + list(argv))
@@ -417,7 +420,7 @@ class MultiTool(object):
         return str(self.colored.magenta('DOWN'))
 
 
-def multi_args(p, cmd='celeryd', append='', prefix='', suffix=''):
+def multi_args(p, cmd='celery worker', append='', prefix='', suffix=''):
     names = p.values
     options = dict(p.options)
     passthrough = p.passthrough
@@ -435,7 +438,7 @@ def multi_args(p, cmd='celeryd', append='', prefix='', suffix=''):
     hostname = options.pop('--hostname',
                    options.pop('-n', socket.gethostname()))
     prefix = options.pop('--prefix', prefix) or ''
-    suffix = options.pop('--suffix', suffix) or '.' + hostname
+    suffix = options.pop('--suffix', suffix) or hostname
     if suffix in ('""', "''"):
         suffix = ''
 
@@ -446,9 +449,18 @@ def multi_args(p, cmd='celeryd', append='', prefix='', suffix=''):
             p.namespaces.pop(ns_name)
 
     for name in names:
-        this_name = options['-n'] = prefix + name + suffix
+        this_suffix = suffix
+        if '@' in name:
+            this_name = options['-n'] = name
+            nodename, this_suffix = nodesplit(name)
+            name = nodename
+        else:
+            nodename = '%s%s' % (prefix, name)
+            this_name = options['-n'] = '%s@%s' % (nodename,  this_suffix)
         expand = abbreviations({'%h': this_name,
-                                '%n': name})
+                                '%n': name,
+                                '%N': nodename,
+                                '%d': this_suffix})
         argv = ([expand(cmd)] +
                 [format_opt(opt, expand(value))
                         for opt, value in items(p.optmerge(name, options))] +

celery/tests/bin/test_celeryd_multi.py

@@ -99,29 +99,29 @@ class test_multi_args(Case):
                 self.assertIn(arg, argv)
 
 
-        assert_line_in('*P*jerry*S*',
+        assert_line_in('*P*jerry@*S*',
             [
-                'COMMAND', '-n *P*jerry*S*', '-Q bar',
+                'COMMAND', '-n *P*jerry@*S*', '-Q bar',
                 '-c 5', '--flag', '--logfile=foo',
                 '-- .disable_rate_limits=1', '*AP*',
             ]
         )
-        assert_line_in('*P*elaine*S*',
+        assert_line_in('*P*elaine@*S*',
             [
-                'COMMAND', '-n *P*elaine*S*', '-Q bar',
+                'COMMAND', '-n *P*elaine@*S*', '-Q bar',
                 '-c 5', '--flag', '--logfile=foo',
                 '-- .disable_rate_limits=1', '*AP*',
             ]
         )
-        assert_line_in('*P*kramer*S*',
+        assert_line_in('*P*kramer@*S*',
             [
-                'COMMAND', '--loglevel=DEBUG', '-n *P*kramer*S*',
+                'COMMAND', '--loglevel=DEBUG', '-n *P*kramer@*S*',
                 '-Q bar', '--flag', '--logfile=foo',
                 '-- .disable_rate_limits=1', '*AP*',
             ]
         )
         expand = names[0][2]
-        self.assertEqual(expand('%h'), '*P*jerry*S*')
+        self.assertEqual(expand('%h'), '*P*jerry@*S*')
         self.assertEqual(expand('%n'), 'jerry')
         names2 = list(multi_args(p, cmd='COMMAND', append='',
                 prefix='*P*', suffix='*S*'))
@@ -131,21 +131,21 @@ class test_multi_args(Case):
         p2 = NamespacedOptionParser(['10', '-c:1', '5'])
         names3 = list(multi_args(p2, cmd='COMMAND'))
         self.assertEqual(len(names3), 10)
-        self.assertEqual(names3[0][0:2], ('celery1.example.com',
-            ['COMMAND', '-n celery1.example.com', '-c 5', '']))
+        self.assertEqual(names3[0][0:2], ('celery1@example.com',
+            ['COMMAND', '-n celery1@example.com', '-c 5', '']))
         for i, worker in enumerate(names3[1:]):
-            self.assertEqual(worker[0:2], ('celery%s.example.com' % (i + 2),
-                ['COMMAND', '-n celery%s.example.com' % (i + 2), '']))
+            self.assertEqual(worker[0:2], ('celery%s@example.com' % (i + 2),
+                ['COMMAND', '-n celery%s@example.com' % (i + 2), '']))
 
         names4 = list(multi_args(p2, cmd='COMMAND', suffix='""'))
         self.assertEqual(len(names4), 10)
-        self.assertEqual(names4[0][0:2], ('celery1',
-            ['COMMAND', '-n celery1', '-c 5', '']))
+        self.assertEqual(names4[0][0:2], ('celery1@',
+            ['COMMAND', '-n celery1@', '-c 5', '']))
 
-        p3 = NamespacedOptionParser(['foo', '-c:foo', '5'])
+        p3 = NamespacedOptionParser(['foo@', '-c:foo', '5'])
         names5 = list(multi_args(p3, cmd='COMMAND', suffix='""'))
-        self.assertEqual(names5[0][0:2], ('foo',
-            ['COMMAND', '-n foo', '-c 5', '']))
+        self.assertEqual(names5[0][0:2], ('foo@',
+            ['COMMAND', '-n foo@', '-c 5', '']))
 
 
 class test_MultiTool(Case):
@@ -212,7 +212,7 @@ class test_MultiTool(Case):
     def test_splash(self):
         self.t.nosplash = False
         self.t.splash()
-        self.assertIn('celeryd-multi', self.fh.getvalue())
+        self.assertIn('celery multi', self.fh.getvalue())
 
     def test_usage(self):
         self.t.usage()
@@ -229,7 +229,7 @@ class test_MultiTool(Case):
 
     def test_restart(self):
         stop = self.t._stop_nodes = Mock()
-        self.t.restart(['jerry', 'george'], 'celeryd')
+        self.t.restart(['jerry', 'george'], 'celery worker')
         waitexec = self.t.waitexec = Mock()
         self.assertTrue(stop.called)
         callback = stop.call_args[1]['callback']
@@ -250,7 +250,7 @@ class test_MultiTool(Case):
         self.t.getpids = Mock()
         self.t.getpids.return_value = [2, 3, 4]
         self.t.shutdown_nodes = Mock()
-        self.t.stop(['a', 'b', '-INT'], 'celeryd')
+        self.t.stop(['a', 'b', '-INT'], 'celery worker')
         self.t.shutdown_nodes.assert_called_with(
             [2, 3, 4], sig=signal.SIGINT, retry=None, callback=None,
 
@@ -265,7 +265,7 @@ class test_MultiTool(Case):
         ]
         sig = self.t.signal_node = Mock()
 
-        self.t.kill(['a', 'b', 'c'], 'celeryd')
+        self.t.kill(['a', 'b', 'c'], 'celery worker')
 
         sigs = sig.call_args_list
         self.assertEqual(len(sigs), 3)
@@ -281,8 +281,8 @@ class test_MultiTool(Case):
 
             def read_pid(self):
                 try:
-                    return {'celeryd@foo.pid': 10,
-                            'celeryd@bar.pid': 11}[self.path]
+                    return {'foo.pid': 10,
+                            'bar.pid': 11}[self.path]
                 except KeyError:
                     raise ValueError()
         Pidfile.side_effect = pids
@@ -295,30 +295,30 @@ class test_MultiTool(Case):
         callback = Mock()
 
         p = NamespacedOptionParser(['foo', 'bar', 'baz'])
-        nodes = self.t.getpids(p, 'celeryd', callback=callback)
+        nodes = self.t.getpids(p, 'celery worker', callback=callback)
         node_0, node_1 = nodes
-        self.assertEqual(node_0[0], 'foo.e.com')
+        self.assertEqual(node_0[0], 'foo@e.com')
         self.assertEqual(sorted(node_0[1]),
-            sorted(('celeryd', '--pidfile=celeryd@foo.pid',
-                    '-n foo.e.com', '')))
+            sorted(('celery worker', '--pidfile=foo.pid',
+                    '-n foo@e.com', '')))
         self.assertEqual(node_0[2], 10)
 
-        self.assertEqual(node_1[0], 'bar.e.com')
+        self.assertEqual(node_1[0], 'bar@e.com')
         self.assertEqual(sorted(node_1[1]),
-            sorted(('celeryd', '--pidfile=celeryd@bar.pid',
-                    '-n bar.e.com', '')))
+            sorted(('celery worker', '--pidfile=bar.pid',
+                    '-n bar@e.com', '')))
         self.assertEqual(node_1[2], 11)
         self.assertTrue(callback.called)
         cargs, _ = callback.call_args
-        self.assertEqual(cargs[0], 'baz.e.com')
+        self.assertEqual(cargs[0], 'baz@e.com')
         self.assertItemsEqual(cargs[1],
-            ['celeryd', '--pidfile=celeryd@baz.pid', '-n baz.e.com', ''],
+            ['celery worker', '--pidfile=baz.pid', '-n baz@e.com', ''],
         )
         self.assertIsNone(cargs[2])
         self.assertIn('DOWN', self.fh.getvalue())
 
         # without callback, should work
-        nodes = self.t.getpids(p, 'celeryd', callback=None)
+        nodes = self.t.getpids(p, 'celery worker', callback=None)
 
     @patch('celery.bin.celeryd_multi.Pidfile')
     @patch('socket.gethostname')
@@ -335,9 +335,9 @@ class test_MultiTool(Case):
         self.t.stop(['foo', 'bar', 'baz'], 'celeryd', callback=callback)
         sigs = sorted(self.t.signal_node.call_args_list)
         self.assertEqual(len(sigs), 2)
-        self.assertIn(('foo.e.com', 10, signal.SIGTERM),
+        self.assertIn(('foo@e.com', 10, signal.SIGTERM),
                 [tup[0] for tup in sigs])
-        self.assertIn(('bar.e.com', 11, signal.SIGTERM),
+        self.assertIn(('bar@e.com', 11, signal.SIGTERM),
                 [tup[0] for tup in sigs])
         self.t.signal_node.return_value = False
         self.assertTrue(callback.called)
@@ -399,23 +399,23 @@ class test_MultiTool(Case):
     @patch('socket.gethostname')
     def test_get(self, gethostname):
         gethostname.return_value = 'e.com'
-        self.t.get(['xuzzy.e.com', 'foo', 'bar', 'baz'], 'celeryd')
+        self.t.get(['xuzzy@e.com', 'foo', 'bar', 'baz'], 'celery worker')
         self.assertFalse(self.fh.getvalue())
-        self.t.get(['foo.e.com', 'foo', 'bar', 'baz'], 'celeryd')
+        self.t.get(['foo@e.com', 'foo', 'bar', 'baz'], 'celery worker')
         self.assertTrue(self.fh.getvalue())
 
     @patch('socket.gethostname')
     def test_names(self, gethostname):
         gethostname.return_value = 'e.com'
         self.t.names(['foo', 'bar', 'baz'], 'celeryd')
-        self.assertIn('foo.e.com\nbar.e.com\nbaz.e.com', self.fh.getvalue())
+        self.assertIn('foo@e.com\nbar@e.com\nbaz@e.com', self.fh.getvalue())
 
     def test_execute_from_commandline(self):
         start = self.t.commands['start'] = Mock()
         self.t.error = Mock()
         self.t.execute_from_commandline(['multi', 'start', 'foo', 'bar'])
         self.assertFalse(self.t.error.called)
-        start.assert_called_with(['foo', 'bar'], 'celeryd')
+        start.assert_called_with(['foo', 'bar'], 'celery worker')
 
         self.t.error = Mock()
         self.t.execute_from_commandline(['multi', 'frob', 'foo', 'bar'])

extra/generic-init.d/celeryd

@@ -20,8 +20,8 @@
 
 #set -e
 
-DEFAULT_PID_FILE="/var/run/celery/%n.pid"
-DEFAULT_LOG_FILE="/var/log/celery/%n.log"
+DEFAULT_PID_FILE="/var/run/celery/%N.pid"
+DEFAULT_LOG_FILE="/var/log/celery/%N.log"
 DEFAULT_LOG_LEVEL="INFO"
 DEFAULT_NODES="celery"
 DEFAULT_CELERYD="-m celery worker --detach"