Use OptionGroup to separate worker arguments

celery/bin/base.py

@@ -80,7 +80,9 @@ import json
 from collections import defaultdict
 from heapq import heappush
 from inspect import getargspec
-from optparse import OptionParser, IndentedHelpFormatter, make_option as Option
+from optparse import (
+    OptionParser, OptionGroup, IndentedHelpFormatter, make_option as Option,
+)
 from pprint import pformat
 
 from celery import VERSION_BANNER, Celery, maybe_patch_concurrency
@@ -328,6 +330,9 @@ class Command(object):
         """Get supported command-line options."""
         return self.option_list
 
+    def prepare_arguments(self, parser):
+        pass
+
     def expanduser(self, value):
         if isinstance(value, string_t):
             return os.path.expanduser(value)
@@ -413,20 +418,21 @@ class Command(object):
         return self.parser.parse_args(arguments)
 
     def create_parser(self, prog_name, command=None):
-        option_list = (
-            self.preload_options +
-            self.get_options() +
-            tuple(self.app.user_options['preload'])
-        )
-        return self.prepare_parser(self.Parser(
+        parser = self.Parser(
             prog=prog_name,
             usage=self.usage(command),
             version=self.version,
             epilog=self.epilog,
             formatter=HelpFormatter(),
             description=self.description,
-            option_list=option_list,
-        ))
+        )
+        parser.option_list.extend(self.preload_options)
+        self.prepare_arguments(parser)
+        option_list = self.get_options()
+        if option_list:
+            parser.option_lisat.extend(option_list)
+        parser.option_list.extend(self.app.user_options['preload'])
+        return self.prepare_parser(parser)
 
     def prepare_parser(self, parser):
         docs = [self.parse_doc(doc) for doc in (self.doc, __doc__) if doc]
@@ -662,12 +668,12 @@ class Command(object):
             self._colored.enabled = not self._no_color
 
 
-def daemon_options(default_pidfile=None, default_logfile=None):
-    return (
-        Option('-f', '--logfile', default=default_logfile),
-        Option('--pidfile', default=default_pidfile),
-        Option('--uid', default=None),
-        Option('--gid', default=None),
-        Option('--umask', default=None),
-        Option('--executable', default=None),
-    )
+def daemon_options(parser, default_pidfile=None, default_logfile=None):
+    group = OptionGroup(parser, "Daemonization Options")
+    group.add_option('-f', '--logfile', default=default_logfile),
+    group.add_option('--pidfile', default=default_pidfile),
+    group.add_option('--uid', default=None),
+    group.add_option('--gid', default=None),
+    group.add_option('--umask', default=None),
+    group.add_option('--executable', default=None),
+    parser.add_option_group(group)

celery/bin/beat.py

@@ -44,7 +44,7 @@ from functools import partial
 
 from celery.platforms import detached, maybe_drop_privileges
 
-from celery.bin.base import Command, Option, daemon_options
+from celery.bin.base import Command, daemon_options
 
 __all__ = ['beat']
 
@@ -78,19 +78,15 @@ class beat(Command):
         else:
             return beat().run()
 
-    def get_options(self):
+    def prepare_arguments(self, parser):
         c = self.app.conf
-
-        return (
-            (Option('--detach', action='store_true'),
-             Option('-s', '--schedule',
-                    default=c.beat_schedule_filename),
-             Option('--max-interval', type='float'),
-             Option('-S', '--scheduler', dest='scheduler_cls'),
-             Option('-l', '--loglevel', default='WARN')) +
-            daemon_options(default_pidfile='celerybeat.pid') +
-            tuple(self.app.user_options['beat'])
-        )
+        parser.add_option('--detach', action='store_true')
+        parser.add_option('-s', '--schedule', default=c.beat_schedule_filename)
+        parser.add_option('--max-interval', type='float')
+        parser.add_option('-S', '--scheduler', dest='scheduler_cls')
+        parser.add_option('-l', '--loglevel', default='WARN')
+        daemon_options(parser, default_pidfile='celerybeat.pid')
+        parser.option_list.extend(self.app.user_options['beat'])
 
 
 def main(app=None):

celery/bin/celery.py

@@ -90,7 +90,7 @@ class multi(Command):
     respects_app_option = False
 
     def get_options(self):
-        return ()
+        pass
 
     def run_from_argv(self, prog_name, argv, command=None):
         from celery.bin.multi import MultiTool

celery/bin/celeryd_detach.py

@@ -21,7 +21,7 @@ from optparse import OptionParser, BadOptionError
 from celery.platforms import EX_FAILURE, detached
 from celery.utils.log import get_logger
 
-from celery.bin.base import daemon_options, Option
+from celery.bin.base import daemon_options
 
 __all__ = ['detached_celeryd', 'detach']
 
@@ -29,13 +29,6 @@ logger = get_logger(__name__)
 
 C_FAKEFORK = os.environ.get('C_FAKEFORK')
 
-OPTION_LIST = daemon_options(default_pidfile='celeryd.pid') + (
-    Option('--workdir', default=None, dest='working_directory'),
-    Option('--fake',
-           default=False, action='store_true', dest='fake',
-           help="Don't fork (for debugging purposes)"),
-)
-
 
 def detach(path, argv, logfile=None, pidfile=None, uid=None,
            gid=None, umask=None, working_directory=None, fake=False, app=None,
@@ -114,7 +107,6 @@ class PartialOptionParser(OptionParser):
 
 
 class detached_celeryd(object):
-    option_list = OPTION_LIST
     usage = '%prog [options] [celeryd options]'
     version = celery.VERSION_BANNER
     description = ('Detaches Celery worker nodes.  See `celery worker --help` '
@@ -128,13 +120,13 @@ class detached_celeryd(object):
 
     def Parser(self, prog_name):
         return PartialOptionParser(prog=prog_name,
-                                   option_list=self.option_list,
                                    usage=self.usage,
                                    description=self.description,
                                    version=self.version)
 
     def parse_options(self, prog_name, argv):
         parser = self.Parser(prog_name)
+        self.prepare_arguments(parser)
         options, values = parser.parse_args(argv)
         if options.logfile:
             parser.leftovers.append('--logfile={0}'.format(options.logfile))
@@ -161,6 +153,15 @@ class detached_celeryd(object):
             **vars(options)
         ))
 
+    def prepare_arguments(self, parser):
+        daemon_options(parser, default_pidfile='celeryd.pid')
+        parser.add_option('--workdir', default=None, dest='working_directory')
+        parser.add_option(
+            '--fake',
+            default=False, action='store_true', dest='fake',
+            help="Don't fork (for debugging purposes)",
+        )
+
 
 def main(app=None):
     detached_celeryd(app).execute_from_commandline()

celery/bin/events.py

@@ -42,7 +42,7 @@ import sys
 from functools import partial
 
 from celery.platforms import detached, set_process_title, strargv
-from celery.bin.base import Command, Option, daemon_options
+from celery.bin.base import Command, daemon_options
 
 __all__ = ['events']
 
@@ -117,18 +117,16 @@ class events(Command):
         info = '{0} {1}'.format(info, strargv(sys.argv))
         return set_process_title(prog, info=info)
 
-    def get_options(self):
-        return (
-            (Option('-d', '--dump', action='store_true'),
-             Option('-c', '--camera'),
-             Option('--detach', action='store_true'),
-             Option('-F', '--frequency', '--freq',
-                    type='float', default=1.0),
-             Option('-r', '--maxrate'),
-             Option('-l', '--loglevel', default='INFO')) +
-            daemon_options(default_pidfile='celeryev.pid') +
-            tuple(self.app.user_options['events'])
-        )
+    def prepare_arguments(self, parser):
+        parser.add_option('-d', '--dump', action='store_true')
+        parser.add_option('-c', '--camera')
+        parser.add_option('--detach', action='store_true')
+        parser.add_option('-F', '--frequency', '--freq',
+                          type='float', default=1.0)
+        parser.add_option('-r', '--maxrate')
+        parser.add_option('-l', '--loglevel', default='INFO')
+        daemon_options(parser, default_pidfile='celeryev.pid')
+        parser.option_list.extend(self.app.user_options['events'])
 
 
 def main():

celery/bin/worker.py

@@ -146,8 +146,10 @@ from __future__ import absolute_import, unicode_literals
 
 import sys
 
+from optparse import OptionGroup
+
 from celery import concurrency
-from celery.bin.base import Command, Option, daemon_options
+from celery.bin.base import Command, daemon_options
 from celery.bin.celeryd_detach import detached_celeryd
 from celery.five import string_t
 from celery.platforms import maybe_drop_privileges
@@ -227,46 +229,102 @@ class worker(Command):
         # that may have to be loaded as early as possible.
         return (['-P'], ['--pool'])
 
-    def get_options(self):
+    def prepare_arguments(self, parser):
         conf = self.app.conf
-        return (
-            Option('-c', '--concurrency',
-                   default=conf.worker_concurrency, type='int'),
-            Option('-P', '--pool', default=conf.worker_pool, dest='pool_cls'),
-            Option('--purge', '--discard', default=False, action='store_true'),
-            Option('-l', '--loglevel', default='WARN'),
-            Option('-n', '--hostname'),
-            Option('-B', '--beat', action='store_true'),
-            Option('-s', '--schedule', dest='schedule_filename',
-                   default=conf.beat_schedule_filename),
-            Option('--scheduler', dest='scheduler_cls'),
-            Option('-S', '--statedb',
-                   default=conf.worker_state_db, dest='state_db'),
-            Option('-E', '--events', default=conf.worker_send_task_events,
-                   action='store_true', dest='send_events'),
-            Option('--time-limit', type='float', dest='task_time_limit',
-                   default=conf.task_time_limit),
-            Option('--soft-time-limit', dest='task_soft_time_limit',
-                   default=conf.task_soft_time_limit, type='float'),
-            Option('--maxtasksperchild', dest='max_tasks_per_child',
-                   default=conf.worker_max_tasks_per_child, type='int'),
-            Option('--prefetch-multiplier', dest='prefetch_multiplier',
-                   default=conf.worker_prefetch_multiplier, type='int'),
-            Option('--maxmemperchild', dest='max_memory_per_child',
-                   default=conf.worker_max_memory_per_child, type='int'),
-            Option('--queues', '-Q', default=[]),
-            Option('--exclude-queues', '-X', default=[]),
-            Option('--include', '-I', default=[]),
-            Option('--autoscale'),
-            Option('--autoreload', action='store_true'),
-            Option('--no-execv', action='store_true', default=False),
-            Option('--without-gossip', action='store_true', default=False),
-            Option('--without-mingle', action='store_true', default=False),
-            Option('--without-heartbeat', action='store_true', default=False),
-            Option('--heartbeat-interval', type='int'),
-            Option('-O', dest='optimization'),
-            Option('-D', '--detach', action='store_true'),
-        ) + daemon_options() + tuple(self.app.user_options['worker'])
+
+        wopts = OptionGroup(parser, 'Worker Options')
+        wopts.add_option('-n', '--hostname')
+        wopts.add_option('-D', '--detach', action='store_true')
+        wopts.add_option(
+            '-S', '--statedb',
+            default=conf.worker_state_db, dest='state_db',
+        )
+        wopts.add_option('-l', '--loglevel', default='WARN')
+        wopts.add_option('-O', dest='optimization')
+        wopts.add_option(
+            '--prefetch-multiplier',
+            dest='prefetch_multiplier', type='int',
+            default=conf.worker_prefetch_multiplier,
+        )
+        parser.add_option_group(wopts)
+
+        topts = OptionGroup(parser, 'Pool Options')
+        topts.add_option(
+            '-c', '--concurrency',
+            default=conf.worker_concurrency, type='int',
+        )
+        topts.add_option(
+            '-P', '--pool',
+            default=conf.worker_pool, dest='pool_cls',
+        )
+        topts.add_option(
+            '-E', '--events',
+            default=conf.worker_send_task_events,
+            action='store_true', dest='send_events',
+        )
+        topts.add_option(
+            '--time-limit',
+            type='float', dest='task_time_limit',
+            default=conf.task_time_limit,
+        )
+        topts.add_option(
+            '--soft-time-limit',
+            dest='task_soft_time_limit', type='float',
+            default=conf.task_soft_time_limit,
+        )
+        topts.add_option(
+            '--maxtasksperchild',
+            dest='max_tasks_per_child', type='int',
+            default=conf.worker_max_tasks_per_child,
+        )
+        topts.add_option(
+            '--maxmemperchild',
+            dest='max_memory_per_child', type='int',
+            default=conf.worker_max_memory_per_child,
+        )
+        parser.add_option_group(topts)
+
+        qopts = OptionGroup(parser, 'Queue Options')
+        qopts.add_option(
+            '--purge', '--discard',
+            default=False, action='store_true',
+        )
+        qopts.add_option('--queues', '-Q', default=[])
+        qopts.add_option('--exclude-queues', '-X', default=[])
+        qopts.add_option('--include', '-I', default=[])
+        parser.add_option_group(qopts)
+
+        fopts = OptionGroup(parser, 'Features')
+        fopts.add_option('--autoscale')
+        fopts.add_option('--autoreload', action='store_true')
+        fopts.add_option(
+            '--without-gossip', action='store_true', default=False,
+        )
+        fopts.add_option(
+            '--without-mingle', action='store_true', default=False,
+        )
+        fopts.add_option(
+            '--without-heartbeat', action='store_true', default=False,
+        )
+        fopts.add_option('--heartbeat-interval', type='int')
+        parser.add_option_group(fopts)
+
+        daemon_options(parser)
+
+        bopts = OptionGroup(parser, 'Embedded Beat Options')
+        bopts.add_option('-B', '--beat', action='store_true')
+        bopts.add_option(
+            '-s', '--schedule', dest='schedule_filename',
+            default=conf.beat_schedule_filename,
+        )
+        bopts.add_option('--scheduler', dest='scheduler_cls')
+        parser.add_option_group(bopts)
+
+        user_options = self.app.user_options['worker']
+        if user_options:
+            uopts = OptionGroup(parser, 'User Options')
+            uopts.options_list.extend(user_options)
+            parser.add_option_group(uopts)
 
 
 def main(app=None):