Commands: Replace optparse with argparse (Closes #793)

celery/bin/base.py

@@ -2,6 +2,7 @@
 """Base command-line interface."""
 from __future__ import absolute_import, print_function, unicode_literals
 
+import argparse
 import os
 import random
 import re
@@ -11,21 +12,30 @@ import json
 
 from collections import defaultdict
 from heapq import heappush
-from optparse import OptionParser, OptionGroup, IndentedHelpFormatter
-from optparse import make_option as Option  # noqa
 from pprint import pformat
 
 from celery import VERSION_BANNER, Celery, maybe_patch_concurrency
 from celery import signals
 from celery.exceptions import CDeprecationWarning, CPendingDeprecationWarning
 from celery.five import (
-    getfullargspec, items, python_2_unicode_compatible, string, string_t,
+    getfullargspec, items, python_2_unicode_compatible,
+    string, string_t, text_t, long_t,
 )
 from celery.platforms import EX_FAILURE, EX_OK, EX_USAGE
 from celery.utils import imports
 from celery.utils import term
 from celery.utils import text
+from celery.utils.functional import dictfilter
 from celery.utils.nodenames import node_format, host_format
+from celery.utils.objects import Bunch
+
+
+# Option is here for backwards compatiblity, as third-party commands
+# may import it from here.
+try:
+    from optparse import Option  # pylint: disable=deprecated-module
+except ImportError:  # pragma: no cover
+    Option = None  # noqa
 
 try:
     input = raw_input
@@ -33,8 +43,7 @@ except NameError:  # pragma: no cover
     pass
 
 __all__ = [
-    'Error', 'UsageError', 'Extensions',
-    'HelpFormatter', 'Command', 'Option', 'daemon_options',
+    'Error', 'UsageError', 'Extensions', 'Command', 'Option', 'daemon_options',
 ]
 
 # always enable DeprecationWarnings, so our users can see them.
@@ -52,6 +61,48 @@ find_rst_ref = re.compile(r':\w+:`(.+?)`')
 find_rst_decl = re.compile(r'^\s*\.\. .+?::.+$')
 
 
+def _optparse_callback_to_type(option, callback):
+    parser = Bunch(values=Bunch())
+
+    def _on_arg(value):
+        callback(option, None, value, parser)
+        return getattr(parser.values, option.dest)
+    return _on_arg
+
+
+def _add_optparse_argument(parser, opt, typemap={
+        'string': text_t,
+        'int': int,
+        'long': long_t,
+        'float': float,
+        'complex': complex,
+        'choice': None}):
+    if opt.callback:
+        opt.type = _optparse_callback_to_type(opt, opt.type)
+    # argparse checks for existence of this kwarg
+    if opt.action == 'callback':
+        opt.action = None
+    parser.add_argument(
+        *opt._long_opts + opt._short_opts,
+        **dictfilter(dict(
+            action=opt.action,
+            type=typemap.get(opt.type, opt.type),
+            dest=opt.dest,
+            nargs=opt.nargs,
+            choices=opt.choices,
+            help=opt.help,
+            metavar=opt.metavar,
+            default=opt.default)))
+
+
+def _add_compat_options(parser, options):
+    for option in options or ():
+        if callable(option):
+            option(parser)
+        else:
+            _add_optparse_argument(parser, option)
+
+
 @python_2_unicode_compatible
 class Error(Exception):
     """Exception raised by commands."""
@@ -91,19 +142,6 @@ class Extensions(object):
         return self.names
 
 
-class HelpFormatter(IndentedHelpFormatter):
-    """Custom help formatter."""
-
-    def format_epilog(self, epilog):
-        if epilog:
-            return '\n{0}\n\n'.format(epilog)
-        return ''
-
-    def format_description(self, description):
-        return text.ensure_newlines(
-            text.fill_paragraphs(text.dedent(description), self.width))
-
-
 class Command(object):
     """Base class for command-line applications.
 
@@ -115,7 +153,7 @@ class Command(object):
 
     Error = Error
     UsageError = UsageError
-    Parser = OptionParser
+    Parser = argparse.ArgumentParser
 
     #: Arg list used in help.
     args = ''
@@ -128,7 +166,7 @@ class Command(object):
     supports_args = True
 
     #: List of options (without preload options).
-    option_list = ()
+    option_list = None
 
     # module Rst documentation to parse help from (if any)
     doc = None
@@ -137,17 +175,6 @@ class Command(object):
     # (Issue #1008).
     respects_app_option = True
 
-    #: List of options to parse before parsing other options.
-    preload_options = (
-        Option('-A', '--app', default=None),
-        Option('-b', '--broker', default=None),
-        Option('--loader', default=None),
-        Option('--config', default=None),
-        Option('--workdir', default=None),
-        Option('--no-color', '-C', action='store_true', default=None),
-        Option('--quiet', '-q', action='store_true'),
-    )
-
     #: Enable if the application should support config from the cmdline.
     enable_config_from_cmdline = False
 
@@ -257,12 +284,26 @@ class Command(object):
             maybe_patch_concurrency(argv, *pool_option)
 
     def usage(self, command):
-        return '%prog {0} [options] {self.args}'.format(command, self=self)
+        return '%(prog)s {0} [options] {self.args}'.format(command, self=self)
+
+    def add_arguments(self, parser):
+        pass
 
     def get_options(self):
-        """Get supported command-line options."""
+        # This is for optparse options, please use add_arguments.
         return self.option_list
 
+    def add_preload_options(self, parser):
+        group = parser.add_argument_group('Global Options')
+        group.add_argument('-A', '--app', default=None)
+        group.add_argument('-b', '--broker', default=None)
+        group.add_argument('--loader', default=None)
+        group.add_argument('--config', default=None)
+        group.add_argument('--workdir', default=None)
+        group.add_argument(
+            '--no-color', '-C', action='store_true', default=None)
+        group.add_argument('--quiet', '-q', action='store_true')
+
     def prepare_arguments(self, parser):
         pass
 
@@ -318,7 +359,7 @@ class Command(object):
         if options:
             options = {
                 k: self.expanduser(v)
-                for k, v in items(vars(options)) if not k.startswith('_')
+                for k, v in items(options) if not k.startswith('_')
             }
         args = [self.expanduser(arg) for arg in args]
         self.check_args(args)
@@ -348,33 +389,50 @@ class Command(object):
         # Don't want to load configuration to just print the version,
         # so we handle --version manually here.
         self.parser = self.create_parser(prog_name, command)
-        return self.parser.parse_args(arguments)
+        options = vars(self.parser.parse_args(arguments))
+        return options, options.pop('args', None) or []
 
     def create_parser(self, prog_name, command=None):
+        usage = self.usage(command)
+        # for compatibility with optparse usage.
+        usage.replace('%prog', '%(prog)s')
         parser = self.Parser(
             prog=prog_name,
             usage=self.usage(command),
             version=self.version,
-            epilog=self.epilog,
-            formatter=HelpFormatter(),
-            description=self.description,
+            epilog=self._format_epilog(self.epilog),
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            description=self._format_description(self.description),
         )
-        parser.add_options(self.preload_options)
-        for typ_ in reversed(type(self).mro()):
-            try:
-                prepare_arguments = typ_.prepare_arguments
-            except AttributeError:
-                continue
-            prepare_arguments(self, parser)
-        parser.add_options(self.get_options() or ())
-        parser.add_options(self.app.user_options['preload'])
+        self.add_preload_options(parser)
+        self.add_arguments(parser)
+        self.add_compat_options(parser, self.get_options())
+        self.add_compat_options(parser, self.app.user_options['preload'])
+
+        if self.supports_args:
+            # for backward compatibility with optparse, we automatically
+            # add arbitrary positional args.
+            parser.add_argument('args', nargs='*')
         return self.prepare_parser(parser)
 
+    def _format_epilog(self, epilog):
+        if epilog:
+            return '\n{0}\n\n'.format(epilog)
+        return ''
+
+    def _format_description(self, description):
+        width = argparse.HelpFormatter('prog')._width
+        return text.ensure_newlines(
+            text.fill_paragraphs(text.dedent(description), width))
+
+    def add_compat_options(self, parser, options):
+        _add_compat_options(parser, options)
+
     def prepare_parser(self, parser):
         docs = [self.parse_doc(doc) for doc in (self.doc, __doc__) if doc]
         for doc in docs:
             for long_opt, help in items(doc):
-                option = parser.get_option(long_opt)
+                option = parser._option_string_actions[long_opt]
                 if option is not None:
                     option.help = ' '.join(help).format(default=option.default)
         return parser
@@ -417,15 +475,17 @@ class Command(object):
         else:
             self.app = Celery(fixups=[])
 
+        self._handle_user_preload_options(argv)
+
+        return argv
+
+    def _handle_user_preload_options(self, argv):
         user_preload = tuple(self.app.user_options['preload'] or ())
         if user_preload:
-            user_options = self.preparse_options(argv, user_preload)
-            for user_option in user_preload:
-                user_options.setdefault(user_option.dest, user_option.default)
+            user_options = self._parse_preload_options(argv, user_preload)
             signals.user_preload_options.send(
                 sender=self, app=self.app, options=user_options,
             )
-        return argv
 
     def find_app(self, app):
         from celery.app.utils import find_app
@@ -445,7 +505,14 @@ class Command(object):
         return argv
 
     def parse_preload_options(self, args):
-        return self.preparse_options(args, self.preload_options)
+        return self._parse_preload_options(args, [self.add_preload_options])
+
+    def _parse_preload_options(self, args, options):
+        args = [arg for arg in args if arg not in ('-h', '--help')]
+        parser = argparse.ArgumentParser()
+        self.add_compat_options(parser, options)
+        namespace, _ = parser.parse_known_args(args)
+        return vars(namespace)
 
     def add_append_opt(self, acc, opt, value):
         default = opt.default or []
@@ -455,53 +522,6 @@ class Command(object):
 
         acc[opt.dest].append(value)
 
-    def preparse_options(self, args, options):
-        acc = {}
-        opts = {}
-        for opt in options:
-            for t in (opt._long_opts, opt._short_opts):
-                opts.update(dict(zip(t, [opt] * len(t))))
-        index = 0
-        length = len(args)
-        while index < length:
-            arg = args[index]
-            if arg.startswith('--'):
-                if '=' in arg:
-                    key, value = arg.split('=', 1)
-                    opt = opts.get(key)
-                    if opt:
-                        if opt.action == 'append':
-                            self.add_append_opt(acc, opt, value)
-                        else:
-                            acc[opt.dest] = value
-                else:
-                    opt = opts.get(arg)
-                    if opt and opt.takes_value():
-                        # optparse also supports ['--opt', 'value']
-                        # (Issue #1668)
-                        if opt.action == 'append':
-                            self.add_append_opt(acc, opt, args[index + 1])
-                        else:
-                            acc[opt.dest] = args[index + 1]
-                        index += 1
-                    elif opt and opt.action == 'store_true':
-                        acc[opt.dest] = True
-            elif arg.startswith('-'):
-                opt = opts.get(arg)
-                if opt:
-                    if opt.takes_value():
-                        try:
-                            acc[opt.dest] = args[index + 1]
-                        except IndexError:
-                            raise ValueError(
-                                'Missing required argument for {0}'.format(
-                                    arg))
-                        index += 1
-                    elif opt.action == 'store_true':
-                        acc[opt.dest] = True
-            index += 1
-        return acc
-
     def parse_doc(self, doc):
         options, in_option = defaultdict(list), None
         for line in doc.splitlines():
@@ -616,12 +636,11 @@ class Command(object):
 
 
 def daemon_options(parser, default_pidfile=None, default_logfile=None):
-    """Add daemon options to optparse parser."""
-    group = OptionGroup(parser, 'Daemonization Options')
-    group.add_option('-f', '--logfile', default=default_logfile),
-    group.add_option('--pidfile', default=default_pidfile),
-    group.add_option('--uid', default=None),
-    group.add_option('--gid', default=None),
-    group.add_option('--umask', default=None),
-    group.add_option('--executable', default=None),
-    parser.add_option_group(group)
+    """Add daemon options to argparse parser."""
+    group = parser.add_argument_group('Daemonization Options')
+    group.add_argument('-f', '--logfile', default=default_logfile),
+    group.add_argument('--pidfile', default=default_pidfile),
+    group.add_argument('--uid', default=None),
+    group.add_argument('--gid', default=None),
+    group.add_argument('--umask', default=None),
+    group.add_argument('--executable', default=None),

celery/bin/beat.py

@@ -74,6 +74,8 @@ from celery.bin.base import Command, daemon_options
 
 __all__ = ['beat']
 
+HELP = __doc__
+
 
 class beat(Command):
     """Start the beat periodic task scheduler.
@@ -89,7 +91,7 @@ class beat(Command):
     package found on PyPI.
     """
 
-    doc = __doc__
+    doc = HELP
     enable_config_from_cmdline = True
     supports_args = False
 
@@ -107,15 +109,22 @@ class beat(Command):
         else:
             return beat().run()
 
-    def prepare_arguments(self, parser):
+    def add_arguments(self, parser):
         c = self.app.conf
-        parser.add_option('--detach', action='store_true')
-        parser.add_option('-s', '--schedule', default=c.beat_schedule_filename)
-        parser.add_option('--max-interval', type='float')
-        parser.add_option('-S', '--scheduler')
-        parser.add_option('-l', '--loglevel', default='WARN')
+        bopts = parser.add_argument_group('Beat Options')
+        bopts.add_argument('--detach', action='store_true')
+        bopts.add_argument(
+            '-s', '--schedule', default=c.beat_schedule_filename)
+        bopts.add_argument('--max-interval', type=float)
+        bopts.add_argument('-S', '--scheduler')
+        bopts.add_argument('-l', '--loglevel', default='WARN')
+
         daemon_options(parser, default_pidfile='celerybeat.pid')
-        parser.add_options(self.app.user_options['beat'])
+
+        user_options = self.app.user_options['beat']
+        if user_options:
+            uopts = parser.add_argument_group('User Options')
+            self.add_compat_options(uopts, user_options)
 
 
 def main(app=None):

celery/bin/celery.py

@@ -277,7 +277,7 @@ from celery.utils.text import str_to_list
 from celery.utils.time import maybe_iso8601
 
 # Cannot use relative imports here due to a Windows issue (#1111).
-from celery.bin.base import Command, Option, Extensions
+from celery.bin.base import Command, Extensions
 
 # Import commands from other modules
 from celery.bin.amqp import amqp
@@ -341,9 +341,6 @@ class multi(Command):
 
     respects_app_option = False
 
-    def get_options(self):
-        pass
-
     def run_from_argv(self, prog_name, argv, command=None):
         from celery.bin.multi import MultiTool
         cmd = MultiTool(quiet=self.quiet, no_color=self.no_color)
@@ -382,7 +379,7 @@ class list_(Command):
         available = ', '.join(topics)
         if not what:
             raise self.UsageError(
-                'You must specify one of {0}'.format(available))
+                'Missing argument, specify one of: {0}'.format(available))
         if what not in topics:
             raise self.UsageError(
                 'unknown topic {0!r} (choose one of: {1})'.format(
@@ -404,18 +401,30 @@ class call(Command):
 
     args = '<task_name>'
 
-    option_list = Command.option_list + (
-        Option('--args', '-a', help='positional arguments (json).'),
-        Option('--kwargs', '-k', help='keyword arguments (json).'),
-        Option('--eta', help='scheduled time (ISO-8601).'),
-        Option('--countdown', type='float',
-               help='eta in seconds from now (float/int).'),
-        Option('--expires', help='expiry time (ISO-8601/float/int).'),
-        Option('--serializer', default='json', help='defaults to json.'),
-        Option('--queue', help='custom queue name.'),
-        Option('--exchange', help='custom exchange name.'),
-        Option('--routing-key', help='custom routing key.'),
-    )
+    def add_arguments(self, parser):
+        group = parser.add_argument_group('Calling Options')
+        group.add_argument('--args', '-a',
+                           help='positional arguments (json).')
+        group.add_argument('--kwargs', '-k',
+                           help='keyword arguments (json).')
+        group.add_argument('--eta',
+                           help='scheduled time (ISO-8601).')
+        group.add_argument(
+            '--countdown', type=float,
+            help='eta in seconds from now (float/int).',
+        )
+        group.add_argument(
+            '--expires',
+            help='expiry time (ISO-8601/float/int).',
+        ),
+        group.add_argument(
+            '--serializer', default='json',
+            help='defaults to json.'),
+
+        ropts = parser.add_argument_group('Routing Options')
+        ropts.add_argument('--queue', help='custom queue name.')
+        ropts.add_argument('--exchange', help='custom exchange name.')
+        ropts.add_argument('--routing-key', help='custom routing key.')
 
     def run(self, name, *_, **kwargs):
         self._send_task(name, **kwargs)
@@ -423,7 +432,7 @@ class call(Command):
     def _send_task(self, name, args=None, kwargs=None,
                    countdown=None, serializer=None,
                    queue=None, exchange=None, routing_key=None,
-                   eta=None, expires=None):
+                   eta=None, expires=None, **_):
         # arguments
         args = loads(args) if isinstance(args, string_t) else args
         kwargs = loads(kwargs) if isinstance(kwargs, string_t) else kwargs
@@ -469,14 +478,20 @@ class purge(Command):
     fmt_purged = 'Purged {mnum} {messages} from {qnum} known task {queues}.'
     fmt_empty = 'No messages purged from {qnum} {queues}'
 
-    option_list = Command.option_list + (
-        Option('--force', '-f', action='store_true',
-               help="Don't prompt for verification"),
-        Option('--queues', '-Q', default=[],
-               help='Comma separated list of queue names to purge.'),
-        Option('--exclude-queues', '-X', default=[],
-               help='Comma separated list of queues names not to purge.')
-    )
+    def add_arguments(self, parser):
+        group = parser.add_argument_group('Purging Options')
+        group.add_argument(
+            '--force', '-f', action='store_true',
+            help="Don't prompt for verification",
+        )
+        group.add_argument(
+            '--queues', '-Q', default=[],
+            help='Comma separated list of queue names to purge.',
+        )
+        group.add_argument(
+            '--exclude-queues', '-X', default=[],
+            help='Comma separated list of queues names not to purge.',
+        )
 
     def run(self, force=False, queues=None, exclude_queues=None, **kwargs):
         queues = set(str_to_list(queues or []))
@@ -522,11 +537,15 @@ class result(Command):
 
     args = '<task_id>'
 
-    option_list = Command.option_list + (
-        Option('--task', '-t', help='name of task (if custom backend)'),
-        Option('--traceback', action='store_true',
-               help='show traceback instead'),
-    )
+    def add_arguments(self, parser):
+        group = parser.add_argument_group('Result Options')
+        group.add_argument(
+            '--task', '-t', help='name of task (if custom backend)',
+        )
+        group.add_argument(
+            '--traceback', action='store_true',
+            help='show traceback instead',
+        )
 
     def run(self, task_id, *args, **kwargs):
         result_cls = self.app.AsyncResult
@@ -549,20 +568,25 @@ class _RemoteControl(Command):
     leaf = False
     control_group = None
 
-    option_list = Command.option_list + (
-        Option('--timeout', '-t', type='float',
-               help='Timeout in seconds (float) waiting for reply'),
-        Option('--destination', '-d',
-               help='Comma separated list of destination node names.'),
-        Option('--json', '-j', action='store_true',
-               help='Use json as output format.'),
-    )
-
     def __init__(self, *args, **kwargs):
         self.show_body = kwargs.pop('show_body', True)
         self.show_reply = kwargs.pop('show_reply', True)
         super(_RemoteControl, self).__init__(*args, **kwargs)
 
+    def add_arguments(self, parser):
+        group = parser.add_argument_group('Remote Control Options')
+        group.add_argument(
+            '--timeout', '-t', type=float,
+            help='Timeout in seconds (float) waiting for reply',
+        )
+        group.add_argument(
+            '--destination', '-d',
+            help='Comma separated list of destination node names.')
+        group.add_argument(
+            '--json', '-j', action='store_true',
+            help='Use json as output format.',
+        )
+
     @classmethod
     def get_command_info(cls, command,
                          indent=0, prefix='', color=None,
@@ -592,7 +616,7 @@ class _RemoteControl(Command):
             for c in sorted(choices))
 
     def usage(self, command):
-        return '%prog {0} [options] {1} <command> [arg1 .. argN]'.format(
+        return '%(prog)s {0} [options] {1} <command> [arg1 .. argN]'.format(
             command, self.args)
 
     def call(self, *args, **kwargs):
@@ -782,24 +806,35 @@ class migrate(Command):
     """
 
     args = '<source_url> <dest_url>'
-
-    option_list = Command.option_list + (
-        Option('--limit', '-n', type='int',
-               help='Number of tasks to consume (int)'),
-        Option('--timeout', '-t', type='float', default=1.0,
-               help='Timeout in seconds (float) waiting for tasks'),
-        Option('--ack-messages', '-a', action='store_true',
-               help='Ack messages from source broker.'),
-        Option('--tasks', '-T',
-               help='List of task names to filter on.'),
-        Option('--queues', '-Q',
-               help='List of queues to migrate.'),
-        Option('--forever', '-F', action='store_true',
-               help='Continually migrate tasks until killed.'),
-    )
-
     progress_fmt = MIGRATE_PROGRESS_FMT
 
+    def add_arguments(self, parser):
+        group = parser.add_argument_group('Migration Options')
+        group.add_argument(
+            '--limit', '-n', type=int,
+            help='Number of tasks to consume (int)',
+        )
+        group.add_argument(
+            '--timeout', '-t', type=float, default=1.0,
+            help='Timeout in seconds (float) waiting for tasks',
+        )
+        group.add_argument(
+            '--ack-messages', '-a', action='store_true',
+            help='Ack messages from source broker.',
+        )
+        group.add_argument(
+            '--tasks', '-T',
+            help='List of task names to filter on.',
+        )
+        group.add_argument(
+            '--queues', '-Q',
+            help='List of queues to migrate.',
+        )
+        group.add_argument(
+            '--forever', '-F', action='store_true',
+            help='Continually migrate tasks until killed.',
+        )
+
     def on_migrate_task(self, state, body, message):
         self.out(self.progress_fmt.format(state=state, body=body))
 
@@ -824,19 +859,31 @@ class shell(Command):  # pragma: no cover
         - all registered tasks.
     """
 
-    option_list = Command.option_list + (
-        Option('--ipython', '-I',
-               action='store_true', help='force iPython.'),
-        Option('--bpython', '-B',
-               action='store_true', help='force bpython.'),
-        Option('--python', '-P',
-               action='store_true', help='force default Python shell.'),
-        Option('--without-tasks', '-T', action='store_true',
-               help="don't add tasks to locals."),
-        Option('--eventlet', action='store_true',
-               help='use eventlet.'),
-        Option('--gevent', action='store_true', help='use gevent.'),
-    )
+    def add_arguments(self, parser):
+        group = parser.add_argument_group('Shell Options')
+        group.add_argument(
+            '--ipython', '-I',
+            action='store_true', help='force iPython.',
+        )
+        group.add_argument(
+            '--bpython', '-B',
+            action='store_true', help='force bpython.',
+        )
+        group.add_argument(
+            '--python',
+            action='store_true', help='force default Python shell.',
+        )
+        group.add_argument(
+            '--without-tasks', '-T',
+            action='store_true', help="don't add tasks to locals.",
+        )
+        group.add_argument(
+            '--eventlet',
+            action='store_true', help='use eventlet.',
+        )
+        group.add_argument(
+            '--gevent', action='store_true', help='use gevent.',
+        )
 
     def run(self, *args, **kwargs):
         if args:
@@ -950,19 +997,25 @@ class shell(Command):  # pragma: no cover
 class upgrade(Command):
     """Perform upgrade between versions."""
 
-    option_list = Command.option_list + (
-        Option('--django', action='store_true',
-               help='Upgrade Django project'),
-        Option('--compat', action='store_true',
-               help='Maintain backwards compatibility'),
-        Option('--no-backup', action='store_true',
-               help='Dont backup original files'),
-    )
-
     choices = {'settings'}
 
+    def add_arguments(self, parser):
+        group = parser.add_argument_group('Upgrading Options')
+        group.add_argument(
+            '--django', action='store_true',
+            help='Upgrade Django project',
+        )
+        group.add_argument(
+            '--compat', action='store_true',
+            help='Maintain backwards compatibility',
+        )
+        group.add_argument(
+            '--no-backup', action='store_true',
+            help='Dont backup original files',
+        )
+
     def usage(self, command):
-        return '%prog <command> settings [filename] [options]'
+        return '%(prog)s <command> settings [filename] [options]'
 
     def run(self, *args, **kwargs):
         try:
@@ -1018,7 +1071,7 @@ class help(Command):
     """Show help screen and exit."""
 
     def usage(self, command):
-        return '%prog <command> [options] {0.args}'.format(self)
+        return '%(prog)s <command> [options] {0.args}'.format(self)
 
     def run(self, *args, **kwargs):
         self.parser.print_help()

celery/bin/celeryd_detach.py

@@ -7,12 +7,11 @@ could have something to do with the threading mutex bug)
 """
 from __future__ import absolute_import, unicode_literals
 
+import argparse
 import celery
 import os
 import sys
 
-from optparse import OptionParser, BadOptionError
-
 from celery.platforms import EX_FAILURE, detached
 from celery.utils.log import get_logger
 from celery.utils.nodenames import default_nodename, node_format
@@ -51,66 +50,10 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None,
         return EX_FAILURE
 
 
-class PartialOptionParser(OptionParser):
-
-    def __init__(self, *args, **kwargs):
-        self.leftovers = []
-        OptionParser.__init__(self, *args, **kwargs)
-
-    def _process_long_opt(self, rargs, values):
-        arg = rargs.pop(0)
-
-        if '=' in arg:
-            opt, next_arg = arg.split('=', 1)
-            rargs.insert(0, next_arg)
-            had_explicit_value = True
-        else:
-            opt = arg
-            had_explicit_value = False
-
-        try:
-            opt = self._match_long_opt(opt)
-            option = self._long_opt.get(opt)
-        except BadOptionError:
-            option = None
-
-        if option:
-            if option.takes_value():
-                nargs = option.nargs
-                if len(rargs) < nargs:
-                    if nargs == 1:
-                        self.error('{0} requires an argument'.format(opt))
-                    else:
-                        self.error('{0} requires {1} arguments'.format(
-                            opt, nargs))
-                elif nargs == 1:
-                    value = rargs.pop(0)
-                else:
-                    value = tuple(rargs[0:nargs])
-                    del rargs[0:nargs]
-
-            elif had_explicit_value:
-                self.error('{0} option does not take a value'.format(opt))
-            else:
-                value = None
-            option.process(opt, value, values, self)
-        else:
-            self.leftovers.append(arg)
-
-    def _process_short_opts(self, rargs, values):
-        arg = rargs[0]
-        try:
-            OptionParser._process_short_opts(self, rargs, values)
-        except BadOptionError:
-            self.leftovers.append(arg)
-            if rargs and not rargs[0][0] == '-':
-                self.leftovers.append(rargs.pop(0))
-
-
 class detached_celeryd(object):
     """Daemonize the celery worker process."""
 
-    usage = '%prog [options] [celeryd options]'
+    usage = '%(prog)s [options] [celeryd options]'
     version = celery.VERSION_BANNER
     description = ('Detaches Celery worker nodes.  See `celery worker --help` '
                    'for the list of supported worker arguments.')
@@ -122,50 +65,63 @@ class detached_celeryd(object):
         self.app = app
 
     def create_parser(self, prog_name):
-        p = PartialOptionParser(
+        p = argparse.ArgumentParser(
             prog=prog_name,
             usage=self.usage,
             description=self.description,
             version=self.version,
         )
-        self.prepare_arguments(p)
+        self.add_arguments(p)
         return p
 
     def parse_options(self, prog_name, argv):
         parser = self.create_parser(prog_name)
-        options, values = parser.parse_args(argv)
+        options, leftovers = parser.parse_known_args(argv)
         if options.logfile:
-            parser.leftovers.append('--logfile={0}'.format(options.logfile))
+            leftovers.append('--logfile={0}'.format(options.logfile))
         if options.pidfile:
-            parser.leftovers.append('--pidfile={0}'.format(options.pidfile))
+            leftovers.append('--pidfile={0}'.format(options.pidfile))
         if options.hostname:
-            parser.leftovers.append('--hostname={0}'.format(options.hostname))
-        return options, values, parser.leftovers
+            leftovers.append('--hostname={0}'.format(options.hostname))
+        return options, leftovers
 
     def execute_from_commandline(self, argv=None):
         argv = sys.argv if argv is None else argv
-        config = []
-        seen_cargs = 0
-        for arg in argv:
-            if seen_cargs:
-                config.append(arg)
-            else:
-                if arg == '--':
-                    seen_cargs = 1
-                    config.append(arg)
         prog_name = os.path.basename(argv[0])
-        options, _, leftovers = self.parse_options(prog_name, argv[1:])
+        config, argv = self._split_command_line_config(argv)
+        options, leftovers = self.parse_options(prog_name, argv[1:])
         sys.exit(detach(
             app=self.app, path=self.execv_path,
             argv=self.execv_argv + leftovers + config,
             **vars(options)
         ))
 
-    def prepare_arguments(self, parser):
+    def _split_command_line_config(self, argv):
+        config = list(self._extract_command_line_config(argv))
+        try:
+            argv = argv[:argv.index('--')]
+        except IndexError:
+            pass
+        return config, argv
+
+    def _extract_command_line_config(self, argv):
+        # Extracts command-line config appearing after '--':
+        #    celery worker -l info -- worker.prefetch_multiplier=10
+        # This to make sure argparse doesn't gobble it up.
+        seen_cargs = 0
+        for arg in argv:
+            if seen_cargs:
+                yield arg
+            else:
+                if arg == '--':
+                    seen_cargs = 1
+                    yield arg
+
+    def add_arguments(self, parser):
         daemon_options(parser, default_pidfile='celeryd.pid')
-        parser.add_option('--workdir', default=None)
-        parser.add_option('-n', '--hostname')
-        parser.add_option(
+        parser.add_argument('--workdir', default=None)
+        parser.add_argument('-n', '--hostname')
+        parser.add_argument(
             '--fake',
             default=False, action='store_true',
             help="Don't fork (for debugging purposes)",

celery/bin/events.py

@@ -76,6 +76,8 @@ from celery.bin.base import Command, daemon_options
 
 __all__ = ['events']
 
+HELP = __doc__
+
 
 class events(Command):
     """Event-stream utilities.
@@ -99,7 +101,7 @@ class events(Command):
             $ celery events -c mod.attr -F 1.0 --detach --maxrate=100/m -l info
     """
 
-    doc = __doc__
+    doc = HELP
     supports_args = False
 
     def run(self, dump=False, camera=None, frequency=1.0, maxrate=None,
@@ -149,16 +151,25 @@ class events(Command):
         info = '{0} {1}'.format(info, strargv(sys.argv))
         return set_process_title(prog, info=info)
 
-    def prepare_arguments(self, parser):
-        parser.add_option('-d', '--dump', action='store_true')
-        parser.add_option('-c', '--camera')
-        parser.add_option('--detach', action='store_true')
-        parser.add_option('-F', '--frequency', '--freq',
-                          type='float', default=1.0)
-        parser.add_option('-r', '--maxrate')
-        parser.add_option('-l', '--loglevel', default='INFO')
+    def add_arguments(self, parser):
+        dopts = parser.add_argument_group('Dumper')
+        dopts.add_argument('-d', '--dump', action='store_true')
+
+        copts = parser.add_argument_group('Snapshot')
+        copts.add_argument('-c', '--camera')
+        copts.add_argument('--detach', action='store_true')
+        copts.add_argument('-F', '--frequency', '--freq',
+                           type=float, default=1.0)
+        copts.add_argument('-r', '--maxrate')
+        copts.add_argument('-l', '--loglevel', default='INFO')
+
         daemon_options(parser, default_pidfile='celeryev.pid')
-        parser.add_options(self.app.user_options['events'])
+
+        user_options = self.app.user_options['events']
+        if user_options:
+            self.add_compat_options(
+                parser.add_argument_group('User Options'),
+                user_options)
 
 
 def main():

celery/bin/worker.py

@@ -22,8 +22,8 @@ The :program:`celery worker` command (previously known as ``celeryd``)
 
 .. cmdoption:: -n, --hostname
 
-    Set custom hostname (e.g., 'w1@%h').  Expands: %h (hostname),
-    %n (name) and %d, (domain).
+    Set custom hostname (e.g., 'w1@%%h').  Expands: %%h (hostname),
+    %%n (name) and %%d, (domain).
 
 .. cmdoption:: -B, --beat
 
@@ -170,8 +170,6 @@ from __future__ import absolute_import, unicode_literals
 
 import sys
 
-from optparse import OptionGroup
-
 from celery import concurrency
 from celery.bin.base import Command, daemon_options
 from celery.bin.celeryd_detach import detached_celeryd
@@ -182,7 +180,7 @@ from celery.utils.nodenames import default_nodename
 
 __all__ = ['worker', 'main']
 
-__MODULE_DOC__ = __doc__
+HELP = __doc__
 
 
 class worker(Command):
@@ -198,7 +196,7 @@ class worker(Command):
             $ celery worker -A proj --concurrency=1000 -P eventlet
     """
 
-    doc = __MODULE_DOC__  # parse help from this too
+    doc = HELP  # parse help from this too
     namespace = 'worker'
     enable_config_from_cmdline = True
     supports_args = False
@@ -255,94 +253,88 @@ class worker(Command):
         # that may have to be loaded as early as possible.
         return (['-P'], ['--pool'])
 
-    def prepare_arguments(self, parser):
+    def add_arguments(self, parser):
         conf = self.app.conf
 
-        wopts = OptionGroup(parser, 'Worker Options')
-        wopts.add_option('-n', '--hostname')
-        wopts.add_option('-D', '--detach', action='store_true')
-        wopts.add_option(
+        wopts = parser.add_argument_group('Worker Options')
+        wopts.add_argument('-n', '--hostname')
+        wopts.add_argument('-D', '--detach', action='store_true')
+        wopts.add_argument(
             '-S', '--statedb',
             default=conf.worker_state_db,
         )
-        wopts.add_option('-l', '--loglevel', default='WARN')
-        wopts.add_option('-O', dest='optimization')
-        wopts.add_option(
+        wopts.add_argument('-l', '--loglevel', default='WARN')
+        wopts.add_argument('-O', dest='optimization')
+        wopts.add_argument(
             '--prefetch-multiplier',
-            type='int', default=conf.worker_prefetch_multiplier,
+            type=int, default=conf.worker_prefetch_multiplier,
         )
-        parser.add_option_group(wopts)
 
-        topts = OptionGroup(parser, 'Pool Options')
-        topts.add_option(
+        topts = parser.add_argument_group('Pool Options')
+        topts.add_argument(
             '-c', '--concurrency',
-            default=conf.worker_concurrency, type='int',
+            default=conf.worker_concurrency, type=int,
         )
-        topts.add_option(
+        topts.add_argument(
             '-P', '--pool',
             default=conf.worker_pool,
         )
-        topts.add_option(
+        topts.add_argument(
             '-E', '--task-events', '--events',
             action='store_true', default=conf.worker_send_task_events,
         )
-        topts.add_option(
+        topts.add_argument(
             '--time-limit',
-            type='float', default=conf.task_time_limit,
+            type=float, default=conf.task_time_limit,
         )
-        topts.add_option(
+        topts.add_argument(
             '--soft-time-limit',
-            type='float', default=conf.task_soft_time_limit,
+            type=float, default=conf.task_soft_time_limit,
         )
-        topts.add_option(
+        topts.add_argument(
             '--max-tasks-per-child', '--maxtasksperchild',
-            type='int', default=conf.worker_max_tasks_per_child,
+            type=int, default=conf.worker_max_tasks_per_child,
         )
-        topts.add_option(
+        topts.add_argument(
             '--max-memory-per-child', '--maxmemperchild',
-            type='int', default=conf.worker_max_memory_per_child,
+            type=int, default=conf.worker_max_memory_per_child,
         )
-        parser.add_option_group(topts)
 
-        qopts = OptionGroup(parser, 'Queue Options')
-        qopts.add_option(
+        qopts = parser.add_argument_group('Queue Options')
+        qopts.add_argument(
             '--purge', '--discard',
             default=False, action='store_true',
         )
-        qopts.add_option('--queues', '-Q', default=[])
-        qopts.add_option('--exclude-queues', '-X', default=[])
-        qopts.add_option('--include', '-I', default=[])
-        parser.add_option_group(qopts)
+        qopts.add_argument('--queues', '-Q', default=[])
+        qopts.add_argument('--exclude-queues', '-X', default=[])
+        qopts.add_argument('--include', '-I', default=[])
 
-        fopts = OptionGroup(parser, 'Features')
-        fopts.add_option(
+        fopts = parser.add_argument_group('Features')
+        fopts.add_argument(
             '--without-gossip', action='store_true', default=False,
         )
-        fopts.add_option(
+        fopts.add_argument(
             '--without-mingle', action='store_true', default=False,
         )
-        fopts.add_option(
+        fopts.add_argument(
             '--without-heartbeat', action='store_true', default=False,
         )
-        fopts.add_option('--heartbeat-interval', type='int')
-        parser.add_option_group(fopts)
+        fopts.add_argument('--heartbeat-interval', type=int)
 
         daemon_options(parser)
 
-        bopts = OptionGroup(parser, 'Embedded Beat Options')
-        bopts.add_option('-B', '--beat', action='store_true')
-        bopts.add_option(
+        bopts = parser.add_argument_group('Embedded Beat Options')
+        bopts.add_argument('-B', '--beat', action='store_true')
+        bopts.add_argument(
             '-s', '--schedule-filename', '--schedule',
             default=conf.beat_schedule_filename,
         )
-        bopts.add_option('--scheduler')
-        parser.add_option_group(bopts)
+        bopts.add_argument('--scheduler')
 
         user_options = self.app.user_options['worker']
         if user_options:
-            uopts = OptionGroup(parser, 'User Options')
-            uopts.option_list.extend(user_options)
-            parser.add_option_group(uopts)
+            uopts = parser.add_argument_group('User Options')
+            self.add_compat_options(uopts, user_options)
 
 
 def main(app=None):

docs/userguide/extending.rst

@@ -698,24 +698,25 @@ You can add additional command-line options to the ``worker``, ``beat``, and
 ``events`` commands by modifying the :attr:`~@user_options` attribute of the
 application instance.
 
-Celery commands uses the :mod:`optparse` module to parse command-line
-arguments, and so you have to use :mod:`optparse` specific option instances created
-using :func:`optparse.make_option`. Please see the :mod:`optparse`
-documentation to read about the fields supported.
+Celery commands uses the :mod:`argparse` module to parse command-line
+arguments, and so to add custom arguments you need to specify a callback
+that takes a :class:`argparse.ArgumentParser` instance - and adds arguments.
+Please see the :mod:`argparse` documentation to read about the fields supported.
 
 Example adding a custom option to the :program:`celery worker` command:
 
 .. code-block:: python
 
     from celery import Celery
-    from celery.bin import Option  # <-- alias to optparse.make_option
 
     app = Celery(broker='amqp://')
 
-    app.user_options['worker'].add(
-        Option('--enable-my-option', action='store_true', default=False,
-               help='Enable custom option.'),
-    )
+    def add_worker_arguments(parser):
+        parser.add_argument(
+            '--enable-my-option', action='store_true', default=False,
+            help='Enable custom option.',
+        ),
+    app.user_options['worker'].add(add_worker_arguments)
 
 
 All bootsteps will now receive this argument as a keyword argument to
@@ -755,10 +756,13 @@ template:
     from celery.bin import Option
 
     app = Celery()
-    app.user_options['preload'].add(
-        Option('-Z', '--template', default='default',
-               help='Configuration template to use.'),
-    )
+
+    def add_preload_options(parser):
+        parser.add_argument(
+            '-Z', '--template', default='default',
+            help='Configuration template to use.',
+        )
+    app.user_options['preload'].add(add_preload_options)
 
     @signals.user_preload_options.connect
     def on_preload_parsed(options, **kwargs):
@@ -816,17 +820,18 @@ something like this:
 
 .. code-block:: python
 
-    from celery.bin.base import Command, Option
+    from celery.bin.base import Command
 
 
     class FlowerCommand(Command):
 
-        def get_options(self):
-            return (
-                Option('--port', default=8888, type='int',
-                    help='Webserver port',
-                ),
-                Option('--debug', action='store_true'),
+        def add_arguments(self, parser):
+            parser.add_argument(
+                '--port', default=8888, type='int',
+                help='Webserver port',
+            ),
+            parser.add_argument(
+                '--debug', action='store_true',
             )
 
         def run(self, port=None, debug=False, **kwargs):

t/unit/bin/test_base.py

@@ -9,10 +9,8 @@ from celery.bin.base import (
     Command,
     Option,
     Extensions,
-    HelpFormatter,
 )
 from celery.five import bytes_if_py2
-from celery.utils.objects import Bunch
 
 
 class MyApp(object):
@@ -25,7 +23,7 @@ class MockCommand(Command):
     mock_args = ('arg1', 'arg2', 'arg3')
 
     def parse_options(self, prog_name, arguments, command=None):
-        options = Bunch(foo='bar', prog_name=prog_name)
+        options = dict(foo='bar', prog_name=prog_name)
         return options, self.mock_args
 
     def run(self, *args, **kwargs):
@@ -61,18 +59,6 @@ class test_Extensions:
                     e.load()
 
 
-class test_HelpFormatter:
-
-    def test_format_epilog(self):
-        f = HelpFormatter()
-        assert f.format_epilog('hello')
-        assert not f.format_epilog('')
-
-    def test_format_description(self):
-        f = HelpFormatter()
-        assert f.format_description('hello')
-
-
 class test_Command:
 
     def test_get_options(self):
@@ -88,6 +74,13 @@ class test_Command:
         c = C()
         assert c.description == 'foo'
 
+    def test_format_epilog(self):
+        assert Command()._format_epilog('hello')
+        assert not Command()._format_epilog('')
+
+    def test_format_description(self):
+        assert Command()._format_description('hello')
+
     def test_register_callbacks(self):
         c = Command(on_error=8, on_usage_error=9)
         assert c.on_error == 8
@@ -304,22 +297,6 @@ class test_Command:
             '.prefetch_multiplier=100'])
         assert cmd.app is cmd.get_app()
 
-    def test_preparse_options__required_short(self, app):
-        cmd = MockCommand(app=app)
-        with pytest.raises(ValueError):
-            cmd.preparse_options(
-                ['a', '-f'], [Option('-f', action='store')])
-
-    def test_preparse_options__longopt_whitespace(self, app):
-        cmd = MockCommand(app=app)
-        cmd.preparse_options(
-            ['a', '--foo', 'val'], [Option('--foo', action='store')])
-
-    def test_preparse_options__shortopt_store_true(self, app):
-        cmd = MockCommand(app=app)
-        cmd.preparse_options(
-            ['a', '--foo'], [Option('--foo', action='store_true')])
-
     def test_get_default_app(self, app, patching):
         patching('celery._state.get_current_app')
         cmd = MockCommand(app=app)
@@ -357,15 +334,22 @@ class test_Command:
             assert cmd.find_app('proj') == 'quick brown fox'
 
     def test_parse_preload_options_shortopt(self):
-        cmd = Command()
-        cmd.preload_options = (Option('-s', action='store', dest='silent'),)
+
+        class TestCommand(Command):
+
+            def add_preload_options(self, parser):
+                parser.add_argument('-s', action='store', dest='silent')
+        cmd = TestCommand()
         acc = cmd.parse_preload_options(['-s', 'yes'])
         assert acc.get('silent') == 'yes'
 
     def test_parse_preload_options_with_equals_and_append(self):
+
+        class TestCommand(Command):
+
+            def add_preload_options(self, parser):
+                parser.add_argument('--zoom', action='append', default=[])
         cmd = Command()
-        opt = Option('--zoom', action='append', default=[])
-        cmd.preload_options = (opt,)
         acc = cmd.parse_preload_options(['--zoom=1', '--zoom=2'])
 
         assert acc, {'zoom': ['1' == '2']}

t/unit/bin/test_beat.py

@@ -144,4 +144,4 @@ class test_div:
         cmd = beat_bin.beat()
         cmd.app = self.app
         options, args = cmd.parse_options('celery beat', ['-s', 'foo'])
-        assert options.schedule == 'foo'
+        assert options['schedule'] == 'foo'

t/unit/bin/test_celeryd_detach.py

@@ -64,14 +64,13 @@ class test_PartialOptionParser:
     def test_parser(self):
         x = detached_celeryd(self.app)
         p = x.create_parser('celeryd_detach')
-        options, values = p.parse_args([
+        options, leftovers = p.parse_known_args([
             '--logfile=foo', '--fake', '--enable',
             'a', 'b', '-c1', '-d', '2',
         ])
         assert options.logfile == 'foo'
-        assert values, ['a' == 'b']
-        assert p.leftovers, ['--enable', '-c1', '-d' == '2']
-        options, values = p.parse_args([
+        assert leftovers, ['--enable', '-c1', '-d' == '2']
+        options, leftovers = p.parse_known_args([
             '--fake', '--enable',
             '--pidfile=/var/pid/foo.pid',
             'a', 'b', '-c1', '-d', '2',
@@ -81,15 +80,14 @@ class test_PartialOptionParser:
         with mock.stdouts():
             with pytest.raises(SystemExit):
                 p.parse_args(['--logfile'])
-            p.get_option('--logfile').nargs = 2
+            p._option_string_actions['--logfile'].nargs = 2
             with pytest.raises(SystemExit):
                 p.parse_args(['--logfile=a'])
             with pytest.raises(SystemExit):
                 p.parse_args(['--fake=abc'])
 
-        assert p.get_option('--logfile').nargs == 2
-        p.parse_args(['--logfile=a', 'b'])
-        p.get_option('--logfile').nargs = 1
+        assert p._option_string_actions['--logfile'].nargs == 2
+        p.parse_args(['--logfile', 'a', 'b'])
 
 
 class test_Command:
@@ -101,7 +99,8 @@ class test_Command:
 
     def test_parse_options(self):
         x = detached_celeryd(app=self.app)
-        o, v, l = x.parse_options('cd', self.argv)
+        _, argv = x._split_command_line_config(self.argv)
+        o, l = x.parse_options('cd', argv)
         assert o.logfile == '/var/log'
         assert l == [
             '--foobar=10,2', '-c', '1',

t/unit/bin/test_worker.py

@@ -399,8 +399,8 @@ class test_funs:
         cmd.app = self.app
         opts, args = cmd.parse_options('worker', ['--concurrency=512',
                                        '--heartbeat-interval=10'])
-        assert opts.concurrency == 512
-        assert opts.heartbeat_interval == 10
+        assert opts['concurrency'] == 512
+        assert opts['heartbeat_interval'] == 10
 
     def test_main(self):
         p, cd.Worker = cd.Worker, Worker