Parse command option doc from docstring

celery/bin/base.py

@@ -1,10 +1,72 @@
 # -*- coding: utf-8 -*-
+"""
+
+.. _preload-options:
+
+Preload Options
+---------------
+
+.. cmdoption:: --app
+
+    Fully qualified name of the app instance to use.
+
+.. cmdoption:: -b, --broker
+
+    Broker URL.  Default is 'amqp://guest:guest@localhost:5672//'
+
+.. cmdoption:: --loader
+
+    Name of the loader class to use.
+    Taken from the environment variable :envvar:`CELERY_LOADER`
+    or 'default' if that is not set.
+
+.. cmdoption:: --config
+
+    Name of the module to read configuration from,
+    default is 'celeryconfig'.
+
+.. _daemon-options:
+
+Daemon Options
+--------------
+
+.. cmdoption:: -f, --logfile
+
+    Path to log file. If no logfile is specified, `stderr` is used.
+
+.. cmdoption:: --pidfile
+
+    Optional file used to store the process pid.
+
+    The program will not start if this file already exists
+    and the pid is still alive.
+
+.. cmdoption:: --uid
+
+    User id, or user name of the user to run as after detaching.
+
+.. cmdoption:: --gid
+
+    Group id, or group name of the main group to change to after
+    detaching.
+
+.. cmdoption:: --umask
+
+    Effective umask of the process after detaching. Default is 0.
+
+.. cmdoption:: --workdir
+
+    Optional directory to change to after detaching.
+
+"""
 from __future__ import absolute_import
 
 import os
+import re
 import sys
 import warnings
 
+from collections import defaultdict
 from optparse import OptionParser, make_option as Option
 
 from celery import Celery, __version__
@@ -22,6 +84,9 @@ Unrecognized command line arguments: %s
 Try --help?
 """
 
+find_long_opt = re.compile(r'.+?(--.+?)(?:\s|,|$)')
+find_rst_ref = re.compile(r':\w+:`(.+?)`')
+
 
 class Command(object):
     """Base class for command line applications.
@@ -30,7 +95,6 @@ class Command(object):
     :keyword get_app: Callable returning the current app if no app provided.
 
     """
-    _default_broker_url = r'amqp://guest:guest@localhost:5672//'
     #: Arg list used in help.
     args = ''
 
@@ -44,24 +108,15 @@ class Command(object):
     #: List of options (without preload options).
     option_list = ()
 
+    # module Rst documentation to parse help from (if any)
+    doc = None
+
     #: List of options to parse before parsing other options.
     preload_options = (
-            Option("--app",
-                    default=None, action="store", dest="app",
-                    help="Name of the app instance to use. "),
-            Option("-b", "--broker",
-                    default=None, action="store", dest="broker",
-                    help="Broker URL.  Default is %s" % (
-                            _default_broker_url, )),
-            Option("--loader",
-                   default=None, action="store", dest="loader",
-                   help="Name of the loader class to use. "
-                        "Taken from the environment variable CELERY_LOADER, "
-                        "or 'default' if that is not set."),
-            Option("--config",
-                    default="celeryconfig", action="store",
-                    dest="config_module",
-                    help="Name of the module to read configuration from."),
+        Option("--app", default=None),
+        Option("-b", "--broker", default=None),
+        Option("--loader", default=None),
+        Option("--config", default="celeryconfig", dest="config_module"),
     )
 
     #: Enable if the application should support config from the cmdline.
@@ -145,11 +200,20 @@ class Command(object):
         return parser.parse_args(arguments)
 
     def create_parser(self, prog_name):
-        return self.Parser(prog=prog_name,
+        return self.prepare_parser(self.Parser(prog=prog_name,
                            usage=self.usage(),
                            version=self.version,
                            option_list=(self.preload_options +
-                                        self.get_options()))
+                                        self.get_options())))
+
+    def prepare_parser(self, parser):
+        docs = [self.parse_doc(doc) for doc in (self.doc, __doc__) if doc]
+        for doc in docs:
+            for long_opt, help in doc.iteritems():
+                option = parser.get_option(long_opt)
+                if option is not None:
+                    option.help = ' '.join(help) % {"default": option.default}
+        return parser
 
     def prepare_preload_options(self, options):
         """Optional handler to do additional processing of preload options.
@@ -219,28 +283,29 @@ class Command(object):
             index += 1
         return acc
 
+    def parse_doc(self, doc):
+        options, in_option = defaultdict(list), None
+        for line in doc.splitlines():
+            if line.startswith(".. cmdoption::"):
+                m = find_long_opt.match(line)
+                if m:
+                    in_option = m.groups()[0].strip()
+                assert in_option, "missing long opt"
+            elif in_option and line.startswith(' ' * 4):
+                options[in_option].append(find_rst_ref.sub(r'\1',
+                    line.strip()).replace('`', ''))
+        return options
+
     def _get_default_app(self, *args, **kwargs):
         return Celery(*args, **kwargs)
 
 
 def daemon_options(default_pidfile=None, default_logfile=None):
     return (
-        Option('-f', '--logfile', default=default_logfile,
-               action="store", dest="logfile",
-               help="Path to the logfile"),
-        Option('--pidfile', default=default_pidfile,
-               action="store", dest="pidfile",
-               help="Path to the pidfile."),
-        Option('--uid', default=None,
-               action="store", dest="uid",
-               help="Effective user id to run as when detached."),
-        Option('--gid', default=None,
-               action="store", dest="gid",
-               help="Effective group id to run as when detached."),
-        Option('--umask', default=0,
-               action="store", type="int", dest="umask",
-               help="Umask of the process when detached."),
-        Option('--workdir', default=None,
-               action="store", dest="working_directory",
-               help="Directory to change to when detached."),
-)
+        Option("-f", "--logfile", default=default_logfile),
+        Option("--pidfile", default=default_pidfile),
+        Option("--uid", default=None),
+        Option("--gid", default=None),
+        Option("--umask", default=0, type="int"),
+        Option("--workdir", default=None, dest="working_directory"),
+    )

celery/bin/celery.py

@@ -54,10 +54,8 @@ class Command(object):
     prog_name = "celery"
 
     option_list = BaseCommand.preload_options + (
-        Option("--quiet", "-q", action="store_true", dest="quiet",
-                default=False),
-        Option("--no-color", "-C", dest="no_color", action="store_true",
-            help="Don't colorize output."),
+        Option("--quiet", "-q", action="store_true"),
+        Option("--no-color", "-C", action="store_true"),
     )
 
     def __init__(self, app=None, no_color=False, stdout=sys.stdout,
@@ -202,15 +200,15 @@ list_ = command(list_, "list")
 class apply(Command):
     args = "<task_name>"
     option_list = Command.option_list + (
-            Option("--args", "-a", dest="args"),
-            Option("--kwargs", "-k", dest="kwargs"),
-            Option("--eta", dest="eta"),
-            Option("--countdown", dest="countdown", type="int"),
-            Option("--expires", dest="expires"),
-            Option("--serializer", dest="serializer", default="json"),
-            Option("--queue", dest="queue"),
-            Option("--exchange", dest="exchange"),
-            Option("--routing-key", dest="routing_key"),
+            Option("--args", "-a"),
+            Option("--kwargs", "-k"),
+            Option("--eta"),
+            Option("--countdown", type="int"),
+            Option("--expires"),
+            Option("--serializer", default="json"),
+            Option("--queue"),
+            Option("--exchange"),
+            Option("--routing-key"),
     )
 
     def run(self, name, *_, **kw):
@@ -265,7 +263,7 @@ purge = command(purge)
 class result(Command):
     args = "<task_id>"
     option_list = Command.option_list + (
-            Option("--task", "-t", dest="task"),
+            Option("--task", "-t"),
     )
 
     def run(self, task_id, *args, **kwargs):
@@ -295,10 +293,9 @@ class inspect(Command):
                "cancel_consumer": 1.0,
                "report": 1.0}
     option_list = Command.option_list + (
-                Option("--timeout", "-t", type="float", dest="timeout",
-                    default=None,
+                Option("--timeout", "-t", type="float",
                     help="Timeout in seconds (float) waiting for reply"),
-                Option("--destination", "-d", dest="destination",
+                Option("--destination", "-d",
                     help="Comma separated list of destination node names."))
     show_body = True
 
@@ -396,24 +393,20 @@ migrate = command(migrate)
 
 class shell(Command):  # pragma: no cover
     option_list = Command.option_list + (
-                Option("--ipython", "-I", action="store_true",
-                    dest="force_ipython", default=False,
+                Option("--ipython", "-I",
+                    action="store_true", dest="force_ipython",
                     help="Force IPython."),
-                Option("--bpython", "-B", action="store_true",
-                    dest="force_bpython", default=False,
+                Option("--bpython", "-B",
+                    action="store_true", dest="force_bpython",
                     help="Force bpython."),
-                Option("--python", "-P", action="store_true",
-                    dest="force_python", default=False,
+                Option("--python", "-P",
+                    action="store_true", dest="force_python",
                     help="Force default Python shell."),
                 Option("--without-tasks", "-T", action="store_true",
-                    dest="without_tasks", default=False,
                     help="Don't add tasks to locals."),
                 Option("--eventlet", action="store_true",
-                    dest="eventlet", default=False,
                     help="Use eventlet."),
-                Option("--gevent", action="store_true",
-                    dest="gevent", default=False,
-                    help="Use gevent."),
+                Option("--gevent", action="store_true", help="Use gevent."),
     )
 
     def run(self, force_ipython=False, force_bpython=False,

celery/bin/celerybeat.py

@@ -3,14 +3,28 @@
 
 .. program:: celerybeat
 
+.. seealso::
+
+    See :ref:`preload-options` and :ref:`daemon-options`.
+
+.. cmdoption:: --detach
+
+    Detach and run in the background as a daemon.
+
 .. cmdoption:: -s, --schedule
 
     Path to the schedule database. Defaults to `celerybeat-schedule`.
-    The extension ".db" will be appended to the filename.
+    The extension ".db" may be appended to the filename.
+    Default is %(default)s.
 
 .. cmdoption:: -S, --scheduler
 
-    Scheduler class to use. Default is celery.beat.PersistentScheduler
+    Scheduler class to use.
+    Default is :class:`celery.beat.PersistentScheduler`.
+
+.. cmdoption:: max-interval
+
+    Max seconds to sleep between schedule iterations.
 
 .. cmdoption:: -f, --logfile
 
@@ -35,6 +49,7 @@ from celery.bin.base import Command, Option, daemon_options
 
 
 class BeatCommand(Command):
+    doc = __doc__
     enable_config_from_cmdline = True
     supports_args = False
     preload_options = (Command.preload_options
@@ -59,30 +74,15 @@ class BeatCommand(Command):
             os.chdir(workdir)
 
     def get_options(self):
-        conf = self.app.conf
+        c = self.app.conf
 
         return (
-            Option('--detach',
-                default=False, action="store_true", dest="detach",
-                help="Detach and run in the background."),
-            Option('-s', '--schedule',
-                default=conf.CELERYBEAT_SCHEDULE_FILENAME,
-                action="store", dest="schedule",
-                help="Path to the schedule database. The extension "
-                    "'.db' will be appended to the filename. Default: %s" % (
-                            conf.CELERYBEAT_SCHEDULE_FILENAME, )),
-            Option('--max-interval',
-                default=None, type="float", dest="max_interval",
-                help="Max. seconds to sleep between schedule iterations."),
-            Option('-S', '--scheduler',
-                default=None,
-                action="store", dest="scheduler_cls",
-                help="Scheduler class. Default is "
-                     "celery.beat:PersistentScheduler"),
-            Option('-l', '--loglevel',
-                default=conf.CELERYBEAT_LOG_LEVEL,
-                action="store", dest="loglevel",
-                help="Loglevel. One of DEBUG/INFO/WARNING/ERROR/CRITICAL."))
+            Option('--detach', action="store_true"),
+            Option('-s', '--schedule', default=c.CELERYBEAT_SCHEDULE_FILENAME),
+            Option('--max-interval', type="float"),
+            Option('-S', '--scheduler', dest="scheduler_cls"),
+            Option('-l', '--loglevel', default=c.CELERYBEAT_LOG_LEVEL),
+        )
 
 
 def main():

celery/bin/celeryd.py

@@ -3,11 +3,21 @@
 
 .. program:: celeryd
 
+.. seealso::
+
+    See :ref:`preload-options`.
+
 .. cmdoption:: -c, --concurrency
 
     Number of child processes processing the queue. The default
     is the number of CPUs available on your system.
 
+.. cmdoption:: -c, --pool
+
+    Pool implementation:
+
+    processes (default), eventlet, gevent, solo or threads.
+
 .. cmdoption:: -f, --logfile
 
     Path to log file. If no logfile is specified, `stderr` is used.
@@ -19,7 +29,7 @@
 
 .. cmdoption:: -n, --hostname
 
-    Set custom hostname.
+    Set custom hostname, e.g. 'foo.example.com'.
 
 .. cmdoption:: -B, --beat
 
@@ -40,18 +50,24 @@
 .. cmdoption:: -s, --schedule
 
     Path to the schedule database if running with the `-B` option.
-    Defaults to `celerybeat-schedule`. The extension ".db" will be
+    Defaults to `celerybeat-schedule`. The extension ".db" may be
     appended to the filename.
 
 .. cmdoption:: --scheduler
 
     Scheduler class to use. Default is celery.beat.PersistentScheduler
 
+.. cmdoption:: -S, --statedb
+
+    Path to the state database. The extension '.db' may
+    be appended to the filename. Default: %(default)s
+
 .. cmdoption:: -E, --events
 
-    Send events that can be captured by monitors like `celerymon`.
+    Send events that can be captured by monitors like :program:`celeryev`,
+    `celerymon`, and others.
 
-.. cmdoption:: --purge, --discard
+.. cmdoption:: --purge
 
     Discard all waiting tasks before the daemon is started.
     **WARNING**: This is unrecoverable, and the tasks will be
@@ -70,6 +86,30 @@
     Maximum number of tasks a pool worker can execute before it's
     terminated and replaced by a new worker.
 
+.. cmdoption:: --pidfile
+
+    Optional file used to store the workers pid.
+
+    The worker will not start if this file already exists
+    and the pid is still alive.
+
+.. cmdoption:: --autoscale
+
+    Enable autoscaling by providing
+    max_concurrency, min_concurrency. Example::
+
+        --autoscale=10,3
+
+    (always keep 3 processes, but grow to 10 if necessary)
+
+.. cmdoption:: --autoreload
+
+    Enable autoreloading.
+
+.. cmdoption:: --no-execv
+
+    Don't do execv after multiprocessing child fork.
+
 """
 from __future__ import absolute_import
 
@@ -81,6 +121,7 @@ from celery.bin.base import Command, Option
 
 
 class WorkerCommand(Command):
+    doc = __doc__  # parse help from this.
     namespace = "celeryd"
     enable_config_from_cmdline = True
     supports_args = False
@@ -98,92 +139,36 @@ class WorkerCommand(Command):
         conf = self.app.conf
         return (
             Option('-c', '--concurrency',
-                default=conf.CELERYD_CONCURRENCY,
-                action="store", dest="concurrency", type="int",
-                help="Number of worker threads/processes"),
-            Option('-P', '--pool',
-                default=conf.CELERYD_POOL,
-                action="store", dest="pool_cls", type="str",
-                help="Pool implementation: "
-                     "processes (default), eventlet, gevent, "
-                     "solo or threads."),
+                default=conf.CELERYD_CONCURRENCY, type="int"),
+            Option('-P', '--pool', default=conf.CELERYD_POOL, dest="pool_cls"),
             Option('--purge', '--discard', default=False,
-                action="store_true", dest="discard",
-                help="Discard all waiting tasks before the server is"
-                     "started. WARNING: There is no undo operation "
-                     "and the tasks will be deleted."),
-            Option('-f', '--logfile', default=conf.CELERYD_LOG_FILE,
-                action="store", dest="logfile",
-                help="Path to log file."),
-            Option('-l', '--loglevel', default=conf.CELERYD_LOG_LEVEL,
-                action="store", dest="loglevel",
-                help="Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL"),
-            Option('-n', '--hostname', default=None,
-                action="store", dest="hostname",
-                help="Set custom host name. E.g. 'foo.example.com'."),
-            Option('-B', '--beat', default=False,
-                action="store_true", dest="embed_clockservice",
-                help="Also run the celerybeat periodic task scheduler. "
-                     "NOTE: Only one instance of celerybeat must be"
-                     "running at any one time."),
-            Option('-s', '--schedule',
-                default=conf.CELERYBEAT_SCHEDULE_FILENAME,
-                action="store", dest="schedule_filename",
-                help="Path to the schedule database if running with the -B "
-                     "option. The extension '.db' will be appended to the "
-                    "filename. Default: %s" % (
-                        conf.CELERYBEAT_SCHEDULE_FILENAME, )),
-            Option('--scheduler',
-                default=None,
-                action="store", dest="scheduler_cls",
-                help="Scheduler class. Default is "
-                     "celery.beat:PersistentScheduler"),
-            Option('-S', '--statedb', default=conf.CELERYD_STATE_DB,
-                action="store", dest="state_db",
-                help="Path to the state database. The extension '.db' will "
-                     "be appended to the filename. Default: %s" % (
-                        conf.CELERYD_STATE_DB, )),
+                action="store_true", dest="discard"),
+            Option('-f', '--logfile', default=conf.CELERYD_LOG_FILE),
+            Option('-l', '--loglevel', default=conf.CELERYD_LOG_LEVEL),
+            Option('-n', '--hostname'),
+            Option('-B', '--beat',
+                action="store_true", dest="embed_clockservice"),
+            Option('-s', '--schedule', dest="schedule_filename",
+                default=conf.CELERYBEAT_SCHEDULE_FILENAME),
+            Option('--scheduler', dest="scheduler_cls"),
+            Option('-S', '--statedb',
+                default=conf.CELERYD_STATE_DB, dest="state_db"),
             Option('-E', '--events', default=conf.CELERY_SEND_EVENTS,
-                action="store_true", dest="send_events",
-                help="Send events so the worker can be monitored by "
-                     "celeryev, celerymon and other monitors.."),
-            Option('--time-limit',
-                default=conf.CELERYD_TASK_TIME_LIMIT,
-                action="store", type="int", dest="task_time_limit",
-                help="Enables a hard time limit (in seconds) for tasks."),
-            Option('--soft-time-limit',
-                default=conf.CELERYD_TASK_SOFT_TIME_LIMIT,
-                action="store", type="int", dest="task_soft_time_limit",
-                help="Enables a soft time limit (in seconds) for tasks."),
-            Option('--maxtasksperchild',
-                default=conf.CELERYD_MAX_TASKS_PER_CHILD,
-                action="store", type="int", dest="max_tasks_per_child",
-                help="Maximum number of tasks a pool worker can execute"
-                     "before it's terminated and replaced by a new worker."),
-            Option('--queues', '-Q', default=[],
-                action="store", dest="queues",
-                help="Comma separated list of queues to consume from. "
-                     "By default all configured queues are used. "
-                     "Example: -Q video,image"),
-            Option('--include', '-I', default=[],
-                action="store", dest="include",
-                help="Comma separated list of additional modules to import. "
-                 "Example: -I foo.tasks,bar.tasks"),
-            Option('--pidfile', dest="pidfile", default=None,
-                help="Optional file used to store the workers pid. "
-                     "The worker will not start if this file already exists "
-                     "and the pid is still alive."),
-            Option('--autoscale', dest="autoscale", default=None,
-                help="Enable autoscaling by providing "
-                     "max_concurrency,min_concurrency. Example: "
-                     "--autoscale=10,3 (always keep 3 processes, "
-                     "but grow to 10 if necessary)."),
-            Option('--autoreload', dest="autoreload",
-                    action="store_true", default=False,
-                help="Enable autoreloading."),
+                action="store_true", dest="send_events"),
+            Option('--time-limit', type="int", dest="task_time_limit",
+                default=conf.CELERYD_TASK_TIME_LIMIT),
+            Option('--soft-time-limit', dest="task_soft_time_limit",
+                default=conf.CELERYD_TASK_SOFT_TIME_LIMIT, type="int"),
+            Option('--maxtasksperchild', dest="max_tasks_per_child",
+                default=conf.CELERYD_MAX_TASKS_PER_CHILD, type="int"),
+            Option('--queues', '-Q', default=[]),
+            Option('--include', '-I', default=[]),
+            Option('--pidfile'),
+            Option('--autoscale'),
+            Option('--autoreload', action="store_true"),
+            Option("--no-execv", action="store_true", default=False),
         )
 
-
 def main():
     # Fix for setuptools generated scripts, so that it will
     # work with multiprocessing fork emulation.

celery/bin/celeryev.py

@@ -1,4 +1,38 @@
 # -*- coding: utf-8 -*-
+"""
+
+.. program:: celeryev
+
+.. seealso::
+
+    See :ref:`preload-options` and :ref:`daemon-options`.
+
+.. cmdoption:: -d, --dump
+
+    Dump events to stdout.
+
+.. cmdoption:: -c, --camera
+
+    Take snapshots of events using this camera.
+
+.. cmdoption:: --detach
+
+    Camera: Detach and run in the background as a daemon.
+
+.. cmdoption:: -F, --freq, --frequency
+
+    Camera: Shutter frequency.  Default is every 1.0 seconds.
+
+.. cmdoption:: -r, --maxrate
+
+    Camera: Optional shutter rate limit (e.g. 10/m).
+
+.. cmdoption:: -l, --loglevel
+
+    Logging level, choose between `DEBUG`, `INFO`, `WARNING`,
+    `ERROR`, `CRITICAL`, or `FATAL`.  Default is INFO.
+
+"""
 from __future__ import absolute_import
 from __future__ import with_statement
 
@@ -12,6 +46,7 @@ from celery.bin.base import Command, Option, daemon_options
 
 
 class EvCommand(Command):
+    doc = __doc__
     supports_args = False
     preload_options = (Command.preload_options
                      + daemon_options(default_pidfile="celeryev.pid"))
@@ -71,25 +106,13 @@ class EvCommand(Command):
 
     def get_options(self):
         return (
-            Option('-d', '--dump',
-                   action="store_true", dest="dump",
-                   help="Dump events to stdout."),
-            Option('-c', '--camera',
-                   action="store", dest="camera",
-                   help="Camera class to take event snapshots with."),
-            Option('--detach',
-                default=False, action="store_true", dest="detach",
-                help="Recording: Detach and run in the background."),
-            Option('-F', '--frequency', '--freq',
-                   action="store", dest="frequency",
-                   type="float", default=1.0,
-                   help="Recording: Snapshot frequency."),
-            Option('-r', '--maxrate',
-                   action="store", dest="maxrate", default=None,
-                   help="Recording: Shutter rate limit (e.g. 10/m)"),
-            Option('-l', '--loglevel',
-                   action="store", dest="loglevel", default="INFO",
-                   help="Loglevel. Default is WARNING."))
+            Option('-d', '--dump', action="store_true"),
+            Option('-c', '--camera'),
+            Option('--detach', action="store_true"),
+            Option('-F', '--frequency', '--freq', type="float", default=1.0),
+            Option('-r', '--maxrate'),
+            Option('-l', '--loglevel', default="INFO"),
+        )
 
 
 def main():