|
|
@@ -35,17 +35,17 @@
|
|
|
Discard all waiting tasks before the daemon is started.
|
|
|
**WARNING**: This is unrecoverable, and the tasks will be
|
|
|
deleted from the messaging server.
|
|
|
-
|
|
|
+
|
|
|
.. cmdoption:: -u, --uid
|
|
|
|
|
|
User-id to run ``celeryd`` as when in daemon mode.
|
|
|
|
|
|
.. cmdoption:: -g, --gid
|
|
|
-
|
|
|
+
|
|
|
Group-id to run ``celeryd`` as when in daemon mode.
|
|
|
|
|
|
.. cmdoption:: --umask
|
|
|
-
|
|
|
+
|
|
|
umask of the process when in daemon mode.
|
|
|
|
|
|
.. cmdoption:: --workdir
|
|
|
@@ -68,6 +68,7 @@ from django.conf import settings
|
|
|
from celery.log import emergency_error
|
|
|
from celery.conf import LOG_LEVELS, DAEMON_LOG_FILE, DAEMON_LOG_LEVEL
|
|
|
from celery.conf import DAEMON_CONCURRENCY, DAEMON_PID_FILE
|
|
|
+from celery.messaging import TaskConsumer
|
|
|
from celery import conf
|
|
|
from celery import discovery
|
|
|
from celery.task import discard_all
|
|
|
@@ -88,12 +89,50 @@ settings.CELERY_STATISTICS = USE_STATISTICS
|
|
|
|
|
|
STARTUP_INFO_FMT = """
|
|
|
* Celery loading with the following configuration
|
|
|
- * Broker -> amqp://%(vhost)s@%(host)s:%(port)s
|
|
|
+ * Broker -> amqp://%(vhost)s@%(host)s:%(port)s
|
|
|
* Exchange -> %(exchange)s (%(exchange_type)s)
|
|
|
* Consumer -> Queue:%(consumer_queue)s Routing:%(consumer_rkey)s
|
|
|
* Concurrency:%(concurrency)s
|
|
|
""".strip()
|
|
|
|
|
|
+OPTION_LIST = (
|
|
|
+ optparse.make_option('-c', '--concurrency', default=DAEMON_CONCURRENCY,
|
|
|
+ action="store", dest="concurrency", type="int",
|
|
|
+ help="Number of child processes processing the queue."),
|
|
|
+ optparse.make_option('--discard', default=False,
|
|
|
+ action="store_true", dest="discard",
|
|
|
+ help="Discard all waiting tasks before the server is started. "
|
|
|
+ "WARNING: This is unrecoverable, and the tasks will be "
|
|
|
+ "deleted from the messaging server."),
|
|
|
+ optparse.make_option('-f', '--logfile', default=DAEMON_LOG_FILE,
|
|
|
+ action="store", dest="logfile",
|
|
|
+ help="Path to log file."),
|
|
|
+ optparse.make_option('-l', '--loglevel', default=DAEMON_LOG_LEVEL,
|
|
|
+ action="store", dest="loglevel",
|
|
|
+ help="Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL/FATAL."),
|
|
|
+ optparse.make_option('-p', '--pidfile', default=DAEMON_PID_FILE,
|
|
|
+ action="store", dest="pidfile",
|
|
|
+ help="Path to pidfile."),
|
|
|
+ optparse.make_option('-d', '--detach', '--daemon', default=False,
|
|
|
+ action="store_true", dest="detach",
|
|
|
+ help="Run in the background as a daemon."),
|
|
|
+ optparse.make_option('-u', '--uid', default=None,
|
|
|
+ action="store", dest="uid",
|
|
|
+ help="User-id to run celeryd as when in daemon mode."),
|
|
|
+ optparse.make_option('-g', '--gid', default=None,
|
|
|
+ action="store", dest="gid",
|
|
|
+ help="Group-id to run celeryd as when in daemon mode."),
|
|
|
+ optparse.make_option('--umask', default=0,
|
|
|
+ action="store", type="int", dest="umask",
|
|
|
+ help="umask of the process when in daemon mode."),
|
|
|
+ optparse.make_option('--workdir', default=None,
|
|
|
+ action="store", dest="working_directory",
|
|
|
+ help="Directory to change to when in daemon mode."),
|
|
|
+ optparse.make_option('--chroot', default=None,
|
|
|
+ action="store", dest="chroot",
|
|
|
+ help="Change root directory to this path when in daemon mode."),
|
|
|
+ )
|
|
|
+
|
|
|
|
|
|
def acquire_pidlock(pidfile):
|
|
|
"""Get the :class:`daemon.pidlockfile.PIDLockFile` handler for
|
|
|
@@ -115,8 +154,8 @@ def acquire_pidlock(pidfile):
|
|
|
except os.error, exc:
|
|
|
if exc.errno == errno.ESRCH:
|
|
|
sys.stderr.write("Stale pidfile exists. Removing it.\n")
|
|
|
- pidlock.release()
|
|
|
- return
|
|
|
+ pidlock.release()
|
|
|
+ return PIDLockFile(pidfile)
|
|
|
else:
|
|
|
raise SystemExit(
|
|
|
"ERROR: Pidfile (%s) already exists.\n"
|
|
|
@@ -125,11 +164,11 @@ def acquire_pidlock(pidfile):
|
|
|
return pidlock
|
|
|
|
|
|
|
|
|
-def run_worker(concurrency=DAEMON_CONCURRENCY, daemon=False,
|
|
|
+def run_worker(concurrency=DAEMON_CONCURRENCY, detach=False,
|
|
|
loglevel=DAEMON_LOG_LEVEL, logfile=DAEMON_LOG_FILE, discard=False,
|
|
|
pidfile=DAEMON_PID_FILE, umask=0, uid=None, gid=None,
|
|
|
working_directory=None, chroot=None, statistics=None, **kwargs):
|
|
|
- """Start a celery worker server."""
|
|
|
+ """Starts the celery worker server."""
|
|
|
|
|
|
print(". Launching celery, please hold on to something...")
|
|
|
|
|
|
@@ -145,17 +184,22 @@ def run_worker(concurrency=DAEMON_CONCURRENCY, daemon=False,
|
|
|
"concurrency. We'll be using a single process only.",
|
|
|
UserWarning)
|
|
|
concurrency = 1
|
|
|
-
|
|
|
+
|
|
|
+ # Setup logging
|
|
|
if not isinstance(loglevel, int):
|
|
|
loglevel = LOG_LEVELS[loglevel.upper()]
|
|
|
+ if not detach:
|
|
|
+ logfile = None # log to stderr when not running in the background.
|
|
|
|
|
|
if discard:
|
|
|
discarded_count = discard_all()
|
|
|
- what = discard_count > 1 and "messages" or "message"
|
|
|
- print("* Discard: Erased %d %s from the queue." % (
|
|
|
+ what = discarded_count > 1 and "messages" or "message"
|
|
|
+ print("discard: Erased %d %s from the queue.\n" % (
|
|
|
discarded_count, what))
|
|
|
-
|
|
|
- startup_info = STARTUP_INFO_FMT % {
|
|
|
+
|
|
|
+ # Dump configuration to screen so we have some basic information
|
|
|
+ # when users sends e-mails.
|
|
|
+ print(STARTUP_INFO_FMT % {
|
|
|
"vhost": settings.AMQP_VHOST,
|
|
|
"host": settings.AMQP_SERVER,
|
|
|
"port": settings.AMQP_PORT,
|
|
|
@@ -168,30 +212,21 @@ def run_worker(concurrency=DAEMON_CONCURRENCY, daemon=False,
|
|
|
"loglevel": loglevel,
|
|
|
"pidfile": pidfile,
|
|
|
}
|
|
|
- print(startup_info)
|
|
|
print("* Reporting of statistics is %s..." % (
|
|
|
settings.CELERY_STATISTICS and "ON" or "OFF"))
|
|
|
|
|
|
- print("* Declaring consumers...")
|
|
|
- conn = DjangoAMQPConnection()
|
|
|
- TaskConsumer(connection=conn).close()
|
|
|
- if settings.CELERY_STATISTICS:
|
|
|
- StatsConsumer(connection=conn).close()
|
|
|
-
|
|
|
-
|
|
|
- context = None
|
|
|
- if daemon:
|
|
|
+ if detach:
|
|
|
# Since without stderr any errors will be silently suppressed,
|
|
|
# we need to know that we have access to the logfile
|
|
|
+ if logfile:
|
|
|
+ open(logfile, "a").close()
|
|
|
pidlock = acquire_pidlock(pidfile)
|
|
|
if not umask:
|
|
|
umask = 0
|
|
|
- if logfile:
|
|
|
- open(logfile, "a").close()
|
|
|
uid = uid and int(uid) or os.geteuid()
|
|
|
gid = gid and int(gid) or os.getegid()
|
|
|
working_directory = working_directory or os.getcwd()
|
|
|
- sys.stderr.write("* Launching celeryd in the background...\n")
|
|
|
+ print("* Launching celeryd in the background...")
|
|
|
context = DaemonContext(chroot_directory=chroot,
|
|
|
working_directory=working_directory,
|
|
|
umask=umask,
|
|
|
@@ -199,16 +234,14 @@ def run_worker(concurrency=DAEMON_CONCURRENCY, daemon=False,
|
|
|
uid=uid,
|
|
|
gid=gid)
|
|
|
context.open()
|
|
|
- else:
|
|
|
- logfile = None # log to stderr when not running as daemon.
|
|
|
|
|
|
discovery.autodiscover()
|
|
|
- celeryd = WorkController(concurrency=concurrency,
|
|
|
- loglevel=loglevel,
|
|
|
- logfile=logfile,
|
|
|
- is_detached=daemon)
|
|
|
+ worker = WorkController(concurrency=concurrency,
|
|
|
+ loglevel=loglevel,
|
|
|
+ logfile=logfile,
|
|
|
+ is_detached=detach)
|
|
|
try:
|
|
|
- celeryd.run()
|
|
|
+ worker.run()
|
|
|
except Exception, e:
|
|
|
emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % (
|
|
|
e.__class__, e, traceback.format_exc()))
|
|
|
@@ -218,49 +251,6 @@ def run_worker(concurrency=DAEMON_CONCURRENCY, daemon=False,
|
|
|
raise
|
|
|
|
|
|
|
|
|
-OPTION_LIST = (
|
|
|
- optparse.make_option('-c', '--concurrency', default=DAEMON_CONCURRENCY,
|
|
|
- action="store", dest="concurrency", type="int",
|
|
|
- help="Number of child processes processing the queue."),
|
|
|
- optparse.make_option('--discard', default=False,
|
|
|
- action="store_true", dest="discard",
|
|
|
- help="Discard all waiting tasks before the daemon is started. "
|
|
|
- "WARNING: This is unrecoverable, and the tasks will be "
|
|
|
- "deleted from the messaging server."),
|
|
|
- optparse.make_option('-f', '--logfile', default=DAEMON_LOG_FILE,
|
|
|
- action="store", dest="logfile",
|
|
|
- help="Path to log file."),
|
|
|
- optparse.make_option('-l', '--loglevel', default=DAEMON_LOG_LEVEL,
|
|
|
- action="store", dest="loglevel",
|
|
|
- help="Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL/FATAL."),
|
|
|
- optparse.make_option('-p', '--pidfile', default=DAEMON_PID_FILE,
|
|
|
- action="store", dest="pidfile",
|
|
|
- help="Path to pidfile."),
|
|
|
- optparse.make_option('-s', '--statistics', default=USE_STATISTICS,
|
|
|
- action="store_true", dest="statistics",
|
|
|
- help="Turn on reporting of statistics (remember to flush the "
|
|
|
- "statistics message queue from time to time)."),
|
|
|
- optparse.make_option('-d', '--detach', '--daemon', default=False,
|
|
|
- action="store_true", dest="daemon",
|
|
|
- help="Run in the background as a daemon."),
|
|
|
- optparse.make_option('-u', '--uid', default=None,
|
|
|
- action="store", dest="uid",
|
|
|
- help="User-id to run celeryd as when in daemon mode."),
|
|
|
- optparse.make_option('-g', '--gid', default=None,
|
|
|
- action="store", dest="gid",
|
|
|
- help="Group-id to run celeryd as when in daemon mode."),
|
|
|
- optparse.make_option('--umask', default=0,
|
|
|
- action="store", type="int", dest="umask",
|
|
|
- help="umask of the process when in daemon mode."),
|
|
|
- optparse.make_option('--workdir', default=None,
|
|
|
- action="store", dest="working_directory",
|
|
|
- help="Directory to change to when in daemon mode."),
|
|
|
- optparse.make_option('--chroot', default=None,
|
|
|
- action="store", dest="chroot",
|
|
|
- help="Change root directory to this path when in daemon mode."),
|
|
|
- )
|
|
|
-
|
|
|
-
|
|
|
def parse_options(arguments):
|
|
|
"""Parse the available options to ``celeryd``."""
|
|
|
parser = optparse.OptionParser(option_list=OPTION_LIST)
|
Ask Solem