Przeglądaj źródła

Merge branch 'mpavlov/centos-init' into 3.0

Ask Solem 12 lat temu
rodzic
commit
77b11e2327

+ 6 - 0
docs/tutorials/daemonizing.rst

@@ -309,3 +309,9 @@ Windows
 See this excellent external tutorial:
 
 http://www.calazan.com/windows-tip-run-applications-in-the-background-using-task-scheduler/
+
+CentOS
+======
+In CentOS we can take advantage of built-in service helpers, such as the
+pid-based status checker function in ``/etc/init.d/functions``.
+See the sample script in http://github.com/celery/celery/tree/3.0/extra/centos/.

+ 243 - 0
extra/centos/celeryd

@@ -0,0 +1,243 @@
+#!/bin/sh
+# ============================================
+# celeryd - Starts the Celery worker daemon.
+# ============================================
+#
+# :Usage: /etc/init.d/celeryd {start|stop|restart|status}
+# :Configuration file: /etc/sysconfig/celeryd
+#
+# See http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html
+
+### BEGIN INIT INFO
+# Provides:          celeryd
+# Required-Start:    $network $local_fs $remote_fs
+# Required-Stop:     $network $local_fs $remote_fs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: celery task worker daemon
+### END INIT INFO
+
+# Source the centos service helper functions
+source /etc/init.d/functions
+# NOTE: "set -e" does not work with the above functions,
+# which use non-zero return codes as non-error return conditions
+
+# some commands work asyncronously, so we'll wait this many seconds
+SLEEP_SECONDS=5
+
+DEFAULT_PID_FILE="/var/run/celery/%n.pid"
+DEFAULT_LOG_FILE="/var/log/celery/%n.log"
+DEFAULT_LOG_LEVEL="INFO"
+DEFAULT_NODES="celery"
+DEFAULT_CELERYD="-m celery.bin.celeryd_detach"
+
+CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/sysconfig/celeryd"}
+
+test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS"
+
+# Set CELERY_CREATE_DIRS to always create log/pid dirs.
+CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0}
+CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS
+CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS
+if [ -z "$CELERYD_PID_FILE" ]; then
+    CELERYD_PID_FILE="$DEFAULT_PID_FILE"
+    CELERY_CREATE_RUNDIR=1
+fi
+if [ -z "$CELERYD_LOG_FILE" ]; then
+    CELERYD_LOG_FILE="$DEFAULT_LOG_FILE"
+    CELERY_CREATE_LOGDIR=1
+fi
+
+CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}}
+CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"}
+CELERYD=${CELERYD:-$DEFAULT_CELERYD}
+CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES}
+
+# This is used to change how Celery loads in the configs.  It does not need to
+# be set to be run.
+export CELERY_LOADER
+
+if [ -n "$2" ]; then
+    CELERYD_OPTS="$CELERYD_OPTS $2"
+fi
+
+CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE`
+CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE`
+
+# Extra start-stop-daemon options, like user/group.
+if [ -n "$CELERYD_USER" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER"
+fi
+if [ -n "$CELERYD_GROUP" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP"
+fi
+
+if [ -n "$CELERYD_CHDIR" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\""
+fi
+
+check_dev_null() {
+    if [ ! -c /dev/null ]; then
+        echo "/dev/null is not a character device!"
+        exit 75  # EX_TEMPFAIL
+    fi
+}
+
+
+maybe_die() {
+    if [ $? -ne 0 ]; then
+        echo "Exiting: $* (errno $?)"
+        exit 77  # EX_NOPERM
+    fi
+}
+
+create_default_dir() {
+    if [ ! -d "$1" ]; then
+        echo "- Creating default directory: '$1'"
+        mkdir -p "$1"
+        maybe_die "Couldn't create directory $1"
+        echo "- Changing permissions of '$1' to 02755"
+        chmod 02755 "$1"
+        maybe_die "Couldn't change permissions for $1"
+        if [ -n "$CELERYD_USER" ]; then
+            echo "- Changing owner of '$1' to '$CELERYD_USER'"
+            chown "$CELERYD_USER" "$1"
+            maybe_die "Couldn't change owner of $1"
+        fi
+        if [ -n "$CELERYD_GROUP" ]; then
+            echo "- Changing group of '$1' to '$CELERYD_GROUP'"
+            chgrp "$CELERYD_GROUP" "$1"
+            maybe_die "Couldn't change group of $1"
+        fi
+    fi
+}
+
+
+check_paths() {
+    if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then
+        create_default_dir "$CELERYD_LOG_DIR"
+    fi
+    if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then
+        create_default_dir "$CELERYD_PID_DIR"
+    fi
+}
+
+create_paths() {
+    create_default_dir "$CELERYD_LOG_DIR"
+    create_default_dir "$CELERYD_PID_DIR"
+}
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
+
+
+_get_pid_files() {
+    [[ ! -d "$CELERYD_PID_DIR" ]] && return
+    echo $(ls -1 "$CELERYD_PID_DIR"/*.pid 2> /dev/null)
+}
+
+stop() {
+    local pid_files=$(_get_pid_files)
+    [[ -z "$pid_files" ]] && echo "celeryd is stopped" && return 0
+
+    local one_failed=
+    for pid_file in $pid_files; do
+        local pid=$(cat "$pid_file")
+        echo -n $"Stopping celeryd (pid $pid): "
+
+        # killproc comes from 'functions' and brings three nice features:
+        #  1. sending TERM, sleeping, then sleeping more if needed, then sending KILL
+        #  2. handling 'success' and 'failure' output
+        #  3. removes stale pid files, if any remain
+        killproc -p "$pid_file" -d "$SLEEP_SECONDS" celeryd || one_failed=true
+        echo
+    done
+
+    [[ "$one_failed" ]] && return 1 || return 0
+}
+
+start() {
+    echo -n $"Starting celeryd: "
+
+    # If Celery is already running, bail out
+    local pid_files=$(_get_pid_files)
+    if [[ "$pid_files" ]]; then
+        echo -n $"celeryd is already running. Use 'restart'."
+        failure
+        echo
+        return 1
+    fi
+
+    $CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS        \
+                         --pidfile="$CELERYD_PID_FILE"      \
+                         --logfile="$CELERYD_LOG_FILE"      \
+                         --loglevel="$CELERYD_LOG_LEVEL"    \
+                         --cmd="$CELERYD"                   \
+                         --quiet                            \
+                         $CELERYD_OPTS
+
+    if [[ "$?" == "0" ]]; then
+        # Sleep a few seconds to give Celery a chance to initialize itself.
+        # This is useful to prevent scripts following this one from trying to
+        # use Celery (or its pid files) too early.
+        sleep $SLEEP_SECONDS
+        pid_files=$(_get_pid_files)
+        if [[ "$pid_files" ]]; then
+            for pid_file in $pid_files; do
+                local node=$(basename "$pid_file" .pid)
+                local pid=$(cat "$pid_file")
+                echo
+                echo -n "    $node (pid $pid):"
+                success
+            done
+            echo
+            return 0
+        else  # celeryd_multi succeeded but no pid files found
+            failure
+        fi
+    else  # celeryd_multi did not succeed
+        failure
+    fi
+    echo
+    return 1
+}
+
+check_status() {
+    local pid_files=$(_get_pid_files)
+    [[ -z "$pid_files" ]] && echo "celeryd is stopped" && return 1
+    for pid_file in $pid_files; do
+        local node=$(basename "$pid_file" .pid)
+        status -p "$pid_file" $"celeryd (node $node)" || return 1  # if one node is down celeryd is down
+    done
+    return 0
+}
+
+case "$1" in
+    start)
+        check_dev_null
+        check_paths
+        start
+    ;;
+
+    stop)
+        check_dev_null
+        check_paths
+        stop
+    ;;
+
+    status)
+        check_status
+    ;;
+
+    restart)
+        check_dev_null
+        check_paths
+        stop && start
+    ;;
+
+    *)
+        echo "Usage: /etc/init.d/celeryd {start|stop|restart|status}"
+        exit 3
+    ;;
+esac
+
+exit $?

+ 0 - 140
extra/centos/celeryd.init

@@ -1,140 +0,0 @@
-#!/bin/sh
-### BEGIN INIT INFO
-# Provides: celeryd
-# Required-Start: $network $local_fs $remote_fs
-# Required-Stop: $network $local_fs $remote_fs
-# Default-Start: 2 3 4 5
-# Default-Stop: 0 1 6
-# Short-Description: celery task worker daemon
-### END INIT INFO
-#
-# ============================================
-# celeryd - Starts the Celery worker daemon.
-# ============================================
-#
-# :Usage: /etc/init.d/${basename $0} {start|stop|restart|status}
-# :Configuration file: /etc/sysconfig/celeryd
-#
-# To implement separate init scripts, do NOT copy this script.  Instead,
-# symlink it.  I.e., if my new application, "little-worker" needs an init, I
-# should just use:
-#
-#   ln -s /etc/init.d/celeryd /etc/init.d/little-worker
-#
-# You can then configure this by manipulating /etc/sysconfig/little-worker.
-#
-
-# Setting `prog` here allows you to symlink this init script, making it easy
-# to run multiple processes on the system.
-prog="$(basename $0)"
-
-# Source the centos stuff
-. /etc/init.d/functions
-
-# Also look at sysconfig; this is where environmental variables should be set
-# on RHEL systems.
-[ -f "/etc/sysconfig/$prog" ] && . /etc/sysconfig/$prog
-
-CELERYD=${CELERYD:-"-m celery.bin.celeryd_detach"}
-CELERYD_MULTI=${CELERYD_MULTI:-"/usr/bin/celeryd-multi"}
-CELERYD_PID_FILE=${CELERYD_PID_FILE:-"/var/run/celeryd/$prog.pid"}
-CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-"/var/log/celeryd/$prog.log"}
-CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-"INFO"}
-
-# This is used to change how Celery loads in the configs.  It does not need to
-# be set to be run.
-export CELERY_LOADER
-
-start_workers () {
-    CELERYD_LOG_DIR=$(dirname $CELERYD_LOG_FILE)
-    CELERYD_PID_DIR=$(dirname $CELERYD_PID_FILE)
-    # Ensure that the directories exist.
-    mkdir -p $CELERYD_LOG_DIR $CELERYD_PID_DIR
-
-    # If we specified a user, and/or a group, chown as needed
-    if [ -n "$CELERYD_USER" ]; then
-        CHOWN_UG="${CELERYD_USER}"
-
-        # If the group is specified, also use that in the chown.
-        [ -n "$CELERYD_GROUP" ] && CHOWN_UG="$CHOWN_UG:$CELERYD_GROUP"
-
-        # Execute the chown on the directory only
-        chown $CHOWN_UG $CELERYD_LOG_DIR $CELERYD_PID_DIR
-
-        CELERYD_OPTS="$CELERYD_OPTS --uid=$CELERYD_USER"
-    fi
-
-    # If we need to be run from a specific location, cd to it before launch
-    if [ -n "$CELERYD_CHDIR" ]; then
-        cd $CELERYD_CHDIR
-    fi
-
-    echo -n $"Starting $prog: "
-    $CELERYD_MULTI start $prog \
-                               --pidfile=$CELERYD_PID_FILE \
-                               --logfile=$CELERYD_LOG_FILE \
-                               --loglevel=$CELERYD_LOG_LEVEL \
-                               --cmd="$CELERYD" \
-                                --quiet \
-                               $CELERYD_OPTS
-    RETVAL=$?
-
-    if [ "$RETVAL" == "0" ]; then
-        touch /var/lock/subsys/$prog
-        success
-    else
-        failure
-    fi
-    echo
-}
-
-stop_workers () {
-    echo -n $"Stopping $prog: "
-
-    # If we haven't ended, explicitly kill it!
-    if [ ! -f $CELERYD_PID_FILE ] || [ ! -e /proc/$(cat $CELERYD_PID_FILE) ]; then
-        failure
-    echo
-        return
-    fi
-
-    # First, try to nicely shut it down.
-    $CELERYD_MULTI stop $prog --pidfile=$CELERYD_PID_FILE --quiet
-    RETVAL=$?
-
-    # SLeep a few seconds. (this was part of the original script; we can't
-    # trust that it will end immediately, or that running the command will
-    # stop it.
-    sleep 3
-
-    # If we haven't ended, explicitly kill it!
-    if [ -f $CELERYD_PID_FILE ] && [ -e /proc/$(cat $CELERYD_PID_FILE) ]; then
-        $CELERYD_MULTI stop $prog -KILL --pidfile=$CELERYD_PID_FILE --quiet
-    fi
-
-    if [ "$RETVAL" == "0" ]; then
-        rm -f /var/lock/sybsys/$prog
-        success
-    else
-        failure
-    fi
-    echo
-}
-
-case "$1" in
-    start)
-        start_workers ;;
-    stop)
-        stop_workers ;;
-    status)
-        status -p $CELERYD_PID_FILE $prog ;;
-    restart)
-        stop_workers
-        start_workers ;;
-    *)
-        echo "Usage: /etc/init.d/$prog {start|stop|restart|status}"
-        exit 1
-    ;;
-esac
-
-exit 0

+ 18 - 15
extra/centos/celeryd.sysconfig

@@ -1,21 +1,24 @@
-# Passed into celeryd multi
-#CELERYD="-m celery.bin.celeryd_detach"
+# In CentOS, contents should be placed in the file /etc/sysconfig/celeryd
 
-# Path to the celerd multi
-#CELERYD_MULTI="/usr/bin/celeryd-multi"
+# Names of nodes to start (space-separated)
+#CELERYD_NODES="my_application-node_1"
 
-# Sets the verbosity of the celeryd logging.
-#CELERYD_LOG_LEVEL="INFO"
+# Where to chdir at start. This could be the root of a virtualenv.
+#CELERYD_CHDIR="/path/to/my_application"
 
-# Define the loader that celeryd should use for loading in configs.
-#CELERY_LOADER=""
+# How to call celeryd-multi
+#CELERYD_MULTI="$CELERYD_CHDIR/bin/celeryd-multi"
 
-# User and group information for directories
-#CELERYD_USER="celery"
-#CELERYD_GROUP="celery"
+# Extra arguments
+#CELERYD_OPTS="--app=my_application.path.to.worker --time-limit=300 --concurrency=8 --loglevel=DEBUG"
 
-# Default arguments to be passed into celeryd.
-#CELERYD_OPTS=""
+# Create log/pid dirs, if they don't already exist
+#CELERY_CREATE_DIRS=1
 
-# Change to this directory first before launching celeryd.
-#CELERYD_CHDIR=""
+# %n will be replaced with the nodename
+#CELERYD_LOG_FILE="/path/to/my_application/log/%n.log"
+#CELERYD_PID_FILE="/var/run/celery/%n.pid"
+
+# Workers run as an unprivileged user
+#CELERYD_USER=celery
+#CELERYD_GROUP=celery

+ 41 - 0
extra/centos/test_celeryd.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+# If you make changes to the celeryd init script,
+# you can use this test script to verify you didn't break the universe
+
+SERVICE="celeryd"
+SERVICE_CMD="sudo /sbin/service $SERVICE"
+
+run_test() {
+    local msg="$1"
+    local cmd="$2"
+    local expected_retval="${3:-0}"
+    local n=${#msg}
+
+    echo
+    echo `printf "%$((${n}+4))s" | tr " " "#"`
+    echo "# $msg #"
+    echo `printf "%$((${n}+4))s" | tr " " "#"`
+
+    $cmd
+    local retval=$?
+    if [[ "$retval" == "$expected_retval" ]]; then
+        echo "[PASSED]"
+    else
+        echo "[FAILED]"
+        echo "Exit status: $retval, but expected: $expected_retval"
+        exit $retval
+    fi
+}
+
+run_test "stop should succeed" "$SERVICE_CMD stop" 0
+run_test "status on a stopped service should return 1" "$SERVICE_CMD status" 1
+run_test "stopping a stopped celery should not fail" "$SERVICE_CMD stop" 0
+run_test "start should succeed" "$SERVICE_CMD start" 0
+run_test "status on a running service should return 0" "$SERVICE_CMD status" 0
+run_test "starting a running service should fail" "$SERVICE_CMD start" 1
+run_test "restarting a running service should succeed" "$SERVICE_CMD restart" 0
+run_test "status on a restarted service should return 0" "$SERVICE_CMD status" 0
+run_test "stop should succeed" "$SERVICE_CMD stop" 0
+
+echo "All tests passed!"