celeryd_multi Must use shlex.split(posix=False) on Windows.

Closes #472.  Thanks to Tempos.

Changelog

@@ -219,6 +219,8 @@ News
 
     Fix contributed by Remy Noel
 
+* celeryd-multi did not work on Windows (Issue #472).
+
 * New-style ``CELERY_REDIS_*`` settings now takes precedence over
   the old ``REDIS_*`` configuration keys (Issue #508).
 

celery/app/base.py

@@ -21,6 +21,7 @@ from functools import wraps
 from threading import Lock
 
 from .. import datastructures
+from .. import platforms
 from ..utils import cached_property, instantiate, lpmerge
 
 from .defaults import DEFAULTS, find_deprecated_settings
@@ -36,20 +37,6 @@ settings -> transport:%(transport)s results:%(results)s
 """
 
 
-def pyimplementation():
-    if hasattr(_platform, "python_implementation"):
-        return _platform.python_implementation()
-    elif sys.platform.startswith("java"):
-        return "Jython %s" % (sys.platform, )
-    elif hasattr(sys, "pypy_version_info"):
-        v = ".".join(map(str, sys.pypy_version_info[:3]))
-        if sys.pypy_version_info[3:]:
-            v += "-" + "".join(map(str, sys.pypy_version_info[3:]))
-        return "PyPy %s" % (v, )
-    else:
-        return "CPython"
-
-
 class LamportClock(object):
     """Lamport's logical clock.
 
@@ -130,9 +117,9 @@ class Settings(datastructures.ConfigurationView):
 
 class BaseApp(object):
     """Base class for apps."""
-    SYSTEM = _platform.system()
-    IS_OSX = SYSTEM == "Darwin"
-    IS_WINDOWS = SYSTEM == "Windows"
+    SYSTEM = platforms.SYSTEM
+    IS_OSX = platforms.IS_OSX
+    IS_WINDOWS = platforms.IS_WINDOWS
 
     amqp_cls = "celery.app.amqp.AMQP"
     backend_cls = None
@@ -365,7 +352,7 @@ class BaseApp(object):
         import kombu
         return BUGREPORT_INFO % {"system": _platform.system(),
                                  "arch": _platform.architecture(),
-                                 "py_i": pyimplementation(),
+                                 "py_i": platforms.pyimplementation(),
                                  "celery_v": celery.__version__,
                                  "kombu_v": kombu.__version__,
                                  "py_v": _platform.python_version(),

celery/bin/celeryd_multi.py

@@ -89,7 +89,6 @@ from __future__ import absolute_import
 
 import errno
 import os
-import shlex
 import signal
 import socket
 import sys
@@ -99,6 +98,7 @@ from subprocess import Popen
 from time import sleep
 
 from .. import __version__
+from ..platforms import shellsplit
 from ..utils import term
 from ..utils.encoding import from_utf8
 
@@ -369,7 +369,7 @@ class MultiTool(object):
 
     def waitexec(self, argv, path=sys.executable):
         args = " ".join([path] + list(argv))
-        argstr = shlex.split(from_utf8(args))
+        argstr = shellsplit(from_utf8(args))
         pipe = Popen(argstr, env=self.env)
         self.info("  %s" % " ".join(argstr))
         retcode = pipe.wait()

celery/platforms.py

@@ -9,10 +9,12 @@ groups, etc.
 """
 from __future__ import absolute_import
 
-import os
-import sys
 import errno
+import os
+import platform as _platform
+import shlex
 import signal as _signal
+import sys
 
 from .local import try_import
 
@@ -21,6 +23,10 @@ resource = try_import("resource")
 pwd = try_import("pwd")
 grp = try_import("grp")
 
+SYSTEM = _platform.system()
+IS_OSX = SYSTEM == "Darwin"
+IS_WINDOWS = SYSTEM == "Windows"
+
 DAEMON_UMASK = 0
 DAEMON_WORKDIR = "/"
 DAEMON_REDIRECT_TO = getattr(os, "devnull", "/dev/null")
@@ -28,7 +34,21 @@ DAEMON_REDIRECT_TO = getattr(os, "devnull", "/dev/null")
 __all__ = ["LockFailed", "get_fdmax", "create_pidlock",
            "DaemonContext", "detached", "parse_uid", "parse_gid",
            "setegid", "seteuid", "set_effective_user", "Signals",
-           "set_process_title", "set_mp_process_title"]
+           "set_process_title", "set_mp_process_title", "pyimplementation"]
+
+
+def pyimplementation():
+    if hasattr(_platform, "python_implementation"):
+        return _platform.python_implementation()
+    elif sys.platform.startswith("java"):
+        return "Jython %s" % (sys.platform, )
+    elif hasattr(sys, "pypy_version_info"):
+        v = ".".join(map(str, sys.pypy_version_info[:3]))
+        if sys.pypy_version_info[3:]:
+            v += "-" + "".join(map(str, sys.pypy_version_info[3:]))
+        return "PyPy %s" % (v, )
+    else:
+        return "CPython"
 
 
 class LockFailed(Exception):
@@ -487,3 +507,11 @@ def set_mp_process_title(progname, info=None, hostname=None):
     else:
         return set_process_title("%s:%s" % (progname,
                                             current_process().name), info=info)
+
+
+def shellsplit(s, posix=True):
+    # posix= option to shlex.split first available in Python 2.6+
+    lexer = shlex.shlex(s, posix=not IS_WINDOWS)
+    lexer.whitespace_split = True
+    lexer.commenters = ''
+    return list(lexer)

celery/tests/test_app/__init__.py

@@ -9,8 +9,9 @@ from mock import Mock
 from celery import Celery
 from celery import app as _app
 from celery.app import defaults
-from celery.app.base import BaseApp, pyimplementation
+from celery.app.base import BaseApp
 from celery.loaders.base import BaseLoader
+from celery.platforms import pyimplementation
 from celery.utils.serialization import pickle
 
 from celery.tests import config