Removes Remote Tasks/celery.contrib.http

celery/task/http.py

@@ -1,224 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    celery.task.http
-    ~~~~~~~~~~~~~~~~
-
-    Webhook task implementation.
-
-"""
-from __future__ import absolute_import, unicode_literals
-
-import sys
-
-try:
-    from urllib.parse import parse_qsl, urlencode, urlparse   # Py3
-except ImportError:  # pragma: no cover
-    from urllib import urlencode              # noqa
-    from urlparse import urlparse, parse_qsl  # noqa
-
-from kombu.utils import json
-from kombu.utils.encoding import bytes_to_str, str_to_bytes
-
-from celery import shared_task, __version__ as celery_version
-from celery.five import items, python_2_unicode_compatible, reraise
-from celery.utils.log import get_task_logger
-
-__all__ = ['InvalidResponseError', 'RemoteExecuteError', 'UnknownStatusError',
-           'HttpDispatch', 'dispatch', 'URL']
-
-GET_METHODS = {'GET', 'HEAD'}
-logger = get_task_logger(__name__)
-
-
-if sys.version_info[0] == 3:  # pragma: no cover
-
-    from urllib.request import Request, urlopen
-
-    def utf8dict(tup):
-        if not isinstance(tup, dict):
-            return dict(tup)
-        return tup
-
-else:
-
-    from urllib2 import Request, urlopen  # noqa
-
-    def utf8dict(tup, enc='utf-8'):  # noqa
-        """With a dict's items() tuple return a new dict with any utf-8
-        keys/values encoded."""
-        return {
-            k.encode(enc): (v.encode(enc) if isinstance(v, unicode) else v)
-            for k, v in tup
-        }
-
-
-class InvalidResponseError(Exception):
-    """The remote server gave an invalid response."""
-
-
-class RemoteExecuteError(Exception):
-    """The remote task gave a custom error."""
-
-
-class UnknownStatusError(InvalidResponseError):
-    """The remote server gave an unknown status."""
-
-
-def extract_response(raw_response, loads=json.loads):
-    """Extract the response text from a raw JSON response."""
-    if not raw_response:
-        raise InvalidResponseError('Empty response')
-    try:
-        payload = loads(raw_response)
-    except ValueError as exc:
-        reraise(InvalidResponseError, InvalidResponseError(
-            str(exc)), sys.exc_info()[2])
-
-    status = payload['status']
-    if status == 'success':
-        return payload['retval']
-    elif status == 'failure':
-        raise RemoteExecuteError(payload.get('reason'))
-    else:
-        raise UnknownStatusError(str(status))
-
-
-@python_2_unicode_compatible
-class MutableURL(object):
-    """Object wrapping a Uniform Resource Locator.
-
-    Supports editing the query parameter list.
-    You can convert the object back to a string, the query will be
-    properly URL-encoded.
-
-    Examples:
-
-    .. code-block:: pycon
-
-        >>> url = URL('http://www.google.com:6580/foo/bar?x=3&y=4#foo')
-        >>> url.query
-        {'x': '3', 'y': '4'}
-        >>> str(url)
-        'http://www.google.com:6580/foo/bar?y=4&x=3#foo'
-        >>> url.query['x'] = 10
-        >>> url.query.update({'George': 'Costanza'})
-        >>> str(url)
-        'http://www.google.com:6580/foo/bar?y=4&x=10&George=Costanza#foo'
-
-    """
-    def __init__(self, url):
-        self.parts = urlparse(url)
-        self.query = dict(parse_qsl(self.parts[4]))
-
-    def __str__(self):
-        scheme, netloc, path, params, query, fragment = self.parts
-        query = urlencode(utf8dict(items(self.query)))
-        components = [scheme + '://', netloc, path or '/',
-                      ';{0}'.format(params) if params else '',
-                      '?{0}'.format(query) if query else '',
-                      '#{0}'.format(fragment) if fragment else '']
-        return ''.join(c for c in components if c)
-
-    def __repr__(self):
-        return '<{0}: {1}>'.format(type(self).__name__, self)
-
-
-class HttpDispatch(object):
-    """Make task HTTP request and collect the task result.
-
-    :param url: The URL to request.
-    :param method: HTTP method used. Currently supported methods are `GET`
-        and `POST`.
-    :param task_kwargs: Task keyword arguments.
-    :param logger: Logger used for user/system feedback.
-
-    """
-    user_agent = 'celery/{version}'.format(version=celery_version)
-    timeout = 5
-
-    def __init__(self, url, method, task_kwargs, **kwargs):
-        self.url = url
-        self.method = method
-        self.task_kwargs = task_kwargs
-        self.logger = kwargs.get('logger') or logger
-
-    def make_request(self, url, method, params):
-        """Perform HTTP request and return the response."""
-        request = Request(url, str_to_bytes(params))
-        for key, val in items(self.http_headers):
-            request.add_header(key, val)
-        response = urlopen(request)  # user catches errors.
-        return response.read()
-
-    def dispatch(self):
-        """Dispatch callback and return result."""
-        url = MutableURL(self.url)
-        params = None
-        if self.method in GET_METHODS:
-            url.query.update(self.task_kwargs)
-        else:
-            params = urlencode(utf8dict(items(self.task_kwargs)))
-        raw_response = self.make_request(str(url), self.method, params)
-        return extract_response(bytes_to_str(raw_response))
-
-    @property
-    def http_headers(self):
-        headers = {'User-Agent': self.user_agent}
-        return headers
-
-
-@shared_task(name='celery.http_dispatch', bind=True, url=None, method=None)
-def dispatch(self, url=None, method='GET', **kwargs):
-    """Task dispatching to a URL.
-
-    :keyword url: The URL location of the HTTP callback task.
-    :keyword method: Method to use when dispatching the callback. Usually
-        `GET` or `POST`.
-    :keyword \*\*kwargs: Keyword arguments to pass on to the HTTP callback.
-
-    .. attribute:: url
-
-        If this is set, this is used as the default URL for requests.
-        Default is to require the user of the task to supply the URL as an
-        argument, as this attribute is intended for subclasses.
-
-    .. attribute:: method
-
-        If this is set, this is the default method used for requests.
-        Default is to require the user of the task to supply the method as an
-        argument, as this attribute is intended for subclasses.
-
-    """
-    return HttpDispatch(
-        url or self.url, method or self.method, kwargs,
-    ).dispatch()
-
-
-class URL(MutableURL):
-    """HTTP Callback URL
-
-    Supports requesting a URL asynchronously.
-
-    :param url: URL to request.
-    :keyword dispatcher: Class used to dispatch the request.
-        By default this is :func:`dispatch`.
-
-    """
-    dispatcher = None
-
-    def __init__(self, url, dispatcher=None, app=None):
-        super(URL, self).__init__(url)
-        self.app = app
-        self.dispatcher = dispatcher or self.dispatcher
-        if self.dispatcher is None:
-            # Get default dispatcher
-            self.dispatcher = (
-                self.app.tasks['celery.http_dispatch'] if self.app
-                else dispatch
-            )
-
-    def get_async(self, **kwargs):
-        return self.dispatcher.delay(str(self), 'GET', **kwargs)
-
-    def post_async(self, **kwargs):
-        return self.dispatcher.delay(str(self), 'POST', **kwargs)

celery/tests/compat_modules/test_http.py

@@ -1,158 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, unicode_literals
-
-from contextlib import contextmanager
-try:
-    from urllib import addinfourl
-except ImportError:  # py3k
-    from urllib.request import addinfourl  # noqa
-
-from kombu.utils.encoding import from_utf8
-from kombu.utils.json import dumps
-from vine.utils import wraps
-
-from celery.five import WhateverIO, items
-from celery.task import http
-from celery.tests.case import AppCase, Case
-
-
-@contextmanager
-def mock_urlopen(response_method):
-
-    urlopen = http.urlopen
-
-    @wraps(urlopen)
-    def _mocked(url, *args, **kwargs):
-        response_data, headers = response_method(url)
-        return addinfourl(WhateverIO(response_data), headers, url)
-
-    http.urlopen = _mocked
-
-    try:
-        yield True
-    finally:
-        http.urlopen = urlopen
-
-
-def _response(res):
-    return lambda r: (res, [])
-
-
-def success_response(value):
-    return _response(dumps({'status': 'success', 'retval': value}))
-
-
-def fail_response(reason):
-    return _response(dumps({'status': 'failure', 'reason': reason}))
-
-
-def unknown_response():
-    return _response(dumps({'status': 'u.u.u.u', 'retval': True}))
-
-
-class test_encodings(Case):
-
-    def test_utf8dict(self):
-        uk = 'foobar'
-        d = {'følelser ær langé': 'ærbadægzaååÆØÅ',
-             from_utf8(uk): from_utf8('xuzzybaz')}
-
-        for key, value in items(http.utf8dict(items(d))):
-            self.assertIsInstance(key, str)
-            self.assertIsInstance(value, str)
-
-
-class test_MutableURL(Case):
-
-    def test_url_query(self):
-        url = http.MutableURL('http://example.com?x=10&y=20&z=Foo')
-        self.assertDictContainsSubset({'x': '10',
-                                       'y': '20',
-                                       'z': 'Foo'}, url.query)
-        url.query['name'] = 'George'
-        url = http.MutableURL(str(url))
-        self.assertDictContainsSubset({'x': '10',
-                                       'y': '20',
-                                       'z': 'Foo',
-                                       'name': 'George'}, url.query)
-
-    def test_url_keeps_everything(self):
-        url = 'https://e.com:808/foo/bar#zeta?x=10&y=20'
-        url = http.MutableURL(url)
-
-        self.assertEqual(
-            str(url).split('?')[0],
-            'https://e.com:808/foo/bar#zeta',
-        )
-
-    def test___repr__(self):
-        url = http.MutableURL('http://e.com/foo/bar')
-        self.assertTrue(repr(url).startswith('<MutableURL: http://e.com'))
-
-    def test_set_query(self):
-        url = http.MutableURL('http://e.com/foo/bar/?x=10')
-        url.query = {'zzz': 'xxx'}
-        url = http.MutableURL(str(url))
-        self.assertEqual(url.query, {'zzz': 'xxx'})
-
-
-class test_HttpDispatch(AppCase):
-
-    def test_dispatch_success(self):
-        with mock_urlopen(success_response(100)):
-            d = http.HttpDispatch('http://example.com/mul', 'GET', {
-                'x': 10, 'y': 10})
-            self.assertEqual(d.dispatch(), 100)
-
-    def test_dispatch_failure(self):
-        with mock_urlopen(fail_response('Invalid moon alignment')):
-            d = http.HttpDispatch('http://example.com/mul', 'GET', {
-                'x': 10, 'y': 10})
-            with self.assertRaises(http.RemoteExecuteError):
-                d.dispatch()
-
-    def test_dispatch_empty_response(self):
-        with mock_urlopen(_response('')):
-            d = http.HttpDispatch('http://example.com/mul', 'GET', {
-                'x': 10, 'y': 10})
-            with self.assertRaises(http.InvalidResponseError):
-                d.dispatch()
-
-    def test_dispatch_non_json(self):
-        with mock_urlopen(_response("{'#{:'''")):
-            d = http.HttpDispatch('http://example.com/mul', 'GET', {
-                'x': 10, 'y': 10})
-            with self.assertRaises(http.InvalidResponseError):
-                d.dispatch()
-
-    def test_dispatch_unknown_status(self):
-        with mock_urlopen(unknown_response()):
-            d = http.HttpDispatch('http://example.com/mul', 'GET', {
-                'x': 10, 'y': 10})
-            with self.assertRaises(http.UnknownStatusError):
-                d.dispatch()
-
-    def test_dispatch_POST(self):
-        with mock_urlopen(success_response(100)):
-            d = http.HttpDispatch('http://example.com/mul', 'POST', {
-                'x': 10, 'y': 10})
-            self.assertEqual(d.dispatch(), 100)
-
-
-class test_URL(AppCase):
-
-    def test_URL_get_async(self):
-        self.app.conf.task_always_eager = True
-        with mock_urlopen(success_response(100)):
-            d = http.URL(
-                'http://example.com/mul', app=self.app,
-            ).get_async(x=10, y=10)
-            self.assertEqual(d.get(), 100)
-
-    def test_URL_post_async(self):
-        self.app.conf.task_always_eager = True
-        with mock_urlopen(success_response(100)):
-            d = http.URL(
-                'http://example.com/mul', app=self.app,
-            ).post_async(x=10, y=10)
-            self.assertEqual(d.get(), 100)

docs/history/changelog-1.0.rst

@@ -864,7 +864,7 @@ News
 * Improved support for webhook tasks.
 
     `celery.task.rest` is now deprecated, replaced with the new and shiny
-    :mod:`celery.task.http`. With more reflective names, sensible interface,
+    `celery.task.http`. With more reflective names, sensible interface,
     and it's possible to override the methods used to perform HTTP requests.
 
 * The results of task sets are now cached by storing it in the result

docs/history/changelog-2.0.rst

@@ -598,7 +598,7 @@ Backward incompatible changes
     `CELERY_AMQP_PUBLISHER_ROUTING_KEY`    `CELERY_DEFAULT_ROUTING_KEY`
     =====================================  =====================================
 
-* The `celery.task.rest` module has been removed, use :mod:`celery.task.http`
+* The `celery.task.rest` module has been removed, use `celery.task.http`
   instead (as scheduled by the :ref:`deprecation-timeline`).
 
 * It's no longer allowed to skip the class name in loader names.

docs/internals/deprecation.rst

@@ -225,5 +225,3 @@ Removals for version 2.0
 
 * :meth:`TaskSet.run`. Use :meth:`celery.task.base.TaskSet.apply_async`
     instead.
-
-* The module :mod:`celery.task.rest`; use :mod:`celery.task.http` instead.

docs/reference/celery.task.http.rst

@@ -1,11 +0,0 @@
-========================================
- ``celery.task.http``
-========================================
-
-.. contents::
-    :local:
-.. currentmodule:: celery.task.http
-
-.. automodule:: celery.task.http
-    :members:
-    :undoc-members:

docs/reference/index.rst

@@ -22,7 +22,6 @@
     celery.app.utils
     celery.bootsteps
     celery.result
-    celery.task.http
     celery.schedules
     celery.signals
     celery.security

docs/userguide/index.rst

@@ -16,7 +16,6 @@
     canvas
     workers
     periodic-tasks
-    remote-tasks
     routing
     monitoring
     security

docs/userguide/remote-tasks.rst

@@ -1,141 +0,0 @@
-.. _guide-webhooks:
-
-================================
- HTTP Callback Tasks (Webhooks)
-================================
-
-.. module:: celery.task.http
-
-.. contents::
-    :local:
-
-.. _webhook-basics:
-
-Basics
-======
-
-If you need to call into another language, framework or similar, you can
-do so by using HTTP callback tasks.
-
-The HTTP callback tasks uses GET/POST data to pass arguments and returns
-result as a JSON response. The scheme to call a task is:
-
-.. code-block:: http
-
-    GET /mytask/?arg1=a&arg2=b&arg3=c HTTP/1.1
-    Host: example.com
-
-or using POST:
-
-.. code-block:: http
-
-    POST /mytask HTTP/1.1
-    Host: example.com
-
-.. note::
-
-    POST data needs to be form encoded.
-
-Whether to use GET or POST is up to you and your requirements.
-
-The web page should then return a response in the following format
-if the execution was successful:
-
-.. code-block:: json
-
-    {"status": "success", "retval": "RETVAL"}
-
-or if there was an error:
-
-.. code-block:: json
-
-    {"status": "failure", "reason": "Invalid moon alignment."}
-
-Enabling the HTTP task
-----------------------
-
-To enable the HTTP dispatch task you have to add :mod:`celery.task.http`
-to :setting:`imports`, or start the worker with
-:option:`-I celery.task.http <celery worker -I>`.
-
-
-.. _webhook-django-example:
-
-Django webhook example
-======================
-
-With this information you could define a simple task in Django:
-
-.. code-block:: python
-
-    from django.http import HttpResponse
-    from json import dumps
-
-
-    def multiply(request):
-        x = int(request.GET['x'])
-        y = int(request.GET['y'])
-        result = x * y
-        response = {'status': 'success', 'retval': result}
-        return HttpResponse(dumps(response), mimetype='application/json')
-
-.. _webhook-rails-example:
-
-Ruby on Rails webhook example
-=============================
-
-or in Ruby on Rails:
-
-.. code-block:: ruby
-
-    def multiply
-        @x = params[:x].to_i
-        @y = params[:y].to_i
-
-        @status = {:status => 'success', :retval => @x * @y}
-
-        render :json => @status
-    end
-
-You can easily port this scheme to any language/framework;
-new examples and libraries are very welcome.
-
-.. _webhook-calling:
-
-Calling webhook tasks
-=====================
-
-To call a task you can use the :class:`~celery.task.http.URL` class:
-
-.. code-block:: pycon
-
-    >>> from celery.task.http import URL
-    >>> res = URL('http://example.com/multiply').get_async(x=10, y=10)
-
-
-:class:`~celery.task.http.URL` is a shortcut to the :class:`HttpDispatchTask`.
-You can subclass this to extend the
-functionality:
-
-.. code-block:: pycon
-
-    >>> from celery.task.http import HttpDispatchTask
-    >>> res = HttpDispatchTask.delay(
-    ...     url='http://example.com/multiply',
-    ...     method='GET', x=10, y=10)
-    >>> res.get()
-    100
-
-The output of :program:`celery worker` (or the log file if enabled) should show the
-task being executed:
-
-.. code-block:: text
-
-    [INFO/MainProcess] Task celery.task.http.HttpDispatchTask
-            [f2cc8efc-2a14-40cd-85ad-f1c77c94beeb] processed: 100
-
-Since calling tasks can be done via HTTP using the
-:func:`djcelery.views.apply` view, calling tasks from other languages is easy.
-For an example service exposing tasks via HTTP you should have a look at
-`examples/celery_http_gateway` in the Celery distribution:
-https://github.com/celery/celery/tree/master/examples/celery_http_gateway/

examples/httpexample/README.rst

@@ -1,33 +0,0 @@
-======================
- Webhook Task Example
-======================
-
-This example is a simple Django HTTP service exposing a single task
-multiplying two numbers:
-
-The multiply http callback task is in `views.py`, mapped to a URL using
-`urls.py`.
-
-There are no models, so to start it do::
-
-    $ python manage.py runserver
-
-To execute the task you could use curl::
-
-    $ curl http://localhost:8000/multiply?x=10&y=10
-
-which then gives the expected JSON response::
-
-    {'status': 'success': 'retval': 100}
-
-
-To execute this http callback task asynchronously you could fire up
-a python shell with a properly configured celery and do::
-
-    >>> from celery.task.http import URL
-    >>> res = URL('http://localhost:8000/multiply').get_async(x=10, y=10)
-    >>> res.wait()
-    100
-
-
-That's all!

examples/httpexample/manage.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-from __future__ import absolute_import, unicode_literals
-
-from django.core.management import execute_manager
-try:
-    from . import settings  # Assumed to be in the same directory.
-except ImportError:
-    import sys
-    sys.stderr.write(
-        "Error: Can't find the file 'settings.py' in the directory "
-        "containing {0!r}.".format(__file__))
-    sys.exit(1)
-
-if __name__ == '__main__':
-    execute_manager(settings)

examples/httpexample/settings.py

@@ -1,91 +0,0 @@
-from __future__ import absolute_import, unicode_literals
-
-# Django settings for httpexample project.
-
-DEBUG = True
-TEMPLATE_DEBUG = DEBUG
-
-ADMINS = (
-    # ('Your Name', 'your_email@domain.com'),
-)
-
-MANAGERS = ADMINS
-# 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
-DATABASE_ENGINE = ''
-
-# Pth to database file if using sqlite3.
-DATABASE_NAME = ''
-
-# Not used with sqlite3.
-DATABASE_USER = ''
-
-# Not used with sqlite3.
-DATABASE_PASSWORD = ''
-
-# Set to empty string for localhost. Not used with sqlite3.
-DATABASE_HOST = ''
-
-# Set to empty string for default. Not used with sqlite3.
-DATABASE_PORT = ''
-
-# Local time zone for this installation. Choices can be found here:
-# https://en.wikipedia.org/wiki/List_of_tz_zones_by_name
-# although not all choices may be available on all operating systems.
-# If running in a Windows environment this must be set to the same as your
-# system time zone.
-TIME_ZONE = 'America/Chicago'
-
-# Language code for this installation. All choices can be found here:
-# http://www.i18nguy.com/unicode/language-identifiers.html
-LANGUAGE_CODE = 'en-us'
-
-SITE_ID = 1
-
-# If you set this to False, Django will make some optimizations so as not
-# to load the internationalization machinery.
-USE_I18N = True
-
-# Absolute path to the directory that holds media.
-# Example: '/home/media/media.lawrence.com/'
-MEDIA_ROOT = ''
-
-# URL that handles the media served from MEDIA_ROOT. Make sure to use a
-# trailing slash if there is a path component (optional in other cases).
-# Examples: 'http://media.lawrence.com', 'http://example.com/media/'
-MEDIA_URL = ''
-
-# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
-# trailing slash.
-# Examples: 'http://foo.com/media/', '/media/'.
-ADMIN_MEDIA_PREFIX = '/media/'
-
-# Make this unique, and don't share it with anybody.
-SECRET_KEY = 'p^@q$@nal#-0+w@v_3bcj2ug(zbh5_m2on8^kkn&!e!b=a@o__'
-
-# List of callables that know how to import templates from various sources.
-TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.load_template_source',
-    'django.template.loaders.app_directories.load_template_source',
-)
-
-MIDDLEWARE_CLASSES = (
-    'django.middleware.common.CommonMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-)
-
-ROOT_URLCONF = 'httpexample.urls'
-
-TEMPLATE_DIRS = (
-    # Put strings here, like '/home/html/django_templates' or
-    # 'C:/www/django/templates'.
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
-)
-
-INSTALLED_APPS = (
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.sites',
-)

examples/httpexample/urls.py

@@ -1,15 +0,0 @@
-from __future__ import absolute_import, unicode_literals
-
-from django.conf.urls.defaults import (  # noqa
-    url, patterns, include, handler500, handler404,
-)
-from . import views
-
-# Uncomment the next two lines to enable the admin:
-# from django.contrib import admin
-# admin.autodiscover()
-
-urlpatterns = patterns(
-    '',
-    url(r'^multiply/', views.multiply, name='multiply'),
-)

examples/httpexample/views.py

@@ -1,14 +0,0 @@
-from __future__ import absolute_import, unicode_literals
-
-from django.http import HttpResponse
-
-from json import dumps
-
-
-def multiply(request):
-    x = int(request.GET['x'])
-    y = int(request.GET['y'])
-
-    retval = x * y
-    response = {'status': 'success', 'retval': retval}
-    return HttpResponse(dumps(response), mimetype='application/json')