Add docker-compose and base dockerfile for development (#4482)

* Add docker-compose and base dockerfile for development

* Change to base jessie docker image and setup pyenv

* Add in aliases for pyenv, fix entrypoint for docker

* Update dockerfile to be non-root and fix encoding problems with tox

* Add convenience method to get redis connection for tests

* Add pypy to install commands

* Force worker to pick up broker url from environment

* Move pypy comment to above apt-get install, default python to 3.6, add in flag to prevent bytecode

* Add documentation

* Fix links

* Update docs

* Add to contributors

* Address feedback: improve documentation, separate dockerfile into scripts, remove redundancy in pyenv setup, add in .env file

* Setup pyenv environments correctly in dockerfile

* Update capitalization

* Change CELERY_USER to ARG in dockerfile and pass build argument in build

* Change default worker loglevel to debug in docker-compose

CONTRIBUTING.rst

@@ -448,6 +448,55 @@ fetch and checkout a remote branch like this::
     https://notes.envato.com/developers/rebasing-merge-commits-in-git/
 .. _`Rebase`: https://help.github.com/rebase/
 
+.. _contributing-docker-development:
+
+Developing and Testing with Docker
+----------------------------------
+
+Because of the many components of Celery, such as a broker and backend,
+`Docker`_ and `docker-compose`_ can be utilized to greatly simplify the
+development and testing cycle. The Docker configuration here requires a
+Docker version of at least 17.09.
+
+The Docker components can be found within the :file:`docker/` folder and the
+Docker image can be built via:
+
+.. code-block:: console
+
+    $ docker-compose build celery
+
+and run via:
+
+.. code-block:: console
+
+    $ docker-compose run --rm celery <command>
+
+where <command> is a command to execute in a Docker container. The `--rm` flag
+indicates that the container should be removed after it is exited and is useful
+to prevent accumulation of unwanted containers.
+
+Some useful commands to run:
+
+* ``bash``
+
+    To enter the Docker container like a normal shell
+
+* ``make test``
+
+    To run the test suite
+
+* ``tox``
+
+    To run tox and test against a variety of configurations
+
+By default, docker-compose will mount the Celery and test folders in the Docker
+container, allowing code changes and testing to be immediately visible inside
+the Docker container. Environment variables, such as the broker and backend to
+use are also defined in the :file:`docker/docker-compose.yml` file.
+
+.. _`Docker`: https://www.docker.com/
+.. _`docker-compose`: https://docs.docker.com/compose/
+
 .. _contributing-testing:
 
 Running the unit test suite

CONTRIBUTORS.txt

@@ -257,3 +257,4 @@ Mikhail Wolfson, 2017/12/11
 Alex Garel, 2018/01/04
 Régis Behmo 2018/01/20
 Igor Kasianov, 2018/01/20
+Chris Mitchell, 2018/02/27

celery/contrib/testing/worker.py

@@ -102,7 +102,7 @@ def _start_worker_thread(app,
     setup_app_for_worker(app, loglevel, logfile)
     assert 'celery.ping' in app.tasks
     # Make sure we can connect to the broker
-    with app.connection() as conn:
+    with app.connection(hostname=os.environ.get('TEST_BROKER')) as conn:
         conn.default_channel.queue_declare
 
     worker = WorkController(

docker/.env

@@ -0,0 +1 @@
+CELERY_USER=developer

docker/Dockerfile

@@ -0,0 +1,78 @@
+FROM debian:jessie
+
+ENV PYTHONIOENCODING UTF-8
+
+# Pypy is installed from a package manager because it takes so long to build.
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    libmemcached-dev \
+    libncurses5-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    pkg-config \
+    pypy \
+    wget \
+    zlib1g-dev
+
+# Setup variables. Even though changing these may cause unnecessary invalidation of
+# unrelated elements, grouping them together makes the Dockerfile read better.
+ENV PROVISIONING /provisioning
+
+# This is provisioned from .env
+ARG CELERY_USER=developer
+
+# Check for mandatory build arguments
+RUN : "${CELERY_USER:?CELERY_USER build argument needs to be set and non-empty.}"
+
+ENV HOME /home/$CELERY_USER
+ENV PATH="$HOME/.pyenv/bin:$PATH"
+
+# Copy and run setup scripts
+WORKDIR $PROVISIONING
+COPY docker/scripts/install-couchbase.sh .
+# Scripts will lose thier executable flags on copy. To avoid the extra instructions
+# we call the shell directly.
+RUN sh install-couchbase.sh
+COPY docker/scripts/create-linux-user.sh .
+RUN sh create-linux-user.sh
+
+# Swap to the celery user so packages and celery are not installed as root.
+USER $CELERY_USER
+
+COPY docker/scripts/install-pyenv.sh .
+RUN sh install-pyenv.sh
+
+# Install celery
+WORKDIR $HOME
+COPY --chown=1000:1000 requirements $HOME/requirements
+COPY --chown=1000:1000 docker/entrypoint /entrypoint
+RUN chmod gu+x /entrypoint
+
+# Define the local pyenvs
+RUN pyenv local python2.7 python3.4 python3.5 python3.6
+
+# Setup one celery environment for basic development use
+RUN pyenv exec pip install \
+  -r requirements/default.txt \
+  -r requirements/pkgutils.txt \
+  -r requirements/test.txt \
+  -r requirements/test-ci-base.txt \
+  -r requirements/test-integration.txt
+
+COPY --chown=1000:1000 MANIFEST.in Makefile setup.py setup.cfg tox.ini $HOME/
+COPY --chown=1000:1000 t $HOME/t
+COPY --chown=1000:1000 celery $HOME/celery
+
+RUN pyenv exec pip install -e .
+
+# the compiled files from earlier steps will cause py.test to fail with
+# an ImportMismatchError
+RUN make clean-pyc
+
+# Setup the entrypoint, this ensures pyenv is initialized when a container is started.
+ENTRYPOINT ["/entrypoint"]

docker/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '2'
+
+services:
+  celery:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+      args:
+        CELERY_USER:
+    environment:
+      TEST_BROKER: pyamqp://rabbit:5672
+      TEST_BACKEND: redis://redis
+      PYTHONUNBUFFERED: 1
+      PYTHONDONTWRITEBYTECODE: 1
+      REDIS_HOST: redis
+      WORKER_LOGLEVEL: DEBUG
+    tty: true
+    volumes:
+      - ../celery:/home/$CELERY_USER/celery
+      # Because pytest fails when it encounters files from alternative python compilations,
+      # __pycache__ and pyc files, PYTHONDONTWRITEBYTECODE must be
+      # set on the host as well or py.test will throw configuration errors.
+#      - ../t:/home/$CELERY_USER/t
+    depends_on:
+      - rabbit
+      - redis
+      - dynamodb
+
+  rabbit:
+    image: rabbitmq:3.7.3
+
+  redis:
+    image: redis:3.2.11
+
+  dynamodb:
+    image: dwmkerr/dynamodb:38

docker/entrypoint

@@ -0,0 +1,4 @@
+#!/bin/bash
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+exec "$@"

docker/scripts/create-linux-user.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+addgroup --gid 1000 $CELERY_USER
+adduser --system --disabled-password --uid 1000 --gid 1000 $CELERY_USER

docker/scripts/install-couchbase.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+wget http://packages.couchbase.com/clients/c/libcouchbase-2.8.4_jessie_amd64.tar
+tar -vxf libcouchbase-2.8.4_jessie_amd64.tar
+dpkg -i libcouchbase-2.8.4_jessie_amd64/libcouchbase2-core_2.8.4-1_amd64.deb
+dpkg -i libcouchbase-2.8.4_jessie_amd64/libcouchbase-dev_2.8.4-1_amd64.deb

docker/scripts/install-pyenv.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# For managing all the local python installations for testing, use pyenv
+curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash
+
+# To enable testing versions like 3.4.8 as 3.4 in tox, we need to alias
+# pyenv python versions
+git clone https://github.com/s1341/pyenv-alias.git $(pyenv root)/plugins/pyenv-alias
+
+# Python versions to test against
+VERSION_ALIAS="python2.7" pyenv install 2.7.14
+VERSION_ALIAS="python3.4" pyenv install 3.4.8
+VERSION_ALIAS="python3.5" pyenv install 3.5.5
+VERSION_ALIAS="python3.6" pyenv install 3.6.4

t/integration/conftest.py

@@ -24,6 +24,11 @@ def flaky(fun):
     return _inner
 
 
+def get_redis_connection():
+    from redis import StrictRedis
+    return StrictRedis(host=os.environ.get('REDIS_HOST'))
+
+
 @pytest.fixture(scope='session')
 def celery_config():
     return {

t/integration/tasks.py

@@ -7,6 +7,8 @@ from celery import chain, group, shared_task
 from celery.exceptions import SoftTimeLimitExceeded
 from celery.utils.log import get_task_logger
 
+from .conftest import get_redis_connection
+
 logger = get_task_logger(__name__)
 
 
@@ -115,17 +117,15 @@ def retry_once(self):
 @shared_task
 def redis_echo(message):
     """Task that appends the message to a redis list"""
-    from redis import StrictRedis
 
-    redis_connection = StrictRedis()
+    redis_connection = get_redis_connection()
     redis_connection.rpush('redis-echo', message)
 
 
 @shared_task(bind=True)
 def second_order_replace1(self, state=False):
-    from redis import StrictRedis
 
-    redis_connection = StrictRedis()
+    redis_connection = get_redis_connection()
     if not state:
         redis_connection.rpush('redis-echo', 'In A')
         new_task = chain(second_order_replace2.s(),
@@ -137,9 +137,7 @@ def second_order_replace1(self, state=False):
 
 @shared_task(bind=True)
 def second_order_replace2(self, state=False):
-    from redis import StrictRedis
-
-    redis_connection = StrictRedis()
+    redis_connection = get_redis_connection()
     if not state:
         redis_connection.rpush('redis-echo', 'In B')
         new_task = chain(redis_echo.s("In/Out C"),

t/integration/test_canvas.py

@@ -4,13 +4,12 @@ from datetime import datetime, timedelta
 from time import sleep
 
 import pytest
-from redis import StrictRedis
 
 from celery import chain, chord, group
 from celery.exceptions import TimeoutError
 from celery.result import AsyncResult, GroupResult
 
-from .conftest import flaky
+from .conftest import flaky, get_redis_connection
 from .tasks import (add, add_chord_to_chord, add_replaced, add_to_all,
                     add_to_all_to_chord, collect_ids, delayed_sum,
                     delayed_sum_with_soft_guard, identity, ids, print_unicode,
@@ -66,7 +65,7 @@ class test_chain:
 
         if not manager.app.conf.result_backend.startswith('redis'):
             raise pytest.skip('Requires redis result backend.')
-        redis_connection = StrictRedis()
+        redis_connection = get_redis_connection()
         redis_connection.delete('redis-echo')
         before = group(redis_echo.si('before {}'.format(i)) for i in range(3))
         connect = redis_echo.si('connect')
@@ -94,7 +93,7 @@ class test_chain:
         if not manager.app.conf.result_backend.startswith('redis'):
             raise pytest.skip('Requires redis result backend.')
 
-        redis_connection = StrictRedis()
+        redis_connection = get_redis_connection()
         redis_connection.delete('redis-echo')
 
         result = second_order_replace1.delay()
@@ -242,7 +241,7 @@ class test_chord:
         if not manager.app.conf.result_backend.startswith('redis'):
             raise pytest.skip('Requires redis result backend.')
 
-        redis_client = StrictRedis()
+        redis_client = get_redis_connection()
         async_result = chord([add.s(5, 6), add.s(6, 7)])(delayed_sum.s())
         for _ in range(TIMEOUT):
             if async_result.state == 'STARTED':

tox.ini

@@ -34,6 +34,7 @@ commands =
     integration: py.test -xsv t/integration
 setenv =
     WORKER_LOGLEVEL = INFO
+    PYTHONIOENCODING = UTF-8
 
     rabbitmq: TEST_BROKER=pyamqp://
     rabbitmq: TEST_BACKEND=rpc