[dist] Use apicheck from sphinx_celery

Makefile

@@ -13,7 +13,7 @@ WORKER_GRAPH_FULL="docs/images/worker_graph_full.png"
 SPHINX_HTMLDIR = "${SPHINX_BUILDDIR}/html"
 
 html:
-	(cd "$(SPHINX_DIR)"; make html)
+	(cd "$(SPHINX_DIR)"; $(MAKE) html)
 	mv "$(SPHINX_HTMLDIR)" Documentation
 
 docsclean:
@@ -23,10 +23,7 @@ htmlclean:
 	-rm -rf "$(SPHINX)"
 
 apicheck:
-	extra/release/doc4allmods "$(PROJ)"
-
-indexcheck:
-	extra/release/verify-reference-index.sh
+	(cd "$(SPHINX_DIR)"; $(MAKE) apicheck)
 
 configcheck:
 	PYTHONPATH=. $(PYTHON) extra/release/verify_config_reference.py $(CONFIGREF_SRC)
@@ -88,7 +85,10 @@ gitclean:
 gitcleanforce:
 	git clean -xdf
 
-distcheck: flakecheck apicheck indexcheck configcheck readmecheck test gitclean
+tox: removepyc
+	tox
+
+distcheck: flakecheck apicheck configcheck readmecheck test gitclean
 
 authorcheck:
 	git shortlog -se | cut -f2 | extra/release/attribution.py

docs/Makefile

@@ -47,6 +47,7 @@ help:
 	@echo "  linkcheck  to check all external links for integrity"
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+	@echo "  apicheck   to verify that all modules are present in autodoc"
 
 .PHONY: clean
 clean:
@@ -210,6 +211,10 @@ coverage:
 	@echo "Testing of coverage in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/coverage/python.txt."
 
+.PHONY: apicheck
+apicheck:
+	$(SPHINXBUILD) -b apicheck $(ALLSPHINXOPTS) $(BUILDDIR)/apicheck
+
 .PHONY: xml
 xml:
 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml

docs/conf.py

@@ -22,4 +22,20 @@ globals().update(conf.build_config(
         'celery.contrib.sphinx',
         'celerydocs',
     ],
+    apicheck_ignore_modules=[
+        'celery.five',
+        'celery.__main__',
+        'celery.task',
+        'celery.task.base',
+        'celery.bin',
+        'celery.bin.celeryd_detach',
+        'celery.contrib',
+        r'celery.fixups.*',
+        'celery.local',
+        'celery.app.base',
+        'celery.apps',
+        'celery.canvas',
+        'celery.concurrency.asynpool',
+        'celery.utils.encoding',
+    ],
 ))

docs/internals/reference/celery.backends.async.rst

@@ -0,0 +1,13 @@
+=====================================
+ celery.backends.async
+=====================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.backends.async
+
+.. automodule:: celery.backends.async
+    :members:
+    :undoc-members:
+
+

docs/internals/reference/index.rst

@@ -24,6 +24,7 @@
     celery.concurrency.threads
     celery.backends
     celery.backends.base
+    celery.backends.async
     celery.backends.rpc
     celery.backends.database
     celery.backends.amqp

docs/reference/celery.worker.consumer.agent.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.agent
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.agent
+
+.. automodule:: celery.worker.consumer.agent
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.connection.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.connection
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.connection
+
+.. automodule:: celery.worker.consumer.connection
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.consumer.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.consumer
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.consumer
+
+.. automodule:: celery.worker.consumer.consumer
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.control.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.control
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.control
+
+.. automodule:: celery.worker.consumer.control
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.events.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.events
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.events
+
+.. automodule:: celery.worker.consumer.events
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.gossip.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.gossip
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.gossip
+
+.. automodule:: celery.worker.consumer.gossip
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.heart.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.heart
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.heart
+
+.. automodule:: celery.worker.consumer.heart
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.mingle.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.mingle
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.mingle
+
+.. automodule:: celery.worker.consumer.mingle
+    :members:
+    :undoc-members:

docs/reference/celery.worker.consumer.tasks.rst

@@ -0,0 +1,11 @@
+==================================================
+ celery.worker.consumer.tasks
+==================================================
+
+.. contents::
+    :local:
+.. currentmodule:: celery.worker.consumer.tasks
+
+.. automodule:: celery.worker.consumer.tasks
+    :members:
+    :undoc-members:

docs/reference/index.rst

@@ -45,10 +45,19 @@
     celery.apps.worker
     celery.apps.beat
     celery.worker
-    celery.worker.consumer
     celery.worker.request
     celery.worker.state
     celery.worker.strategy
+    celery.worker.consumer
+    celery.worker.consumer.agent
+    celery.worker.consumer.connection
+    celery.worker.consumer.consumer
+    celery.worker.consumer.control
+    celery.worker.consumer.events
+    celery.worker.consumer.gossip
+    celery.worker.consumer.heart
+    celery.worker.consumer.mingle
+    celery.worker.consumer.tasks
     celery.bin.base
     celery.bin.celery
     celery.bin.worker

extra/release/doc4allmods

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-PACKAGE="$1"
-SKIP_PACKAGES="$PACKAGE tests management urls"
-SKIP_FILES="celery.five.rst
-            celery.__main__.rst
-            celery.task.rst
-            celery.task.base.rst
-            celery.task.sets.rst
-            celery.bin.rst
-            celery.bin.celeryd_detach.rst
-            celery.contrib.rst
-            celery.fixups.rst
-            celery.fixups.django.rst
-            celery.local.rst
-            celery.app.base.rst
-            celery.apps.rst
-            celery.canvas.rst
-            celery.concurrency.asynpool.rst
-            celery.utils.encoding.rst"
-
-modules=$(find "$PACKAGE" -name "*.py")
-
-failed=0
-for module in $modules; do
-    dotted=$(echo $module | sed 's/\//\./g')
-    name=${dotted%.__init__.py}
-    name=${name%.py}
-    rst=$name.rst
-    skip=0
-    for skip_package in $SKIP_PACKAGES; do
-        [ $(echo "$name" | cut -d. -f 2) == "$skip_package" ] && skip=1
-    done
-    for skip_file in $SKIP_FILES; do
-        [ "$skip_file" == "$rst" ] && skip=1
-    done
-
-    if [ $skip -eq 0 ]; then
-        if [ ! -f "docs/reference/$rst" ]; then
-            if [ ! -f "docs/internals/reference/$rst" ]; then
-                echo $rst :: FAIL
-                failed=1
-            fi
-        fi
-    fi
-done
-
-exit $failed

extra/release/removepyc.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-(cd "${1:-.}";
-        find . -name "*.pyc" | xargs rm -- 2>/dev/null) || echo "ok"

extra/release/verify-reference-index.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-RETVAL=0
-
-verify_index() {
-    retval=0
-    for refdir in $*; do
-        verify_modules_in_index "$refdir/index.rst"
-        verify_files "$refdir"
-    done
-    return $RETVAL
-}
-
-verify_files() {
-    for path in $1/*.rst; do
-        rst=${path##*/}
-        modname=${rst%*.rst}
-        if [ $modname != "index" ]; then
-            modpath=$(echo $modname | tr . /)
-            pkg="$modpath/__init__.py"
-            mod="$modpath.py"
-            if [ ! -f "$pkg" ]; then
-                if [ ! -f "$mod" ]; then
-                    echo "*** NO MODULE $modname for reference '$path'"
-                    RETVAL=1
-                fi
-            fi
-        fi
-    done
-}
-
-verify_modules_in_index() {
-    modules=$(grep "celery." "$1" | \
-                perl -ple's/^\s*|\s*$//g;s{\.}{/}g;')
-    for module in $modules; do
-        if [ ! -f "$module.py" ]; then
-            if [ ! -f "$module/__init__.py" ]; then
-                echo "*** IN INDEX BUT NO MODULE: $module"
-                RETVAL=1
-            fi
-        fi
-    done
-}
-
-verify_index docs/reference docs/internals/reference

requirements/docs.txt

@@ -1,3 +1,3 @@
-sphinx_celery
+sphinx_celery>=1.1
 -r extras/sqlalchemy.txt
 -r dev.txt

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = 2.7,pypy,3.4,3.5,pypy3,flake8,flakeplus
+envlist = 2.7,pypy,3.4,3.5,pypy3,flake8,flakeplus,apicheck
 
 [testenv]
 deps=
@@ -10,33 +10,35 @@ deps=
     3.4,3.5: -r{toxinidir}/requirements/test-ci-default.txt
     pypy,pypy3: -r{toxinidir}/requirements/test-ci-base.txt
 
+    linkcheck,apicheck: -r{toxinidir}/requirements/docs.txt
+    flake8,flakeplus: -r{toxinidir}/requirements/pkgutils.txt
 sitepackages = False
 recreate = False
-commands = {toxinidir}/extra/release/removepyc.sh {toxinidir}
-           pip install -U -r{toxinidir}/requirements/dev.txt
+commands = pip install -U -r{toxinidir}/requirements/dev.txt
            nosetests -xsv --with-coverage \
                 --cover-inclusive --cover-min-percentage=94 --cover-erase []
 
 basepython =
-    2.7: python2.7
+    2.7,flake8,flakeplus,linkcheck,apicheck python2.7
     3.4: python3.4
     3.5: python3.5
     pypy: pypy
     pypy3: pypy3
-    docs,flake8,flakeplus: python2.7
 
-[testenv:docs]
-deps = -r{toxinidir}/requirements/docs.txt
+[testenv:apicheck]
+commands =
+    pip install -U -r{toxinidir}/requirements/dev.txt
+    sphinx-build -W -b apicheck -d {envtmpdir}/doctrees docs docs/_build/apicheck
+
+[testenv:linkcheck]
 commands =
     pip install -U -r{toxinidir}/requirements/dev.txt
     sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees docs docs/_build/linkcheck
 
 [testenv:flake8]
-deps = -r{toxinidir}/requirements/pkgutils.txt
 commands =
     flake8 {toxinidir}/celery
 
 [testenv:flakeplus]
-deps = -r{toxinidir}/requirements/pkgutils.txt
 commands =
-    flakeplus --2.6 {toxinidir}/celery
+    flakeplus --2.7 {toxinidir}/celery