Update CONTRIBUTING.rst, fix tests and update Dockerfile. Related to c…

…elery#5096. (celery#5143)

* Update couchbase install steps

Fix integration test for channel leak. We have to inspect the channels created before the tasks are done to ensure we're looking at the correct data

Running multiple times test_parallel_chords to make sure test works

Add pytest fixtures to t/integration/conftest.py so we don't have to install package to use fixtures

Update sphinx test to use sphinx_testing instead of running sphinx-build

Bump setuptools

Install reqs for all python versions and upgrade pip

Update docker-compose to create a tagged image and update PYTHONPATH

Add bandit to pkgutils

Update contributing documentation and changelog to show 4.3 version

Add pkgutils and docs requirements to run sphinx unit tests and use cyanide when running travis

Forgot to fix flake8 issues on tests. Remove bandit from pkgutils.txt since tox already installs it. Update CONTRIBUTING.rst to show how to install bandit to run it

Fix flake8 issues on test_sphinx and add shared task to the test

Update wording for CONTRIBUTING.rst

Make python3.6 default python version, mount the entire celery folder so everything can be done inside the container and bump setuptools

Update label definitions.

Remove cyanide from requirements for now and add bumpversion information.

* Update celery.contrib.sphinx. Checking if the object to document is a subclass of
BaseTask and has the attribute __wrapped__ should be enough to know if it's a Celery task.
Checking if the object is also an instance of Proxy/PromiseProxy makes the extension not work correctly.
Probably becuase of how sphinx loads objects and the dunder overrides that the Proxy class does,depending on how sphinx-doc is ran a celery task might be or not a
instance of Proxy.

* Update Test Case details.

Changelog

@@ -8,6 +8,12 @@ This document contains change notes for bugfix releases in
 the 4.x series, please see :ref:`whatsnew-4.2` for
 an overview of what's new in Celery 4.2.
 
+4.3.0
+=====
+:release-date: TBA
+:status: DEVELOPMENT
+:branch: dev (git calls this master)
+
 4.2.1
 =====
 :release-date: 2018-07-18 11:00 AM IST

celery/contrib/sphinx.py

@@ -35,7 +35,6 @@
 from sphinx.ext.autodoc import FunctionDocumenter
 
 from celery.app.task import BaseTask
-from celery.local import Proxy
 
 try:  # pragma: no cover
     from inspect import formatargspec, getfullargspec
@@ -72,10 +71,9 @@ def check_module(self):
         # given by *self.modname*. But since functions decorated with the @task
         # decorator are instances living in the celery.local, we have to check
         # the wrapped function instead.
-        if isinstance(self.object, Proxy):
-            wrapped = getattr(self.object, '__wrapped__', None)
-            if wrapped and getattr(wrapped, '__module__') == self.modname:
-                return True
+        wrapped = getattr(self.object, '__wrapped__', None)
+        if wrapped and getattr(wrapped, '__module__') == self.modname:
+            return True
         return super(TaskDocumenter, self).check_module()
 
 
@@ -94,7 +92,7 @@ def autodoc_skip_member_handler(app, what, name, obj, skip, options):
     # suppress repetition of class documentation in an instance of the
     # class. This overrides that behavior.
     if isinstance(obj, BaseTask) and getattr(obj, '__wrapped__'):
-        if skip and isinstance(obj, Proxy):
+        if skip:
             return False
     return None
 

docker/Dockerfile

@@ -17,7 +17,8 @@ RUN apt-get update && apt-get install -y \
     pkg-config \
     pypy \
     wget \
-    zlib1g-dev
+    zlib1g-dev \
+    lsb-release
 
 # Setup variables. Even though changing these may cause unnecessary invalidation of
 # unrelated elements, grouping them together makes the Dockerfile read better.
@@ -53,25 +54,49 @@ COPY --chown=1000:1000 docker/entrypoint /entrypoint
 RUN chmod gu+x /entrypoint
 
 # Define the local pyenvs
-RUN pyenv local python2.7 python3.4 python3.5 python3.6
+RUN pyenv local python3.6 python3.5 python3.4 python2.7
+
+RUN pyenv exec python2.7 -m pip install --upgrade pip && \
+    pyenv exec python3.4 -m pip install --upgrade pip && \
+    pyenv exec python3.5 -m pip install --upgrade pip && \
+    pyenv exec python3.6 -m pip install --upgrade pip
 
 # Setup one celery environment for basic development use
-RUN pyenv exec pip install \
+RUN pyenv exec python3.6 -m pip install \
+  -r requirements/default.txt \
+  -r requirements/test.txt \
+  -r requirements/test-ci-default.txt \
+  -r requirements/docs.txt \
+  -r requirements/test-integration.txt \
+  -r requirements/pkgutils.txt && \
+  pyenv exec python3.5 -m pip install \
   -r requirements/default.txt \
+  -r requirements/test.txt \
+  -r requirements/test-ci-default.txt \
   -r requirements/docs.txt \
-  -r requirements/pkgutils.txt \
+  -r requirements/test-integration.txt \
+  -r requirements/pkgutils.txt && \
+  pyenv exec python3.4 -m pip install \
+  -r requirements/default.txt \
   -r requirements/test.txt \
-  -r requirements/test-ci-base.txt \
-  -r requirements/test-integration.txt
+  -r requirements/test-ci-default.txt \
+  -r requirements/docs.txt \
+  -r requirements/test-integration.txt \
+  -r requirements/pkgutils.txt && \
+  pyenv exec python2.7 -m pip install \
+  -r requirements/default.txt \
+  -r requirements/test.txt \
+  -r requirements/test-ci-default.txt \
+  -r requirements/docs.txt \
+  -r requirements/test-integration.txt \
+  -r requirements/pkgutils.txt
 
-COPY --chown=1000:1000 MANIFEST.in Makefile setup.py setup.cfg tox.ini $HOME/
-COPY --chown=1000:1000 docs $HOME/docs
-COPY --chown=1000:1000 t $HOME/t
-COPY --chown=1000:1000 celery $HOME/celery
+COPY --chown=1000:1000 . $HOME/celery
 
-RUN pyenv exec pip install -e .
+WORKDIR $HOME/celery
 
 # Setup the entrypoint, this ensures pyenv is initialized when a container is started
 # and that any compiled files from earlier steps or from moutns are removed to avoid
 # py.test failing with an ImportMismatchError
 ENTRYPOINT ["/entrypoint"]
+

docker/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2'
+version: '3'
 
 services:
   celery:
@@ -7,6 +7,7 @@ services:
       dockerfile: docker/Dockerfile
       args:
         CELERY_USER: developer
+    image: celery/celery:dev
     environment:
       TEST_BROKER: pyamqp://rabbit:5672
       TEST_BACKEND: redis://redis
@@ -15,11 +16,10 @@ services:
       REDIS_HOST: redis
       WORKER_LOGLEVEL: DEBUG
       AZUREBLOCKBLOB_URL: azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;
+      PYTHONPATH: /home/developer/celery
     tty: true
     volumes:
-      - ../docs:/home/developer/docs
-      - ../celery:/home/developer/celery
-      - ../t:/home/developer/t
+      - ../.:/home/developer/celery
     depends_on:
       - rabbit
       - redis

docker/entrypoint

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-make --quiet --directory="$HOME" clean-pyc
+make --quiet --directory="$HOME/celery" clean-pyc
 
 eval "$(pyenv init -)"
 eval "$(pyenv virtualenv-init -)"

docker/scripts/install-couchbase.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-wget http://packages.couchbase.com/clients/c/libcouchbase-2.8.4_jessie_amd64.tar
-tar -vxf libcouchbase-2.8.4_jessie_amd64.tar
-dpkg -i libcouchbase-2.8.4_jessie_amd64/libcouchbase2-core_2.8.4-1_amd64.deb
-dpkg -i libcouchbase-2.8.4_jessie_amd64/libcouchbase-dev_2.8.4-1_amd64.deb
+wget http://packages.couchbase.com/releases/couchbase-release/couchbase-release-1.0-4-amd64.deb
+dpkg -i couchbase-release-1.0-4-amd64.deb
+apt-get update
+apt-get install libcouchbase-dev build-essential

requirements/docs.txt

@@ -1,5 +1,6 @@
 git+https://github.com/celery/sphinx_celery.git
 Sphinx==1.7.1
+sphinx-testing==0.7.2
 typing
 -r extras/sqlalchemy.txt
 -r test.txt

requirements/pkgutils.txt

@@ -1,9 +1,10 @@
-setuptools>=20.6.7
+setuptools>=30.0.0
 wheel>=0.29.0
 flake8>=2.5.4
 flakeplus>=1.1
 pydocstyle==1.1.1
 tox>=2.3.1
 sphinx2rst>=1.0
-cyanide>=1.0.1
+# Disable cyanide until it's fully updated.
+# cyanide>=1.0.1
 bumpversion

t/integration/conftest.py

@@ -6,10 +6,26 @@
 import pytest
 
 from celery.contrib.testing.manager import Manager
+# we have to import the pytest plugin fixtures here,
+# in case user did not do the `python setup.py develop` yet,
+# that installs the pytest plugin into the setuptools registry.
+from celery.contrib.pytest import (
+    celery_app,
+    celery_session_worker,
+)
 
 TEST_BROKER = os.environ.get('TEST_BROKER', 'pyamqp://')
 TEST_BACKEND = os.environ.get('TEST_BACKEND', 'redis://')
 
+# Tricks flake8 into silencing redefining fixtures warnings.
+__all__ = (
+    'celery_app',
+    'celery_session_worker',
+    'flaky',
+    'get_active_redis_channels',
+    'get_redis_connection',
+)
+
 
 def flaky(fun):
     @wraps(fun)

t/integration/test_canvas.py

@@ -284,7 +284,6 @@ def assert_ids(r, expected_value, expected_root_id, expected_parent_id):
 
 
 class test_chord:
-
     @flaky
     def test_redis_subscribed_channels_leak(self, manager):
         if not manager.app.conf.result_backend.startswith('redis'):
@@ -293,18 +292,16 @@ def test_redis_subscribed_channels_leak(self, manager):
         manager.app.backend.result_consumer.on_after_fork()
         initial_channels = get_active_redis_channels()
         initial_channels_count = len(initial_channels)
-
         total_chords = 10
         async_results = [
             chord([add.s(5, 6), add.s(6, 7)])(delayed_sum.s())
             for _ in range(total_chords)
         ]
 
+        channels_before = get_active_redis_channels()
         manager.assert_result_tasks_in_progress_or_completed(async_results)
 
-        channels_before = get_active_redis_channels()
         channels_before_count = len(channels_before)
-
         assert set(channels_before) != set(initial_channels)
         assert channels_before_count > initial_channels_count
 
@@ -602,6 +599,7 @@ def test_chord_on_error(self, manager):
         assert len([cr for cr in chord_results if cr[2] != states.SUCCESS]
                    ) == 1
 
+    @flaky
     def test_parallel_chords(self, manager):
         try:
             manager.app.backend.ensure_chords_allowed()

t/unit/contrib/proj/conf.py

@@ -3,7 +3,7 @@
 import os
 import sys
 
-extensions = ['celery.contrib.sphinx']
+extensions = ['sphinx.ext.autodoc', 'celery.contrib.sphinx']
 autodoc_default_flags = ['members']
 
-sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

t/unit/contrib/proj/contents.rst

@@ -1 +1,7 @@
+Documentation
+===============
+.. toctree::
+   :maxdepth: 2
+
 .. automodule:: foo
+   :members:

t/unit/contrib/proj/foo.py

@@ -1,11 +1,24 @@
 from __future__ import absolute_import, unicode_literals
 
-from celery import Celery
+from celery import Celery, shared_task
 from xyzzy import plugh  # noqa
 
 app = Celery()
 
 
 @app.task
 def bar():
-    """This task has a docstring!"""
+    """Task.
+
+    This is a sample Task.
+    """
+    pass
+
+
+@shared_task
+def baz():
+    """Shared Task.
+
+    This is a sample Shared Task.
+    """
+    pass

t/unit/contrib/test_sphinx.py

@@ -1,20 +1,30 @@
 from __future__ import absolute_import, unicode_literals
 
-import pkg_resources
+import os
 import pytest
 
 try:
-    sphinx_build = pkg_resources.load_entry_point(
-        'sphinx', 'console_scripts', 'sphinx-build')
-except pkg_resources.DistributionNotFound:
-    sphinx_build = None
+    from sphinx_testing import TestApp
+    from sphinx.application import Sphinx  # noqa: F401
+    sphinx_installed = True
+except ImportError:
+    sphinx_installed = False
 
 
-@pytest.mark.skipif(sphinx_build is None, reason='Sphinx is not installed')
-def test_sphinx(tmpdir):
-    srcdir = pkg_resources.resource_filename(__name__, 'proj')
-    sphinx_build([srcdir, str(tmpdir)])
-    with open(tmpdir / 'contents.html', 'r') as f:
-        contents = f.read()
-    assert 'This task has a docstring!' in contents
-    assert 'This task is in a different module!' not in contents
+SRCDIR = os.path.join(os.path.dirname(__file__), 'proj')
+
+
+@pytest.mark.skipif(
+    sphinx_installed is False,
+    reason='Sphinx is not installed'
+)
+def test_sphinx():
+    app = TestApp(srcdir=SRCDIR, confdir=SRCDIR)
+    app.build()
+    contents = (app.outdir / 'contents.html').read_text(encoding='UTF-8')
+    assert 'This is a sample Task' in contents
+    assert 'This is a sample Shared Task' in contents
+    assert (
+        'This task is in a different module!'
+        not in contents
+    )

tox.ini

@@ -14,6 +14,8 @@ envlist =
 deps=
     -r{toxinidir}/requirements/default.txt
     -r{toxinidir}/requirements/test.txt
+    -r{toxinidir}/requirements/docs.txt
+    -r{toxinidir}/requirements/pkgutils.txt
 
     2.7: -r{toxinidir}/requirements/test-ci-default.txt
     3.4,3.5,3.6: -r{toxinidir}/requirements/test-ci-default.txt