WIP: 4.3 Release (celery#5331)

* Initial effort for completing 4.3.

* Fix documentation fixers list.

* Added a note about riak not supported on Python 3.7.

* Lot's of changelog...

* Fix typo.

* More changelog.

* isort.

* isort.

* Added a note about likely drop of 3.4 support.

* Rephrase Kombu section and mention billiard bump,

* Mention fixes for several memory leaks.

* Added categories. Minor fixes...

* Fix style.

* Fix styling.

* Upgrade sphinx_celery.

* Fix wrong usage of quotes.

* Autopep8.

* isort.

* Updated changelog.

* Elaborate.

* Fix.

* Elaborate.

* More changelog.

* Elaborate.

* Elaborate.

* Elaborate.

* Elaborate.

* Elaborate.

* Elaborate.

* Elaborate.

* Fix warning.

* Fix style.

README.rst

@@ -2,7 +2,7 @@
 
 |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge|
 
-:Version: 4.2.1 (windowlicker)
+:Version: 4.2.1 (rhubarb)
 :Web: http://celeryproject.org/
 :Download: https://pypi.org/project/celery/
 :Source: https://github.com/celery/celery/
@@ -58,10 +58,11 @@ in such a way that the client enqueues an URL to be requested by a worker.
 What do I need?
 ===============
 
-Celery version 4.2 runs on,
+Celery version 4.3 runs on,
 
-- Python (2.7, 3.4, 3.5, 3.6)
-- PyPy (6.0)
+- Python (2.7, 3.4, 3.5, 3.6, 3.7)
+- PyPy2.7 (6.0)
+- PyPy3.5 (6.0)
 
 
 This is the last version to support Python 2.7,

celery/__init__.py

@@ -12,7 +12,7 @@
 import sys
 from collections import namedtuple
 
-SERIES = 'windowlicker'
+SERIES = 'rhubarb'
 
 __version__ = '4.2.0'
 __author__ = 'Ask Solem'

celery/app/base.py

@@ -42,7 +42,7 @@
 from . import builtins  # noqa
 from . import backends
 from .annotations import prepare as prepare_annotations
-from .defaults import find_deprecated_settings, DEFAULT_SECURITY_DIGEST
+from .defaults import DEFAULT_SECURITY_DIGEST, find_deprecated_settings
 from .registry import TaskRegistry
 from .utils import (AppPickler, Settings, _new_key_to_old, _old_key_to_new,
                     _unpickle_app, _unpickle_app_v2, appstr, bugreport,

celery/app/routes.py

@@ -8,11 +8,7 @@
 import re
 import string
 from collections import OrderedDict
-try:
-    from collections.abc import Mapping
-except ImportError:
-    # TODO: Remove this when we drop Python 2.7 support
-    from collections import Mapping
+
 from kombu import Queue
 
 from celery.exceptions import QueueNotFound
@@ -21,6 +17,13 @@
 from celery.utils.functional import maybe_evaluate, mlazy
 from celery.utils.imports import symbol_by_name
 
+try:
+    from collections.abc import Mapping
+except ImportError:
+    # TODO: Remove this when we drop Python 2.7 support
+    from collections import Mapping
+
+
 try:
     Pattern = re._pattern_type
 except AttributeError:  # pragma: no cover

celery/app/utils.py

@@ -6,11 +6,6 @@
 import platform as _platform
 import re
 from collections import namedtuple
-try:
-    from collections.abc import Mapping
-except ImportError:
-    # TODO: Remove this when we drop Python 2.7 support
-    from collections import Mapping
 from copy import deepcopy
 from types import ModuleType
 
@@ -26,6 +21,13 @@
 from .defaults import (_OLD_DEFAULTS, _OLD_SETTING_KEYS, _TO_NEW_KEY,
                        _TO_OLD_KEY, DEFAULTS, SETTING_KEYS, find)
 
+try:
+    from collections.abc import Mapping
+except ImportError:
+    # TODO: Remove this when we drop Python 2.7 support
+    from collections import Mapping
+
+
 __all__ = (
     'Settings', 'appstr', 'bugreport',
     'filter_hidden_settings', 'find_app',

celery/backends/redis.py

@@ -3,7 +3,6 @@
 from __future__ import absolute_import, unicode_literals
 
 import time
-
 from functools import partial
 from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
 

celery/backends/s3.py

@@ -2,16 +2,17 @@
 """s3 result store backend."""
 from __future__ import absolute_import, unicode_literals
 
+from celery.exceptions import ImproperlyConfigured
+
+from .base import KeyValueStoreBackend
+
 try:
     import boto3
     import botocore
 except ImportError:
     boto3 = None
     botocore = None
 
-from celery.exceptions import ImproperlyConfigured
-from .base import KeyValueStoreBackend
-
 
 __all__ = ('S3Backend',)
 

celery/events/state.py

@@ -19,11 +19,6 @@
 import sys
 import threading
 from collections import defaultdict
-try:
-    from collections.abc import Callable
-except ImportError:
-    # TODO: Remove this when we drop Python 2.7 support
-    from collections import Callable
 from datetime import datetime
 from decimal import Decimal
 from itertools import islice
@@ -39,6 +34,13 @@
 from celery.utils.functional import LRUCache, memoize, pass1
 from celery.utils.log import get_logger
 
+try:
+    from collections.abc import Callable
+except ImportError:
+    # TODO: Remove this when we drop Python 2.7 support
+    from collections import Callable
+
+
 __all__ = ('Worker', 'Task', 'State', 'heartbeat_expires')
 
 # pylint: disable=redefined-outer-name

celery/schedules.py

@@ -6,11 +6,6 @@
 import re
 from bisect import bisect, bisect_left
 from collections import namedtuple
-try:
-    from collections.abc import Iterable
-except ImportError:
-    # TODO: Remove this when we drop Python 2.7 support
-    from collections import Iterable
 from datetime import datetime, timedelta
 
 from kombu.utils.objects import cached_property
@@ -21,6 +16,13 @@
 from .utils.time import (ffwd, humanize_seconds, localize, maybe_make_aware,
                          maybe_timedelta, remaining, timezone, weekday)
 
+try:
+    from collections.abc import Iterable
+except ImportError:
+    # TODO: Remove this when we drop Python 2.7 support
+    from collections import Iterable
+
+
 __all__ = (
     'ParseException', 'schedule', 'crontab', 'crontab_parser',
     'maybe_schedule', 'solar',

celery/security/certificate.py

@@ -2,14 +2,13 @@
 """X.509 certificates."""
 from __future__ import absolute_import, unicode_literals
 
+import datetime
 import glob
 import os
-import datetime
-
 
-from cryptography.x509 import load_pem_x509_certificate
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.asymmetric import padding
+from cryptography.x509 import load_pem_x509_certificate
 from kombu.utils.encoding import bytes_to_str, ensure_bytes
 
 from celery.exceptions import SecurityError

celery/security/key.py

@@ -2,10 +2,10 @@
 """Private keys for the security serializer."""
 from __future__ import absolute_import, unicode_literals
 
-from kombu.utils.encoding import ensure_bytes
+from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import padding
-from cryptography.hazmat.backends import default_backend
+from kombu.utils.encoding import ensure_bytes
 
 from .utils import reraise_errors
 

celery/security/serialization.py

@@ -5,8 +5,8 @@
 from kombu.serialization import dumps, loads, registry
 from kombu.utils.encoding import bytes_to_str, ensure_bytes, str_to_bytes
 
-from celery.utils.serialization import b64decode, b64encode
 from celery.app.defaults import DEFAULT_SECURITY_DIGEST
+from celery.utils.serialization import b64decode, b64encode
 
 from .certificate import Certificate, FSCertStore
 from .key import PrivateKey

celery/security/utils.py

@@ -5,12 +5,12 @@
 import sys
 from contextlib import contextmanager
 
-from cryptography.hazmat.primitives import hashes
 import cryptography.exceptions
+from cryptography.hazmat.primitives import hashes
+
 from celery.exceptions import SecurityError
 from celery.five import reraise
 
-
 __all__ = ('get_digest_algorithm', 'reraise_errors',)
 
 

celery/utils/abstract.py

@@ -3,13 +3,15 @@
 from __future__ import absolute_import, unicode_literals
 
 from abc import ABCMeta, abstractmethod, abstractproperty
+
+from celery.five import with_metaclass
+
 try:
     from collections.abc import Callable
 except ImportError:
     # TODO: Remove this when we drop Python 2.7 support
     from collections import Callable
 
-from celery.five import with_metaclass
 
 __all__ = ('CallableTask', 'CallableSignature')
 

celery/utils/collections.py

@@ -5,13 +5,6 @@
 import sys
 from collections import OrderedDict as _OrderedDict
 from collections import deque
-try:
-    from collections.abc import Callable, Mapping, MutableMapping, MutableSet
-    from collections.abc import Sequence
-except ImportError:
-    # TODO: Remove this when we drop Python 2.7 support
-    from collections import Callable, Mapping, MutableMapping, MutableSet
-    from collections import Sequence
 from heapq import heapify, heappop, heappush
 from itertools import chain, count
 
@@ -21,6 +14,15 @@
 from .functional import first, uniq
 from .text import match_case
 
+try:
+    from collections.abc import Callable, Mapping, MutableMapping, MutableSet
+    from collections.abc import Sequence
+except ImportError:
+    # TODO: Remove this when we drop Python 2.7 support
+    from collections import Callable, Mapping, MutableMapping, MutableSet
+    from collections import Sequence
+
+
 try:
     # pypy: dicts are ordered in recent versions
     from __pypy__ import reversed_dict as _dict_is_ordered

celery/utils/text.py

@@ -3,16 +3,18 @@
 from __future__ import absolute_import, unicode_literals
 
 import re
+from functools import partial
+from pprint import pformat
+from textwrap import fill
+
+from celery.five import string_t
+
 try:
     from collections.abc import Callable
 except ImportError:
     # TODO: Remove this when we drop Python 2.7 support
     from collections import Callable
-from functools import partial
-from pprint import pformat
-from textwrap import fill
 
-from celery.five import string_t
 
 __all__ = (
     'abbr', 'abbrtask', 'dedent', 'dedent_initial',

celery/worker/consumer/consumer.py

@@ -16,7 +16,7 @@
 from billiard.common import restart_state
 from billiard.exceptions import RestartFreqExceeded
 from kombu.asynchronous.semaphore import DummyLock
-from kombu.exceptions import DecodeError, ContentDisallowed
+from kombu.exceptions import ContentDisallowed, DecodeError
 from kombu.utils.compat import _detect_environment
 from kombu.utils.encoding import bytes_t, safe_repr
 from kombu.utils.limits import TokenBucket

celery/worker/consumer/gossip.py

@@ -8,7 +8,7 @@
 
 from kombu import Consumer
 from kombu.asynchronous.semaphore import DummyLock
-from kombu.exceptions import DecodeError, ContentDisallowed
+from kombu.exceptions import ContentDisallowed, DecodeError
 
 from celery import bootsteps
 from celery.five import values