Skip to content

Commit

Permalink
build: Revamp test/CI configuration
Browse files Browse the repository at this point in the history 
Reduce tox matrix to one env per python version, with two extra builds
for lint and docs. Delegate to tox from travis-ci.

Add 3.8 to testing. Simplify by dropping coverage reporting and
"no-deps" test runs.
  • Loading branch information
@bdarnell
bdarnell committed Nov 3, 2019
1 parent 467dfb0 commit e145a36
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 152 deletions.
86 changes: 19 additions & 67 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,79 +17,31 @@ jobs:
# "provisional feature" rules there are significant differences between patch
# versions for asyncio and typing.
- python: 3.5.2
env: TOX_ENV=py35-full
- python: '3.5'
env: TOX_ENV=py35-full
- python: '3.6'
env: TOX_ENV=py36-full
- python: '3.7'
env: TOX_ENV=py37-full
- python: '3.8'
env: TOX_ENV=py38-full
- python: pypy3.5-5.10.1
python: nightly
# Pypy is a lot slower due to jit warmup costs, so don't run the "full"
# test config there.
env: TOX_ENV=pypy3
- python: nightly
env: TOX_ENV=py3-full
# Docs and lint python versions must be synced with those in tox.ini
- python: '3.7'
env: TOX_ENV=docs
- python: '3.6'
env: TOX_ENV=lint

install:
# On nightly, upgrade setuptools first to work around
# https://github.com/pypa/setuptools/issues/1257
- if [[ $TRAVIS_PYTHON_VERSION == 'nightly' ]]; then travis_retry pip install -U setuptools; fi
# TODO(bdarnell): pycares tests are currently disabled on travis due to ipv6 issues.
#- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycares; fi
- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycurl; fi
# Twisted is flaky on pypy (TODO: still? this note is quite old)
# It also sometimes has problems on nightly.
- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* && $TRAVIS_PYTHON_VERSION != 'nightly' ]]; then travis_retry pip install Twisted; fi
# Ideally we'd run the lint stuff on the latest Python
# version supported. But Python 3.7 requires slower-to-start VMs,
# so we run it on 3.6 to minimize total CI run time.
- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then travis_retry pip install flake8 mypy==0.701 black==19.3b0; fi
# We run docs on py37 because we need some bug fixes for introspection of annotations.
- if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then travis_retry pip install -r docs/requirements.txt; fi
# On travis the extension should always be built
- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export TORNADO_EXTENSION=1; fi
- travis_retry python setup.py install
- travis_retry pip install codecov virtualenv
# Create a separate no-dependencies virtualenv to make sure all imports
# of optional-dependencies are guarded.
- virtualenv /tmp/nodeps
- /tmp/nodeps/bin/python -VV
- /tmp/nodeps/bin/python setup.py install
- travis_retry pip install tox tox-venv
- python -VV
- curl-config --version; pip freeze

script:
# Run the tests once from the source directory to detect issues
# involving relative __file__ paths; see
# https://github.com/tornadoweb/tornado/issues/1780
- unset TORNADO_EXTENSION && python -m tornado.test
# For all other test variants, get out of the source directory before
# running tests to ensure that we get the installed speedups module
# instead of the source directory which doesn't have it.
- cd maint
# Copy the coveragerc down so coverage.py can find it.
- cp ../.coveragerc .
- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export TORNADO_EXTENSION=1; fi
- export TARGET="-m tornado.test.runtests"
# Travis workers are often overloaded and cause our tests to exceed
# the default timeout of 5s.
- export ASYNC_TEST_TIMEOUT=15
# We use "python -m coverage" instead of the "bin/coverage" script
# so we can pass additional arguments to python.
# coverage needs a function that was removed in python 3.6 so we can't
# run it with nightly cpython. Coverage is very slow on pypy.
- if [[ $TRAVIS_PYTHON_VERSION != nightly && $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export RUN_COVERAGE=1; fi
- if [[ "$RUN_COVERAGE" == 1 ]]; then export TARGET="-m coverage run $TARGET"; fi
# See comments in tox.ini. Disabled on py35 because ResourceWarnings are too noisy there.
- if [[ $TRAVIS_PYTHON_VERSION != '3.5'* ]]; then export PYTHONWARNINGS=error,ignore:::site; fi
- python -bb $TARGET
- python -O $TARGET
- LANG=C python $TARGET
- LANG=en_US.utf-8 python $TARGET
#- if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then python $TARGET --resolver=tornado.platform.caresresolver.CaresResolver; fi
- if [[ $TRAVIS_PYTHON_VERSION != 'pypy3' ]]; then /tmp/nodeps/bin/python -m tornado.test.runtests; fi
# make coverage reports for Codecov to find
- if [[ "$RUN_COVERAGE" == 1 ]]; then coverage xml; fi
- export TORNADO_EXTENSION=0
- unset PYTHONWARNINGS
- if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]]; then (cd ../docs && mkdir sphinx-out && sphinx-build -E -n -W -b html . sphinx-out); fi
- if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]]; then (cd ../docs && mkdir sphinx-doctest-out && sphinx-build -E -n -b doctest . sphinx-out); fi
- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && flake8); fi
- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && black --check --diff tornado demos); fi
- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && mypy tornado); fi

after_success:
# call codecov from project root
- if [[ "$RUN_COVERAGE" == 1 ]]; then cd ../ && codecov; fi
- "tox -e $TOX_ENV -- $TOX_ARGS"
149 changes: 64 additions & 85 deletions tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,108 +11,93 @@
# 'port install curl +universal' to get both 32- and 64-bit versions of
# libcurl.
[tox]
# When tox is run from a venv (instead of a virtualenv), it can get confused
# unless tox-venv is also installed.
requires = tox-venv
envlist =
# Basic configurations: Run the tests in both minimal installations
# and with all optional dependencies.
# (pypy3 doesn't have any optional deps yet)
{py35,py36,py37,pypy3},
{py35,py36,py37}-full,
# Basic configurations: Run the tests for each python version.
py35-full,py36-full,py37-full,py38-full,pypy3-full

# Also run the tests with each possible replacement of a default
# component (only one 3.x version unless there are known differences).
# Build and test the docs with sphinx.
docs

# Alternate HTTP clients.
py3-curl,
# Run the linters.
lint

# Alternate Resolvers.
py3-full-caresresolver,

# Other configurations; see comments below.
py3-opt,
py3-{lang_c,lang_utf8},

# Ensure the sphinx build has no errors or warnings
py3-sphinx-docs,
# Run the doctests via sphinx (which covers things not run
# in the regular test suite and vice versa)
py3-sphinx-doctest,

py3-lint
py3-mypy

# Tox doesn't like that we specify py3==py37
ignore_basepython_conflict=True
# Allow shell commands in tests
whitelist_externals = /bin/sh

[testenv]
# Most of these are defaults, but if you specify any you can't fall back
# defaults for the others.
basepython =
py3: python3
py35: python3.5
py36: python3.6
py37: python3.7
pypy: pypy
py38: python3.8
pypy3: pypy3
py2: python2.7
py3: python3.7
# In theory, it doesn't matter which python version is used here.
# In practice, things like changes to the ast module can alter
# the outputs of the tools (especially where exactly the
# linter warning-supression comments go), so we specify a
# python version for these builds.
docs: python3.7
lint: python3.6

deps =
# cpython-only deps: pycurl installs but curl_httpclient doesn't work;
# twisted mostly works but is a bit flaky under pypy.
{py35,py36,py37}-full: pycurl
py3: pycurl>=7.19.3.1
# twisted is cpython only.
{py35,py36,py37}-full: twisted
py3: twisted
{py3,py35,py36,py37}-full: pycares
sphinx: -r{toxinidir}/docs/requirements.txt
full: pycurl
full: twisted
full: pycares
docs: -r{toxinidir}/docs/requirements.txt
lint: flake8
lint: black==19.3b0
mypy: mypy==0.701
lint: mypy==0.701

setenv =
# The extension is mandatory on cpython.
{py3,py35,py36,py37}: TORNADO_EXTENSION=1
# In python 3, opening files in text mode uses a
# system-dependent encoding by default. Run the tests with "C"
# (ascii) and "utf-8" locales to ensure we don't have hidden
# dependencies on this setting.
lang_c: LANG=C
lang_utf8: LANG=en_US.utf-8
# tox's parser chokes if all the setenv entries are conditional.
DUMMY=dummy
{py3,py35,py36,py37}-no-ext: TORNADO_EXTENSION=0
# Treat the extension as mandatory in testing (but not on pypy)
{py3,py36,py37,py38}: TORNADO_EXTENSION=1
# CI workers are often overloaded and can cause our tests to exceed
# the default timeout of 5s.
ASYNC_TEST_TIMEOUT=15
# Treat warnings as errors by default. We have a whitelist of
# allowed warnings in runtests.py, but we want to be strict
# about any import-time warnings before that setup code is
# reached. Note that syntax warnings are only reported in
# -opt builds because regular builds reuse pycs created
# during sdist installation (and it doesn't seem to be
# possible to set environment variables during that phase of
# tox).
# ResourceWarnings are too noisy on py35 so don't enable
# warnings-as-errors there.
{py3,py36,py37,py38,pypy3}: PYTHONWARNINGS=error:::tornado


# All non-comment lines but the last must end in a backslash.
# Tox filters line-by-line based on the environment name.
commands =
python \
# py3*: -b turns on an extra warning when calling
# str(bytes), and -bb makes it an error.
-bb \
# Treat warnings as errors by default. We have a whitelist of
# allowed warnings in runtests.py, but we want to be strict
# about any import-time warnings before that setup code is
# reached. Note that syntax warnings are only reported in
# -opt builds because regular builds reuse pycs created
# during sdist installation (and it doesn't seem to be
# possible to set environment variables during that phase of
# tox).
"-Werror" \
# Virtualenv hits a deprecation warning very early which we must
# suppress here.
"-Wignore:::site" \
python -bb -m tornado.test {posargs:}
# Python's optimized mode disables the assert statement, so
# run the tests in this mode to ensure we haven't fallen into
# the trap of relying on an assertion's side effects or using
# them for things that should be runtime errors.
opt: -O \
-m tornado.test.runtests \
full: python -O -m tornado.test
# In python 3, opening files in text mode uses a
# system-dependent encoding by default. Run the tests with "C"
# (ascii) and "utf-8" locales to ensure we don't have hidden
# dependencies on this setting.
full: sh -c 'LANG=C python -m tornado.test'
full: sh -c 'LANG=en_US.utf-8 python -m tornado.test'
# Note that httpclient_test is always run with both client
# implementations; this flag controls which client all the
# other tests use.
curl: --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient \
caresresolver: --resolver=tornado.platform.caresresolver.CaresResolver \
{posargs:}
full: python -m tornado.test --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient
full: python -m tornado.test --resolver=tornado.platform.caresresolver.CaresResolver
# Run the tests once from the source directory to detect issues
# involving relative __file__ paths; see
# https://github.com/tornadoweb/tornado/issues/1780
full: sh -c '(cd {toxinidir} && unset TORNADO_EXTENSION && python -m tornado.test)'


# python will import relative to the current working directory by default,
# so cd into the tox working directory to avoid picking up the working
Expand All @@ -124,27 +109,21 @@ changedir = {toxworkdir}
# Remove it (it's not a part of {opts}) to only install real releases.
install_command = pip install {opts} {packages}

[testenv:py3-sphinx-docs]
[testenv:docs]
changedir = docs
# For some reason the extension fails to load in this configuration,
# but it's not really needed for docs anyway.
setenv = TORNADO_EXTENSION=0
commands =
# Build the docs
sphinx-build -q -E -n -W -b html . {envtmpdir}/html
# Run the doctests. No -W for doctests because that disallows tests
# with empty output.
sphinx-build -q -E -n -b doctest . {envtmpdir}/doctest

[testenv:py3-sphinx-doctest]
changedir = docs
setenv = TORNADO_EXTENSION=0
# No -W for doctests because that disallows tests with empty output.
commands =
sphinx-build -q -E -n -b doctest . {envtmpdir}/doctest

[testenv:py3-lint]
[testenv:lint]
commands =
flake8 {posargs:}
black --check --diff {posargs:tornado demos}
changedir = {toxinidir}

[testenv:py3-mypy]
commands = mypy {posargs:tornado}
mypy {posargs:tornado}
changedir = {toxinidir}

0 comments on commit e145a36

Please sign in to comment.