From 59a3e3229b0ff1bd575c41ce3fd451febf3bea33 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Feb 2022 14:25:52 -0600 Subject: [PATCH] MOTOR-896 Set up pre-commit (#145) --- .evergreen/run-enterprise-auth-tests.sh | 2 +- .evergreen/run-tox.sh | 2 +- .git-blame-ignore-revs | 2 + .github/workflows/test-python.yml | 45 +++++++++++++++++ .pre-commit-config.yaml | 49 +++++++++++++++++++ CONTRIBUTING.rst | 17 +++++++ LICENSE | 1 - MANIFEST.in | 17 +++++++ doc/__init__.py | 1 - doc/api-asyncio/index.rst | 1 - doc/api-tornado/motor_client_encryption.rst | 2 +- doc/changelog.rst | 2 +- .../tornado_change_stream_example.rst | 2 +- doc/tutorial-asyncio.rst | 1 - motor/docstrings.py | 20 ++++---- test/certificates/ca.pem | 2 +- test/test_environment.py | 2 +- tox.ini | 28 ++++++++++- 18 files changed, 173 insertions(+), 23 deletions(-) create mode 100644 .git-blame-ignore-revs create mode 100644 .github/workflows/test-python.yml create mode 100644 .pre-commit-config.yaml diff --git a/.evergreen/run-enterprise-auth-tests.sh b/.evergreen/run-enterprise-auth-tests.sh index dd95671c..6056df0e 100644 --- a/.evergreen/run-enterprise-auth-tests.sh +++ b/.evergreen/run-enterprise-auth-tests.sh @@ -23,7 +23,7 @@ export GSSAPI_PORT=${SASL_PORT} export GSSAPI_PRINCIPAL=${PRINCIPAL} # Pass needed env variables to the test environment. -export TOX_TESTENV_PASSENV=* +export TOX_TESTENV_PASSENV="*" # --sitepackages allows use of pykerberos without a test dep. /opt/python/3.6/bin/python3 -m tox -e "$TOX_ENV" --sitepackages -- -x test.test_auth diff --git a/.evergreen/run-tox.sh b/.evergreen/run-tox.sh index 31b0ed1f..421c652d 100755 --- a/.evergreen/run-tox.sh +++ b/.evergreen/run-tox.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..206e9730 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Initial pre-commit reformat +1e62b868ea58afeb42b3d0346e33776561c16ab6 diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml new file mode 100644 index 00000000..1f1f14cf --- /dev/null +++ b/.github/workflows/test-python.yml @@ -0,0 +1,45 @@ +name: Python Tests + +on: + push: + pull_request: + +jobs: + + pre-commit: + name: pre-commit + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 + with: + extra_args: --all-files --hook-stage=manual + + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.6", "3.10"] + fail-fast: false + name: CPython ${{ matrix.python-version }}-${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: 'setup.py' + - name: Start MongoDB with Custom Options + run: | + mkdir data + mongod --fork --dbpath=$(pwd)/data --logpath=$PWD/mongo.log --setParameter enableTestCommands=1 + - name: Install Python dependencies + run: | + python -m pip install -U pip tox tox-gh-actions + - name: Run tests + run: | + tox diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..2bb18452 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,49 @@ + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + exclude: WHEEL + exclude_types: [json] + - id: forbid-new-submodules + - id: trailing-whitespace + exclude: .patch + exclude_types: [json] + +- repo: https://github.com/psf/black + rev: 22.1.0 + hooks: + - id: black + files: \.py$ + args: [--line-length=100] + +- repo: https://github.com/PyCQA/isort + rev: 5.7.0 + hooks: + - id: isort + files: \.py$ + args: [--profile=black] + +# We use the Python version instead of the original version which seems to require Docker +# https://github.com/koalaman/shellcheck-precommit +- repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.8.0.1 + hooks: + - id: shellcheck + name: shellcheck + args: ["--severity=warning"] + +- repo: https://github.com/sirosen/check-jsonschema + rev: 0.10.2 + hooks: + - id: check-jsonschema + name: "Check GitHub Workflows" + files: ^\.github/workflows/ + types: [yaml] + args: ["--schemafile", "https://json.schemastore.org/github-workflow"] diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1710c1c5..bee5c252 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -40,6 +40,23 @@ port 27017: .. _tox: https://testrun.org/tox/ +Running Linters +--------------- + +Motor uses `pre-commit `_ +for managing linting of the codebase. +``pre-commit`` performs various checks on all files in Motor and uses tools +that help follow a consistent code style within the codebase. + +To set up ``pre-commit`` locally, run:: + + pip install pre-commit + pre-commit install + +To run ``pre-commit`` manually, run:: + + > tox -e lint + General Guidelines ------------------ diff --git a/LICENSE b/LICENSE index 57bc88a1..261eeb9e 100644 --- a/LICENSE +++ b/LICENSE @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/MANIFEST.in b/MANIFEST.in index 64c1bb10..b04d00b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,25 @@ include README.rst include LICENSE +include tox.ini +include doc/Makefile +include doc/examples/tornado_change_stream_templates/index.html recursive-include doc *.rst recursive-include doc *.py +recursive-include doc *.png recursive-include test *.py +recursive-include test *.pem recursive-include doc *.conf recursive-include doc *.css recursive-include doc *.js +recursive-include doc *.txt +recursive-include doc *.bat +recursive-include synchro *.py + +exclude .readthedocs.yaml +exclude .git-blame-ignore-revs +exclude .pre-commit-config.yaml +exclude release.sh +exclude ez_setup.py +exclude RELEASE.rst +exclude CONTRIBUTING.rst +exclude .evergreen/* diff --git a/doc/__init__.py b/doc/__init__.py index 8b137891..e69de29b 100644 --- a/doc/__init__.py +++ b/doc/__init__.py @@ -1 +0,0 @@ - diff --git a/doc/api-asyncio/index.rst b/doc/api-asyncio/index.rst index 9181fc79..f4ad4c3d 100644 --- a/doc/api-asyncio/index.rst +++ b/doc/api-asyncio/index.rst @@ -17,4 +17,3 @@ Motor asyncio API This page describes using Motor with asyncio. For Tornado integration, see :doc:`../api-tornado/index`. - diff --git a/doc/api-tornado/motor_client_encryption.rst b/doc/api-tornado/motor_client_encryption.rst index 3601a304..e6222681 100644 --- a/doc/api-tornado/motor_client_encryption.rst +++ b/doc/api-tornado/motor_client_encryption.rst @@ -4,4 +4,4 @@ .. currentmodule:: motor.motor_tornado .. autoclass:: MotorClientEncryption - :members: \ No newline at end of file + :members: diff --git a/doc/changelog.rst b/doc/changelog.rst index 4c7b533c..ed96ede7 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -838,7 +838,7 @@ accessible, Motor collections now allow dict-style access, the same as Motor clients and databases always have:: # New in Motor 0.6 - subcollection = collection['_subcollection'] + subcollection = collection['_subcollection'] These changes solve problems with iPython code completion and the Python 3 :class:`ABC` abstract base class. diff --git a/doc/examples/tornado_change_stream_example.rst b/doc/examples/tornado_change_stream_example.rst index 207f945e..0a3a8d58 100644 --- a/doc/examples/tornado_change_stream_example.rst +++ b/doc/examples/tornado_change_stream_example.rst @@ -6,7 +6,7 @@ Tornado Change Stream Example .. currentmodule:: motor.motor_tornado Watch a collection for changes with :meth:`MotorCollection.watch` and display -each change notification on a web page using web sockets. +each change notification on a web page using web sockets. Instructions ------------ diff --git a/doc/tutorial-asyncio.rst b/doc/tutorial-asyncio.rst index b09e0a2a..5dd0c4c2 100644 --- a/doc/tutorial-asyncio.rst +++ b/doc/tutorial-asyncio.rst @@ -457,4 +457,3 @@ Learning to use the MongoDB driver is just the beginning, of course. For in-depth instruction in MongoDB itself, see `The MongoDB Manual`_. .. _The MongoDB Manual: http://docs.mongodb.org/manual/ - diff --git a/motor/docstrings.py b/motor/docstrings.py index 3c48d0b0..f2010ba0 100644 --- a/motor/docstrings.py +++ b/motor/docstrings.py @@ -374,7 +374,7 @@ async def clear_collection(): Added session parameter. """ -find_one_and_delete_doc = """Finds a single document and deletes it, returning +find_one_and_delete_doc = """Finds a single document and deletes it, returning the document. If we have a collection with 2 documents like ``{'x': 1}``, then this code @@ -532,7 +532,7 @@ async def set_done(): {'_id': 665}, {'$inc': {'count': 1}, '$set': {'done': True}})) This outputs:: - + {'_id': 665, 'done': False, 'count': 25}} To return the updated version of the document instead, use the @@ -855,12 +855,12 @@ async def insert_x(): {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} - + We can add 3 to each "x" field:: async def add_3_to_x(): result = await db.test.update_many({'x': 1}, {'$inc': {'x': 3}}) - print('matched %d, modified %d' % + print('matched %d, modified %d' % (result.matched_count, result.modified_count)) print('collection:') @@ -1076,24 +1076,24 @@ async def coro(): # the doc we just inserted, even reading from a secondary. async for doc in secondary.find(session=s): print(doc) - + # Run a multi-document transaction: async with await client.start_session() as s: # Note, start_transaction doesn't require "await". async with s.start_transaction(): await collection.delete_one({'x': 1}, session=s) await collection.insert_one({'x': 2}, session=s) - + # Exiting the "with s.start_transaction()" block while throwing an # exception automatically aborts the transaction, exiting the block # normally automatically commits it. - # You can run additional transactions in the same session, so long as + # You can run additional transactions in the same session, so long as # you run them one at a time. async with s.start_transaction(): await collection.insert_one({'x': 3}, session=s) await collection.insert_many({'x': {'$gte': 2}}, - {'$inc': {'x': 1}}, + {'$inc': {'x': 1}}, session=s) @@ -1144,12 +1144,12 @@ async def coro(): create_data_key_doc = """Create and insert a new data key into the key vault collection. -Takes the same arguments as +Takes the same arguments as :class:`pymongo.encryption.ClientEncryption.create_data_key`, with only the following slight difference using async syntax. The following example shows creating and referring to a data key by alternate name:: - + await client_encryption.create_data_key("local", keyAltNames=["name1"]) # reference the key with the alternate name await client_encryption.encrypt("457-55-5462", keyAltName="name1", diff --git a/test/certificates/ca.pem b/test/certificates/ca.pem index 6ac86cfc..24beea2d 100644 --- a/test/certificates/ca.pem +++ b/test/certificates/ca.pem @@ -18,4 +18,4 @@ gT564CmvkUat8uXPz6olOCdwkMpJ9Sj62i0mpgXJdBfxKQ6TZ9yGz6m3jannjZpN LchB7xSAEWtqUgvNusq0dApJsf4n7jZ+oBZVaQw2+tzaMfaLqHgMwcu1FzA8UKCD sxCgIsZUs8DdxaD418Ot6nPfheOTqe24n+TTa+Z6O0W0QtnofJBx7tmAo1aEc57i 77s89pfwIJetpIlhzNSMKurCAocFCJMJLAASJFuu6dyDvPo= ------END CERTIFICATE----- \ No newline at end of file +-----END CERTIFICATE----- diff --git a/test/test_environment.py b/test/test_environment.py index ece4c013..080a7c7b 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -19,9 +19,9 @@ import sys import warnings from functools import wraps -from unittest import SkipTest from test.utils import create_user from test.version import Version +from unittest import SkipTest import pymongo.errors diff --git a/tox.ini b/tox.ini index 66669177..efa1f40a 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ envlist = tornado6-{pypy35,pypy36,py35,py36,py37,py38,py39,py310}, # Test Tornado's dev version in a few configurations. - tornado_git-{py36,py37}, + tornado_git-{py37,py38}, # Ensure the sphinx build has no errors or warnings. py3-sphinx-docs, @@ -29,6 +29,12 @@ envlist = # Apply PyMongo's test suite to Motor via Synchro. synchro37 + # Run pre-commit on all files. + lint + + # Check the sdist integrity. + manifest + [testenv] passenv = DB_IP @@ -49,7 +55,7 @@ basepython = pypy36: {env:PYTHON_BINARY:pypy3} # Default Python 3 when we don't care about minor version. - py3: {env:PYTHON_BINARY:python3.7} + py3,lint,manifest: {env:PYTHON_BINARY:python3} deps = tornado5: tornado>=5,<6 @@ -100,3 +106,21 @@ setenv = commands = git clone --depth 1 --branch master https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver python3 -m synchro.synchrotest --with-xunit --xunit-file=xunit-synchro-results -v -w {envtmpdir}/mongo-python-driver {posargs} + +[testenv:lint] +deps = + pre-commit +commands = + pre-commit run --all-files + +[testenv:manifest] +deps = + check-manifest +commands = + check-manifest -v + +[gh-actions] +# Map GitHub Actions python-version to environment using tox-github-actions. +python = + 3.6: py36 + 3.10: py310,manifest