MOTOR-896 Set up pre-commit (mongodb#145)

.evergreen/run-enterprise-auth-tests.sh

@@ -23,7 +23,7 @@ export GSSAPI_PORT=${SASL_PORT}
 export GSSAPI_PRINCIPAL=${PRINCIPAL}
 
 # Pass needed env variables to the test environment.
-export TOX_TESTENV_PASSENV=*
+export TOX_TESTENV_PASSENV="*"
 
 # --sitepackages allows use of pykerberos without a test dep.
 /opt/python/3.6/bin/python3 -m tox -e "$TOX_ENV" --sitepackages -- -x test.test_auth

.evergreen/run-tox.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail
 

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Initial pre-commit reformat
+1e62b868ea58afeb42b3d0346e33776561c16ab6

.github/workflows/test-python.yml

@@ -0,0 +1,45 @@
+name: Python Tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+
+  pre-commit:
+    name: pre-commit
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: pre-commit/[email protected]
+        with:
+          extra_args: --all-files --hook-stage=manual
+
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.6", "3.10"]
+      fail-fast: false
+    name: CPython ${{ matrix.python-version }}-${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'setup.py'
+      - name: Start MongoDB with Custom Options
+        run: |
+          mkdir data
+          mongod --fork --dbpath=$(pwd)/data --logpath=$PWD/mongo.log --setParameter enableTestCommands=1
+      - name: Install Python dependencies
+        run: |
+          python -m pip install -U pip tox tox-gh-actions
+      - name: Run tests
+        run: |
+          tox

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.4.0
+  hooks:
+  - id: check-added-large-files
+  - id: check-case-conflict
+  - id: check-toml
+  - id: check-yaml
+  - id: debug-statements
+  - id: end-of-file-fixer
+    exclude: WHEEL
+    exclude_types: [json]
+  - id: forbid-new-submodules
+  - id: trailing-whitespace
+    exclude: .patch
+    exclude_types: [json]
+
+- repo: https://github.com/psf/black
+  rev: 22.1.0
+  hooks:
+  - id: black
+    files: \.py$
+    args: [--line-length=100]
+
+- repo: https://github.com/PyCQA/isort
+  rev: 5.7.0
+  hooks:
+  - id: isort
+    files: \.py$
+    args: [--profile=black]
+
+# We use the Python version instead of the original version which seems to require Docker
+# https://github.com/koalaman/shellcheck-precommit
+- repo: https://github.com/shellcheck-py/shellcheck-py
+  rev: v0.8.0.1
+  hooks:
+    - id: shellcheck
+      name: shellcheck
+      args: ["--severity=warning"]
+
+- repo: https://github.com/sirosen/check-jsonschema
+  rev: 0.10.2
+  hooks:
+    - id: check-jsonschema
+      name: "Check GitHub Workflows"
+      files: ^\.github/workflows/
+      types: [yaml]
+      args: ["--schemafile", "https://json.schemastore.org/github-workflow"]

CONTRIBUTING.rst

@@ -40,6 +40,23 @@ port 27017:
 
 .. _tox: https://testrun.org/tox/
 
+Running Linters
+---------------
+
+Motor uses `pre-commit <https://pypi.org/project/pre-commit/>`_
+for managing linting of the codebase.
+``pre-commit`` performs various checks on all files in Motor and uses tools
+that help follow a consistent code style within the codebase.
+
+To set up ``pre-commit`` locally, run::
+
+    pip install pre-commit
+    pre-commit install
+
+To run ``pre-commit`` manually, run::
+
+    > tox -e lint
+
 General Guidelines
 ------------------
 

LICENSE

@@ -199,4 +199,3 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-

MANIFEST.in

@@ -1,8 +1,25 @@
 include README.rst
 include LICENSE
+include tox.ini
+include doc/Makefile
+include doc/examples/tornado_change_stream_templates/index.html
 recursive-include doc *.rst
 recursive-include doc *.py
+recursive-include doc *.png
 recursive-include test *.py
+recursive-include test *.pem
 recursive-include doc *.conf
 recursive-include doc *.css
 recursive-include doc *.js
+recursive-include doc *.txt
+recursive-include doc *.bat
+recursive-include synchro *.py
+
+exclude .readthedocs.yaml
+exclude .git-blame-ignore-revs
+exclude .pre-commit-config.yaml
+exclude release.sh
+exclude ez_setup.py
+exclude RELEASE.rst
+exclude CONTRIBUTING.rst
+exclude .evergreen/*

doc/__init__.py

@@ -1 +0,0 @@
-

doc/api-asyncio/index.rst

@@ -17,4 +17,3 @@ Motor asyncio API
 
 This page describes using Motor with asyncio. For Tornado integration, see
 :doc:`../api-tornado/index`.
-

doc/api-tornado/motor_client_encryption.rst

@@ -4,4 +4,4 @@
 .. currentmodule:: motor.motor_tornado
 
 .. autoclass:: MotorClientEncryption
-  :members:
+  :members:

doc/changelog.rst

@@ -838,7 +838,7 @@ accessible, Motor collections now allow dict-style access, the same as Motor
 clients and databases always have::
 
     # New in Motor 0.6
-    subcollection = collection['_subcollection']    
+    subcollection = collection['_subcollection']
 
 These changes solve problems with iPython code completion and the Python 3
 :class:`ABC` abstract base class.

doc/examples/tornado_change_stream_example.rst

@@ -6,7 +6,7 @@ Tornado Change Stream Example
 .. currentmodule:: motor.motor_tornado
 
 Watch a collection for changes with :meth:`MotorCollection.watch` and display
-each change notification on a web page using web sockets. 
+each change notification on a web page using web sockets.
 
 Instructions
 ------------

doc/tutorial-asyncio.rst

@@ -457,4 +457,3 @@ Learning to use the MongoDB driver is just the beginning, of course. For
 in-depth instruction in MongoDB itself, see `The MongoDB Manual`_.
 
 .. _The MongoDB Manual: http://docs.mongodb.org/manual/
-

motor/docstrings.py

@@ -374,7 +374,7 @@ async def clear_collection():
    Added session parameter.
 """
 
-find_one_and_delete_doc = """Finds a single document and deletes it, returning 
+find_one_and_delete_doc = """Finds a single document and deletes it, returning
 the document.
 
 If we have a collection with 2 documents like ``{'x': 1}``, then this code
@@ -532,7 +532,7 @@ async def set_done():
           {'_id': 665}, {'$inc': {'count': 1}, '$set': {'done': True}}))
 
 This outputs::
-  
+
   {'_id': 665, 'done': False, 'count': 25}}
 
 To return the updated version of the document instead, use the
@@ -855,12 +855,12 @@ async def insert_x():
   {'x': 1, '_id': 0}
   {'x': 1, '_id': 1}
   {'x': 1, '_id': 2}
-  
+
 We can add 3 to each "x" field::
 
   async def add_3_to_x():
     result = await db.test.update_many({'x': 1}, {'$inc': {'x': 3}})
-    print('matched %d, modified %d' % 
+    print('matched %d, modified %d' %
           (result.matched_count, result.modified_count))
 
     print('collection:')
@@ -1076,24 +1076,24 @@ async def coro():
           # the doc we just inserted, even reading from a secondary.
           async for doc in secondary.find(session=s):
               print(doc)
-              
+
       # Run a multi-document transaction:
       async with await client.start_session() as s:
           # Note, start_transaction doesn't require "await".
           async with s.start_transaction():
               await collection.delete_one({'x': 1}, session=s)
               await collection.insert_one({'x': 2}, session=s)
-          
+
           # Exiting the "with s.start_transaction()" block while throwing an
           # exception automatically aborts the transaction, exiting the block
           # normally automatically commits it.
 
-          # You can run additional transactions in the same session, so long as 
+          # You can run additional transactions in the same session, so long as
           # you run them one at a time.
           async with s.start_transaction():
               await collection.insert_one({'x': 3}, session=s)
               await collection.insert_many({'x': {'$gte': 2}},
-                                           {'$inc': {'x': 1}}, 
+                                           {'$inc': {'x': 1}},
                                            session=s)
 
 
@@ -1144,12 +1144,12 @@ async def coro():
 
 create_data_key_doc = """Create and insert a new data key into the key vault collection.
 
-Takes the same arguments as 
+Takes the same arguments as
 :class:`pymongo.encryption.ClientEncryption.create_data_key`,
 with only the following slight difference using async syntax.
 The following example shows creating and referring to a data
 key by alternate name::
-   
+
     await client_encryption.create_data_key("local", keyAltNames=["name1"])
     # reference the key with the alternate name
     await client_encryption.encrypt("457-55-5462", keyAltName="name1",

test/certificates/ca.pem

@@ -18,4 +18,4 @@ gT564CmvkUat8uXPz6olOCdwkMpJ9Sj62i0mpgXJdBfxKQ6TZ9yGz6m3jannjZpN
 LchB7xSAEWtqUgvNusq0dApJsf4n7jZ+oBZVaQw2+tzaMfaLqHgMwcu1FzA8UKCD
 sxCgIsZUs8DdxaD418Ot6nPfheOTqe24n+TTa+Z6O0W0QtnofJBx7tmAo1aEc57i
 77s89pfwIJetpIlhzNSMKurCAocFCJMJLAASJFuu6dyDvPo=
------END CERTIFICATE-----
+-----END CERTIFICATE-----

test/test_environment.py

@@ -19,9 +19,9 @@
 import sys
 import warnings
 from functools import wraps
-from unittest import SkipTest
 from test.utils import create_user
 from test.version import Version
+from unittest import SkipTest
 
 import pymongo.errors
 

tox.ini

@@ -12,7 +12,7 @@ envlist =
     tornado6-{pypy35,pypy36,py35,py36,py37,py38,py39,py310},
 
     # Test Tornado's dev version in a few configurations.
-    tornado_git-{py36,py37},
+    tornado_git-{py37,py38},
 
     # Ensure the sphinx build has no errors or warnings.
     py3-sphinx-docs,
@@ -29,6 +29,12 @@ envlist =
     # Apply PyMongo's test suite to Motor via Synchro.
     synchro37
 
+    # Run pre-commit on all files.
+    lint
+
+    # Check the sdist integrity.
+    manifest
+
 [testenv]
 passenv =
     DB_IP
@@ -49,7 +55,7 @@ basepython =
     pypy36: {env:PYTHON_BINARY:pypy3}
 
     # Default Python 3 when we don't care about minor version.
-    py3: {env:PYTHON_BINARY:python3.7}
+    py3,lint,manifest: {env:PYTHON_BINARY:python3}
 
 deps =
     tornado5: tornado>=5,<6
@@ -100,3 +106,21 @@ setenv =
 commands =
     git clone --depth 1 --branch master https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
     python3 -m synchro.synchrotest --with-xunit --xunit-file=xunit-synchro-results -v -w {envtmpdir}/mongo-python-driver {posargs}
+
+[testenv:lint]
+deps =
+    pre-commit
+commands =
+    pre-commit run --all-files
+
+[testenv:manifest]
+deps =
+    check-manifest
+commands =
+    check-manifest -v
+
+[gh-actions]
+# Map GitHub Actions python-version to environment using tox-github-actions.
+python =
+    3.6: py36
+    3.10: py310,manifest