ARROW-11870: [Dev] Automatically run merge script in virtual environment

Closes apache#9637 from pitrou/ARROW-11870-merge-script-venv

Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>

dev/.gitignore

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Python virtual environments for dev tools
+.venv*/
+

dev/README.md

@@ -33,31 +33,35 @@ committer.
 
 ## How to merge a Pull request
 
+Please don't merge PRs using the Github Web interface.  Instead, set up
+your git clone such as to have a remote named ``apache`` pointing to the
+official Arrow repository:
 ```
 git remote add apache [email protected]:apache/arrow.git
 ```
 
-run the following command
-
+and then run the following command:
 ```
-dev/merge_arrow_pr.py
+./dev/merge_arrow_pr.sh
 ```
 
-This script uses requests and jira libraries.  Before running this script,
-run the following command to prepare them:
+This creates a new Python virtual environment under `dev/.venv[PY_VERSION]`
+and installs all the necessary dependencies to run the Arrow merge script.
+After installed, it runs the merge script.
 
-```
-pip install requests jira
-```
+(we don't provide a wrapper script for Windows yet, so under Windows you'll
+have to install Python dependencies yourself and then run `dev/merge_arrow_pr.py`
+directly)
 
-This uses the GitHub REST API; if you encounter rate limit issues, you may set
-a `ARROW_GITHUB_API_TOKEN` environment variable to use a Personal Access Token.
+The merge script uses the GitHub REST API; if you encounter rate limit issues,
+you may set a `ARROW_GITHUB_API_TOKEN` environment variable to use a Personal
+Access Token.
 
 You can specify the username and the password of your JIRA account in
 `APACHE_JIRA_USERNAME` and `APACHE_JIRA_PASSWORD` environment variables.
 If these aren't supplied, the script will ask you the values of them.
 
-Note that the directory name of your Arrow git clone must be called arrow.
+Note that the directory name of your Arrow git clone must be called `arrow`.
 
 example output:
 ```

dev/archery/archery/utils/lint.py

@@ -162,12 +162,15 @@ def python_linter(src, fix=False):
             "Please run `pip install -r dev/archery/requirements-lint.txt`")
         return
 
-    yield LintResult.from_cmd(flake8(setup_py, src.pyarrow,
-                                     os.path.join(src.python, "examples"),
-                                     src.dev, check=False))
+    flake8_exclude = ['.venv*']
+
+    yield LintResult.from_cmd(
+        flake8("--extend-exclude=" + ','.join(flake8_exclude),
+               setup_py, src.pyarrow, os.path.join(src.python, "examples"),
+               src.dev, check=False))
     config = os.path.join(src.python, ".flake8.cython")
-    yield LintResult.from_cmd(flake8("--config=" + config, src.pyarrow,
-                                     check=False))
+    yield LintResult.from_cmd(
+        flake8("--config=" + config, src.pyarrow, check=False))
 
 
 def python_numpydoc(symbols=None, allow_rules=None, disallow_rules=None):

dev/merge_arrow_pr.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Wrapper script that automatically creates a Python virtual environment
+# and runs merge_arrow_pr.py inside it.
+
+set -e
+
+PYTHON=$(which python3)
+PYVER=$($PYTHON -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+ENV_DIR=$GIT_ROOT/dev/.venv$PYVER
+
+ENV_PYTHON=$ENV_DIR/bin/python3
+ENV_PIP="$ENV_PYTHON -m pip --no-input"
+
+check_venv() {
+    [ -x $ENV_PYTHON ] || {
+        echo "Virtual environment broken: $ENV_PYTHON not an executable"
+        exit 1
+    }
+}
+
+create_venv() {
+    echo ""
+    echo "Creating Python virtual environment in $ENV_DIR ..."
+    echo ""
+    $PYTHON -m venv $ENV_DIR
+    $ENV_PIP install -q -r $GIT_ROOT/dev/requirements_merge_arrow_pr.txt || {
+        echo "Failed to setup virtual environment"
+        echo "Please delete directory '$ENV_DIR' and try again"
+        exit $?
+    }
+}
+
+[ -d $ENV_DIR ] || create_venv
+check_venv
+
+$ENV_PYTHON $GIT_ROOT/dev/merge_arrow_pr.py "$@"

dev/release/rat_exclude_files.txt

@@ -43,6 +43,7 @@ cpp/src/plasma/thirdparty/dlmalloc.c
 cpp/thirdparty/flatbuffers/include/flatbuffers/base.h
 cpp/thirdparty/flatbuffers/include/flatbuffers/flatbuffers.h
 cpp/thirdparty/flatbuffers/include/flatbuffers/stl_emulation.h
+dev/requirements*.txt
 dev/archery/MANIFEST.in
 dev/archery/requirements*.txt
 dev/archery/archery/tests/fixtures/*

dev/requirements_merge_arrow_pr.txt

@@ -0,0 +1,3 @@
+jira
+requests
+six