Add option flag --incremental to ci_sanity.sh

Speed up sanity check during CL builds by running pylint and pep8 checks on only the Python files changed in the last non-merge git commit.
Change: 128431639

Author: caisq

tensorflow/tools/ci_build/ci_sanity.sh

@@ -19,6 +19,8 @@
 # Options:
 #           run sanity checks: python 2&3 pylint checks and bazel nobuild
 #  --pep8   run pep8 test only
+#  --incremental  Performs checks incrementally, by using the files changed in
+#                 the latest commit
 
 # Current script directory
 SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
@@ -39,11 +41,52 @@ num_cpus() {
   echo ${N_CPUS}
 }
 
+# Helper functions for examining changed files in the last non-merge git
+# commit.
+
+# Get the hash of the last non-merge git commit on the current branch.
+# Usage: get_last_non_merge_git_commit
+get_last_non_merge_git_commit() {
+  echo $(git rev-list --no-merges -n 1 HEAD)
+}
+
+# List files changed (i.e., added, removed or revised) in the last non-merge
+# git commit.
+# Usage: get_changed_files_in_last_non_merge_git_commit
+get_changed_files_in_last_non_merge_git_commit() {
+  git diff-tree --no-commit-id --name-only -r $(get_last_non_merge_git_commit)
+}
+
+# List Python files changed in the last non-merge git commit that still exist,
+# i.e., not removed.
+# Usage: get_py_files_to_check [--incremental]
+get_py_files_to_check() {
+  if [[ "$1" == "--incremental" ]]; then
+    CHANGED_PY_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
+                       grep '.*\.py$')
+
+    # Do not include files removed in the last non-merge commit.
+    PY_FILES=""
+    for PY_FILE in ${CHANGED_PY_FILES}; do
+      if [[ -f "${PY_FILE}" ]]; then
+        PY_FILES="${PY_FILES} ${PY_FILE}"
+      fi
+    done
+
+    echo "${PY_FILES}"
+  else
+    echo $(find tensorflow -name '*.py')
+  fi
+}
 
 # Subfunctions for substeps
 # Run pylint
 do_pylint() {
-  # Usage: do_pylint (PYTHON2 | PYTHON3)
+  # Usage: do_pylint (PYTHON2 | PYTHON3) [--incremental]
+  #
+  # Options:
+  #   --incremental  Performs check on only the python files changed in the
+  #                  last non-merge git commit.
 
   # Use this list to whitelist pylint errors
   ERROR_WHITELIST="^tensorflow/python/framework/function_test\.py.*\[E1123.*noinline "\
@@ -53,9 +96,9 @@ do_pylint() {
 
   echo "ERROR_WHITELIST=\"${ERROR_WHITELIST}\""
 
-  if [[ $# != "1" ]]; then
+  if [[ $# != "1" ]] && [[ $# != "2" ]]; then
     echo "Invalid syntax when invoking do_pylint"
-    echo "Usage: do_pylint (PYTHON2 | PYTHON3)"
+    echo "Usage: do_pylint (PYTHON2 | PYTHON3) [--incremental]"
     return 1
   fi
 
@@ -68,7 +111,27 @@ do_pylint() {
     return 1
   fi
 
-  PYTHON_SRC_FILES=$(find tensorflow -name '*.py')
+  if [[ "$2" == "--incremental" ]]; then
+    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)
+    NUM_PYTHON_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)
+
+    echo "do_pylint will perform checks on only the ${NUM_PYTHON_SRC_FILES} "\
+"Python file(s) changed in the last non-merge git commit due to the "\
+"--incremental flag:"
+    echo "${PYTHON_SRC_FILES}"
+    echo ""
+  elif [[ -z "$2" ]]; then
+    PYTHON_SRC_FILES=$(get_py_files_to_check)
+  else
+    echo "Invalid syntax when invoking do_pylint"
+    echo "Usage: do_pylint (PYTHON2 | PYTHON3) [--incremental]"
+    return 1
+  fi
+
+  if [[ -z ${PYTHON_SRC_FILES} ]]; then
+    echo "do_pylint found no Python files to check. Returning."
+    return 0
+  fi
 
   PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc"
 
@@ -134,12 +197,32 @@ do_pylint() {
 
 # Run pep8 check
 do_pep8() {
-  # Usage: do_pep8
+  # Usage: do_pep8 [--incremental]
+  # Options:
+  #   --incremental  Performs check on only the python files changed in the
+  #                  last non-merge git commit.
 
   PEP8_BIN="/usr/local/bin/pep8"
-  PYTHON_SRC_FILES=$(find tensorflow -name '*.py')
   PEP8_CONFIG_FILE="${SCRIPT_DIR}/pep8"
 
+  if [[ "$1" == "--incremental" ]]; then
+    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)
+    NUM_PYTHON_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)
+
+    echo "do_pep8 will perform checks on only the ${NUM_PYTHON_SRC_FILES} "\
+"Python file(s) changed in the last non-merge git commit due to the "\
+"--incremental flag:"
+    echo "${PYTHON_SRC_FILES}"
+    echo ""
+  else
+    PYTHON_SRC_FILES=$(get_py_files_to_check)
+  fi
+
+  if [[ -z ${PYTHON_SRC_FILES} ]]; then
+    echo "do_pep8 found no Python files to check. Returning."
+    return 0
+  fi
+
   if [[ ! -f "${PEP8_CONFIG_FILE}" ]]; then
     die "ERROR: Cannot find pep8 config file at ${PEP8_CONFIG_FILE}"
   fi
@@ -197,11 +280,22 @@ do_bazel_nobuild() {
 SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_bazel_nobuild")
 SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "bazel nobuild")
 
-# Only run pep8 test if "--pep8" option supplied
-if [[ "$1" == "--pep8" ]]; then
-  SANITY_STEPS=("do_pep8")
-  SANITY_STEPS_DESC=("pep8 test")
-fi
+INCREMENTAL_FLAG=""
+
+# Parse command-line arguments
+for arg in "$@"; do
+  if [[ "${arg}" == "--pep8" ]]; then
+    # Only run pep8 test if "--pep8" option supplied
+    SANITY_STEPS=("do_pep8")
+    SANITY_STEPS_DESC=("pep8 test")
+  elif [[ "${arg}" == "--incremental" ]]; then
+    INCREMENTAL_FLAG="--incremental"
+  else
+    echo "ERROR: Unrecognized command-line flag: $1"
+    exit 1
+  fi
+done
+
 
 FAIL_COUNTER=0
 PASS_COUNTER=0
@@ -218,7 +312,7 @@ while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
 "${SANITY_STEPS[COUNTER]} (${SANITY_STEPS_DESC[COUNTER]}) ==="
   echo ""
 
-  ${SANITY_STEPS[COUNTER]}
+  ${SANITY_STEPS[COUNTER]} ${INCREMENTAL_FLAG}
   RESULT=$?
 
   if [[ ${RESULT} != "0" ]]; then