HOTFIX for broken CI, by SPARK-1336

Learnt about `set -o pipefail` is very useful. Author: Prashant Sharma <[email protected]> Author: Prashant Sharma <[email protected]> Closes apache#321 from ScrapCodes/hf-SPARK-1336 and squashes the following commits: 9d22bc2 [Prashant Sharma] added comment why echo -e q exists. f865951 [Prashant Sharma] made error to match with word boundry so errors does not match. This is there to make sure build fails if provided SparkBuild has compile errors. 7fffdf2 [Prashant Sharma] Removed a stray line. 97379d8 [Prashant Sharma] HOTFIX for broken CI, by SPARK-1336
xingwudao · Apr 5, 2014 · 7c18428 · 7c18428
1 parent 0acc7a0
commit 7c18428
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 7 deletions.
diff --git a/.rat-excludes b/.rat-excludes
@@ -39,3 +39,4 @@ work
 .*\.q
 golden
 test.out/*
+.*iml
diff --git a/dev/run-tests b/dev/run-tests
@@ -26,13 +26,12 @@ rm -rf ./work
 
 # Fail fast
 set -e
-
+set -o pipefail
 if test -x "$JAVA_HOME/bin/java"; then
     declare java_cmd="$JAVA_HOME/bin/java"
 else 
     declare java_cmd=java
 fi
-
 JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
 [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8."
 
@@ -49,7 +48,9 @@ dev/scalastyle
 echo "========================================================================="
 echo "Running Spark unit tests"
 echo "========================================================================="
-sbt/sbt assembly test
+# echo "q" is needed because sbt on encountering a build file with failure (either resolution or compilation)
+# prompts the user for input either q, r, etc to quit or retry. This echo is there to make it not block.
+echo -e "q\n" | sbt/sbt assembly test |  grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
 
 echo "========================================================================="
 echo "Running PySpark tests"
@@ -63,5 +64,4 @@ echo "========================================================================="
 echo "Detecting binary incompatibilites with MiMa"
 echo "========================================================================="
 ./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore
-sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving" 
-
+echo -e "q\n" | sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving" 
diff --git a/dev/scalastyle b/dev/scalastyle
@@ -17,8 +17,8 @@
 # limitations under the License.
 #
 
-sbt/sbt clean scalastyle > scalastyle.txt
-ERRORS=$(cat scalastyle.txt | grep -e "error file")
+echo -e "q\n" | sbt/sbt clean scalastyle > scalastyle.txt
+ERRORS=$(cat scalastyle.txt | grep -e "\<error\>")
 if test ! -z "$ERRORS"; then
     echo -e "Scalastyle checks failed at following occurrences:\n$ERRORS"
     exit 1
-Original file line number
+Diff line change
@@ Expand Up / @@ -39,3 +39,4 @@ work @@
     .*\.q
     golden
     test.out/*
+    .*iml