Skip to content

Commit 628932b

Browse files
ScrapCodespwendell
ScrapCodes
authored and
pwendell
committed
[SPARK-1776] Have Spark's SBT build read dependencies from Maven.
Patch introduces the new way of working also retaining the existing ways of doing things. For example build instruction for yarn in maven is `mvn -Pyarn -PHadoop2.2 clean package -DskipTests` in sbt it can become `MAVEN_PROFILES="yarn, hadoop-2.2" sbt/sbt clean assembly` Also supports `sbt/sbt -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 clean assembly` Author: Prashant Sharma <[email protected]> Author: Patrick Wendell <[email protected]> Closes apache#772 from ScrapCodes/sbt-maven and squashes the following commits: a8ac951 [Prashant Sharma] Updated sbt version. 62b09bb [Prashant Sharma] Improvements. fa6221d [Prashant Sharma] Excluding sql from mima 4b8875e [Prashant Sharma] Sbt assembly no longer builds tools by default. 72651ca [Prashant Sharma] Addresses code reivew comments. acab73d [Prashant Sharma] Revert "Small fix to run-examples script." ac4312c [Prashant Sharma] Revert "minor fix" 6af91ac [Prashant Sharma] Ported oldDeps back. + fixes issues with prev commit. 65cf06c [Prashant Sharma] Servelet API jars mess up with the other servlet jars on the class path. 446768e [Prashant Sharma] minor fix 89b9777 [Prashant Sharma] Merge conflicts d0a02f2 [Prashant Sharma] Bumped up pom versions, Since the build now depends on pom it is better updated there. + general cleanups. dccc8ac [Prashant Sharma] updated mima to check against 1.0 a49c61b [Prashant Sharma] Fix for tools jar a2f5ae1 [Prashant Sharma] Fixes a bug in dependencies. cf88758 [Prashant Sharma] cleanup 9439ea3 [Prashant Sharma] Small fix to run-examples script. 96cea1f [Prashant Sharma] SPARK-1776 Have Spark's SBT build read dependencies from Maven. 36efa62 [Patrick Wendell] Set project name in pom files and added eclipse/intellij plugins. 4973dbd [Patrick Wendell] Example build using pom reader.
1 parent c2babc0 commit 628932b

File tree

32 files changed

+317
-598
lines changed

32 files changed

+317
-598
lines changed

‎assembly/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
<packaging>pom</packaging>
3333

3434
<properties>
35+
<sbt.project.name>assembly</sbt.project.name>
3536
<spark.jar.dir>scala-${scala.binary.version}</spark.jar.dir>
3637
<spark.jar.basename>spark-assembly-${project.version}-hadoop${hadoop.version}.jar</spark.jar.basename>
3738
<spark.jar>${project.build.directory}/${spark.jar.dir}/${spark.jar.basename}</spark.jar>

‎bagel/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-bagel_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>bagel</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project Bagel</name>
3235
<url>http://spark.apache.org/</url>

‎bin/spark-class

+2-2
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,9 @@ export JAVA_OPTS
110110

111111
TOOLS_DIR="$FWDIR"/tools
112112
SPARK_TOOLS_JAR=""
113-
if [ -e "$TOOLS_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar ]; then
113+
if [ -e "$TOOLS_DIR"/target/scala-$SCALA_VERSION/spark-tools*[0-9Tg].jar ]; then
114114
# Use the JAR from the SBT build
115-
export SPARK_TOOLS_JAR=`ls "$TOOLS_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar`
115+
export SPARK_TOOLS_JAR=`ls "$TOOLS_DIR"/target/scala-$SCALA_VERSION/spark-tools*[0-9Tg].jar`
116116
fi
117117
if [ -e "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar ]; then
118118
# Use the JAR from the Maven build

‎core/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-core_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>core</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project Core</name>
3235
<url>http://spark.apache.org/</url>

‎dev/run-tests

+2-2
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,10 @@ echo "========================================================================="
6666
# (either resolution or compilation) prompts the user for input either q, r,
6767
# etc to quit or retry. This echo is there to make it not block.
6868
if [ -n "$_RUN_SQL_TESTS" ]; then
69-
echo -e "q\n" | SPARK_HIVE=true sbt/sbt clean assembly test | \
69+
echo -e "q\n" | SPARK_HIVE=true sbt/sbt clean package assembly/assembly test | \
7070
grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
7171
else
72-
echo -e "q\n" | sbt/sbt clean assembly test | \
72+
echo -e "q\n" | sbt/sbt clean package assembly/assembly test | \
7373
grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
7474
fi
7575

‎examples/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-examples_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>examples</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project Examples</name>
3235
<url>http://spark.apache.org/</url>

‎external/flume/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-streaming-flume_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>streaming-flume</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project External Flume</name>
3235
<url>http://spark.apache.org/</url>

‎external/kafka/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-streaming-kafka_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>streaming-kafka</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project External Kafka</name>
3235
<url>http://spark.apache.org/</url>

‎external/mqtt/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-streaming-mqtt_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>streaming-mqtt</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project External MQTT</name>
3235
<url>http://spark.apache.org/</url>

‎external/twitter/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-streaming-twitter_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>streaming-twitter</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project External Twitter</name>
3235
<url>http://spark.apache.org/</url>

‎external/zeromq/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-streaming-zeromq_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>streaming-zeromq</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project External ZeroMQ</name>
3235
<url>http://spark.apache.org/</url>

‎extras/java8-tests/pom.xml

+5-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@
2828
<artifactId>java8-tests_2.10</artifactId>
2929
<packaging>pom</packaging>
3030
<name>Spark Project Java8 Tests POM</name>
31-
31+
32+
<properties>
33+
<sbt.project.name>java8-tests</sbt.project.name>
34+
</properties>
35+
3236
<dependencies>
3337
<dependency>
3438
<groupId>org.apache.spark</groupId>

‎extras/spark-ganglia-lgpl/pom.xml

+5-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,11 @@
2929
<artifactId>spark-ganglia-lgpl_2.10</artifactId>
3030
<packaging>jar</packaging>
3131
<name>Spark Ganglia Integration</name>
32-
32+
33+
<properties>
34+
<sbt.project.name>ganglia-lgpl</sbt.project.name>
35+
</properties>
36+
3337
<dependencies>
3438
<dependency>
3539
<groupId>org.apache.spark</groupId>

‎graphx/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-graphx_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>graphx</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project GraphX</name>
3235
<url>http://spark.apache.org/</url>

‎mllib/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
<groupId>org.apache.spark</groupId>
2929
<artifactId>spark-mllib_2.10</artifactId>
30+
<properties>
31+
<sbt.project.name>mllib</sbt.project.name>
32+
</properties>
3033
<packaging>jar</packaging>
3134
<name>Spark Project ML Library</name>
3235
<url>http://spark.apache.org/</url>

‎pom.xml

+9-1
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@
110110
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
111111

112112
<java.version>1.6</java.version>
113-
113+
<sbt.project.name>spark</sbt.project.name>
114114
<scala.version>2.10.4</scala.version>
115115
<scala.binary.version>2.10</scala.binary.version>
116116
<mesos.version>0.18.1</mesos.version>
@@ -535,6 +535,10 @@
535535
<groupId>org.mortbay.jetty</groupId>
536536
<artifactId>servlet-api-2.5</artifactId>
537537
</exclusion>
538+
<exclusion>
539+
<groupId>javax.servlet</groupId>
540+
<artifactId>servlet-api</artifactId>
541+
</exclusion>
538542
<exclusion>
539543
<groupId>junit</groupId>
540544
<artifactId>junit</artifactId>
@@ -618,6 +622,10 @@
618622
<artifactId>hadoop-yarn-api</artifactId>
619623
<version>${yarn.version}</version>
620624
<exclusions>
625+
<exclusion>
626+
<groupId>javax.servlet</groupId>
627+
<artifactId>servlet-api</artifactId>
628+
</exclusion>
621629
<exclusion>
622630
<groupId>asm</groupId>
623631
<artifactId>asm</artifactId>

‎project/MimaBuild.scala

+15-7
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,16 @@
1515
* limitations under the License.
1616
*/
1717

18+
import sbt._
19+
import sbt.Keys.version
20+
1821
import com.typesafe.tools.mima.core._
1922
import com.typesafe.tools.mima.core.MissingClassProblem
2023
import com.typesafe.tools.mima.core.MissingTypesProblem
2124
import com.typesafe.tools.mima.core.ProblemFilters._
2225
import com.typesafe.tools.mima.plugin.MimaKeys.{binaryIssueFilters, previousArtifact}
2326
import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
24-
import sbt._
27+
2528

2629
object MimaBuild {
2730

@@ -53,7 +56,7 @@ object MimaBuild {
5356
excludePackage("org.apache.spark." + packageName)
5457
}
5558

56-
def ignoredABIProblems(base: File) = {
59+
def ignoredABIProblems(base: File, currentSparkVersion: String) = {
5760

5861
// Excludes placed here will be used for all Spark versions
5962
val defaultExcludes = Seq()
@@ -77,11 +80,16 @@ object MimaBuild {
7780
}
7881

7982
defaultExcludes ++ ignoredClasses.flatMap(excludeClass) ++
80-
ignoredMembers.flatMap(excludeMember) ++ MimaExcludes.excludes
83+
ignoredMembers.flatMap(excludeMember) ++ MimaExcludes.excludes(currentSparkVersion)
84+
}
85+
86+
def mimaSettings(sparkHome: File, projectRef: ProjectRef) = {
87+
val organization = "org.apache.spark"
88+
val previousSparkVersion = "1.0.0"
89+
val fullId = "spark-" + projectRef.project + "_2.10"
90+
mimaDefaultSettings ++
91+
Seq(previousArtifact := Some(organization % fullId % previousSparkVersion),
92+
binaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value))
8193
}
8294

83-
def mimaSettings(sparkHome: File) = mimaDefaultSettings ++ Seq(
84-
previousArtifact := None,
85-
binaryIssueFilters ++= ignoredABIProblems(sparkHome)
86-
)
8795
}

‎project/MimaExcludes.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ import com.typesafe.tools.mima.core._
3131
* MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap")
3232
*/
3333
object MimaExcludes {
34-
val excludes =
35-
SparkBuild.SPARK_VERSION match {
34+
def excludes(version: String) =
35+
version match {
3636
case v if v.startsWith("1.1") =>
3737
Seq(
3838
MimaBuild.excludeSparkPackage("deploy"),

0 commit comments

Comments
 (0)