Skip to content

Commit efd7eed

Browse files
marmbrusrxin
authored andcommitted
[BUILD] Runner for spark packages
This is a convenience method added to the SBT build for developers, though if people think its useful we could consider adding a official script that runs using the assembly instead of compiling on demand. It simply compiles spark (without requiring an assembly), and invokes Spark Submit to download / run the package. Example Usage: ``` $ build/sbt > sparkPackage com.databricks:spark-sql-perf_2.10:0.2.4 com.databricks.spark.sql.perf.RunBenchmark --help ``` Author: Michael Armbrust <[email protected]> Closes apache#10834 from marmbrus/sparkPackageRunner.
1 parent c6f971b commit efd7eed

File tree

1 file changed

+15
-0
lines changed

1 file changed

+15
-0
lines changed

project/SparkBuild.scala

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,11 @@ object SparkBuild extends PomBuild {
274274
* Usage: `build/sbt sparkShell`
275275
*/
276276
val sparkShell = taskKey[Unit]("start a spark-shell.")
277+
val sparkPackage = inputKey[Unit](
278+
s"""
279+
|Download and run a spark package.
280+
|Usage `builds/sbt "sparkPackage <group:artifact:version> <MainClass> [args]
281+
""".stripMargin)
277282
val sparkSql = taskKey[Unit]("starts the spark sql CLI.")
278283

279284
enable(Seq(
@@ -287,6 +292,16 @@ object SparkBuild extends PomBuild {
287292
(runMain in Compile).toTask(" org.apache.spark.repl.Main -usejavacp").value
288293
},
289294

295+
sparkPackage := {
296+
import complete.DefaultParsers._
297+
val packages :: className :: otherArgs = spaceDelimited("<group:artifact:version> <MainClass> [args]").parsed.toList
298+
val scalaRun = (runner in run).value
299+
val classpath = (fullClasspath in Runtime).value
300+
val args = Seq("--packages", packages, "--class", className, (Keys.`package` in Compile in "core").value.getCanonicalPath) ++ otherArgs
301+
println(args)
302+
scalaRun.run("org.apache.spark.deploy.SparkSubmit", classpath.map(_.data), args, streams.value.log)
303+
},
304+
290305
javaOptions in Compile += "-Dspark.master=local",
291306

292307
sparkSql := {

0 commit comments

Comments
 (0)