Switch the bootstrap build over to sbt

All of the individual ant builds that occured during `bootstrap` are replaced by equivalent sbt builds. - Allow extra dashes in version suffix when using SPLIT - Clean up ScriptCommands - Building an extra `locker` for stability testing with ant was not necessary but sbt also drops `strap`, so we need to build again with `quick` to get the equivalent of `strap`. The script for checking stability is invoked directly from the bootstrap script, not from sbt. - `STARR` and `locker` build output is still logged to `logs/builds`, the main build runs log directly to the main console with colored output. - Allow `—show-log` option on partest command line in sbt - Normalize inferred LUB in `run/t7747-repl.scala` - Add `normalize` feature from `ReplTest` to `InteractiveTest` - Normalize inferred LUBs in `presentation/callcc-interpreter`
fx88 · Jul 15, 2016 · 36e2ae2 · 36e2ae2
1 parent 3c43a7b
commit 36e2ae2
Show file tree

Hide file tree

Showing 8 changed files with 164 additions and 70 deletions.
diff --git a/project/PartestUtil.scala b/project/PartestUtil.scala
@@ -29,7 +29,7 @@ object PartestUtil {
     val knownUnaryOptions = List(
       "--pos", "--neg", "--run", "--jvm", "--res", "--ant", "--scalap", "--specialized",
       "--scalacheck", "--instrumented", "--presentation", "--failed", "--update-check",
-      "--show-diff", "--verbose", "--terse", "--debug", "--version", "--self-test", "--help")
+      "--show-diff", "--show-log", "--verbose", "--terse", "--debug", "--version", "--self-test", "--help")
     val srcPathOption = "--srcpath"
     val grepOption = "--grep"
 

diff --git a/project/ScriptCommands.scala b/project/ScriptCommands.scala
@@ -4,30 +4,116 @@ import BuildSettings.autoImport._
 
 /** Custom commands for use by the Jenkins scripts. This keeps the surface area and call syntax small. */
 object ScriptCommands {
-  def all = Seq(setupPublishCore, setupValidateTest)
+  def all = Seq(
+    setupPublishCore,
+    setupValidateTest,
+    setupBootstrapStarr, setupBootstrapLocker, setupBootstrapQuick, setupBootstrapPublish,
+    testAll
+  )
 
   /** Set up the environment for `validate/publish-core`. The argument is the Artifactory snapshot repository URL. */
-  def setupPublishCore = Command.single("setupPublishCore") { case (state, url) =>
-      Project.extract(state).append(Seq(
-        baseVersionSuffix in Global := "SHA-SNAPSHOT",
-        // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088):
-        publishTo in Global := Some("scala-pr" at url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis),
-        publishArtifact in (Compile, packageDoc) in ThisBuild := false,
-        scalacOptions in Compile in ThisBuild += "-opt:l:classpath",
-        logLevel in ThisBuild := Level.Info,
-        logLevel in update in ThisBuild := Level.Warn
-      ), state)
-    }
+  def setupPublishCore = setup("setupPublishCore") { case Seq(url) =>
+    Seq(
+      baseVersionSuffix in Global := "SHA-SNAPSHOT"
+    ) ++ publishTarget(url) ++ noDocs ++ enableOptimizer
+  }
 
   /** Set up the environment for `validate/test`. The argument is the Artifactory snapshot repository URL. */
-  def setupValidateTest = Command.single("setupValidateTest") { case (state, url) =>
+  def setupValidateTest = setup("setupValidateTest") { case Seq(url) =>
     //TODO When ant is gone, pass starr version as an argument to this command instead of using version.properties
-    Project.extract(state).append(Seq(
+    Seq(
+      resolvers in Global += "scala-pr" at url,
+      testOptions in IntegrationTest in LocalProject("test") ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
+    ) ++ enableOptimizer
+  }
+
+  /** Set up the environment for building STARR in `validate/bootstrap`. The arguments are:
+    * - Repository URL for publishing
+    * - Version number to publish */
+  def setupBootstrapStarr = setup("setupBootstrapStarr") { case Seq(url, ver) =>
+    Seq(
+      baseVersion in Global := ver,
+      baseVersionSuffix in Global := "SPLIT"
+    ) ++ publishTarget(url) ++ noDocs ++ enableOptimizer
+  }
+
+  /** Set up the environment for building locker in `validate/bootstrap`. The arguments are:
+    * - Repository URL for publishing locker and resolving STARR
+    * - Version number to publish */
+  def setupBootstrapLocker = setup("setupBootstrapLocker") { case Seq(url, ver) =>
+    Seq(
+      baseVersion in Global := ver,
+      baseVersionSuffix in Global := "SPLIT",
+      resolvers in Global += "scala-pr" at url
+    ) ++ publishTarget(url) ++ noDocs ++ enableOptimizer
+  }
+
+  /** Set up the environment for building quick in `validate/bootstrap`. The arguments are:
+    * - Repository URL for publishing
+    * - Version number to publish */
+  def setupBootstrapQuick = setup("setupBootstrapQuick") { case Seq(url, ver) =>
+    Seq(
+      baseVersion in Global := ver,
+      baseVersionSuffix in Global := "SPLIT",
+      resolvers in Global += "scala-pr" at url,
+      testOptions in IntegrationTest in LocalProject("test") ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
+    ) ++ publishTarget(url) ++ enableOptimizer
+  }
+
+  /** Set up the environment for publishing in `validate/bootstrap`. The arguments are:
+    * - Temporary bootstrap repository URL for resolving modules
+    * - Version number to publish
+    * All artifacts are published to Sonatype. GPG signing has to be configured from the
+    * shell script after `setupBootstrapPublish` because we don't pull the GPG plugin in
+    * by default, so we cannot reference its keys statically. */
+  def setupBootstrapPublish = setup("setupBootstrapPublish") { case Seq(url, ver) =>
+    // Define a copy of the setting key here in case the plugin is not part of the build
+    val pgpPassphrase = SettingKey[Option[Array[Char]]]("pgp-passphrase", "The passphrase associated with the secret used to sign artifacts.", KeyRanks.BSetting)
+    Seq(
+      baseVersion in Global := ver,
+      baseVersionSuffix in Global := "SPLIT",
       resolvers in Global += "scala-pr" at url,
-      scalacOptions in Compile in ThisBuild += "-opt:l:classpath",
-      testOptions in IntegrationTest in LocalProject("test") ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff")),
-      logLevel in ThisBuild := Level.Info,
-      logLevel in update in ThisBuild := Level.Warn
-    ), state)
+      publishTo in Global := Some("sonatype-releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2"),
+      credentials in Global += Credentials(Path.userHome / ".credentials-sonatype"),
+      pgpPassphrase in Global := Some(Array.empty)
+    ) ++ enableOptimizer
+  }
+
+  private[this] def setup(name: String)(f: Seq[String] => Seq[Setting[_]]) =
+    Command.args(name, name) { case (state, seq) => Project.extract(state).append(f(seq) ++ resetLogLevels, state) }
+
+  private[this] val resetLogLevels = Seq(
+    logLevel in ThisBuild := Level.Info,
+    logLevel in update in ThisBuild := Level.Warn
+  )
+
+  private[this] val enableOptimizer = Seq(
+    scalacOptions in Compile in ThisBuild += "-opt:l:classpath"
+  )
+
+  private[this] val noDocs = Seq(
+    publishArtifact in (Compile, packageDoc) in ThisBuild := false
+  )
+
+  private[this] def publishTarget(url: String) = Seq(
+    // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088):
+    publishTo in Global := Some("scala-pr-publish" at url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis)
+  )
+
+  def testAll = Command.command("testAll") { state =>
+    val cmds = Seq(
+      "test",
+      "partest run pos neg jvm",
+      "partest res scalap specialized scalacheck",
+      "partest instrumented presentation",
+      "partest --srcpath scaladoc",
+      "osgiTestFelix/test",
+      "osgiTestEclipse/test",
+      "library/mima",
+      "reflect/mima",
+      "doc"
+    )
+    state.log.info(cmds.mkString("Running all tests: \"", "\", \"", "\""))
+    cmds ::: state
   }
 }
diff --git a/project/VersionUtil.scala b/project/VersionUtil.scala
@@ -61,7 +61,7 @@ object VersionUtil {
     val (base, suffix) = {
       val (b, s) = (baseVersion.value, baseVersionSuffix.value)
       if(s == "SPLIT") {
-        val split = """([\w+\.]+)(-[\w+\.]+)??""".r
+        val split = """([\w+\.]+)(-[\w+\.-]+)??""".r
         val split(b2, sOrNull) = b
         (b2, Option(sOrNull).map(_.drop(1)).getOrElse(""))
       } else (b, s)

diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap
@@ -92,7 +92,7 @@ publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set t
 
 forceRebuild=${forceRebuild-no}
 
-antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak)
+sbtBuildTask="testAll" # TESTING leave empty to avoid the sanity check
 clean="clean" # TESTING leave empty to speed up testing
 
 baseDir=${WORKSPACE-`pwd`}
@@ -174,20 +174,20 @@ function st_stagingRepoClose() {
 # the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir
 # the new version of sbt-extras ignores sbt-dir, so we pass it in as -Dsbt.global.base
 # need to set sbt-dir to one that has the gpg.sbt plugin config
-sbtArgs="-no-colors -ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13"
+sbtArgs="-ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13"
 
 sbtBuild() {
-  echo "### sbtBuild: "$SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
-  $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1
+  echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
+  $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1
 }
 
 sbtResolve() {
   cd $baseDir/resolutionScratch_
   touch build.sbt
   # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to.
   cross=${4-binary}
-  echo "### sbtResolve: $SBT_CMD $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
-  $SBT_CMD $sbtArgs "${scalaVersionTasks[@]}" \
+  echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
+  $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \
     "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \
       'show update' >> $baseDir/logs/resolution 2>&1
 }
@@ -490,13 +490,7 @@ bootstrap() {
         git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR
         cd $STARR_DIR
         git co $STARR_REF
-        ant -Dmaven.version.number=$STARR_VER\
-            -Dremote.snapshot.repository=NOPE\
-            -Dremote.release.repository=$releaseTempRepoUrl\
-            -Drepository.credentials.id=$releaseTempRepoCred\
-            -Ddocs.skip=1\
-            -Dlocker.skip=1\
-            $publishStarrPrivateTask >> $baseDir/logs/builds 2>&1
+        $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $releaseTempRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1
       )
   fi
 
@@ -510,14 +504,7 @@ bootstrap() {
   # publish more than just core: partest needs scalap
   # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler
   if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi
-  ant -Dmaven.version.number=$SCALA_VER\
-      -Dremote.snapshot.repository=NOPE\
-      $SET_STARR\
-      -Dremote.release.repository=$releaseTempRepoUrl\
-      -Drepository.credentials.id=$releaseTempRepoCred\
-      -Ddocs.skip=1\
-      -Dlocker.skip=1\
-      $publishLockerPrivateTask >> $baseDir/logs/builds 2>&1
+  $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $releaseTempRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1
 
   echo "### Building modules using locker"
 
@@ -534,31 +521,43 @@ bootstrap() {
   echo "### Bootstrapping Scala using locker"
 
   # # TODO: close all open staging repos so that we can be reasonably sure the only open one we see after publishing below is ours
-  # # the ant call will create a new one
+  # # the sbt call will create a new one
   #
   # Rebuild Scala with these modules so that all binary versions are consistent.
   # Update versions.properties to new modules.
   # Sanity check: make sure the Scala test suite passes / docs can be generated with these modules.
   # don't skip locker (-Dlocker.skip=1), or stability will fail
   # overwrite "locker" version of scala at private-repo with bootstrapped version
   cd $baseDir
-  rm -rf build/ # must leave everything else in $baseDir for downstream jobs
+  rm -rf build/
 
-  ant -Dstarr.version=$SCALA_VER\
-      -Dextra.repo.url=$releaseTempRepoUrl\
-      -Dmaven.version.suffix=$SCALA_VER_SUFFIX\
+  $SBT_CMD $sbtArgs \
+      --warn \
+      -Dstarr.version=$SCALA_VER \
       ${updatedModuleVersions[@]} \
-      -Dupdate.versions=1\
-      -Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\
-      -Dremote.snapshot.repository=NOPE\
-      -Dremote.release.repository=$releaseTempRepoUrl\
-      -Drepository.credentials.id=$releaseTempRepoCred\
-      -Dscalac.args.optimise=-opt:l:classpath\
-      $antBuildTask $publishPrivateTask
+      "setupBootstrapQuick $releaseTempRepoUrl $SCALA_VER" \
+      $clean \
+      $sbtBuildTask \
+      dist/mkQuick \
+      publish | grep -v "was too long to be displayed in the webview, and will be left out"
 
   # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala
   rm -rf $baseDir/ivy2
 
+  # Run stability tests using the just built version as "quick" and a new version as "strap"
+  mv build/quick quick1
+  rm -rf build/
+  $SBT_CMD $sbtArgs \
+      --warn \
+      -Dstarr.version=$SCALA_VER \
+      ${updatedModuleVersions[@]} \
+      "setupBootstrapQuick $releaseTempRepoUrl $SCALA_VER" \
+      $clean \
+      dist/mkQuick
+  mv build/quick build/strap
+  mv quick1 build/quick
+  tools/stability-test.sh
+
   # TODO: create PR with following commit (note that release will have been tagged already)
   # git commit versions.properties -m"Bump versions.properties for $SCALA_VER."
 }
@@ -571,7 +570,12 @@ publishSonatype() {
   # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary,
   # since we're just publishing an existing build
   echo "### Publishing core to sonatype"
-  ant -Dmaven.version.number=$SCALA_VER $publishSonatypeTaskCore
+  $SBT_CMD $sbtArgs \
+      --warn \
+      -Dstarr.version=$SCALA_VER \
+      ${updatedModuleVersions[@]} \
+      "setupBootstrapPublish $releaseTempRepoUrl $SCALA_VER" \
+      publishSigned | grep -v "was too long to be displayed in the webview, and will be left out"
 
   echo "### Publishing modules to sonatype"
   # build/test/publish scala core modules to sonatype (this will start a new staging repo)

diff --git a/scripts/jobs/validate/test b/scripts/jobs/validate/test
@@ -20,16 +20,7 @@ case $prDryRun in
        --warn \
        "setupValidateTest $prRepoUrl" \
        $testExtraArgs \
-       "test" \
-       "partest run pos neg jvm" \
-       "partest res scalap specialized scalacheck" \
-       "partest instrumented presentation" \
-       "partest --srcpath scaladoc" \
-       osgiTestFelix/test \
-       osgiTestEclipse/test \
-       library/mima \
-       reflect/mima \
-       doc
+       testAll
 
     ;;
 

diff --git a/src/interactive/scala/tools/nsc/interactive/tests/InteractiveTest.scala b/src/interactive/scala/tools/nsc/interactive/tests/InteractiveTest.scala
@@ -78,10 +78,16 @@ abstract class InteractiveTest
   }
 
   protected def execute(): Unit = {
-    loadSources()
-    runDefaultTests()
+    util.stringFromStream { ostream =>
+      Console.withOut(ostream) {
+        loadSources()
+        runDefaultTests()
+      }
+    }.lines.map(normalize).foreach(println)
   }
 
+  protected def normalize(s: String) = s
+
   /** Load all sources before executing the test. */
   protected def loadSources() {
     // ask the presentation compiler to track all sources. We do

diff --git a/test/files/presentation/callcc-interpreter/Runner.scala b/test/files/presentation/callcc-interpreter/Runner.scala
@@ -1,3 +1,6 @@
 import scala.tools.nsc.interactive.tests._
 
-object Test extends InteractiveTest
+object Test extends InteractiveTest {
+  // Normalize ordering of LUB
+  override def normalize(s: String) = s.replace("Serializable with Product", "Product with Serializable")
+}
diff --git a/test/files/run/t7747-repl.scala b/test/files/run/t7747-repl.scala
@@ -8,8 +8,12 @@ object Test extends ReplTest {
     s
   }
 
-  // replace indylambda function names by <function0>
-  override def normalize(s: String) = """\$Lambda.*""".r.replaceAllIn(s, "<function0>")
+  override def normalize(s: String) = {
+    // replace indylambda function names by <function0>
+    val s2 = """\$Lambda.*""".r.replaceAllIn(s, "<function0>")
+    // Normalize ordering of LUB
+    s2.replace("Serializable with Product", "Product with Serializable")
+  }
 
   def code = """
     |var x = 10