Updating Editions (enso-org#1944)

Yanndd1 · Aug 12, 2021 · fac0405 · fac0405
1 parent ad0b677
commit fac0405
Show file tree

Hide file tree

Showing 58 changed files with 833 additions and 269 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -17,6 +17,8 @@ env:
   sbtVersion: 1.5.2
   # Please ensure that this is in sync with rustVersion in build.sbt
   rustToolchain: nightly-2021-05-12
+  # Specifies how many nightly releases should be kept. Any older releases are removed.
+  NIGHTLIES_TO_KEEP: 20
 
 concurrency: "releases"
 
@@ -566,6 +568,32 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      # This is done regardless of the release-publish-edition workflow, because this workflow seems to not be triggered by Actions automatically.
+      - name: Prepare AWS Session
+        shell: bash
+        run: |
+          aws configure --profile s3-upload <<-EOF > /dev/null 2>&1
+          ${{ secrets.ARTEFACT_S3_ACCESS_KEY_ID }}
+          ${{ secrets.ARTEFACT_S3_SECRET_ACCESS_KEY }}
+          eu-central-1
+          text
+          EOF
+
+      - name: Update the Editions Bucket
+        shell: bash
+        working-directory: repo
+        run: sbt edition-uploader/run
+
+      - name: Teardown AWS Session
+        shell: bash
+        run: |
+          aws configure --profile s3-upload <<-EOF > /dev/null 2>&1
+          null
+          null
+          null
+          text
+          EOF
+
       # TODO enable this once IDE supports triggers
       #      - name: Trigger IDE Build
       #        shell: bash
@@ -576,7 +604,7 @@ jobs:
       - uses: dev-drprasad/[email protected]
         name: Remove Old Releases
         with:
-          keep_latest: 3
+          keep_latest: ${{ env.NIGHTLIES_TO_KEEP }}
           delete_tag_pattern: SNAPSHOT
           delete_tags: true
         env:

diff --git a/.github/workflows/release-publish-edition.yml b/.github/workflows/release-publish-edition.yml
@@ -0,0 +1,76 @@
+name: Publish Edition to S3
+
+on:
+  release:
+    types: [published]
+
+env:
+  # Please ensure that this is in sync with graalVersion in build.sbt
+  graalVersion: 21.1.0
+  # Please ensure that this is in sync with javaVersion in build.sbt
+  javaVersion: 11
+  # Please ensure that this is in sync with project/build.properties
+  sbtVersion: 1.5.2
+  # Please ensure that this is in sync with NIGHTLIES_TO_KEEP in nightly.yml
+  NIGHTLIES_TO_KEEP: 20
+
+concurrency: "releases"
+
+jobs:
+  update-editions-on-s3:
+    name: Update Editions on S3
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: repo
+      - name: Setup GraalVM Environment
+        uses: ayltai/setup-graalvm@v1
+        with:
+          graalvm-version: ${{ env.graalVersion }}
+          java-version: ${{ env.javaVersion }}
+          native-image: true
+      - name: Set Up SBT
+        shell: bash
+        run: |
+          curl -fsSL -o sbt.tgz https://github.com/sbt/sbt/releases/download/v${{env.sbtVersion}}/sbt-${{env.sbtVersion}}.tgz
+          tar -xzf sbt.tgz
+          echo $GITHUB_WORKSPACE/sbt/bin/ >> $GITHUB_PATH
+
+      # Caches
+      - name: Cache SBT
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.sbt
+            ~/.ivy2/cache
+            ~/.cache
+          key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }}
+          restore-keys: ${{ runner.os }}-sbt-
+
+      - name: Prepare AWS Session
+        shell: bash
+        run: |
+          aws configure --profile s3-upload <<-EOF > /dev/null 2>&1
+          ${{ secrets.ARTEFACT_S3_ACCESS_KEY_ID }}
+          ${{ secrets.ARTEFACT_S3_SECRET_ACCESS_KEY }}
+          eu-central-1
+          text
+          EOF
+
+      - name: Update the Editions Bucket
+        shell: bash
+        working-directory: repo
+        run: sbt edition-uploader/run
+
+      - name: Teardown AWS Session
+        shell: bash
+        run: |
+          aws configure --profile s3-upload <<-EOF > /dev/null 2>&1
+          null
+          null
+          null
+          text
+          EOF
diff --git a/RELEASES.md b/RELEASES.md
@@ -16,6 +16,9 @@
   subcommand to the Launcher.
 - Implemented an HTTP endpoint that resets the time that the language server has
   spent idle ([#1938](https://github.com/enso-org/enso/pull/1938)).
+- Implemented the mechanism for updating the editions cache
+  ([#1944](https://github.com/enso-org/enso/pull/1944)), allowing the engine to
+  download and use editions other than the one bundled with a given release.
 
 ## Libraries
 

diff --git a/build.sbt b/build.sbt
@@ -250,6 +250,7 @@ lazy val enso = (project in file("."))
     editions,
     `distribution-manager`,
     `edition-updater`,
+    `edition-uploader`,
     `library-manager`,
     `library-manager-test`,
     syntax.jvm,
@@ -818,6 +819,7 @@ lazy val `project-manager` = (project in file("lib/scala/project-manager"))
   .dependsOn(`akka-native`)
   .dependsOn(`version-output`)
   .dependsOn(editions)
+  .dependsOn(`edition-updater`)
   .dependsOn(cli)
   .dependsOn(`task-progress-notifications`)
   .dependsOn(`polyglot-api`)
@@ -1017,6 +1019,7 @@ lazy val `language-server` = (project in file("engine/language-server"))
   .dependsOn(`json-rpc-server`)
   .dependsOn(`task-progress-notifications`)
   .dependsOn(`library-manager`)
+  .dependsOn(`edition-updater`)
   .dependsOn(`logging-service`)
   .dependsOn(`polyglot-api`)
   .dependsOn(`searcher`)
@@ -1187,6 +1190,7 @@ lazy val runtime = (project in file("engine/runtime"))
   .dependsOn(`polyglot-api`)
   .dependsOn(`text-buffer`)
   .dependsOn(searcher)
+  .dependsOn(`edition-updater`)
   .dependsOn(`library-manager`)
   .dependsOn(testkit % Test)
   .dependsOn(`logging-utils`)
@@ -1391,6 +1395,13 @@ lazy val `edition-updater` = project
   )
   .dependsOn(editions)
   .dependsOn(downloader)
+  .dependsOn(`distribution-manager`)
+  .dependsOn(`library-manager-test` % Test)
+
+lazy val `edition-uploader` = project
+  .in(file("lib/scala/edition-uploader"))
+  .dependsOn(editions)
+  .dependsOn(`version-output`)
 
 lazy val `library-manager` = project
   .in(file("lib/scala/library-manager"))
@@ -1441,6 +1452,7 @@ lazy val `runtime-version-manager` = project
   .dependsOn(`logging-service`)
   .dependsOn(cli)
   .dependsOn(`version-output`)
+  .dependsOn(`edition-updater`)
   .dependsOn(`distribution-manager`)
 
 lazy val `runtime-version-manager-test` = project

diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala b/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala
@@ -1,13 +1,9 @@
 package org.enso.languageserver.boot
 
 import akka.actor.ActorSystem
-import org.enso.distribution.{
-  DistributionManager,
-  EditionManager,
-  Environment,
-  LanguageHome
-}
+import org.enso.distribution.{DistributionManager, Environment, LanguageHome}
 import org.enso.editions.EditionResolver
+import org.enso.editions.updater.EditionManager
 import org.enso.jsonrpc.JsonRpcServer
 import org.enso.languageserver.boot.DeploymentType.{Azure, Desktop}
 import org.enso.languageserver.capability.CapabilityRouter

diff --git a/...ge-server/src/main/scala/org/enso/languageserver/libraries/EditionReferenceResolver.scala b/...ge-server/src/main/scala/org/enso/languageserver/libraries/EditionReferenceResolver.scala
@@ -22,7 +22,7 @@ class EditionReferenceResolver(
     editionReference: EditionReference
   ): Try[Editions.RawEdition] = editionReference match {
     case EditionReference.NamedEdition(editionName) =>
-      editionProvider.findEditionForName(editionName)
+      editionProvider.findEditionForName(editionName).toTry
     case EditionReference.CurrentProjectEdition =>
       Try {
         projectPackage.config.edition.getOrElse {

diff --git a/...c/main/scala/org/enso/languageserver/libraries/handler/EditionsListAvailableHandler.scala b/...c/main/scala/org/enso/languageserver/libraries/handler/EditionsListAvailableHandler.scala
@@ -2,7 +2,7 @@ package org.enso.languageserver.libraries.handler
 
 import akka.actor.{Actor, Props}
 import com.typesafe.scalalogging.LazyLogging
-import org.enso.distribution.EditionManager
+import org.enso.editions.updater.EditionManager
 import org.enso.jsonrpc.{Request, ResponseError, ResponseResult}
 import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError
 import org.enso.languageserver.libraries.LibraryApi._
@@ -11,9 +11,6 @@ import org.enso.languageserver.util.UnhandledLogging
 import scala.util.{Failure, Success, Try}
 
 /** A request handler for the `editions/listAvailable` endpoint.
-  *
-  * It is a partial implementation - it already allows to list existing
-  * editions, but updating is not yet implemented.
   *
   * @param editionManager an edition manager instance
   */
@@ -22,14 +19,17 @@ class EditionsListAvailableHandler(editionManager: EditionManager)
     with LazyLogging
     with UnhandledLogging {
   override def receive: Receive = {
-    case Request(EditionsListAvailable, id, _: EditionsListAvailable.Params) =>
-      // TODO [RW] once updating editions is implemented this should be made asynchronous
-      Try(editionManager.findAllAvailableEditions()) match {
+    case Request(
+          EditionsListAvailable,
+          id,
+          EditionsListAvailable.Params(update)
+        ) =>
+      Try(editionManager.findAllAvailableEditions(update)) match {
         case Success(editions) =>
           sender() ! ResponseResult(
             EditionsListAvailable,
             id,
-            EditionsListAvailable.Result(editions)
+            EditionsListAvailable.Result(editions.sorted)
           )
         case Failure(exception) =>
           sender() ! ResponseError(

diff --git a/...erver/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala b/...erver/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala
@@ -6,7 +6,7 @@ import akka.util.Timeout
 import com.typesafe.scalalogging.LazyLogging
 import org.enso.cli.task.ProgressUnit
 import org.enso.cli.task.notifications.TaskNotificationApi
-import org.enso.distribution.EditionManager
+import org.enso.editions.updater.EditionManager
 import org.enso.jsonrpc._
 import org.enso.languageserver.boot.resource.InitializationComponent
 import org.enso.languageserver.capability.CapabilityApi.{

diff --git a/...rc/main/scala/org/enso/languageserver/protocol/json/JsonConnectionControllerFactory.scala b/...rc/main/scala/org/enso/languageserver/protocol/json/JsonConnectionControllerFactory.scala
@@ -1,7 +1,7 @@
 package org.enso.languageserver.protocol.json
 
 import akka.actor.{ActorRef, ActorSystem}
-import org.enso.distribution.EditionManager
+import org.enso.editions.updater.EditionManager
 import org.enso.jsonrpc.ClientControllerFactory
 import org.enso.languageserver.boot.resource.InitializationComponent
 import org.enso.languageserver.data.Config

diff --git a/...anguage-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala b/...anguage-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala
@@ -5,8 +5,9 @@ import io.circe.literal._
 import io.circe.parser.parse
 import io.circe.syntax.EncoderOps
 import org.apache.commons.io.FileUtils
-import org.enso.distribution.{DistributionManager, EditionManager, LanguageHome}
+import org.enso.distribution.{DistributionManager, LanguageHome}
 import org.enso.editions.EditionResolver
+import org.enso.editions.updater.EditionManager
 import org.enso.jsonrpc.test.JsonRpcServerTestKit
 import org.enso.jsonrpc.{ClientControllerFactory, Protocol}
 import org.enso.languageserver.TestClock

diff --git a/...language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala b/...language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala
@@ -4,7 +4,10 @@ import io.circe.literal._
 import io.circe.{Json, JsonObject}
 import org.enso.languageserver.libraries.LibraryEntry
 import org.enso.languageserver.libraries.LibraryEntry.PublishedLibraryVersion
-import org.enso.librarymanager.published.repository.EmptyRepository
+import org.enso.librarymanager.published.repository.{
+  EmptyRepository,
+  ExampleRepository
+}
 
 import java.nio.file.Files
 
@@ -105,8 +108,7 @@ class LibrariesTest extends BaseServerTest {
           """)
 
       val repoRoot = getTestDirectory.resolve("libraries_repo_root")
-      val server   = EmptyRepository.startServer(port, repoRoot, uploads = true)
-      try {
+      EmptyRepository.withServer(port, repoRoot, uploads = true) {
         val uploadUrl = s"http://localhost:$port/upload"
         client.send(json"""
           { "jsonrpc": "2.0",
@@ -147,9 +149,6 @@ class LibrariesTest extends BaseServerTest {
           .resolve("0.0.1")
         val mainPackage = libraryRoot.resolve("main.tgz")
         assert(Files.exists(mainPackage))
-      } finally {
-        server.kill(killDescendants    = true)
-        server.join(waitForDescendants = true)
       }
     }
   }
@@ -225,7 +224,33 @@ class LibrariesTest extends BaseServerTest {
     }
 
     "update the list of editions if requested" ignore {
-      // TODO [RW] updating editions
+      val repo     = new ExampleRepository
+      val repoPath = getTestDirectory.resolve("repo_root")
+      repo.createRepository(repoPath)
+      repo.withServer(43707, repoPath) {
+        val client = getInitialisedWsClient()
+
+        client.send(json"""
+          { "jsonrpc": "2.0",
+            "method": "editions/listAvailable",
+            "id": 0,
+            "params": {
+              "update": true
+            }
+          }
+          """)
+        client.expectJson(json"""
+          { "jsonrpc": "2.0",
+            "id": 0,
+            "result": {
+              "editionNames": [
+                ${buildinfo.Info.currentEdition},
+                "testlocal"
+              ]
+            }
+          }
+          """)
+      }
     }
   }
 

diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala
@@ -4,12 +4,10 @@ import java.nio.file.Path
 import com.typesafe.scalalogging.Logger
 import io.circe.Json
 import nl.gn0s1s.bump.SemVer
-import org.enso.distribution.EditionManager
+import org.enso.distribution.config.DefaultVersion
+import org.enso.editions.updater.EditionManager
 import org.enso.runtimeversionmanager.CurrentVersion
-import org.enso.runtimeversionmanager.config.{
-  DefaultVersion,
-  GlobalConfigurationManager
-}
+import org.enso.runtimeversionmanager.config.GlobalRunnerConfigurationManager
 import org.enso.runtimeversionmanager.runner.{
   JVMSettings,
   LanguageServerOptions,
@@ -45,7 +43,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
     manager
   }
   private lazy val configurationManager =
-    new GlobalConfigurationManager(componentsManager, distributionManager)
+    new GlobalRunnerConfigurationManager(componentsManager, distributionManager)
   private lazy val editionManager = EditionManager(distributionManager)
   private lazy val projectManager = new ProjectManager
   private lazy val runner =

diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala
@@ -10,7 +10,7 @@ import org.enso.cli.arguments.Opts.implicits._
 import org.enso.distribution.FileSystem
 import org.enso.runtimeversionmanager.CurrentVersion
 import org.enso.distribution.FileSystem.PathSyntax
-import org.enso.runtimeversionmanager.cli.Arguments._
+import org.enso.distribution.config.DefaultVersion._
 import org.enso.launcher.distribution.LauncherEnvironment
 import org.enso.launcher.upgrade.LauncherUpgrader
 import org.enso.launcher.releases.LauncherRepository