Skip to content

Commit

Permalink
ENSO_LAUNCHER can be native,test,debug,fast,-ls (#12117)
Browse files Browse the repository at this point in the history 
Fixes another issue related to #12014 - `runEngineDistribution` has to generate _native image_ launchers when in `ENSO_LAUNCHER=native` mode. This PR also introduces additional comma separated values to the `ENSO_LAUNCHER` variable:
- `shell` uses shell launchers and has no additional sub options
- `native` generates some form of _native image_ launchers
- concatenations are possible:
- `fast,native` disables code optimizations to speed the build up
- `test,native` enables assertions
- `debug,native` enables debug information
- combinations up to `native,test,debug,fast` are also possible
  • Loading branch information
@JaroslavTulach
JaroslavTulach authored Jan 24, 2025
1 parent 78febd5 commit 8cc8c4f
Show file tree
Hide file tree
Showing 7 changed files with 277 additions and 131 deletions.
132 changes: 92 additions & 40 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3691,9 +3691,19 @@ lazy val `engine-runner` = project
val epbLang =
(`runtime-language-epb` / Compile / fullClasspath).value
.map(_.data.getAbsolutePath)
val langServer =
(`language-server` / Compile / fullClasspath).value
def langServer = {
val log = streams.value.log
val path = (`language-server` / Compile / fullClasspath).value
.map(_.data.getAbsolutePath)
if (GraalVM.EnsoLauncher.disableLanguageServer) {
log.info(
s"Skipping language server in native image build as ${GraalVM.EnsoLauncher.VAR_NAME} env variable is ${GraalVM.EnsoLauncher.toString}"
)
Seq()
} else {
path
}
}
val core = (
runnerDeps ++
runtimeDeps ++
Expand Down Expand Up @@ -3822,7 +3832,7 @@ lazy val `engine-runner` = project
.dependsOn(NativeImage.additionalCp)
.dependsOn(NativeImage.smallJdk)
.dependsOn(
createEnginePackage
createEnginePackageNoIndex
)
.value,
buildNativeImage := Def.taskDyn {
Expand Down Expand Up @@ -5106,6 +5116,7 @@ lazy val createEnginePackage =
taskKey[Unit]("Creates the engine distribution package")
createEnginePackage := {
updateLibraryManifests.value
buildEngineDistributionNoIndex.value
val modulesToCopy = componentModulesPaths.value
val root = engineDistributionRoot.value
val log = streams.value.log
Expand All @@ -5131,48 +5142,49 @@ ThisBuild / createEnginePackage := {
createEnginePackage.result.value
}

lazy val buildEngineDistribution =
taskKey[Unit]("Builds the engine distribution and optionally native image")
buildEngineDistribution := Def.taskIf {
lazy val createEnginePackageNoIndex =
taskKey[Unit]("Creates the engine distribution package")
createEnginePackageNoIndex := {
updateLibraryManifests.value
val modulesToCopy = componentModulesPaths.value
val root = engineDistributionRoot.value
val log = streams.value.log
val cacheFactory = streams.value.cacheStoreFactory
DistributionPackage.createEnginePackage(
distributionRoot = root,
cacheFactory = cacheFactory,
log = log,
jarModulesToCopy = modulesToCopy,
graalVersion = graalMavenPackagesVersion,
javaVersion = graalVersion,
ensoVersion = ensoVersion,
editionName = currentEdition,
sourceStdlibVersion = stdLibVersion,
targetStdlibVersion = targetStdlibVersion,
targetDir = (`syntax-rust-definition` / rustParserTargetDirectory).value,
generateIndex = false
)
log.info(s"Engine package created at $root")
}

ThisBuild / createEnginePackageNoIndex := {
createEnginePackageNoIndex.result.value
}

lazy val buildEngineDistributionNoIndex =
taskKey[Unit](
"Builds the engine distribution without generating indexes and optionally generating native image"
)
buildEngineDistributionNoIndex := Def.taskIf {
createEnginePackageNoIndex.value
if (shouldBuildNativeImage.value) {
createEnginePackage.value
(`engine-runner` / buildNativeImage).value
} else {
createEnginePackage.value
}
}.value

// This makes the buildEngineDistribution task usable as a dependency
// of other tasks.
ThisBuild / buildEngineDistribution := {
buildEngineDistribution.result.value
}

lazy val shouldBuildNativeImage = taskKey[Boolean](
"Whether native image should be build within buildEngineDistribution task"
)

ThisBuild / shouldBuildNativeImage := {
val prop = System.getenv("ENSO_LAUNCHER")
prop == "native" || prop == "debugnative"
}

ThisBuild / NativeImage.additionalOpts := {
val prop = System.getenv("ENSO_LAUNCHER")
if (prop == "native") {
Seq("-O3")
} else {
Seq("-ea", "-Ob", "-H:GenerateDebugInfo=1")
}
}

ThisBuild / engineDistributionRoot := {
engineDistributionRoot.value
}

lazy val buildEngineDistributionNoIndex =
taskKey[Unit]("Builds the engine distribution without generating indexes")
buildEngineDistributionNoIndex := {
ThisBuild / buildEngineDistributionNoIndex := {
updateLibraryManifests.value
val modulesToCopy = componentModulesPaths.value
val root = engineDistributionRoot.value
Expand All @@ -5195,10 +5207,49 @@ buildEngineDistributionNoIndex := {
log.info(s"Engine package created at $root")
}

lazy val shouldBuildNativeImage = taskKey[Boolean](
"Whether native image should be build within buildEngineDistribution task"
)

ThisBuild / shouldBuildNativeImage := {
GraalVM.EnsoLauncher.native
}

ThisBuild / NativeImage.additionalOpts := {
if (GraalVM.EnsoLauncher.shell) {
Seq()
} else {
var opts = if (GraalVM.EnsoLauncher.release) {
Seq("-O3")
} else {
Seq("-Ob")
}

if (GraalVM.EnsoLauncher.debug) {
opts = opts ++ Seq("-H:GenerateDebugInfo=1")
}
if (GraalVM.EnsoLauncher.test) {
opts = opts ++ Seq("-ea")
}
opts
}
}

ThisBuild / engineDistributionRoot := {
engineDistributionRoot.value
}

lazy val buildEngineDistribution =
taskKey[Unit]("Builds the engine distribution")
buildEngineDistribution := {
buildEngineDistributionNoIndex.value
createEnginePackage.value
}

// This makes the buildEngineDistributionNoIndex task usable as a dependency
// of other tasks.
ThisBuild / buildEngineDistributionNoIndex := {
buildEngineDistributionNoIndex.result.value
ThisBuild / buildEngineDistribution := {
buildEngineDistribution.result.value
}

lazy val runEngineDistribution =
Expand Down Expand Up @@ -5284,6 +5335,7 @@ buildStdLib := Def.inputTaskDyn {

lazy val pkgStdLibInternal = inputKey[Unit]("Use `buildStdLib`")
pkgStdLibInternal := Def.inputTask {
buildEngineDistributionNoIndex.value
val cmd = allStdBits.parsed
val root = engineDistributionRoot.value
val log: sbt.Logger = streams.value.log
Expand Down
34 changes: 12 additions & 22 deletions build_tools/build/src/engine.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,43 +180,33 @@ pub enum EngineLauncher {
/// The binary inside the engine distribution will be built as an optimized native image
Native,
/// The binary inside the engine distribution will be built as native image with assertions
/// enabled but no debug information
TestNative,
/// The binary inside the engine distribution will be built as native image with assertions
/// enabled and debug information
DebugNative,
TestDebugNative,
/// The binary inside the engine distribution will be a shell script
#[default]
Shell,
}

impl FromStr for EngineLauncher {
type Err = anyhow::Error;

fn from_str(s: &str) -> Result<Self> {
match s {
"native" => Ok(Self::Native),
"debugnative" => Ok(Self::DebugNative),
"shell" => Ok(Self::Shell),
_ => bail!("Invalid Engine Launcher type: {}", s),
}
bail!("Parsing of ENSO_LAUNCHER isn't needed: {}", s)
}
}

impl From<EngineLauncher> for String {
fn from(value: EngineLauncher) -> Self {
match value {
impl Display for EngineLauncher {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
let str = match self {
EngineLauncher::Native => "native".to_string(),
EngineLauncher::DebugNative => "debugnative".to_string(),
EngineLauncher::TestNative => "native,test".to_string(),
EngineLauncher::TestDebugNative => "native,test,debug".to_string(),
EngineLauncher::Shell => "shell".to_string(),
}
}
}
};

impl Display for EngineLauncher {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
match *self {
EngineLauncher::Native => write!(f, "native"),
EngineLauncher::DebugNative => write!(f, "debugnative"),
EngineLauncher::Shell => write!(f, "shell"),
}
write!(f, "{}", str)
}
}

Expand Down
2 changes: 1 addition & 1 deletion build_tools/build/src/engine/context.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ impl RunContext {
sbt.call_arg("syntax-rust-definition/Runtime/managedClasspath").await?;
}
if self.config.build_native_runner {
env::ENSO_LAUNCHER.set(&engine::EngineLauncher::DebugNative)?;
env::ENSO_LAUNCHER.set(&engine::EngineLauncher::TestNative)?;
}

// TODO: Once the native image is production ready, we should switch to
Expand Down
33 changes: 19 additions & 14 deletions docs/infrastructure/native-image.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,25 +206,30 @@ one of the following:

- `shell`: The default value. `buildEngineDistribution` command does not build
the native image.
- `debugnative`: `buildEngineDistribution` command builds native image with
assertions enabled (`-ea`). Useful for running tests on the CI.
- `native`: `buildEngineDistribution` command builds native image with
assertions disabled (`-ea`). Turns on maximal optimizations which may increase
the build time.

To generate the Native Image for runner either explicitly execute

```bash
sbt> engine-runner/buildNativeImage
```

or
- `native`: `buildEngineDistribution` command builds native image in _release
mode_ - e.g. turns on maximal optimizations increasing the build time.
- There are additional variants of `native` useful for _development_. They are
specified as comma separated attributes following `native`:
- using `native,fast` turns on _native image_ build, but disables
optimizations - e.g. produces build similar to _release mode_, but more
quickly
- using `native,test` _enables assertions_ - e.g. it instructs
`buildEngineDistribution` command to build native image with assertions
enabled (`-ea`). Useful for running Enso tests in the _native mode_.
- using `native,debug` generates _debugging informations_ for VSCode _native
image debugger_
- using `native,-ls` disables support for _language server_ in the generated
binary
- it is possible to combine all features - e.g. use `debug,fast,test,native`

To test _native image_ launcher choose one of the `native` configurations and
invoke:

```bash
$ ENSO_LAUNCHER=native sbt buildEngineDistribution
```

and execute any program with that binary - for example `test/Base_Tests`
then execute any program with that binary - for example `test/Base_Tests`

```bash
$ ./built-distribution/enso-engine-*/enso-*/bin/enso --run test/Base_Tests
Expand Down
3 changes: 2 additions & 1 deletion engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.LinkedHashSet;
import java.util.TreeSet;
import org.enso.compiler.core.EnsoParser;
Expand Down Expand Up @@ -110,7 +111,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs());
for (var nativeLib : nativeLibs) {
var out = new File(nativeLibDir, nativeLib.getName());
Files.copy(nativeLib.toPath(), out.toPath());
Files.copy(nativeLib.toPath(), out.toPath(), StandardCopyOption.REPLACE_EXISTING);
nativeLibPaths.add(out.getAbsolutePath());
}
}
Expand Down
Loading

0 comments on commit 8cc8c4f

Please sign in to comment.