Skip to content

Commit 79d9afb

Browse files
romanowskiromanowski
authored
Graal scala3 (#830)
* Add tests for native image generation for Scala 3 * Create processor and runtime component for transforamtion * hook it up in the build * Add fixes for native image with Scala 3 * Add support for pathing jars * review tweaks * Do not use coursier cache Switch scala-cli to use TempCache and create DirCache for our build Move some classes from BytecodeProcessor to dedicated files
1 parent bc12b78 commit 79d9afb

File tree

11 files changed

+529
-39
lines changed

11 files changed

+529
-39
lines changed

‎build.sc‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import $file.project.settings, settings.{
1212
PublishLocalNoFluff,
1313
ScalaCliCrossSbtModule,
1414
ScalaCliScalafixModule,
15+
ScalaCliCompile,
1516
localRepoResourcePath,
1617
platformExecutableJarExtension,
1718
workspaceDirName
@@ -35,7 +36,7 @@ implicit def millModuleBasePath: define.BasePath =
3536

3637
object cli extends Cli
3738
// remove once migrate to Scala 3
38-
object cli3 extends Cli { override def myScalaVersion = Scala.scala3 }
39+
object cli3 extends Cli3
3940
object `cli-options` extends CliOptions
4041
object `build-macros` extends Cross[BuildMacros](Scala.defaultInternal, Scala.scala3)
4142
object options extends Cross[Options](Scala.defaultInternal, Scala.scala3)
@@ -47,6 +48,12 @@ object runner extends Cross[Runner](Scala.all: _*)
4748
object `test-runner` extends Cross[TestRunner](Scala.all: _*)
4849
object `bloop-rifle` extends Cross[BloopRifle](Scala.all: _*)
4950
object `tasty-lib` extends Cross[TastyLib](Scala.all: _*)
51+
// Runtime classes used within native image on Scala 3 replacing runtime from Scala
52+
object `scala3-runtime` extends Scala3Runtime
53+
// Logic to process classes that is shared between build and the scala-cli itself
54+
object `scala3-graal` extends Cross[Scala3Graal](Scala.defaultInternal, Scala.scala3)
55+
// Main app used to process classpath within build itself
56+
object `scala3-graal-processor` extends Scala3GraalProcessor
5057

5158
object stubs extends JavaModule with ScalaCliPublishModule {
5259
def javacOptions = T {
@@ -472,11 +479,41 @@ class Options(val crossScalaVersion: String) extends BuildLikeModule {
472479
}
473480
}
474481

475-
trait ScalaParse extends SbtModule with ScalaCliPublishModule with settings.ScalaCliCompile {
482+
trait ScalaParse extends SbtModule with ScalaCliPublishModule with ScalaCliCompile {
476483
def ivyDeps = super.ivyDeps() ++ Agg(Deps.scalaparse)
477484
def scalaVersion = Scala.defaultInternal
478485
}
479486

487+
trait Scala3Runtime extends SbtModule with ScalaCliPublishModule with ScalaCliCompile {
488+
def ivyDeps = super.ivyDeps()
489+
def scalaVersion = Scala.scala3
490+
}
491+
492+
class Scala3Graal(val crossScalaVersion: String) extends BuildLikeModule {
493+
def ivyDeps = super.ivyDeps() ++ Agg(
494+
Deps.asm,
495+
Deps.osLib
496+
)
497+
498+
def resources = T.sources {
499+
val extraResourceDir = T.dest / "extra"
500+
// scala3RuntimeFixes.jar is also used within
501+
// resource-config.json and BytecodeProcessor.scala
502+
os.copy.over(
503+
`scala3-runtime`.jar().path,
504+
extraResourceDir / "scala3RuntimeFixes.jar",
505+
createFolders = true
506+
)
507+
super.resources() ++ Seq(mill.PathRef(extraResourceDir))
508+
}
509+
}
510+
511+
trait Scala3GraalProcessor extends ScalaModule {
512+
def moduleDeps = Seq(`scala3-graal`(Scala.scala3))
513+
def scalaVersion = Scala.scala3
514+
def finalMainClass = "scala.cli.graal.CoursierCacheProcessor"
515+
}
516+
480517
class Build(val crossScalaVersion: String) extends BuildLikeModule {
481518
def moduleDeps = Seq(
482519
`options`(),
@@ -541,7 +578,7 @@ class Build(val crossScalaVersion: String) extends BuildLikeModule {
541578
}
542579
}
543580

544-
trait CliOptions extends SbtModule with ScalaCliPublishModule with settings.ScalaCliCompile {
581+
trait CliOptions extends SbtModule with ScalaCliPublishModule with ScalaCliCompile {
545582
def ivyDeps = super.ivyDeps() ++ Agg(
546583
Deps.caseApp,
547584
Deps.jsoniterCore,
@@ -571,7 +608,8 @@ trait Cli extends SbtModule with ProtoBuildModule with CliLaunchers
571608
def moduleDeps = Seq(
572609
build(myScalaVersion),
573610
`cli-options`,
574-
`test-runner`(myScalaVersion)
611+
`test-runner`(myScalaVersion),
612+
`scala3-graal`(myScalaVersion)
575613
)
576614

577615
def repositories = super.repositories ++ customRepositories
@@ -602,6 +640,24 @@ trait Cli extends SbtModule with ProtoBuildModule with CliLaunchers
602640
}
603641
}
604642

643+
trait Cli3 extends Cli {
644+
override def myScalaVersion = Scala.scala3
645+
646+
override def nativeImageClassPath = T {
647+
val classpath = super.nativeImageClassPath().map(_.path).mkString(File.pathSeparator)
648+
val cache = T.dest / "native-cp"
649+
// `scala3-graal-processor`.run() do not give me output and I cannot pass dynamically computed values like classpath
650+
val res = mill.modules.Jvm.callSubprocess(
651+
mainClass = `scala3-graal-processor`.finalMainClass(),
652+
classPath = `scala3-graal-processor`.runClasspath().map(_.path),
653+
mainArgs = Seq(cache.toNIO.toString, classpath),
654+
workingDir = os.pwd
655+
)
656+
val cp = res.out.text
657+
cp.split(File.pathSeparator).toSeq.map(p => mill.PathRef(os.Path(p)))
658+
}
659+
}
660+
605661
trait CliIntegrationBase extends SbtModule with ScalaCliPublishModule with HasTests
606662
with ScalaCliScalafixModule {
607663
def scalaVersion = sv

‎modules/cli/src/main/resources/META-INF/native-image/org.virtuslab/scala-cli-core/resource-config.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818
},
1919
{
2020
"pattern": "\\Qcommons-logging.properties\\E"
21+
},
22+
{
23+
"pattern": ".*scala3RuntimeFixes.jar$"
2124
}
2225
]
2326
},

‎modules/cli/src/main/scala/scala/cli/packaging/NativeImage.scala‎

Lines changed: 63 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import scala.annotation.tailrec
66
import scala.build.internal.{NativeBuilderHelper, Runner}
77
import scala.build.{Build, Logger}
88
import scala.cli.errors.GraalVMNativeImageError
9+
import scala.cli.graal.{BytecodeProcessor, TempCache}
910
import scala.util.Properties
1011

1112
object NativeImage {
@@ -231,46 +232,75 @@ object NativeImage {
231232
maybeWithManifestClassPath(
232233
createManifest = Properties.isWin,
233234
classPath = originalClasspath.map(os.Path(_, os.pwd))
234-
) { classPath =>
235-
val args = extraOptions ++ Seq(
236-
s"-H:Path=${dest / os.up}",
237-
s"-H:Name=${dest.last.stripSuffix(".exe")}", // FIXME Case-insensitive strip suffix?
238-
"-cp",
239-
classPath.map(_.toString).mkString(File.pathSeparator),
240-
mainClass
241-
)
235+
) { processedClassPath =>
236+
val (classPath, toClean, scala3extraOptions) =
237+
if (!build.scalaParams.scalaBinaryVersion.startsWith("3"))
238+
(processedClassPath, Seq[os.Path](), Seq[String]())
239+
else {
240+
val cpString = processedClassPath.mkString(File.pathSeparator)
241+
val processed = BytecodeProcessor.processClassPath(cpString, TempCache).toSeq
242+
val nativeConfigFile = os.temp(suffix = ".json")
243+
os.write.over(
244+
nativeConfigFile,
245+
"""[
246+
| {
247+
| "name": "sun.misc.Unsafe",
248+
| "allDeclaredConstructors": true,
249+
| "allPublicConstructors": true,
250+
| "allDeclaredMethods": true,
251+
| "allDeclaredFields": true
252+
| }
253+
|]
254+
|""".stripMargin
255+
)
256+
val cp = processed.map(_.path)
257+
val options = Seq(s"-H:ReflectionConfigurationFiles=$nativeConfigFile")
242258

243-
maybeWithShorterGraalvmHome(javaHome.javaHome, logger) { graalVMHome =>
259+
(cp, nativeConfigFile +: BytecodeProcessor.toClean(processed), options)
260+
}
244261

245-
val nativeImageCommand = ensureHasNativeImageCommand(graalVMHome, logger)
246-
val command = nativeImageCommand.toString +: args
262+
try {
263+
val args = extraOptions ++ scala3extraOptions ++ Seq(
264+
s"-H:Path=${dest / os.up}",
265+
s"-H:Name=${dest.last.stripSuffix(".exe")}", // FIXME Case-insensitive strip suffix?
266+
"-cp",
267+
classPath.map(_.toString).mkString(File.pathSeparator),
268+
mainClass
269+
)
247270

248-
val exitCode =
249-
if (Properties.isWin)
250-
vcvarsOpt match {
251-
case Some(vcvars) =>
252-
runFromVcvarsBat(command, vcvars, nativeImageWorkDir, logger)
253-
case None =>
254-
Runner.run("unused", command, logger).waitFor()
255-
}
256-
else
257-
Runner.run("unused", command, logger).waitFor()
258-
if (exitCode == 0) {
259-
val actualDest =
271+
maybeWithShorterGraalvmHome(javaHome.javaHome, logger) { graalVMHome =>
272+
273+
val nativeImageCommand = ensureHasNativeImageCommand(graalVMHome, logger)
274+
val command = nativeImageCommand.toString +: args
275+
276+
val exitCode =
260277
if (Properties.isWin)
261-
if (dest.last.endsWith(".exe")) dest
262-
else dest / os.up / s"${dest.last}.exe"
278+
vcvarsOpt match {
279+
case Some(vcvars) =>
280+
runFromVcvarsBat(command, vcvars, nativeImageWorkDir, logger)
281+
case None =>
282+
Runner.run("unused", command, logger).waitFor()
283+
}
263284
else
264-
dest
265-
NativeBuilderHelper.updateProjectAndOutputSha(
266-
actualDest,
267-
nativeImageWorkDir,
268-
cacheData.projectSha
269-
)
285+
Runner.run("unused", command, logger).waitFor()
286+
if (exitCode == 0) {
287+
val actualDest =
288+
if (Properties.isWin)
289+
if (dest.last.endsWith(".exe")) dest
290+
else dest / os.up / s"${dest.last}.exe"
291+
else
292+
dest
293+
NativeBuilderHelper.updateProjectAndOutputSha(
294+
actualDest,
295+
nativeImageWorkDir,
296+
cacheData.projectSha
297+
)
298+
}
299+
else
300+
throw new GraalVMNativeImageError
270301
}
271-
else
272-
throw new GraalVMNativeImageError
273302
}
303+
finally util.Try(toClean.foreach(os.remove.all))
274304
}
275305
}
276306
else

‎modules/integration/src/test/scala/scala/cli/integration/TestNativeImageOnScala3.scala‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package scala.cli.integration
2+
3+
import com.eed3si9n.expecty.Expecty.expect
4+
5+
import scala.util.Properties
6+
7+
class TestNativeImageOnScala3 extends munit.FunSuite {
8+
9+
def runTest(args: String*)(expectedLines: String*)(code: String): Unit = {
10+
val dest =
11+
if (Properties.isWin) "testApp.exe"
12+
else "testApp"
13+
14+
val inputs = TestInputs(Seq(os.rel / "Hello.scala" -> code))
15+
inputs.fromRoot { root =>
16+
os.proc(
17+
TestUtil.cli,
18+
"package",
19+
".",
20+
"--native-image",
21+
"-o",
22+
dest,
23+
"--",
24+
"--no-fallback"
25+
).call(
26+
cwd = root,
27+
stdin = os.Inherit,
28+
stdout = os.Inherit
29+
)
30+
31+
expect(os.isFile(root / dest))
32+
33+
// FIXME Check that dest is indeed a binary?
34+
35+
val res = os.proc(root / dest, args).call(cwd = root)
36+
val outputLines = res.out.text().trim.linesIterator.to(Seq)
37+
expect(expectedLines == outputLines)
38+
}
39+
}
40+
41+
test("lazy vals") {
42+
runTest("1")("2") {
43+
"""//> using scala "3.1.1"
44+
|class A(a: String) { lazy val b = a.toInt + 1 }
45+
|@main def add1(i: String) = println(A(i).b)
46+
|""".stripMargin
47+
}
48+
}
49+
50+
test("lazy vals and enums with default scala version") {
51+
runTest("1")("2", "A") {
52+
"""class A(a: String) { lazy val b = a.toInt + 1 }
53+
|enum Ala:
54+
| case A
55+
| case B
56+
|@main def add1(i: String) =
57+
| println(A(i).b)
58+
| println(Ala.valueOf("A"))
59+
|""".stripMargin
60+
}
61+
}
62+
}

‎modules/scala3-graal-processor/src/scala/cli/graal/CoursierCacheProcessor.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package scala.cli.graal
2+
3+
import java.io.File
4+
import java.nio.channels.FileChannel
5+
6+
object CoursierCacheProcessor {
7+
def main(args: Array[String]) = {
8+
val List(cacheDir, classpath) = args.toList
9+
val cache = DirCache(os.Path(cacheDir, os.pwd))
10+
11+
val newCp = BytecodeProcessor.processClassPath(classpath, cache).map(_.nioPath)
12+
13+
println(newCp.mkString(File.pathSeparator))
14+
}
15+
}

0 commit comments

Comments
 (0)