Add embedding resources support for Scala Native

jchyb · lwronski · commit 7145b1fb623a · 2022-09-12T09:29:07.000+02:00
Works automatically when a resourceDir option or using Directory is
added. Having that working effectively required fixing duplicated
classpath in Scala Native packaging. Will work only on Scala Native
0.4.4 and up. Caching was also improved, as previously only the cli
option resources would be cached, and using directive resources would
be ignored.

A --embed-resources option was also added, allowing to disable resource
embedding (which will also make the resources API not work). The use
case for this is using c interop without having unnecessary libraries
metadata embedded, taking up space in a binary file.
diff --git a/modules/build/src/main/scala/scala/build/Build.scala b/modules/build/src/main/scala/scala/build/Build.scala
@@ -472,34 +472,60 @@ object Build {
 
   def scalaNativeSupported(
     options: BuildOptions,
-    inputs: Inputs
+    inputs: Inputs,
+    logger: Logger
   ): Either[BuildException, Option[ScalaNativeCompatibilityError]] =
     either {
       val scalaParamsOpt = value(options.scalaParams)
       scalaParamsOpt.flatMap { scalaParams =>
-        val scalaVersion  = scalaParams.scalaVersion
-        val nativeVersion = options.scalaNativeOptions.numeralVersion
-        val isCompatible = nativeVersion match {
-          case Some(snNumeralVer) =>
-            if (snNumeralVer < SNNumeralVersion(0, 4, 1) && Properties.isWin)
-              false
-            else if (scalaVersion.startsWith("3.0"))
-              false
-            else if (scalaVersion.startsWith("3") || scalaVersion.startsWith("2.12"))
-              snNumeralVer >= SNNumeralVersion(0, 4, 3)
-            else if (scalaVersion.startsWith("2.13"))
-              true
-            else false
-          case None => false
-        }
-        if (isCompatible) None
-        else
-          Some(
+        val scalaVersion       = scalaParams.scalaVersion
+        val nativeVersionMaybe = options.scalaNativeOptions.numeralVersion
+        def snCompatError =
+          Left(
             new ScalaNativeCompatibilityError(
               scalaVersion,
               options.scalaNativeOptions.finalVersion
             )
           )
+        def warnIncompatibleNativeOptions(numeralVersion: SNNumeralVersion) =
+          if (
+            numeralVersion < SNNumeralVersion(0, 4, 4)
+            && options.scalaNativeOptions.embedResources.isDefined
+          )
+            logger.diagnostic(
+              "This Scala Version cannot embed resources, regardless of the options used."
+            )
+
+        val numeralOrError: Either[ScalaNativeCompatibilityError, SNNumeralVersion] =
+          nativeVersionMaybe match {
+            case Some(snNumeralVer) =>
+              if (snNumeralVer < SNNumeralVersion(0, 4, 1) && Properties.isWin)
+                snCompatError
+              else if (scalaVersion.startsWith("3.0"))
+                snCompatError
+              else if (scalaVersion.startsWith("3"))
+                if (snNumeralVer >= SNNumeralVersion(0, 4, 3)) Right(snNumeralVer)
+                else snCompatError
+              else if (scalaVersion.startsWith("2.13"))
+                Right(snNumeralVer)
+              else if (scalaVersion.startsWith("2.12"))
+                if (
+                  inputs.sourceFiles().forall {
+                    case _: Inputs.AnyScript => snNumeralVer >= SNNumeralVersion(0, 4, 3)
+                    case _                   => true
+                  }
+                ) Right(snNumeralVer)
+                else snCompatError
+              else snCompatError
+            case None => snCompatError
+          }
+
+        numeralOrError match {
+          case Left(compatError) => Some(compatError)
+          case Right(snNumeralVersion) =>
+            warnIncompatibleNativeOptions(snNumeralVersion)
+            None
+        }
       }
     }
 
@@ -981,7 +1007,7 @@ object Build {
   ): Either[BuildException, Build] = either {
 
     if (options.platform.value == Platform.Native)
-      value(scalaNativeSupported(options, inputs)) match {
+      value(scalaNativeSupported(options, inputs, logger)) match {
         case None        =>
         case Some(error) => value(Left(error))
       }
diff --git a/modules/build/src/main/scala/scala/build/Inputs.scala b/modules/build/src/main/scala/scala/build/Inputs.scala
@@ -114,11 +114,6 @@ final case class Inputs(
           case dirInput: Inputs.Directory =>
             Seq("dir:") ++ Inputs.singleFilesFromDirectory(dirInput, enableMarkdown)
               .map(file => s"${file.path}:" + os.read(file.path))
-          case resDirInput: Inputs.ResourceDirectory =>
-            // Resource changes for SN require relinking, so they should also be hashed
-            Seq("resource-dir:") ++ os.walk(resDirInput.path)
-              .filter(os.isFile(_))
-              .map(filePath => s"$filePath:" + os.read(filePath))
           case _ => Seq(os.read(elem.path))
         }
         (Iterator(elem.path.toString) ++ content.iterator ++ Iterator("\n")).map(bytes)
diff --git a/modules/cli-options/src/main/scala/scala/cli/commands/ScalaNativeOptions.scala b/modules/cli-options/src/main/scala/scala/cli/commands/ScalaNativeOptions.scala
@@ -43,7 +43,12 @@ final case class ScalaNativeOptions(
   @Group("Scala Native")
   @Hidden
   @HelpMessage("Use default compile options")
-    nativeCompileDefaults: Option[Boolean] = None //TODO does it even work when we default it to true while handling?
+    nativeCompileDefaults: Option[Boolean] = None, //TODO does it even work when we default it to true while handling?
+
+  @Group("Scala Native")
+  @HelpMessage("Embed resources into the Scala Native binary (can be read with the Java resources API)")
+    embedResources: Option[Boolean] = None
+
 )
 // format: on
 
diff --git a/modules/cli/src/main/scala/scala/cli/commands/Package.scala b/modules/cli/src/main/scala/scala/cli/commands/Package.scala
@@ -912,7 +912,8 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
     logger: Logger
   ): Either[BuildException, Unit] = either {
 
-    val cliOptions = build.options.scalaNativeOptions.configCliOptions()
+    val cliOptions =
+      build.options.scalaNativeOptions.configCliOptions(!build.sources.resourceDirs.isEmpty)
 
     val setupPython = build.options.notForBloopOptions.doSetupPython.getOrElse(false)
     val pythonLdFlags =
@@ -943,7 +944,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
     if (cacheData.changed)
       Library.withLibraryJar(build, dest.last.stripSuffix(".jar")) { mainJar =>
 
-        val classpath = build.fullClassPath.map(_.toString) :+ mainJar.toString
+        val classpath = mainJar.toString +: build.artifacts.classPath.map(_.toString)
         val args =
           allCliOptions ++
             logger.scalaNativeCliInternalLoggerOptions ++
diff --git a/modules/cli/src/main/scala/scala/cli/internal/CachedBinary.scala b/modules/cli/src/main/scala/scala/cli/internal/CachedBinary.scala
@@ -4,7 +4,7 @@ import java.math.BigInteger
 import java.nio.charset.StandardCharsets
 import java.security.MessageDigest
 
-import scala.build.Build
+import scala.build.{Build, Inputs}
 import scala.build.internal.Constants
 
 object CachedBinary {
@@ -23,10 +23,48 @@ object CachedBinary {
     String.format(s"%040x", calculatedSum)
   }
 
+  private def hashResources(build: Build.Successful) = {
+    def hashResourceDir(path: os.Path) =
+      os.walk(path)
+        .filter(os.isFile(_))
+        .map { filePath =>
+          val md = MessageDigest.getInstance("SHA-1")
+          md.update(os.read.bytes(filePath))
+          s"$filePath:" + new BigInteger(1, md.digest()).toString()
+        }
+
+    val classpathResourceDirsIt =
+      build.options
+        .classPathOptions
+        .resourcesDir
+        .flatMap(dir => hashResourceDir(dir))
+        .iterator ++
+        Iterator("\n")
+
+    val projectResourceDirsIt = build.inputs.elements.iterator.flatMap {
+      case elem: Inputs.OnDisk =>
+        val content = elem match {
+          case resDirInput: Inputs.ResourceDirectory =>
+            hashResourceDir(resDirInput.path)
+          case _ => List.empty
+        }
+        Iterator(elem.path.toString) ++ content.iterator ++ Iterator("\n")
+      case _ =>
+        Iterator.empty
+    }
+
+    (classpathResourceDirsIt ++ projectResourceDirsIt)
+      .map(_.getBytes(StandardCharsets.UTF_8))
+  }
+
   private def projectSha(build: Build.Successful, config: List[String]) = {
     val md      = MessageDigest.getInstance("SHA-1")
     val charset = StandardCharsets.UTF_8
     md.update(build.inputs.sourceHash().getBytes(charset))
+    md.update("<resources>".getBytes())
+    // Resource changes for SN require relinking, so they should also be hashed
+    hashResources(build).foreach(md.update)
+    md.update("</resources>".getBytes())
     md.update(0: Byte)
     md.update("<config>".getBytes(charset))
     for (elem <- config) {
diff --git a/modules/cli/src/test/scala/cli/tests/CachedBinaryTests.scala b/modules/cli/src/test/scala/cli/tests/CachedBinaryTests.scala
@@ -48,7 +48,7 @@ class CachedBinaryTests extends munit.FunSuite {
         (_, _, maybeBuild) =>
           val build = maybeBuild.successfulOpt.get
 
-          val config        = build.options.scalaNativeOptions.configCliOptions()
+          val config = build.options.scalaNativeOptions.configCliOptions(resourcesExist = false)
           val nativeWorkDir = build.inputs.nativeWorkDir
           val destPath      = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}"
           // generate dummy output
@@ -65,7 +65,7 @@ class CachedBinaryTests extends munit.FunSuite {
         (_, _, maybeBuild) =>
           val build = maybeBuild.successfulOpt.get
 
-          val config        = build.options.scalaNativeOptions.configCliOptions()
+          val config = build.options.scalaNativeOptions.configCliOptions(resourcesExist = false)
           val nativeWorkDir = build.inputs.nativeWorkDir
           val destPath      = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}"
           // generate dummy output
@@ -91,7 +91,7 @@ class CachedBinaryTests extends munit.FunSuite {
         (_, _, maybeBuild) =>
           val build = maybeBuild.successfulOpt.get
 
-          val config        = build.options.scalaNativeOptions.configCliOptions()
+          val config = build.options.scalaNativeOptions.configCliOptions(resourcesExist = false)
           val nativeWorkDir = build.inputs.nativeWorkDir
           val destPath      = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}"
           // generate dummy output
@@ -118,7 +118,7 @@ class CachedBinaryTests extends munit.FunSuite {
         (_, _, maybeBuild) =>
           val build = maybeBuild.successfulOpt.get
 
-          val config        = build.options.scalaNativeOptions.configCliOptions()
+          val config = build.options.scalaNativeOptions.configCliOptions(resourcesExist = false)
           val nativeWorkDir = build.inputs.nativeWorkDir
           val destPath      = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}"
           // generate dummy output
@@ -145,7 +145,7 @@ class CachedBinaryTests extends munit.FunSuite {
         (root, _, maybeBuild) =>
           val build = maybeBuild.successfulOpt.get
 
-          val config        = build.options.scalaNativeOptions.configCliOptions()
+          val config = build.options.scalaNativeOptions.configCliOptions(resourcesExist = false)
           val nativeWorkDir = build.inputs.nativeWorkDir
           val destPath      = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}"
           os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true)
@@ -171,7 +171,7 @@ class CachedBinaryTests extends munit.FunSuite {
         (_, _, maybeBuild) =>
           val build = maybeBuild.successfulOpt.get
 
-          val config        = build.options.scalaNativeOptions.configCliOptions()
+          val config = build.options.scalaNativeOptions.configCliOptions(resourcesExist = false)
           val nativeWorkDir = build.inputs.nativeWorkDir
           val destPath      = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}"
           os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true)
@@ -192,7 +192,8 @@ class CachedBinaryTests extends munit.FunSuite {
               )
             )
           )
-          val updatedConfig = updatedBuild.options.scalaNativeOptions.configCliOptions()
+          val updatedConfig =
+            updatedBuild.options.scalaNativeOptions.configCliOptions(resourcesExist = false)
 
           val cacheAfterConfigUpdate =
             CachedBinary.getCacheData(
diff --git a/modules/directives/src/main/scala/scala/build/preprocessing/directives/UsingScalaNativeOptionsDirectiveHandler.scala b/modules/directives/src/main/scala/scala/build/preprocessing/directives/UsingScalaNativeOptionsDirectiveHandler.scala
@@ -25,7 +25,9 @@ case object UsingScalaNativeOptionsDirectiveHandler extends UsingDirectiveHandle
       |
       |`//> using nativeClang` _value_
       |
-      |`//> using nativeClangPP` _value_""".stripMargin
+      |`//> using nativeClangPP` _value_
+      |
+      |`//> using nativeEmbedResources` _true|false_""".stripMargin
 
   override def examples: Seq[String] = Seq(
     "//> using nativeVersion \"0.4.0\""
@@ -42,13 +44,15 @@ case object UsingScalaNativeOptionsDirectiveHandler extends UsingDirectiveHandle
       "native-linking",
       "native-clang",
       "native-clang-pp",
+      "native-no-embed",
       "nativeGc",
       "nativeMode",
       "nativeVersion",
       "nativeCompile",
       "nativeLinking",
       "nativeClang",
-      "nativeClangPP"
+      "nativeClangPP",
+      "nativeEmbedResources"
     )
 
   override def getValueNumberBounds(key: String): UsingDirectiveValueNumberBounds = key match {
@@ -61,9 +65,15 @@ case object UsingScalaNativeOptionsDirectiveHandler extends UsingDirectiveHandle
     case "native-compile" | "nativeCompile" => UsingDirectiveValueNumberBounds(1, Int.MaxValue)
   }
 
+  def getBooleanOption(groupedValues: GroupedScopedValuesContainer): Option[Boolean] =
+    groupedValues.scopedBooleanValues.map(_.positioned.value.toBoolean).headOption.orElse(Some(
+      true
+    ))
+
   override def getSupportedTypes(key: String): Set[UsingDirectiveValueKind] = Set(
     UsingDirectiveValueKind.STRING,
-    UsingDirectiveValueKind.NUMERIC
+    UsingDirectiveValueKind.NUMERIC,
+    UsingDirectiveValueKind.BOOLEAN
   )
 
   def handleValues(
@@ -101,6 +111,10 @@ case object UsingScalaNativeOptionsDirectiveHandler extends UsingDirectiveHandle
           ScalaNativeOptions(
             clangpp = Some(values.head.positioned.value)
           )
+        case "native-embed-resources" | "nativeEmbedResources" =>
+          ScalaNativeOptions(
+            embedResources = getBooleanOption(groupedValuesContainer)
+          )
       }
       val options = BuildOptions(scalaNativeOptions = scalaNativeOptions)
       ProcessedDirective(Some(options), Nil)
diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala
@@ -258,6 +258,40 @@ abstract class RunTestDefinitions(val scalaVersionOpt: Option[String])
     }
   }
 
+  test("Resource embedding in Scala Native") {
+    val projectDir       = "nativeres"
+    val resourceContent  = "resource contents"
+    val resourceFileName = "embeddedfile.txt"
+    val inputs = TestInputs(
+      os.rel / projectDir / "main.scala" ->
+        s"""|//> using platform "scala-native"
+            |//> using resourceDir "resources"
+            |
+            |import java.nio.charset.StandardCharsets
+            |import java.io.{BufferedReader, InputStreamReader}
+            |
+            |object Main {
+            |  def main(args: Array[String]): Unit = {
+            |    val inputStream = getClass().getResourceAsStream("/$resourceFileName")
+            |    val nativeResourceText = new BufferedReader(
+            |      new InputStreamReader(inputStream, StandardCharsets.UTF_8)
+            |    ).readLine()
+            |    println(nativeResourceText)
+            |  }
+            |}
+            |""".stripMargin,
+      os.rel / projectDir / "resources" / resourceFileName -> resourceContent
+    )
+    inputs.fromRoot { root =>
+      val output =
+        os.proc(TestUtil.cli, extraOptions, projectDir, "-q")
+          .call(cwd = root)
+          .out.trim()
+      println(output)
+      expect(output == resourceContent)
+    }
+  }
+
   if (actualScalaVersion.startsWith("3.1"))
     test("Scala 3 in Scala Native") {
       val message  = "using Scala 3 Native"
diff --git a/modules/options/src/main/scala/scala/build/options/ScalaNativeOptions.scala b/modules/options/src/main/scala/scala/build/options/ScalaNativeOptions.scala
@@ -17,7 +17,8 @@ final case class ScalaNativeOptions(
   linkingOptions: List[String] = Nil,
   linkingDefaults: Option[Boolean] = None,
   compileOptions: List[String] = Nil,
-  compileDefaults: Option[Boolean] = None
+  compileDefaults: Option[Boolean] = None,
+  embedResources: Option[Boolean] = None
 ) {
 
   def finalVersion = version.map(_.trim).filter(_.nonEmpty).getOrElse(Constants.scalaNativeVersion)
@@ -65,6 +66,15 @@ final case class ScalaNativeOptions(
   private def compileCliOptions(): List[String] =
     finalCompileOptions().flatMap(option => List("--compile-option", option))
 
+  private def resourcesCliOptions(resourcesExist: Boolean): List[String] =
+    if (embedResources.getOrElse(true))
+      (numeralVersion, resourcesExist) match {
+        case (Some(numeralVersion), true) if numeralVersion >= SNNumeralVersion(0, 4, 4) =>
+          List("--embed-resources")
+        case _ => Nil
+      }
+    else Nil
+
   def platformSuffix: String =
     "native" + ScalaVersion.nativeBinary(finalVersion).getOrElse(finalVersion)
 
@@ -100,13 +110,14 @@ final case class ScalaNativeOptions(
       output = None
     )
 
-  def configCliOptions(): List[String] =
+  def configCliOptions(resourcesExist: Boolean): List[String] =
     gcCliOption() ++
       modeCliOption() ++
       clangCliOption() ++
       clangppCliOption() ++
       linkingCliOptions() ++
-      compileCliOptions()
+      compileCliOptions() ++
+      resourcesCliOptions(resourcesExist)
 
 }
 
diff --git a/website/docs/reference/cli-options.md b/website/docs/reference/cli-options.md
diff --git a/website/docs/reference/directives.md b/website/docs/reference/directives.md
