Generate native launcher of scala-cli for M1 (#1396)

lwronski · web-flow · commit 562ed30a9342 · 2022-09-27T19:25:43.000+02:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -285,6 +285,57 @@ jobs:
         SCALA_CLI_IT_GROUP: 3
         SCALA_CLI_SODIUM_JNI_ALLOW: false
 
+  generate-macos-m1-launcher:
+    timeout-minutes: 120
+    runs-on: "macOS-m1"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: coursier/setup-action@f883d08305acbc28e5e5363bf5ec086397627021
+        with:
+          apps: ""
+          jvm: "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.2.0/graalvm-ce-java17-darwin-aarch64-22.2.0.tar.gz"
+      - name: Generate native launcher
+        run: .github/scripts/generate-native-image.sh
+      #      - run: ./mill -i ci.setShouldPublish
+      #      - name: Build OS packages TODO generate os packages for M1
+      #        if: env.SHOULD_PUBLISH == 'true'
+      #        run: .github/scripts/generate-os-packages.sh
+      - name: Copy artifacts
+        run: ./mill -i copyDefaultLauncher artifacts/
+      - uses: actions/upload-artifact@v3
+        with:
+          name: macos-m1-launchers
+          path: artifacts/
+          if-no-files-found: error
+          retention-days: 2
+
+  native-macos-m1-tests:
+    needs: generate-macos-m1-launcher
+    timeout-minutes: 120
+    runs-on: "macOS-m1"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: coursier/setup-action@f883d08305acbc28e5e5363bf5ec086397627021
+        with:
+          apps: ""
+          jvm: "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.2.0/graalvm-ce-java17-darwin-aarch64-22.2.0.tar.gz"
+      - uses: actions/download-artifact@v3
+        with:
+          name: macos-m1-launchers
+          path: artifacts/
+      - name: Native integration tests
+        run: ./mill -i nativeIntegrationTests
+        env:
+          UPDATE_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SCALA_CLI_IT_FORCED_LAUNCHER_DIRECTORY: artifacts/
+          SCALA_CLI_SODIUM_JNI_ALLOW: false
+
   generate-windows-launcher:
     timeout-minutes: 120
     runs-on: "windows-latest"
@@ -809,6 +860,7 @@ jobs:
       - native-macos-tests-1
       - native-macos-tests-2
       - native-macos-tests-3
+      - native-macos-m1-tests
       - native-windows-tests-1
       - native-windows-tests-2
       - native-windows-tests-3
@@ -843,6 +895,11 @@ jobs:
       with:
         name: macos-launchers
         path: artifacts/
+    - uses: actions/download-artifact@v3
+      if: env.SHOULD_PUBLISH == 'true'
+      with:
+        name: macos-m1-launchers
+        path: artifacts/
     - uses: actions/download-artifact@v3
       if: env.SHOULD_PUBLISH == 'true'
       with:
diff --git a/modules/cli/src/main/scala/scala/cli/commands/github/LibSodiumJni.scala b/modules/cli/src/main/scala/scala/cli/commands/github/LibSodiumJni.scala
@@ -87,7 +87,7 @@ object LibSodiumJni {
       else sys.error(s"Unrecognized operating system: ${sys.props("os.name")}")
 
     val dep =
-      dep"io.github.alexarchambault.tmp.libsodiumjni:libsodiumjni:$libsodiumjniVersion,intransitive,classifier=$classifier,ext=$ext,type=$ext"
+      dep"org.virtuslab.scala-cli:libsodiumjni:$libsodiumjniVersion,intransitive,classifier=$classifier,ext=$ext,type=$ext"
     val fetch = coursier.Fetch()
       .addDependencies(dep.toCs)
       .addArtifactTypes(Type(ext))
diff --git a/modules/core/src/main/scala/scala/build/Os.scala b/modules/core/src/main/scala/scala/build/Os.scala
@@ -1,5 +1,7 @@
 package scala.build
 
+import java.util.Locale
+
 import scala.util.Properties
 
 object Os {
@@ -8,4 +10,6 @@ object Os {
       os.Path(os.pwd.toIO.getCanonicalFile)
     else
       os.pwd
+  lazy val isArmArchitecture: Boolean =
+    sys.props.getOrElse("os.arch", "").toLowerCase(Locale.ROOT) == "aarch64"
 }
diff --git a/modules/core/src/main/scala/scala/build/internals/OsLibc.scala b/modules/core/src/main/scala/scala/build/internals/OsLibc.scala
@@ -5,6 +5,7 @@ import coursier.jvm.{JavaHome, JvmIndex}
 import java.io.IOException
 import java.nio.charset.Charset
 
+import scala.build.Os
 import scala.build.blooprifle.VersionUtil.parseJavaVersion
 import scala.util.{Properties, Try}
 
@@ -68,7 +69,8 @@ object OsLibc {
       .forall(_ >= 17)
     if (os == "linux-musl") s"liberica:$jvmVersion" // zulu could work too
     else if (java17OrHigher) s"temurin:$jvmVersion"
-    else s"adopt:$jvmVersion"
+    else if (Os.isArmArchitecture) s"zulu:$jvmVersion" // adopt doesn't support Java 8 on macOS arm
+    else s"temurin:$jvmVersion"
   }
 
   def defaultJvm(os: String): String = {
diff --git a/modules/integration/src/test/scala/scala/cli/integration/NativePackagerTests.scala b/modules/integration/src/test/scala/scala/cli/integration/NativePackagerTests.scala
@@ -48,7 +48,7 @@ class NativePackagerTests extends ScalaCliSuite {
         val pkgAppPath = root / pkgAppFile
         expect(os.isFile(pkgAppPath))
 
-        if (TestUtil.isCI) {
+        if (TestUtil.isCI && !TestUtil.isArmArchitecture) {
           os.proc("installer", "-pkg", pkgAppFile, "-target", "CurrentUserHomeDirectory").call(
             cwd = root,
             stdin = os.Inherit,
diff --git a/modules/integration/src/test/scala/scala/cli/integration/SparkTests212.scala b/modules/integration/src/test/scala/scala/cli/integration/SparkTests212.scala
@@ -105,7 +105,7 @@ class SparkTests212 extends SparkTestDefinitions {
         .call(cwd = root)
 
       val java8Home =
-        os.Path(os.proc(TestUtil.cs, "java-home", "--jvm", "8").call().out.trim(), os.pwd)
+        os.Path(os.proc(TestUtil.cs, "java-home", "--jvm", "zulu:8").call().out.trim(), os.pwd)
 
       val ext = if (Properties.isWin) ".cmd" else ""
       val res =
diff --git a/modules/integration/src/test/scala/scala/cli/integration/TestUtil.scala b/modules/integration/src/test/scala/scala/cli/integration/TestUtil.scala
@@ -3,6 +3,7 @@ package scala.cli.integration
 import os.{CommandResult, Path}
 
 import java.io.File
+import java.util.Locale
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.concurrent.{ExecutorService, Executors, ScheduledExecutorService, ThreadFactory}
 
@@ -20,6 +21,9 @@ object TestUtil {
   val detectCliPath                = if (TestUtil.isNativeCli) TestUtil.cliPath else "scala-cli"
   val cli: Seq[String]             = cliCommand(cliPath)
 
+  lazy val isArmArchitecture: Boolean =
+    sys.props.getOrElse("os.arch", "").toLowerCase(Locale.ROOT) == "aarch64"
+
   def cliCommand(cliPath: String): Seq[String] =
     if (isNativeCli)
       Seq(cliPath)
diff --git a/modules/options/src/main/scala/scala/build/options/JavaOptions.scala b/modules/options/src/main/scala/scala/build/options/JavaOptions.scala
@@ -8,7 +8,7 @@ import dependency.AnyDependency
 import scala.build.internal.CsLoggerUtil._
 import scala.build.internal.OsLibc
 import scala.build.options.BuildOptions.JavaHomeInfo
-import scala.build.{Position, Positioned}
+import scala.build.{Os, Position, Positioned}
 import scala.concurrent.ExecutionContextExecutorService
 import scala.util.control.NonFatal
 
@@ -75,9 +75,14 @@ final case class JavaOptions(
             val enforceLiberica =
               finalJvmIndexOs == "linux-musl" &&
               jvmId.forall(c => c.isDigit || c == '.' || c == '-')
+            val enforceZulu =
+              Os.isArmArchitecture &&
+              jvmId.forall(c => c.isDigit || c == '.' || c == '-')
             val jvmId0 =
               if (enforceLiberica)
                 s"liberica:$jvmId" // FIXME Workaround, until this is automatically handled by coursier-jvm
+              else if (enforceZulu) // default jvmId adoptium doesn't support java 8 for M1
+                s"zulu:$jvmId"
               else
                 jvmId
             val javaHomeManager0 = javaHomeManager(archiveCache, cache, verbosity)
diff --git a/project/deps.sc b/project/deps.sc
@@ -2,6 +2,7 @@ import coursier.mavenRepositoryString
 import mill._, scalalib._
 
 import scala.util.Properties
+import $file.utils, utils.isArmArchitecture
 
 object Scala {
   def scala212     = "2.12.16"
@@ -60,6 +61,7 @@ object Deps {
     // jni-utils version may need to be sync-ed when bumping the coursier version
     def coursier           = "2.1.0-M6-53-gb4f448130"
     def coursierCli        = "2.1.0-M5-18-gfebf9838c"
+    def coursierM1Cli      = "2.1.0-M6-53-gb4f448130"
     def jsoniterScala      = "2.17.3"
     def jsoniterScalaJava8 = "2.13.5"
     def scalaMeta          = "4.5.13"
@@ -103,7 +105,7 @@ object Deps {
   def jsoniterMacrosJava8 =
     ivy"com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-macros:${Versions.jsoniterScalaJava8}"
   def libdaemonjvm  = ivy"io.github.alexarchambault.libdaemon::libdaemon:0.0.10"
-  def libsodiumjni  = ivy"io.github.alexarchambault.tmp.libsodiumjni:libsodiumjni:0.0.3"
+  def libsodiumjni  = ivy"org.virtuslab.scala-cli:libsodiumjni:0.0.3"
   def macroParadise = ivy"org.scalamacros:::paradise:2.1.1"
   def metaconfigTypesafe =
     ivy"com.geirsson::metaconfig-typesafe-config:0.11.1"
@@ -170,7 +172,8 @@ def graalVmJvmId       = s"graalvm-java$graalVmJavaVersion:$graalVmVersion"
 
 def csDockerVersion = Deps.Versions.coursierCli
 
-def buildCsVersion = Deps.Versions.coursierCli
+def buildCsVersion   = Deps.Versions.coursierCli
+def buildCsM1Version = Deps.Versions.coursierM1Cli
 
 // Native library used to encrypt GitHub secrets
 def libsodiumVersion = "1.0.18"
diff --git a/project/settings.sc b/project/settings.sc
@@ -2,7 +2,9 @@ import $ivy.`com.goyeau::mill-scalafix::0.2.8`
 import $ivy.`io.github.alexarchambault.mill::mill-native-image::0.1.19`
 import $ivy.`io.github.alexarchambault.mill::mill-scala-cli::0.1.0`
 
-import $file.deps, deps.{BuildDeps, Deps, Docker, alpineVersion, buildCsVersion, libsodiumVersion}
+import $file.deps,
+deps.{BuildDeps, Deps, Docker, alpineVersion, buildCsVersion, buildCsM1Version, libsodiumVersion}
+import $file.utils, utils.isArmArchitecture
 
 import com.goyeau.mill.scalafix.ScalafixModule
 import de.tobiasroeser.mill.vcs.version.VcsVersion
@@ -49,30 +51,35 @@ def fromPath(name: String): String =
 
 def cs: T[String] = T.persistent {
 
-  val ext  = if (Properties.isWin) ".exe" else ""
-  val dest = T.dest / s"cs-$buildCsVersion$ext"
+  val arch      = sys.props.getOrElse("os.arch", "").toLowerCase(Locale.ROOT)
+  val ext       = if (Properties.isWin) ".exe" else ""
+  val csVersion = if (arch == "aarch64" && Properties.isMac) buildCsM1Version else buildCsVersion
+  val dest      = T.dest / s"cs-$csVersion$ext"
 
   def downloadOpt(): Option[String] = {
-    val arch = sys.props.getOrElse("os.arch", "").toLowerCase(Locale.ROOT)
     val urlOpt = arch match {
       case "x86_64" | "amd64" =>
         if (Properties.isWin)
           Some(
-            s"https://github.com/coursier/coursier/releases/download/v$buildCsVersion/cs-x86_64-pc-win32.zip"
+            s"https://github.com/coursier/coursier/releases/download/v$csVersion/cs-x86_64-pc-win32.zip"
           )
         else if (Properties.isMac)
           Some(
-            s"https://github.com/coursier/coursier/releases/download/v$buildCsVersion/cs-x86_64-apple-darwin.gz"
+            s"https://github.com/coursier/coursier/releases/download/v$csVersion/cs-x86_64-apple-darwin.gz"
           )
         else if (Properties.isLinux)
           Some(
-            s"https://github.com/coursier/coursier/releases/download/v$buildCsVersion/cs-x86_64-pc-linux.gz"
+            s"https://github.com/coursier/coursier/releases/download/v$csVersion/cs-x86_64-pc-linux.gz"
           )
         else None
       case "aarch64" =>
         if (Properties.isLinux)
           Some(
-            s"https://github.com/coursier/coursier/releases/download/v$buildCsVersion/cs-aarch64-pc-linux.gz"
+            s"https://github.com/coursier/coursier/releases/download/v$csVersion/cs-aarch64-pc-linux.gz"
+          )
+        else if (Properties.isMac)
+          Some(
+            s"https://github.com/VirtusLab/coursier-m1/releases/download/v$csVersion/cs-aarch64-apple-darwin.gz"
           )
         else None
       case _ =>
@@ -196,6 +203,7 @@ trait CliLaunchers extends SbtModule { self =>
         case Some("x86_64" | "amd64") =>
           if (Properties.isWin) ("x86_64-pc-win32", "lib")
           else if (Properties.isLinux) ("x86_64-pc-linux", "a")
+          else if (Properties.isMac && isArmArchitecture) ("aarch64-apple-darwin", "a")
           else if (Properties.isMac) ("x86_64-apple-darwin", "a")
           else sys.error(s"Unsupported OS for x86_64 platform: ${sys.props("os.name")}")
         case Some("aarch64") =>
@@ -210,7 +218,7 @@ trait CliLaunchers extends SbtModule { self =>
         cs,
         "fetch",
         "--intransitive",
-        s"io.github.alexarchambault.tmp.libsodiumjni:libsodiumjni:$libsodiumjniVersion,classifier=$classifier,ext=$ext,type=$ext",
+        s"org.virtuslab.scala-cli:libsodiumjni:$libsodiumjniVersion,classifier=$classifier,ext=$ext,type=$ext",
         "-A",
         ext
       ).call()
diff --git a/project/utils.sc b/project/utils.sc
@@ -0,0 +1,4 @@
+import java.util.Locale
+
+lazy val isArmArchitecture: Boolean =
+  sys.props.getOrElse("os.arch", "").toLowerCase(Locale.ROOT) == "aarch64"
