Merge pull request #1086 from alexarchambault/package-spark-job

Add --spark option to package sub-command

Author: romanowski

modules/cli-options/src/main/scala/scala/cli/commands/PackageOptions.scala

@@ -42,6 +42,10 @@ final case class PackageOptions(
   @HelpMessage("For assembly JAR, whether to add a bash / bat preamble")
     preamble: Boolean = true,
   @Group("Package")
+  @Hidden
+  @HelpMessage("Generate an assembly JAR for Spark (assembly that doesn't contain Spark, nor any of its dependencies)")
+    spark: Boolean = false,
+  @Group("Package")
   @HelpMessage("Package standalone JARs")
     standalone: Option[Boolean] = None,
   @Recurse

modules/cli/src/main/scala/scala/cli/commands/Package.scala

@@ -30,6 +30,7 @@ import scala.build.internal.{Runner, ScalaJsLinkerConfig}
 import scala.build.options.{PackageType, Platform}
 import scala.cli.CurrentParams
 import scala.cli.commands.OptionsHelper._
+import scala.cli.commands.packaging.Spark
 import scala.cli.commands.util.MainClassOptionsUtil._
 import scala.cli.commands.util.PackageOptionsUtil._
 import scala.cli.commands.util.SharedOptionsUtil._
@@ -197,6 +198,7 @@ object Package extends ScalaCommand[PackageOptions] {
         case PackageType.SourceJar                              => ".jar"
         case PackageType.DocJar                                 => ".jar"
         case _: PackageType.Assembly                            => ".jar"
+        case PackageType.Spark                                  => ".jar"
         case PackageType.Js                                     => ".js"
         case PackageType.Debian                                 => ".deb"
         case PackageType.Dmg                                    => ".dmg"
@@ -214,6 +216,7 @@ object Package extends ScalaCommand[PackageOptions] {
         case PackageType.SourceJar                              => "source.jar"
         case PackageType.DocJar                                 => "scaladoc.jar"
         case _: PackageType.Assembly                            => "app.jar"
+        case PackageType.Spark                                  => "job.jar"
         case PackageType.Js                                     => "app.js"
         case PackageType.Debian                                 => "app.deb"
         case PackageType.Dmg                                    => "app.dmg"
@@ -294,12 +297,30 @@ object Package extends ScalaCommand[PackageOptions] {
               build,
               destPath,
               value(mainClass),
+              Nil,
               withPreamble = a.addPreamble,
               () => alreadyExistsCheck(),
               logger
             )
           }
           destPath
+        case PackageType.Spark =>
+          value {
+            assembly(
+              build,
+              destPath,
+              value(mainClass),
+              // The Spark modules are assumed to be already on the class path,
+              // along with all their transitive dependencies (originating from
+              // the Spark distribution), so we don't include any of them in the
+              // assembly.
+              Spark.sparkModules,
+              withPreamble = false,
+              () => alreadyExistsCheck(),
+              logger
+            )
+          }
+          destPath
 
         case PackageType.Js =>
           value(buildJs(build, destPath, value(mainClass), logger))
@@ -696,6 +717,7 @@ object Package extends ScalaCommand[PackageOptions] {
     build: Build.Successful,
     destPath: os.Path,
     mainClass: String,
+    extraProvided: Seq[dependency.AnyModule],
     withPreamble: Boolean,
     alreadyExistsCheck: () => Unit,
     logger: Logger
@@ -712,7 +734,7 @@ object Package extends ScalaCommand[PackageOptions] {
         (ent, content)
       }
 
-    val provided = build.options.notForBloopOptions.packageOptions.provided
+    val provided = build.options.notForBloopOptions.packageOptions.provided ++ extraProvided
     val allFiles = build.artifacts.artifacts.map(_._2)
     val files =
       if (provided.isEmpty) allFiles

modules/cli/src/main/scala/scala/cli/commands/packaging/Spark.scala

@@ -0,0 +1,26 @@
+package scala.cli.commands.packaging
+
+import dependency._
+
+object Spark {
+
+  private def names = Seq(
+    // FIXME Add more?
+    // (see "cs complete-dependency org.apache.spark: | grep '_2\.12$'"
+    // or `ls "$(cs get https://archive.apache.org/dist/spark/spark-2.4.2/spark-2.4.2-bin-hadoop2.7.tgz --archive)"/*/jars | grep '^spark-'`)
+    "core",
+    "graphx",
+    "hive",
+    "hive-thriftserver",
+    "kubernetes",
+    "mesos",
+    "mllib",
+    "repl",
+    "sql",
+    "streaming",
+    "yarn"
+  )
+
+  def sparkModules: Seq[AnyModule] =
+    names.map(name => mod"org.apache.spark::spark-$name")
+}

modules/cli/src/main/scala/scala/cli/commands/util/PackageOptionsUtil.scala

@@ -20,6 +20,7 @@ object PackageOptionsUtil {
         if (v.library) Some(PackageType.LibraryJar)
         else if (source) Some(PackageType.SourceJar)
         else if (assembly) Some(PackageType.Assembly(addPreamble = preamble))
+        else if (spark) Some(PackageType.Spark)
         else if (deb) Some(PackageType.Debian)
         else if (dmg) Some(PackageType.Dmg)
         else if (pkg) Some(PackageType.Pkg)
@@ -90,8 +91,9 @@ object PackageOptionsUtil {
           )
         ),
         internal = baseOptions.internal.copy(
-          // computing the provided modules sub-graph needs the final Resolution instance
-          keepResolution = provided.nonEmpty
+          // computing the provided modules sub-graph need the final Resolution instance
+          // Spark packaging adds provided modules, so it needs it too
+          keepResolution = provided.nonEmpty || packageTypeOpt.contains(PackageType.Spark)
         ),
         internalDependencies = baseOptions.internalDependencies.copy(
           addRunnerDependencyOpt = Some(false)

modules/integration/src/test/scala/scala/cli/integration/SparkTestDefinitions.scala

@@ -0,0 +1,7 @@
+package scala.cli.integration
+
+class SparkTestDefinitions extends munit.FunSuite {
+
+  protected lazy val extraOptions: Seq[String] = TestUtil.extraOptions
+
+}

modules/integration/src/test/scala/scala/cli/integration/SparkTests212.scala

@@ -0,0 +1,98 @@
+package scala.cli.integration
+
+import com.eed3si9n.expecty.Expecty.expect
+
+import java.io.File
+
+import scala.util.Properties
+
+object SparkTests212 {
+
+  private final case class Spark(sparkVersion: String, scalaVersion: String) {
+    def sparkHome(): os.Path = {
+      val url =
+        s"https://archive.apache.org/dist/spark/spark-$sparkVersion/spark-$sparkVersion-bin-hadoop2.7.tgz"
+      val dirName = url.drop(url.lastIndexOf('/') + 1).stripSuffix(".tgz")
+      val baseDir =
+        os.Path(os.proc(TestUtil.cs, "get", "--archive", url).call().out.text().trim, os.pwd)
+      baseDir / dirName
+    }
+  }
+
+  private val spark30 = Spark(
+    "3.0.3",
+    // The spark distribution actually ships with Scala 2.12.10, but we run into #1092 if we use it here
+    "2.12.15"
+  )
+
+  private val spark24 = Spark(
+    "2.4.2",
+    // The spark distribution actually ships with Scala 2.12.8, but we run into #1092 if we use it here
+    "2.12.15"
+  )
+
+}
+
+class SparkTests212 extends SparkTestDefinitions {
+
+  import SparkTests212._
+
+  def simplePackageSparkJobTest(spark: Spark): Unit = {
+    val master = "local[4]"
+    val inputs = TestInputs(
+      Seq(
+        os.rel / "SparkJob.scala" ->
+          s"""//> using lib "org.apache.spark::spark-sql:${spark.sparkVersion}"
+             |//> using scala "${spark.scalaVersion}"
+             |
+             |import org.apache.spark._
+             |import org.apache.spark.sql._
+             |
+             |object SparkJob {
+             |  def main(args: Array[String]): Unit = {
+             |    val spark = SparkSession.builder()
+             |      .appName("Test job")
+             |      .getOrCreate()
+             |    import spark.implicits._
+             |    def sc    = spark.sparkContext
+             |    val accum = sc.longAccumulator
+             |    sc.parallelize(1 to 10).foreach(x => accum.add(x))
+             |    println("Result: " + accum.value)
+             |  }
+             |}
+             |""".stripMargin
+      )
+    )
+    inputs.fromRoot { root =>
+      val dest = os.rel / "SparkJob.jar"
+      os.proc(TestUtil.cli, "package", extraOptions, "--spark", "--jvm", "8", ".", "-o", dest)
+        .call(cwd = root)
+
+      val java8Home =
+        os.Path(os.proc(TestUtil.cs, "java-home", "--jvm", "8").call().out.trim(), os.pwd)
+
+      val ext = if (Properties.isWin) ".cmd" else ""
+      val res =
+        os.proc(spark.sparkHome() / "bin" / s"spark-submit$ext", "--master", master, dest).call(
+          cwd = root,
+          env = Map(
+            "JAVA_HOME" -> java8Home.toString,
+            "PATH" -> ((java8Home / "bin").toString + File.pathSeparator + System.getenv("PATH"))
+          )
+        )
+
+      val expectedOutput = "Result: 55"
+
+      expect(res.out.trim() == expectedOutput)
+    }
+  }
+
+  test("spark 2.4") {
+    simplePackageSparkJobTest(spark24)
+  }
+
+  test("spark 3.0") {
+    simplePackageSparkJobTest(spark30)
+  }
+
+}

modules/integration/src/test/scala/scala/cli/integration/SparkTests213.scala

@@ -0,0 +1,3 @@
+package scala.cli.integration
+
+class SparkTests213 extends SparkTestDefinitions

modules/options/src/main/scala/scala/build/options/PackageType.scala

@@ -19,6 +19,9 @@ object PackageType {
   final case class Assembly(addPreamble: Boolean) extends PackageType {
     override def runnable = Some(addPreamble)
   }
+  case object Spark extends PackageType {
+    override def runnable = Some(false)
+  }
   case object Js extends PackageType
   case object Native extends PackageType {
     override def runnable = Some(true)
@@ -42,6 +45,7 @@ object PackageType {
     "library"      -> LibraryJar,
     "source"       -> SourceJar,
     "doc"          -> DocJar,
+    "spark"        -> Spark,
     "js"           -> Js,
     "native"       -> Native,
     "docker"       -> Docker,

website/docs/reference/cli-options.md

@@ -866,6 +866,10 @@ Generate an assembly JAR
 
 For assembly JAR, whether to add a bash / bat preamble
 
+#### `--spark`
+
+Generate an assembly JAR for Spark (assembly that doesn't contain Spark, nor any of its dependencies)
+
 #### `--standalone`
 
 Package standalone JARs

website/src/pages/spark.md

@@ -0,0 +1,53 @@
+# Experimental Spark features
+
+import {ChainedSnippets, GiflikeVideo} from "../../src/components/MarkdownComponents.js";
+
+## Packaging
+
+The `package` sub-commands offers to package Scala CLI projects as JARs ready to be passed
+to `spark-submit`, and optimized for it.
+
+<ChainedSnippets>
+
+```scala title=SparkJob.scala
+//> using lib "org.apache.spark::spark-sql:3.0.3"
+//> using scala "2.12.15"
+
+import org.apache.spark._
+import org.apache.spark.sql._
+
+object SparkJob {
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession.builder()
+      .appName("Test job")
+      .getOrCreate()
+    import spark.implicits._
+    def sc    = spark.sparkContext
+    val accum = sc.longAccumulator
+    sc.parallelize(1 to 10).foreach(x => accum.add(x))
+    println("Result: " + accum.value)
+  }
+}
+```
+
+```bash
+scala-cli package --spark SparkJob.scala -o spark-job.jar
+```
+
+```text
+Compiling project (Scala 2.12.15, JVM)
+Compiled project (Scala 2.12.15, JVM)
+Wrote spark-job.jar
+```
+
+```bash
+spark-submit spark-job.jar
+```
+
+```text
+…
+Result: 55
+…
+```
+
+</ChainedSnippets>