Skip to content

Commit e9066e5

Browse files
GedochaoGedochao
committed
Support running the test scope with --spark and --spark-standalone 
1 parent 71ab4dd commit e9066e5

File tree

10 files changed

+160
-116
lines changed

10 files changed

+160
-116
lines changed

‎modules/cli/src/main/scala/scala/cli/commands/package0/Package.scala

Lines changed: 22 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
316316
value(bootstrap(build, destPath, value(mainClass), () => alreadyExistsCheck(), logger))
317317
destPath
318318
case PackageType.LibraryJar =>
319-
val libraryJar = Library.libraryJar(build)
319+
val libraryJar = Library.libraryJar(Seq(build))
320320
value(alreadyExistsCheck())
321321
if (force) os.copy.over(libraryJar, destPath, createFolders = true)
322322
else os.copy(libraryJar, destPath, createFolders = true)
@@ -550,7 +550,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
550550
outputStream = os.write.outputStream(dest, createFolders = true)
551551
Library.writeLibraryJarTo(
552552
outputStream,
553-
build,
553+
Seq(build),
554554
hasActualManifest = false,
555555
contentDirOverride = Some(contentDir)
556556
)
@@ -807,38 +807,34 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
807807
* the whole dependency graph.
808808
*/
809809
def providedFiles(
810-
build: Build.Successful,
810+
builds: Seq[Build.Successful],
811811
provided: Seq[dependency.AnyModule],
812812
logger: Logger
813813
): Either[BuildException, Seq[os.Path]] = either {
814814

815815
logger.debug(s"${provided.length} provided dependencies")
816-
val res = build.artifacts.resolution.getOrElse {
816+
val res = builds.map(_.artifacts.resolution.getOrElse {
817817
sys.error("Internal error: expected resolution to have been kept")
818-
}
819-
val modules = value {
818+
})
819+
val modules: Seq[coursier.Module] = value {
820820
provided
821-
.map(_.toCs(build.scalaParams))
821+
.map(_.toCs(builds.head.scalaParams)) // Scala params should be the same for all scopes
822822
.sequence
823823
.left.map(CompositeBuildException(_))
824824
}
825825
val modulesSet = modules.toSet
826826
val providedDeps = res
827-
.dependencyArtifacts
828-
.map(_._1)
827+
.flatMap(_.dependencyArtifacts.map(_._1))
829828
.filter(dep => modulesSet.contains(dep.module))
830-
val providedRes = res.subset(providedDeps)
831-
val fileMap = build.artifacts.detailedRuntimeArtifacts
832-
.map {
833-
case (_, _, artifact, path) =>
834-
artifact -> path
835-
}
829+
val providedRes = res.map(_.subset(providedDeps))
830+
val fileMap = builds.flatMap(_.artifacts.detailedRuntimeArtifacts).distinct
831+
.map { case (_, _, artifact, path) => artifact -> path }
836832
.toMap
837-
val providedFiles = coursier.Artifacts.artifacts(providedRes, Set.empty, None, None, true)
833+
val providedFiles = providedRes
834+
.flatMap(r => coursier.Artifacts.artifacts(r, Set.empty, None, None, true))
835+
.distinct
838836
.map(_._3)
839-
.map { a =>
840-
fileMap.getOrElse(a, sys.error(s"should not happen (missing: $a)"))
841-
}
837+
.map(a => fileMap.getOrElse(a, sys.error(s"should not happen (missing: $a)")))
842838
logger.debug {
843839
val it = Iterator(s"${providedFiles.size} provided JAR(s)") ++
844840
providedFiles.toVector.map(_.toString).sorted.iterator.map(f => s" $f")
@@ -857,9 +853,9 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
857853
logger: Logger
858854
): Either[BuildException, Unit] = either {
859855
val compiledClasses = os.walk(build.output).filter(os.isFile(_))
860-
val (extraClasseFolders, extraJars) =
856+
val (extraClassesFolders, extraJars) =
861857
build.options.classPathOptions.extraClassPath.partition(os.isDir(_))
862-
val extraClasses = extraClasseFolders.flatMap(os.walk(_)).filter(os.isFile(_))
858+
val extraClasses = extraClassesFolders.flatMap(os.walk(_)).filter(os.isFile(_))
863859

864860
val byteCodeZipEntries = (compiledClasses ++ extraClasses).map { path =>
865861
val name = path.relativeTo(build.output).toString
@@ -876,7 +872,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
876872
val jars =
877873
if (provided.isEmpty) allJars
878874
else {
879-
val providedFilesSet = value(providedFiles(build, provided, logger)).toSet
875+
val providedFilesSet = value(providedFiles(Seq(build), provided, logger)).toSet
880876
allJars.filterNot(providedFilesSet.contains)
881877
}
882878

@@ -957,8 +953,8 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
957953
logger: Logger,
958954
scratchDirOpt: Option[os.Path] = None
959955
): Either[BuildException, os.Path] = {
960-
val jars = builds.map(Library.libraryJar(_))
961-
val classPath = jars ++ builds.flatMap(_.artifacts.classPath)
956+
val jar = Library.libraryJar(builds)
957+
val classPath = Seq(jar) ++ builds.flatMap(_.artifacts.classPath)
962958
val input = ScalaJsLinker.LinkJSInput(
963959
options = builds.head.options.notForBloopOptions.scalaJsLinkerOptions,
964960
javaCommand =
@@ -1090,8 +1086,8 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
10901086

10911087
if (cacheData.changed) {
10921088
builds.foreach(build => NativeResourceMapper.copyCFilesToScalaNativeDir(build, nativeWorkDir))
1093-
val jars = builds.map(Library.libraryJar(_))
1094-
val classpath = (jars ++ builds.flatMap(_.artifacts.classPath)).map(_.toString).distinct
1089+
val jar = Library.libraryJar(builds)
1090+
val classpath = (Seq(jar) ++ builds.flatMap(_.artifacts.classPath)).map(_.toString).distinct
10951091
val args =
10961092
allCliOptions ++
10971093
logger.scalaNativeCliInternalLoggerOptions ++

‎modules/cli/src/main/scala/scala/cli/commands/publish/Publish.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -565,7 +565,7 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers {
565565
case Right(cls) => Some(cls)
566566
}
567567
}
568-
val libraryJar = Library.libraryJar(build, mainClassOpt)
568+
val libraryJar = Library.libraryJar(Seq(build), mainClassOpt)
569569
val dest = workingDir / org / s"$moduleName-$ver.jar"
570570
os.copy.over(libraryJar, dest, createFolders = true)
571571
dest

‎modules/cli/src/main/scala/scala/cli/commands/repl/Repl.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ object Repl extends ScalaCommand[ReplOptions] with BuildCommandHelpers {
176176
buildOptions = builds.head.options,
177177
allArtifacts = builds.map(_.artifacts),
178178
mainJarsOrClassDirs =
179-
if (asJar) builds.map(Library.libraryJar(_)) else builds.map(_.output),
179+
if asJar then Seq(Library.libraryJar(builds)) else builds.map(_.output),
180180
allowExit = allowExit,
181181
runMode = runMode,
182182
successfulBuilds = builds

‎modules/cli/src/main/scala/scala/cli/commands/run/Run.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
158158
runMode: RunMode,
159159
showCommand: Boolean,
160160
scratchDirOpt: Option[os.Path]
161-
): Either[BuildException, Option[(Process, CompletableFuture[_])]] = either {
161+
): Either[BuildException, Option[(Process, CompletableFuture[?])]] = either {
162162
val potentialMainClasses = builds.flatMap(_.foundMainClasses()).distinct
163163
if (options.sharedRun.mainClass.mainClassLs.contains(true))
164164
value {
@@ -620,7 +620,7 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
620620
case mode: RunMode.SparkSubmit =>
621621
value {
622622
RunSpark.run(
623-
builds.head, // TODO: handle multiple builds
623+
builds,
624624
mainClass,
625625
args,
626626
mode.submitArgs,
@@ -633,7 +633,7 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
633633
case mode: RunMode.StandaloneSparkSubmit =>
634634
value {
635635
RunSpark.runStandalone(
636-
builds.head, // TODO: handle multiple builds
636+
builds,
637637
mainClass,
638638
args,
639639
mode.submitArgs,

‎modules/cli/src/main/scala/scala/cli/commands/util/RunSpark.scala

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import scala.util.Properties
1414
object RunSpark {
1515

1616
def run(
17-
build: Build.Successful,
17+
builds: Seq[Build.Successful],
1818
mainClass: String,
1919
args: Seq[String],
2020
submitArgs: Seq[String],
@@ -27,11 +27,11 @@ object RunSpark {
2727
// FIXME Get Spark.sparkModules via provided settings?
2828
val providedModules = Spark.sparkModules
2929
val providedFiles =
30-
value(PackageCmd.providedFiles(build, providedModules, logger)).toSet
31-
val depCp = build.dependencyClassPath.filterNot(providedFiles)
32-
val javaHomeInfo = build.options.javaHome().value
33-
val javaOpts = build.options.javaOptions.javaOpts.toSeq.map(_.value.value)
34-
val ext = if (Properties.isWin) ".cmd" else ""
30+
value(PackageCmd.providedFiles(builds, providedModules, logger)).toSet
31+
val depCp = builds.flatMap(_.dependencyClassPath).distinct.filterNot(providedFiles)
32+
val javaHomeInfo = builds.head.options.javaHome().value
33+
val javaOpts = builds.head.options.javaOptions.javaOpts.toSeq.map(_.value.value)
34+
val ext = if Properties.isWin then ".cmd" else ""
3535
val submitCommand: String =
3636
EnvVar.Spark.sparkHome.valueOpt
3737
.map(os.Path(_, os.pwd))
@@ -44,7 +44,7 @@ object RunSpark {
4444
else Seq("--jars", depCp.mkString(","))
4545

4646
scratchDirOpt.foreach(os.makeDir.all(_))
47-
val library = Library.libraryJar(build)
47+
val library = Library.libraryJar(builds)
4848

4949
val finalCommand =
5050
Seq(submitCommand, "--class", mainClass) ++
@@ -54,24 +54,23 @@ object RunSpark {
5454
Seq(library.toString) ++
5555
args
5656
val envUpdates = javaHomeInfo.envUpdates(sys.env)
57-
if (showCommand)
58-
Left(Runner.envCommand(envUpdates) ++ finalCommand)
57+
if showCommand then Left(Runner.envCommand(envUpdates) ++ finalCommand)
5958
else {
6059
val proc =
61-
if (allowExecve)
60+
if allowExecve then
6261
Runner.maybeExec("spark-submit", finalCommand, logger, extraEnv = envUpdates)
63-
else
64-
Runner.run(finalCommand, logger, extraEnv = envUpdates)
62+
else Runner.run(finalCommand, logger, extraEnv = envUpdates)
6563
Right((
6664
proc,
67-
if (scratchDirOpt.isEmpty) Some(() => os.remove(library, checkExists = true))
65+
if scratchDirOpt.isEmpty then
66+
Some(() => os.remove(library, checkExists = true))
6867
else None
6968
))
7069
}
7170
}
7271

7372
def runStandalone(
74-
build: Build.Successful,
73+
builds: Seq[Build.Successful],
7574
mainClass: String,
7675
args: Seq[String],
7776
submitArgs: Seq[String],
@@ -83,18 +82,20 @@ object RunSpark {
8382

8483
// FIXME Get Spark.sparkModules via provided settings?
8584
val providedModules = Spark.sparkModules
86-
val sparkClassPath = value(PackageCmd.providedFiles(build, providedModules, logger))
85+
val sparkClassPath: Seq[os.Path] = value(PackageCmd.providedFiles(
86+
builds,
87+
providedModules,
88+
logger
89+
))
8790

8891
scratchDirOpt.foreach(os.makeDir.all(_))
89-
val library = Library.libraryJar(build)
92+
val library = Library.libraryJar(builds)
9093

9194
val finalMainClass = "org.apache.spark.deploy.SparkSubmit"
92-
val depCp = build.dependencyClassPath.filterNot(sparkClassPath.toSet)
93-
val javaHomeInfo = build.options.javaHome().value
94-
val javaOpts = build.options.javaOptions.javaOpts.toSeq.map(_.value.value)
95-
val jarsArgs =
96-
if (depCp.isEmpty) Nil
97-
else Seq("--jars", depCp.mkString(","))
95+
val depCp = builds.flatMap(_.dependencyClassPath).distinct.filterNot(sparkClassPath.toSet)
96+
val javaHomeInfo = builds.head.options.javaHome().value
97+
val javaOpts = builds.head.options.javaOptions.javaOpts.toSeq.map(_.value.value)
98+
val jarsArgs = if depCp.isEmpty then Nil else Seq("--jars", depCp.mkString(","))
9899
val finalArgs =
99100
Seq("--class", mainClass) ++
100101
jarsArgs ++
@@ -103,19 +104,19 @@ object RunSpark {
103104
Seq(library.toString) ++
104105
args
105106
val envUpdates = javaHomeInfo.envUpdates(sys.env)
106-
if (showCommand) {
107-
val command = Runner.jvmCommand(
108-
javaHomeInfo.javaCommand,
109-
javaOpts,
110-
sparkClassPath,
111-
finalMainClass,
112-
finalArgs,
113-
extraEnv = envUpdates,
114-
useManifest = build.options.notForBloopOptions.runWithManifest,
115-
scratchDirOpt = scratchDirOpt
116-
)
117-
Left(command)
118-
}
107+
if showCommand then
108+
Left {
109+
Runner.jvmCommand(
110+
javaHomeInfo.javaCommand,
111+
javaOpts,
112+
sparkClassPath,
113+
finalMainClass,
114+
finalArgs,
115+
extraEnv = envUpdates,
116+
useManifest = builds.head.options.notForBloopOptions.runWithManifest,
117+
scratchDirOpt = scratchDirOpt
118+
)
119+
}
119120
else {
120121
val proc = Runner.runJvm(
121122
javaHomeInfo.javaCommand,
@@ -126,13 +127,12 @@ object RunSpark {
126127
logger,
127128
allowExecve = allowExecve,
128129
extraEnv = envUpdates,
129-
useManifest = build.options.notForBloopOptions.runWithManifest,
130+
useManifest = builds.head.options.notForBloopOptions.runWithManifest,
130131
scratchDirOpt = scratchDirOpt
131132
)
132133
Right((
133134
proc,
134-
if (scratchDirOpt.isEmpty) Some(() => os.remove(library, checkExists = true))
135-
else None
135+
if scratchDirOpt.isEmpty then Some(() => os.remove(library, checkExists = true)) else None
136136
))
137137
}
138138
}

‎modules/cli/src/main/scala/scala/cli/packaging/Library.scala

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,21 @@ import scala.build.Build
1010
import scala.cli.internal.CachedBinary
1111

1212
object Library {
13-
1413
def libraryJar(
15-
build: Build.Successful,
14+
builds: Seq[Build.Successful],
1615
mainClassOpt: Option[String] = None
1716
): os.Path = {
18-
19-
val workDir = build.inputs.libraryJarWorkDir
17+
val workDir = builds.head.inputs.libraryJarWorkDir
2018
val dest = workDir / "library.jar"
2119
val cacheData =
2220
CachedBinary.getCacheData(
23-
Seq(build),
21+
builds,
2422
mainClassOpt.toList.flatMap(c => List("--main-class", c)),
2523
dest,
2624
workDir
2725
)
2826

29-
if (cacheData.changed) {
27+
if cacheData.changed then {
3028
var outputStream: OutputStream = null
3129
try {
3230
outputStream = os.write.outputStream(
@@ -36,13 +34,12 @@ object Library {
3634
)
3735
writeLibraryJarTo(
3836
outputStream,
39-
build,
37+
builds,
4038
mainClassOpt
4139
)
4240
}
4341
finally
44-
if (outputStream != null)
45-
outputStream.close()
42+
if outputStream != null then outputStream.close()
4643

4744
CachedBinary.updateProjectAndOutputSha(dest, workDir, cacheData.projectSha)
4845
}
@@ -52,7 +49,7 @@ object Library {
5249

5350
def writeLibraryJarTo(
5451
outputStream: OutputStream,
55-
build: Build.Successful,
52+
builds: Seq[Build.Successful],
5653
mainClassOpt: Option[String] = None,
5754
hasActualManifest: Boolean = true,
5855
contentDirOverride: Option[os.Path] = None
@@ -61,16 +58,21 @@ object Library {
6158
val manifest = new java.util.jar.Manifest
6259
manifest.getMainAttributes.put(JarAttributes.Name.MANIFEST_VERSION, "1.0")
6360

64-
if (hasActualManifest)
65-
for (mainClass <- mainClassOpt.orElse(build.sources.defaultMainClass) if mainClass.nonEmpty)
66-
manifest.getMainAttributes.put(JarAttributes.Name.MAIN_CLASS, mainClass)
61+
if hasActualManifest then
62+
for {
63+
mainClass <- mainClassOpt.orElse(builds.flatMap(_.sources.defaultMainClass).headOption)
64+
if mainClass.nonEmpty
65+
} manifest.getMainAttributes.put(JarAttributes.Name.MAIN_CLASS, mainClass)
6766

6867
var zos: ZipOutputStream = null
69-
val contentDir = contentDirOverride.getOrElse(build.output)
68+
val contentDirs = builds.map(b => contentDirOverride.getOrElse(b.output))
7069

7170
try {
7271
zos = new JarOutputStream(outputStream, manifest)
73-
for (path <- os.walk(contentDir) if os.isFile(path)) {
72+
for {
73+
contentDir <- contentDirs
74+
path <- os.walk(contentDir) if os.isFile(path)
75+
} {
7476
val name = path.relativeTo(contentDir).toString
7577
val lastModified = os.mtime(path)
7678
val ent = new ZipEntry(name)
@@ -88,10 +90,10 @@ object Library {
8890
}
8991

9092
extension (build: Build.Successful) {
91-
def fullClassPathAsJar: Seq[os.Path] =
92-
Seq(libraryJar(build)) ++ build.dependencyClassPath
93+
private def fullClassPathAsJar: Seq[os.Path] =
94+
Seq(libraryJar(Seq(build))) ++ build.dependencyClassPath
9395
def fullClassPathMaybeAsJar(asJar: Boolean): Seq[os.Path] =
94-
if (asJar) fullClassPathAsJar else build.fullClassPath
96+
if asJar then fullClassPathAsJar else build.fullClassPath
9597
}
9698

9799
}

0 commit comments

Comments
 (0)