Skip to content

Commit 0b7b6d7

Browse files
lolgablolgab
authored
Incremental Scala.js Linking (#2928)
* Incremental Scala.js Linking * Cleanup code * Rename function * Use released version * Close streams before destroying process
1 parent b24faad commit 0b7b6d7

File tree

4 files changed

+202
-53
lines changed

4 files changed

+202
-53
lines changed

‎modules/build/src/main/scala/scala/build/internal/Runner.scala

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,25 @@ object Runner {
3030
logger,
3131
allowExecve = true,
3232
cwd,
33-
extraEnv
33+
extraEnv,
34+
inheritStreams = true
3435
)
3536

3637
def run(
3738
command: Seq[String],
3839
logger: Logger,
3940
cwd: Option[os.Path] = None,
40-
extraEnv: Map[String, String] = Map.empty
41+
extraEnv: Map[String, String] = Map.empty,
42+
inheritStreams: Boolean = true
4143
): Process =
4244
run0(
4345
"unused",
4446
command,
4547
logger,
4648
allowExecve = false,
4749
cwd,
48-
extraEnv
50+
extraEnv,
51+
inheritStreams
4952
)
5053

5154
def run0(
@@ -54,7 +57,8 @@ object Runner {
5457
logger: Logger,
5558
allowExecve: Boolean,
5659
cwd: Option[os.Path],
57-
extraEnv: Map[String, String]
60+
extraEnv: Map[String, String],
61+
inheritStreams: Boolean
5862
): Process = {
5963

6064
import logger.{log, debug}
@@ -81,6 +85,12 @@ object Runner {
8185
else {
8286
val b = new ProcessBuilder(command: _*)
8387
.inheritIO()
88+
89+
if (!inheritStreams) {
90+
b.redirectInput(ProcessBuilder.Redirect.PIPE)
91+
b.redirectOutput(ProcessBuilder.Redirect.PIPE)
92+
}
93+
8494
if (extraEnv.nonEmpty) {
8595
val env = b.environment()
8696
for ((k, v) <- extraEnv)

‎modules/cli/src/main/scala/scala/cli/commands/package0/Package.scala

Lines changed: 49 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -913,6 +913,37 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
913913
finally os.remove(jar)
914914
}
915915

916+
private object LinkingDir {
917+
case class Input(linkJsInput: ScalaJsLinker.LinkJSInput, scratchDirOpt: Option[os.Path])
918+
private var currentInput: Option[Input] = None
919+
private var currentLinkingDir: Option[os.Path] = None
920+
def getOrCreate(
921+
linkJsInput: ScalaJsLinker.LinkJSInput,
922+
scratchDirOpt: Option[os.Path]
923+
): os.Path =
924+
val input = Input(linkJsInput, scratchDirOpt)
925+
currentLinkingDir match {
926+
case Some(linkingDir) if currentInput.contains(input) =>
927+
linkingDir
928+
case _ =>
929+
scratchDirOpt.foreach(os.makeDir.all(_))
930+
931+
currentLinkingDir.foreach(dir => os.remove.all(dir))
932+
currentLinkingDir = None
933+
934+
val linkingDirectory = os.temp.dir(
935+
dir = scratchDirOpt.orNull,
936+
prefix = "scala-cli-js-linking",
937+
deleteOnExit = scratchDirOpt.isEmpty
938+
)
939+
940+
currentInput = Some(input)
941+
currentLinkingDir = Some(linkingDirectory)
942+
943+
linkingDirectory
944+
}
945+
}
946+
916947
def linkJs(
917948
build: Build.Successful,
918949
dest: os.Path,
@@ -926,30 +957,29 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
926957
): Either[BuildException, os.Path] = {
927958
val mainJar = Library.libraryJar(build)
928959
val classPath = mainJar +: build.artifacts.classPath
929-
val delete = scratchDirOpt.isEmpty
930-
scratchDirOpt.foreach(os.makeDir.all(_))
931-
val linkingDir =
932-
os.temp.dir(
933-
dir = scratchDirOpt.orNull,
934-
prefix = "scala-cli-js-linking",
935-
deleteOnExit = delete
936-
)
960+
val input = ScalaJsLinker.LinkJSInput(
961+
options = build.options.notForBloopOptions.scalaJsLinkerOptions,
962+
javaCommand =
963+
build.options.javaHome().value.javaCommand, // FIXME Allow users to use another JVM here?
964+
classPath = classPath,
965+
mainClassOrNull = mainClassOpt.orNull,
966+
addTestInitializer = addTestInitializer,
967+
config = config,
968+
fullOpt = fullOpt,
969+
noOpt = noOpt,
970+
scalaJsVersion = build.options.scalaJsOptions.finalVersion
971+
)
972+
973+
val linkingDir = LinkingDir.getOrCreate(input, scratchDirOpt)
974+
937975
either {
938976
value {
939977
ScalaJsLinker.link(
940-
build.options.notForBloopOptions.scalaJsLinkerOptions,
941-
build.options.javaHome().value.javaCommand, // FIXME Allow users to use another JVM here?
942-
classPath,
943-
mainClassOpt.orNull,
944-
addTestInitializer,
945-
config,
978+
input,
946979
linkingDir,
947-
fullOpt,
948-
noOpt,
949980
logger,
950981
build.options.finalCache,
951-
build.options.archiveCache,
952-
build.options.scalaJsOptions.finalVersion
982+
build.options.archiveCache
953983
)
954984
}
955985
val relMainJs = os.rel / "main.js"
@@ -988,8 +1018,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
9881018
os.copy(sourceMapJs, sourceMapDest, replaceExisting = true)
9891019
logger.message(s"Emitted js source maps to: $sourceMapDest")
9901020
}
991-
if (delete)
992-
os.remove.all(linkingDir)
1021+
9931022
dest
9941023
}
9951024
else {

‎modules/cli/src/main/scala/scala/cli/commands/pgp/PgpExternalCommand.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ abstract class PgpExternalCommand extends ExternalCommand {
6262
logger,
6363
allowExecve = allowExecve,
6464
cwd = None,
65-
extraEnv = extraEnv
65+
extraEnv = extraEnv,
66+
inheritStreams = true
6667
).waitFor()
6768
}
6869

‎modules/cli/src/main/scala/scala/cli/internal/ScalaJsLinker.scala

Lines changed: 137 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,31 @@ import coursier.util.Task
77
import dependency._
88
import org.scalajs.testing.adapter.{TestAdapterInitializer => TAI}
99

10-
import java.io.File
10+
import java.io.{File, InputStream, OutputStream}
1111

1212
import scala.build.EitherCps.{either, value}
1313
import scala.build.errors.{BuildException, ScalaJsLinkingError}
1414
import scala.build.internal.Util.{DependencyOps, ModuleOps}
1515
import scala.build.internal.{ExternalBinaryParams, FetchExternalBinary, Runner, ScalaJsLinkerConfig}
1616
import scala.build.options.scalajs.ScalaJsLinkerOptions
1717
import scala.build.{Logger, Positioned}
18+
import scala.io.Source
1819
import scala.util.Properties
1920

2021
object ScalaJsLinker {
2122

23+
case class LinkJSInput(
24+
options: ScalaJsLinkerOptions,
25+
javaCommand: String,
26+
classPath: Seq[os.Path],
27+
mainClassOrNull: String,
28+
addTestInitializer: Boolean,
29+
config: ScalaJsLinkerConfig,
30+
fullOpt: Boolean,
31+
noOpt: Boolean,
32+
scalaJsVersion: String
33+
)
34+
2235
private def linkerMainClass = "org.scalajs.cli.Scalajsld"
2336

2437
private def linkerCommand(
@@ -98,61 +111,157 @@ object ScalaJsLinker {
98111
}
99112
}
100113

101-
def link(
102-
options: ScalaJsLinkerOptions,
103-
javaCommand: String,
104-
classPath: Seq[os.Path],
105-
mainClassOrNull: String,
106-
addTestInitializer: Boolean,
107-
config: ScalaJsLinkerConfig,
114+
private def getCommand(
115+
input: LinkJSInput,
108116
linkingDir: os.Path,
109-
fullOpt: Boolean,
110-
noOpt: Boolean,
111117
logger: Logger,
112118
cache: FileCache[Task],
113119
archiveCache: ArchiveCache[Task],
114-
scalaJsVersion: String
115-
): Either[BuildException, Unit] = either {
116-
120+
useLongRunning: Boolean
121+
) = either {
117122
val command = value {
118-
linkerCommand(options, javaCommand, logger, cache, archiveCache, scalaJsVersion)
123+
linkerCommand(
124+
input.options,
125+
input.javaCommand,
126+
logger,
127+
cache,
128+
archiveCache,
129+
input.scalaJsVersion
130+
)
119131
}
120132

121133
val allArgs = {
122-
val outputArgs = Seq("--outputDir", linkingDir.toString)
134+
val outputArgs = Seq("--outputDir", linkingDir.toString)
135+
val longRunning = if (useLongRunning) Seq("--longRunning") else Seq.empty[String]
123136
val mainClassArgs =
124-
Option(mainClassOrNull).toSeq.flatMap(mainClass => Seq("--mainMethod", mainClass + ".main"))
137+
Option(input.mainClassOrNull).toSeq.flatMap(mainClass =>
138+
Seq("--mainMethod", mainClass + ".main")
139+
)
125140
val testInitializerArgs =
126-
if (addTestInitializer)
141+
if (input.addTestInitializer)
127142
Seq("--mainMethodWithNoArgs", TAI.ModuleClassName + "." + TAI.MainMethodName)
128143
else
129144
Nil
130145
val optArg =
131-
if (noOpt) "--noOpt"
132-
else if (fullOpt) "--fullOpt"
146+
if (input.noOpt) "--noOpt"
147+
else if (input.fullOpt) "--fullOpt"
133148
else "--fastOpt"
134149

135150
Seq[os.Shellable](
136151
outputArgs,
137152
mainClassArgs,
138153
testInitializerArgs,
139154
optArg,
140-
config.linkerCliArgs,
141-
classPath.map(_.toString)
155+
input.config.linkerCliArgs,
156+
input.classPath.map(_.toString),
157+
longRunning
142158
)
143159
}
144160

145-
val cmd = command ++ allArgs.flatMap(_.value)
146-
val res = Runner.run(cmd, logger)
147-
val retCode = res.waitFor()
161+
command ++ allArgs.flatMap(_.value)
162+
}
163+
164+
def link(
165+
input: LinkJSInput,
166+
linkingDir: os.Path,
167+
logger: Logger,
168+
cache: FileCache[Task],
169+
archiveCache: ArchiveCache[Task]
170+
): Either[BuildException, Unit] = either {
171+
val useLongRunning = !input.fullOpt
148172

149-
if (retCode == 0)
150-
logger.debug("Scala.js linker ran successfully")
173+
if (useLongRunning)
174+
longRunningProcess.startOrReuse(input, linkingDir, logger, cache, archiveCache)
151175
else {
152-
logger.debug(s"Scala.js linker exited with return code $retCode")
153-
value(Left(new ScalaJsLinkingError))
176+
val cmd =
177+
value(getCommand(input, linkingDir, logger, cache, archiveCache, useLongRunning = false))
178+
val res = Runner.run(cmd, logger)
179+
val retCode = res.waitFor()
180+
181+
if (retCode == 0)
182+
logger.debug("Scala.js linker ran successfully")
183+
else {
184+
logger.debug(s"Scala.js linker exited with return code $retCode")
185+
value(Left(new ScalaJsLinkingError))
186+
}
154187
}
155188
}
189+
190+
private object longRunningProcess {
191+
case class Proc(process: Process, stdin: OutputStream, stdout: InputStream) {
192+
val stdoutLineIterator: Iterator[String] = Source.fromInputStream(stdout).getLines()
193+
}
194+
case class Input(input: LinkJSInput, linkingDir: os.Path)
195+
var currentInput: Option[Input] = None
196+
var currentProc: Option[Proc] = None
197+
198+
def startOrReuse(
199+
linkJsInput: LinkJSInput,
200+
linkingDir: os.Path,
201+
logger: Logger,
202+
cache: FileCache[Task],
203+
archiveCache: ArchiveCache[Task]
204+
) = either {
205+
val input = Input(linkJsInput, linkingDir)
206+
207+
def createProcess(): Proc = {
208+
val cmd =
209+
value(getCommand(
210+
linkJsInput,
211+
linkingDir,
212+
logger,
213+
cache,
214+
archiveCache,
215+
useLongRunning = true
216+
))
217+
val process = Runner.run(cmd, logger, inheritStreams = false)
218+
val stdin = process.getOutputStream()
219+
val stdout = process.getInputStream()
220+
val proc = Proc(process, stdin, stdout)
221+
currentProc = Some(proc)
222+
currentInput = Some(input)
223+
proc
224+
}
225+
226+
def loop(proc: Proc): Unit =
227+
if (proc.stdoutLineIterator.hasNext) {
228+
val line = proc.stdoutLineIterator.next()
229+
230+
if (line == "SCALA_JS_LINKING_DONE")
231+
logger.debug("Scala.js linker ran successfully")
232+
else {
233+
// inherit other stdout from Scala.js
234+
println(line)
235+
236+
loop(proc)
237+
}
238+
}
239+
else {
240+
val retCode = proc.process.waitFor()
241+
logger.debug(s"Scala.js linker exited with return code $retCode")
242+
value(Left(new ScalaJsLinkingError))
243+
}
244+
245+
val proc = currentProc match {
246+
case Some(proc) if currentInput.contains(input) && proc.process.isAlive() =>
247+
// trigger new linking
248+
proc.stdin.write('\n')
249+
proc.stdin.flush()
250+
251+
proc
252+
case Some(proc) =>
253+
proc.stdin.close()
254+
proc.stdout.close()
255+
proc.process.destroy()
256+
createProcess()
257+
case _ =>
258+
createProcess()
259+
}
260+
261+
loop(proc)
262+
}
263+
}
264+
156265
def updateSourceMappingURL(mainJsPath: os.Path) =
157266
val content = os.read(mainJsPath)
158267
content.replace(

0 commit comments

Comments
 (0)