Skip to content

Commit 2a59ce5

Browse files
lihaoyilihaoyi
authored
Isolate jarjarabrams in its own classloader (#5059)
Further cleaning up of the `build.mill` classpath. Covered by the existing unit test ` mill.scalalib.ScalaAssemblyExcludeTests.relocate.withDeps`
1 parent f36dac8 commit 2a59ce5

File tree

8 files changed

+96
-45
lines changed

8 files changed

+96
-45
lines changed

‎core/define/package.mill‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ object `package` extends build.MillStableScalaModule {
2828
// Necessary so we can share the JNA classes throughout the build process
2929
build.Deps.jna,
3030
build.Deps.jnaPlatform,
31-
build.Deps.jarjarabrams,
3231
build.Deps.mainargs,
3332
build.Deps.scalaparse,
3433
mvn"org.apache.commons:commons-lang3:3.16.0"

‎integration/ide/gen-idea/resources/extended/idea/mill_modules/mill-build.iml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,6 @@
1212
<orderEntry type="sourceFolder" forTests="false"/>
1313
<orderEntry type="library" name="scala-SDK-3.6.4" level="project"/>
1414
<orderEntry type="library" name="aircompressor-0.27.jar" level="project"/>
15-
<orderEntry type="library" name="asm-9.6.jar" level="project"/>
16-
<orderEntry type="library" name="asm-commons-9.6.jar" level="project"/>
17-
<orderEntry type="library" name="asm-tree-9.6.jar" level="project"/>
1815
<orderEntry type="library" name="cache-util-2.1.25-M11.jar" level="project"/>
1916
<orderEntry type="library" name="commons-codec-1.17.0.jar" level="project"/>
2017
<orderEntry type="library" name="commons-compress-1.26.2.jar" level="project"/>
@@ -40,8 +37,6 @@
4037
<orderEntry type="library" name="interface-1.0.29-M1.jar" level="project"/>
4138
<orderEntry type="library" name="is-terminal-0.1.2.jar" level="project"/>
4239
<orderEntry type="library" name="jansi-2.4.1.jar" level="project"/>
43-
<orderEntry type="library" name="jarjar-1.14.1.jar" level="project"/>
44-
<orderEntry type="library" name="jarjar-abrams-core_2.13-1.14.1.jar" level="project"/>
4540
<orderEntry type="library" name="javax.inject-1.jar" level="project"/>
4641
<orderEntry type="library" name="jcl-over-slf4j-1.7.30.jar" level="project"/>
4742
<orderEntry type="library" name="jgrapht-core-1.4.0.jar" level="project"/>

‎integration/ide/gen-idea/resources/extended/idea/mill_modules/mill-build.mill-build.iml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
<orderEntry type="library" name="scala-SDK-3.6.4" level="project"/>
1616
<orderEntry type="library" name="aircompressor-0.27.jar" level="project"/>
1717
<orderEntry type="library" name="asm-9.8.jar" level="project"/>
18-
<orderEntry type="library" name="asm-commons-9.6.jar" level="project"/>
1918
<orderEntry type="library" name="asm-tree-9.8.jar" level="project"/>
2019
<orderEntry type="library" name="cache-util-2.1.25-M11.jar" level="project"/>
2120
<orderEntry type="library" name="commons-codec-1.17.0.jar" level="project"/>
@@ -41,8 +40,6 @@
4140
<orderEntry type="library" name="interface-1.0.29-M1.jar" level="project"/>
4241
<orderEntry type="library" name="is-terminal-0.1.2.jar" level="project"/>
4342
<orderEntry type="library" name="jansi-2.4.1.jar" level="project"/>
44-
<orderEntry type="library" name="jarjar-1.14.1.jar" level="project"/>
45-
<orderEntry type="library" name="jarjar-abrams-core_2.13-1.14.1.jar" level="project"/>
4643
<orderEntry type="library" name="javax.inject-1.jar" level="project"/>
4744
<orderEntry type="library" name="jcl-over-slf4j-1.7.30.jar" level="project"/>
4845
<orderEntry type="library" name="jgrapht-core-1.4.0.jar" level="project"/>

‎integration/ide/gen-idea/resources/hello-idea/idea/mill_modules/mill-build.iml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,6 @@
1212
<orderEntry type="sourceFolder" forTests="false"/>
1313
<orderEntry type="library" name="scala-SDK-3.6.4" level="project"/>
1414
<orderEntry type="library" name="aircompressor-0.27.jar" level="project"/>
15-
<orderEntry type="library" name="asm-9.6.jar" level="project"/>
16-
<orderEntry type="library" name="asm-commons-9.6.jar" level="project"/>
17-
<orderEntry type="library" name="asm-tree-9.6.jar" level="project"/>
1815
<orderEntry type="library" name="cache-util-2.1.25-M11.jar" level="project"/>
1916
<orderEntry type="library" name="commons-codec-1.17.0.jar" level="project"/>
2017
<orderEntry type="library" name="commons-compress-1.26.2.jar" level="project"/>
@@ -39,8 +36,6 @@
3936
<orderEntry type="library" name="interface-1.0.29-M1.jar" level="project"/>
4037
<orderEntry type="library" name="is-terminal-0.1.2.jar" level="project"/>
4138
<orderEntry type="library" name="jansi-2.4.1.jar" level="project"/>
42-
<orderEntry type="library" name="jarjar-1.14.1.jar" level="project"/>
43-
<orderEntry type="library" name="jarjar-abrams-core_2.13-1.14.1.jar" level="project"/>
4439
<orderEntry type="library" name="javax.inject-1.jar" level="project"/>
4540
<orderEntry type="library" name="jcl-over-slf4j-1.7.30.jar" level="project"/>
4641
<orderEntry type="library" name="jgrapht-core-1.4.0.jar" level="project"/>

‎libs/scalalib/jarjarabrams-worker/src/mill/scalalib/jarjarabrams/impl/JarJarAbramsWorkerImpl.scala‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package mill.scalalib.jarjarabrams.impl
2+
3+
import com.eed3si9n.jarjarabrams.{ShadePattern, Shader}
4+
5+
import java.io.{ByteArrayInputStream, InputStream}
6+
7+
object JarJarAbramsWorkerImpl {
8+
type UnopenedInputStream = () => InputStream
9+
10+
def apply(
11+
relocates: Seq[(String, String)],
12+
name: String,
13+
inputStream: UnopenedInputStream
14+
): Option[(String, UnopenedInputStream)] = {
15+
val shadeRules = relocates.map {
16+
case (from, to) => ShadePattern.Rename(List(from -> to)).inAll
17+
}
18+
if (shadeRules.isEmpty) Some(name -> inputStream)
19+
else {
20+
val shader = Shader.bytecodeShader(shadeRules, verbose = false, skipManifest = true)
21+
val is = inputStream()
22+
shader(is.readAllBytes(), name).map {
23+
case (bytes, name) =>
24+
name ->
25+
(() => new ByteArrayInputStream(bytes) { override def close(): Unit = is.close() })
26+
}
27+
}
28+
}
29+
}

‎libs/scalalib/package.mill‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ object `package` extends build.MillStableScalaModule {
3131
def transitiveLocalTestOverrides =
3232
super.transitiveLocalTestOverrides() ++ Seq(
3333
worker.localTestOverride(),
34-
`classgraph-worker`.localTestOverride()
34+
`classgraph-worker`.localTestOverride(),
35+
`jarjarabrams-worker`.localTestOverride()
3536
)
3637
def testForkEnv = {
3738
val locale = if (Properties.isMac) "en_US.UTF-8" else "C.utf8"
@@ -108,4 +109,10 @@ object `package` extends build.MillStableScalaModule {
108109
def moduleDeps = Seq(api, build.core.util)
109110
def mvnDeps = Agg(build.Deps.classgraph)
110111
}
112+
113+
object `jarjarabrams-worker` extends build.MillPublishScalaModule {
114+
def moduleDeps = Seq(api, build.core.util, build.libs.scalalib)
115+
def mvnDeps = Agg(build.Deps.jarjarabrams)
116+
}
117+
111118
}

‎libs/scalalib/src/mill/scalalib/Assembly.scala‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package mill.scalalib
22

3-
import com.eed3si9n.jarjarabrams.{ShadePattern, Shader}
43
import mill.define.PathRef
54
import mill.util.JarManifest
65
import os.Generator
@@ -111,28 +110,9 @@ object Assembly {
111110

112111
def loadShadedClasspath(
113112
inputPaths: Seq[os.Path],
114-
assemblyRules: Seq[Assembly.Rule]
113+
shader: (String, UnopenedInputStream) => Option[(String, UnopenedInputStream)] =
114+
(n, is) => Some((n, is))
115115
): (Generator[(String, UnopenedInputStream)], ResourceCloser) = {
116-
val shadeRules = assemblyRules.collect {
117-
case Rule.Relocate(from, to) => ShadePattern.Rename(List(from -> to)).inAll
118-
}
119-
val shader =
120-
if (shadeRules.isEmpty)
121-
(name: String, inputStream: UnopenedInputStream) => Some(name -> inputStream)
122-
else {
123-
val shader = Shader.bytecodeShader(shadeRules, verbose = false, skipManifest = true)
124-
(name: String, inputStream: UnopenedInputStream) => {
125-
val is = inputStream()
126-
shader(is.readAllBytes(), name).map {
127-
case (bytes, name) =>
128-
name -> (() =>
129-
new ByteArrayInputStream(bytes) {
130-
override def close(): Unit = is.close()
131-
}
132-
)
133-
}
134-
}
135-
}
136116

137117
val pathsWithResources = inputPaths.filter(os.exists).map { path =>
138118
if (os.isFile(path)) path -> Some(new JarFile(path.toIO))
@@ -181,7 +161,12 @@ object Assembly {
181161
manifest: JarManifest = JarManifest.MillDefault,
182162
prependShellScript: Option[String] = None,
183163
base: Option[Assembly] = None,
184-
assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules
164+
assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules,
165+
shader: (
166+
Seq[(String, String)],
167+
String,
168+
UnopenedInputStream
169+
) => Option[(String, UnopenedInputStream)] = (_, n, is) => Some((n, is))
185170
): Assembly = {
186171
val rawJar = os.temp("out-tmp", deleteOnExit = false)
187172
// we create the file later
@@ -206,7 +191,9 @@ object Assembly {
206191
manifest.build.write(manifestOut)
207192
}
208193

209-
val (mappings, resourceCleaner) = Assembly.loadShadedClasspath(inputPaths, assemblyRules)
194+
val relocates = assemblyRules.collect { case Rule.Relocate(from, to) => (from -> to) }
195+
val (mappings, resourceCleaner) =
196+
Assembly.loadShadedClasspath(inputPaths, shader(relocates, _, _))
210197
try {
211198
Assembly.groupAssemblyEntries(mappings, assemblyRules).foreach {
212199
case (mapping, entry) =>

‎libs/scalalib/src/mill/scalalib/AssemblyModule.scala‎

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package mill.scalalib
22

3-
import mill.define.{PathRef}
4-
import mill.api.{Result}
3+
import mill.define.PathRef
4+
import mill.api.Result
55
import mill.util.JarManifest
6-
import mill.define.{Target => T, _}
6+
import mill.define.{Target as T, *}
7+
import mill.scalalib.Assembly.UnopenedInputStream
78
import mill.util.Jvm
89

910
/**
@@ -90,7 +91,8 @@ trait AssemblyModule extends mill.define.Module {
9091
destJar = Task.dest / "out.jar",
9192
inputPaths = upstreamIvyAssemblyClasspath().map(_.path),
9293
manifest = manifest(),
93-
assemblyRules = assemblyRules
94+
assemblyRules = assemblyRules,
95+
shader = AssemblyModule.jarjarabramsWorker()
9496
)
9597
}
9698

@@ -107,7 +109,8 @@ trait AssemblyModule extends mill.define.Module {
107109
inputPaths = upstreamLocalAssemblyClasspath().map(_.path),
108110
manifest = manifest(),
109111
base = Some(resolvedIvyAssembly()),
110-
assemblyRules = assemblyRules
112+
assemblyRules = assemblyRules,
113+
shader = AssemblyModule.jarjarabramsWorker()
111114
)
112115
}
113116

@@ -125,7 +128,8 @@ trait AssemblyModule extends mill.define.Module {
125128
manifest = manifest(),
126129
prependShellScript = prependScript,
127130
base = Some(upstream),
128-
assemblyRules = assemblyRules
131+
assemblyRules = assemblyRules,
132+
shader = AssemblyModule.jarjarabramsWorker()
129133
)
130134
// See https://github.com/com-lihaoyi/mill/pull/2655#issuecomment-1672468284
131135
val problematicEntryCount = 65535
@@ -144,3 +148,41 @@ trait AssemblyModule extends mill.define.Module {
144148
}
145149
}
146150
}
151+
object AssemblyModule extends ExternalModule with CoursierModule with OfflineSupportModule {
152+
153+
def jarjarabramsWorkerClasspath: T[Seq[PathRef]] = T {
154+
defaultResolver().classpath(Seq(
155+
Dep.millProjectModule("mill-libs-scalalib-jarjarabrams-worker")
156+
))
157+
}
158+
159+
override def prepareOffline(all: mainargs.Flag): Command[Seq[PathRef]] = Task.Command {
160+
(
161+
super.prepareOffline(all)() ++
162+
jarjarabramsWorkerClasspath()
163+
).distinct
164+
}
165+
166+
private[mill] def jarjarabramsWorkerClassloader: Worker[ClassLoader] = Task.Worker {
167+
Jvm.createClassLoader(
168+
classPath = jarjarabramsWorkerClasspath().map(_.path),
169+
parent = getClass().getClassLoader()
170+
)
171+
}
172+
173+
def jarjarabramsWorker: Worker[(Seq[(String, String)], String, UnopenedInputStream) => Option[(
174+
String,
175+
UnopenedInputStream
176+
)]] = Task.Worker {
177+
(relocates: Seq[(String, String)], name: String, is: UnopenedInputStream) =>
178+
jarjarabramsWorkerClassloader()
179+
.loadClass("mill.scalalib.jarjarabrams.impl.JarJarAbramsWorkerImpl")
180+
.getMethods
181+
.filter(_.getName == "apply")
182+
.head
183+
.invoke(null, relocates, name, is)
184+
.asInstanceOf[Option[(String, UnopenedInputStream)]]
185+
}
186+
187+
override protected def millDiscover: Discover = Discover[this.type]
188+
}

0 commit comments

Comments
 (0)