Skip to content

Commit c0b6f18

Browse files
authored
Add example docs for classloader workers (#3796)
Fixes #3794 It's still pretty messy, but i'm just documenting the existing APIs that already exist, and at least it provides a paved path for people to get _something_ working. Cleaning them up can come separately in #3775
1 parent 4a074b1 commit c0b6f18

File tree

11 files changed

+148
-5
lines changed

11 files changed

+148
-5
lines changed

docs/modules/ROOT/pages/extending/running-jvm-code.adoc

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,8 @@ include::partial$example/extending/jvmcode/1-subprocess.adoc[]
3030

3131
== In-process Isolated Classloaders
3232

33-
include::partial$example/extending/jvmcode/2-inprocess.adoc[]
33+
include::partial$example/extending/jvmcode/2-classloader.adoc[]
34+
35+
== Classloader Worker Tasks
36+
37+
include::partial$example/extending/jvmcode/3-worker.adoc[]

example/extending/jvmcode/2-inprocess/build.mill renamed to example/extending/jvmcode/2-classloader/build.mill

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ object foo extends JavaModule {
1414
def groovyScript = Task.Source(millSourcePath / "generate.groovy")
1515

1616
def groovyGeneratedResources = Task{
17-
Jvm.runInprocess(classPath = groovyClasspath().map(_.path)){ classLoader =>
17+
Jvm.runClassloader(classPath = groovyClasspath().map(_.path)){ classLoader =>
1818
classLoader
1919
.loadClass("groovy.ui.GroovyMain")
2020
.getMethod("main", classOf[Array[String]])
@@ -34,7 +34,7 @@ object foo extends JavaModule {
3434
def resources = super.resources() ++ Seq(groovyGeneratedResources())
3535
}
3636

37-
// Note that unlike `Jvm.runSubprocess`, `Jvm.runInprocess` does not take a `workingDir`
37+
// Note that unlike `Jvm.runSubprocess`, `Jvm.runClassloader` does not take a `workingDir`
3838
// on `mainArgs`: it instead provides you an in-memory `classLoader` that contains the
3939
// classpath you gave it. From there, you can use `.loadClass` and `.getMethod` to fish out
4040
// the classes and methods you want, and `.invoke` to call them.
@@ -45,7 +45,7 @@ object foo extends JavaModule {
4545
Contents of groovy-generated.html is <html><body><h1>Hello!</h1><p>Groovy!</p></body></html>
4646
*/
4747

48-
// `Jvm.runInprocess` has significantly less overhead than `Jvm.runSubprocess`: both in terms
48+
// `Jvm.runClassloader` has significantly less overhead than `Jvm.runSubprocess`: both in terms
4949
// of wall-clock time and in terms of memory footprint. However, it does have somewhat less
5050
// isolation, as the code is running inside your JVM and cannot be configured to have a separate
5151
// working directory, environment variables, and other process-global configs. Which one is
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
def htmlContent = "<html><body><h1>Hello!</h1><p>" + args[0] + "</p></body></html>"
2+
3+
def outputFile = new File(args[1])
4+
outputFile.write(htmlContent)
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package bar;
2+
3+
import java.io.IOException;
4+
import java.io.InputStream;
5+
6+
public class Bar {
7+
8+
// Read `file.txt` from classpath
9+
public static String groovyGeneratedHtml() throws IOException {
10+
// Get the resource as an InputStream
11+
try (InputStream inputStream = Bar.class.getClassLoader().getResourceAsStream("groovy-generated.html")) {
12+
return new String(inputStream.readAllBytes());
13+
}
14+
}
15+
16+
public static void main(String[] args) throws IOException{
17+
String appClasspathResourceText = Bar.groovyGeneratedHtml();
18+
System.out.println("Contents of groovy-generated.html is " + appClasspathResourceText);
19+
}
20+
}
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Althought running JVM bytecode via a one-off isolated classloader has less overhead
2+
// than running it in a subprocess, the fact that the classloader needs to be created
3+
// each time adds overhead: newly-created classloaders contain code that is not yet
4+
// optimized by the JVM. When performance matters, you can put the classloader in a
5+
// `Task.Worker` to keep it around, allowing the code internally to be optimized and
6+
// stay optimized without being thrown away each time
7+
8+
// This example is similar to the earlier example running the Groovy interpreter in
9+
// a subprocess, but instead of using `Jvm.runSubprocess` we use `Jvm.inprocess` to
10+
// load the Groovy interpreter classpath files into an in-memory in-process classloader:
11+
12+
package build
13+
import mill._, javalib._
14+
import mill.util.Jvm
15+
16+
object coursierModule extends CoursierModule
17+
18+
def groovyClasspath: Task[Agg[PathRef]] = Task{
19+
coursierModule.defaultResolver().resolveDeps(Agg(ivy"org.codehaus.groovy:groovy:3.0.9"))
20+
}
21+
22+
def groovyWorker: Worker[java.net.URLClassLoader] = Task.Worker{
23+
mill.api.ClassLoader.create(groovyClasspath().map(_.path.toIO.toURL).toSeq, parent = null)
24+
}
25+
26+
trait GroovyGenerateJavaModule extends JavaModule {
27+
def groovyScript = Task.Source(millSourcePath / "generate.groovy")
28+
29+
def groovyGeneratedResources = Task{
30+
val oldCl = Thread.currentThread().getContextClassLoader
31+
Thread.currentThread().setContextClassLoader(groovyWorker())
32+
try {
33+
groovyWorker()
34+
.loadClass("groovy.ui.GroovyMain")
35+
.getMethod("main", classOf[Array[String]])
36+
.invoke(
37+
null,
38+
Array[String](
39+
groovyScript().path.toString,
40+
groovyGenerateArg(),
41+
(Task.dest / "groovy-generated.html").toString
42+
)
43+
)
44+
} finally Thread.currentThread().setContextClassLoader(oldCl)
45+
PathRef(Task.dest)
46+
}
47+
48+
def groovyGenerateArg: T[String]
49+
def resources = super.resources() ++ Seq(groovyGeneratedResources())
50+
}
51+
52+
object foo extends GroovyGenerateJavaModule{
53+
def groovyGenerateArg = "Foo Groovy!"
54+
}
55+
object bar extends GroovyGenerateJavaModule{
56+
def groovyGenerateArg = "Bar Groovy!"
57+
}
58+
59+
// Here we have two modules `foo` and `bar`, each of which makes use of `groovyWorker`
60+
// to evaluate a groovy script to generate some resources. In this case, we invoke the `main`
61+
// method of `groovy.ui.GroovyMain`, which also happens to require us to set the
62+
// `ContextClassLoader` to work.
63+
64+
65+
/** Usage
66+
67+
> ./mill foo.run
68+
Contents of groovy-generated.html is <html><body><h1>Hello!</h1><p>Foo Groovy!</p></body></html>
69+
70+
> ./mill bar.run
71+
Contents of groovy-generated.html is <html><body><h1>Hello!</h1><p>Bar Groovy!</p></body></html>
72+
*/
73+
74+
75+
// Because the `URLClassLoader` within `groovyWorker` is long-lived, the code within the
76+
// classloader can be optimized by the JVM runtime, and would have less overhead than if
77+
// run in separate classloaders via `Jvm.runClassloader`. And because `URLClassLoader`
78+
// already extends `AutoCloseable`, `groovyWorker` gets treated as an
79+
// xref:fundamentals/tasks.adoc#_autoclosable_workers[Autocloseable Worker] automatically.
80+
81+
// NOTE: As mentioned in documentation for xref:fundamentals/tasks.adoc#_workers[Worker Tasks],
82+
// the classloader contained within `groovyWorker` above is *initialized* in a single-thread,
83+
// but it may be *used* concurrently in a multi-threaded environment. Practically, that means
84+
// that the classes and methods you are invoking within the classloader do not make use of
85+
// un-synchronized global mutable variables.
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
def htmlContent = "<html><body><h1>Hello!</h1><p>" + args[0] + "</p></body></html>"
2+
3+
def outputFile = new File(args[1])
4+
outputFile.write(htmlContent)
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package foo;
2+
3+
import java.io.IOException;
4+
import java.io.InputStream;
5+
6+
public class Foo {
7+
8+
// Read `file.txt` from classpath
9+
public static String groovyGeneratedHtml() throws IOException {
10+
// Get the resource as an InputStream
11+
try (InputStream inputStream = Foo.class.getClassLoader().getResourceAsStream("groovy-generated.html")) {
12+
return new String(inputStream.readAllBytes());
13+
}
14+
}
15+
16+
public static void main(String[] args) throws IOException{
17+
String appClasspathResourceText = Foo.groovyGeneratedHtml();
18+
System.out.println("Contents of groovy-generated.html is " + appClasspathResourceText);
19+
}
20+
}

example/fundamentals/tasks/6-workers/build.mill

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ def compressBytes(input: Array[Byte]) = {
6363
// 2. Classloaders containing plugin code, to avoid classpath conflicts while
6464
// also avoiding classloading cost every time the code is executed
6565
//
66+
// NOTE: The _initialization_ of a `Task.Worker`'s value is single threaded,
67+
// but _usage_ of the worker's value may be done concurrently. The user of
68+
// `Task.Worker` is responsible for ensuring it's value is safe to use in a
69+
// multi-threaded environment via techniques like locks, atomics, or concurrent data
70+
// structures
71+
//
6672
// Workers live as long as the Mill process. By default, consecutive `mill`
6773
// commands in the same folder will re-use the same Mill process and workers,
6874
// unless `--no-server` is passed which will terminate the Mill process and

0 commit comments

Comments
 (0)