Inline scala compiler worker classloader in mill daemon (#5087)

Discussed in #4872

Rather than going through this whole repeated
resolve-dependencies-populate-classloader thing, we just directly depend
on and instantiate the `MillParserImpl` in the main `runner.daemon` code
and pass it down through all the layers of classloaders via a shared
`DynamicVariable`. `runner.compilerworker.api` was simplified and merged
into `core.api` so the basic `MillScalaParser` types can be shared
across all parts of Mill

Apart from simplifying the code considerably, this should reduce the
number of duplicate classloaders we have, which should also help with
#4522

Author: lihaoyi

core/api/src/mill/api/internal/MillScalaParser.scala

@@ -0,0 +1,39 @@
+package mill.api.internal
+
+import scala.util.DynamicVariable
+
+trait MillScalaParser {
+  def splitScript(rawCode: String, fileName: String): Either[String, (Seq[String], Seq[String])]
+
+  /* not sure if this is the right way, in case needs change, or if we should accept some
+   * "generic" visitor over some "generic" trees?
+   */
+  def parseObjectData(rawCode: String): Seq[MillScalaParser.ObjectData]
+}
+
+object MillScalaParser {
+  trait ObjectData {
+    def obj: Snip
+
+    def name: Snip
+
+    def parent: Snip
+
+    def endMarker: Option[Snip]
+
+    def finalStat: Option[(String, Snip)]
+  }
+
+  trait Snip {
+    def text: String | Null
+
+    def start: Int
+
+    def end: Int
+
+    final def applyTo(s: String, replacement: String): String =
+      s.patch(start, replacement.padTo(end - start, ' '), end - start)
+  }
+
+  val current = new DynamicVariable[MillScalaParser](null)
+}

core/define/src/mill/define/RootModule0.scala

@@ -29,18 +29,15 @@ abstract class RootModule0()(implicit
 @internal
 object RootModule0 {
   class Info(
-      val compilerWorkerClasspath: Seq[os.Path],
       val projectRoot: os.Path,
       val output: os.Path,
       val topLevelProjectRoot: os.Path
   ) {
     def this(
-        compilerWorkerClasspath0: Seq[String],
         projectRoot0: String,
         output0: String,
         topLevelProjectRoot0: String
     ) = this(
-      compilerWorkerClasspath0.map(os.Path(_)),
       os.Path(projectRoot0),
       os.Path(output0),
       os.Path(topLevelProjectRoot0)

integration/feature/leak-hygiene/src/LeakHygieneTests.scala

@@ -42,7 +42,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
         checkClassloaders(tester)(
           "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
           "mill.codesig.ExternalSummary.apply upstreamClassloader" -> 1,
-          "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
           "mill.scalalib.JvmWorkerModule#worker cl" -> 1,
           "mill.scalalib.worker.JvmWorkerImpl#getCachedClassLoader cl" -> 1
         )
@@ -67,7 +66,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
             "mill.codesig.ExternalSummary.apply upstreamClassloader" -> 1,
             "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
             "mill.kotlinlib.KotlinWorkerFactory#setup cl" -> 1,
-            "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
             "mill.scalalib.JvmWorkerModule#worker cl" -> 2,
             "mill.scalalib.worker.JvmWorkerImpl#getCachedClassLoader cl" -> 2
           )
@@ -93,7 +91,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
             "mill.codesig.ExternalSummary.apply upstreamClassloader" -> 1,
             "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
             "mill.kotlinlib.KotlinWorkerFactory#setup cl" -> 1,
-            "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
             "mill.scalalib.JvmWorkerModule#worker cl" -> 2,
             "mill.scalalib.worker.JvmWorkerImpl#getCachedClassLoader cl" -> 2
           )
@@ -117,7 +114,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
         tester.eval(("shutdown"))
         checkClassloaders(tester)(
           "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
-          "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
           "mill.scalalib.JvmWorkerModule#worker cl" -> 1
         )
         checkThreads(tester)(
@@ -140,7 +136,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
           checkClassloaders(tester)(
             "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
             "mill.kotlinlib.KotlinWorkerFactory#setup cl" -> 1,
-            "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
             "mill.scalalib.JvmWorkerModule#worker cl" -> 2,
             "mill.scalalib.worker.JvmWorkerImpl#getCachedClassLoader cl" -> 1
           )
@@ -166,7 +161,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
           checkClassloaders(tester)(
             "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
             "mill.kotlinlib.KotlinWorkerFactory#setup cl" -> 1,
-            "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
             "mill.scalalib.JvmWorkerModule#worker cl" -> 2,
             "mill.scalalib.worker.JvmWorkerImpl#getCachedClassLoader cl" -> 1
           )
@@ -194,7 +188,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
           checkClassloaders(tester)(
             "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
             "mill.kotlinlib.KotlinWorkerFactory#setup cl" -> 1,
-            "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
             "mill.scalalib.JvmWorkerModule#worker cl" -> 2,
             "mill.scalalib.worker.JvmWorkerImpl#getCachedClassLoader cl" -> 1
           )
@@ -219,7 +212,6 @@ object LeakHygieneTests extends UtestIntegrationTestSuite {
           "leaked classloader" -> 1,
           "mill.daemon.MillBuildBootstrap#processRunClasspath classLoader cl" -> 1,
           "mill.kotlinlib.KotlinWorkerFactory#setup cl" -> 1,
-          "mill.meta.ScalaCompilerWorker.reflectUnsafe cl" -> 1,
           "mill.scalalib.JvmWorkerModule#worker cl" -> 2,
           "mill.scalalib.worker.JvmWorkerImpl#getCachedClassLoader cl" -> 1
         )

integration/ide/gen-idea/resources/extended/idea/mill_modules/mill-build.mill-build.iml

@@ -79,7 +79,6 @@
         <orderEntry type="library" name="mill-libs_3.jar" level="project"/>
         <orderEntry type="library" name="mill-moduledefs_3-0.11.4.jar" level="project"/>
         <orderEntry type="library" name="mill-runner-codesig_3.jar" level="project"/>
-        <orderEntry type="library" name="mill-runner-compilerworker-api_3.jar" level="project"/>
         <orderEntry type="library" name="mill-runner-meta_3.jar" level="project"/>
         <orderEntry type="library" name="native-terminal-no-ffm-0.0.9.1.jar" level="project"/>
         <orderEntry type="library" name="os-lib_3-0.11.5-M2.jar" level="project"/>