Clean up mill-dist generation (#5014)

- Rename `trait InstallModule` to `DistModule`
- Rename `testDep` to `localTestOverrides`
- No longer include `mill.runner` in `dist.localTestOverrides`, since
those are bundled with the executable classpath and should never be
resolved via coursier
- `DistModule` is now a bare `Module`, not a `MillPublishJavaModule`.
`build.dist` is a non-publish module which defines `def
publishArtifacts` itself rather than going through `PublishModule`
- `DistModule` defines `def publishArtifacts` directly rather than
inheriting/overriding it from `MillPublishJavaModule`. The various
publishing `ExternalModule`s all just look for a duck-typed `def
publishArtifacts: T[PublishModule.PublishData]`, so we shouldn't
actually need to inherit from `PublishModule` for publishing to work
- `def uploadToGithub` since we distribute all artifacts on maven
central now
- Remove `dist0` since it's no longer necessary
- Consolidate handling of `def transitiveLocalTestOverrides` in
`dist/package.mill`


Overall this narrows down the logic around `dist` publishing
considerably: we no longer publish them as full `JavaModule`s with
associated poms and sourcejars and javadoc and metadata, and instead
publish only the executable assembly jar or native image for people to
use. Which is what we want, since the `dist` files are all standalone
and should never be part of a larger JVM dependency graph.

Author: lihaoyi

build.mill

@@ -412,7 +412,8 @@ val bridgeVersion = "0.0.1"
 trait MillJavaModule extends JavaModule {
 
   // Test setup
-  def testDep = Task { (s"com.lihaoyi-${artifactId()}", testDepPaths().map(_.path).mkString("\n")) }
+  def localTestOverride =
+    Task { (s"com.lihaoyi-${artifactId()}", localTestOverridePaths().map(_.path).mkString("\n")) }
 
   def testArgs: T[Seq[String]] = Task {
     // Workaround for Zinc/JNA bug
@@ -423,14 +424,15 @@ trait MillJavaModule extends JavaModule {
       else Nil
     jnaArgs ++ userLang
   }
-  def testDepPaths = Task { upstreamAssemblyClasspath() ++ Seq(compile().classes) ++ resources() }
+  def localTestOverridePaths =
+    Task { upstreamAssemblyClasspath() ++ Seq(compile().classes) ++ resources() }
 
-  def testTransitiveDeps: T[Map[String, String]] = Task {
+  def transitiveLocalTestOverrides: T[Map[String, String]] = Task {
     val upstream = Task.traverse(moduleDeps ++ compileModuleDeps) {
-      case m: MillJavaModule => m.testTransitiveDeps.map(Some(_))
+      case m: MillJavaModule => m.transitiveLocalTestOverrides.map(Some(_))
       case _ => Task.Anon(None)
     }().flatten.flatten
-    val current = Seq(testDep())
+    val current = Seq(localTestOverride())
     upstream.toMap ++ current
   }
 
@@ -441,7 +443,7 @@ trait MillJavaModule extends JavaModule {
     else Seq(this, build.libs.main.test)
 
   def localTestOverridesClasspath = Task {
-    for ((k, v) <- testTransitiveDeps()) {
+    for ((k, v) <- transitiveLocalTestOverrides()) {
       os.write(Task.dest / "mill/local-test-overrides" / k, v, createFolders = true)
     }
     PathRef(Task.dest)

contrib/package.mill

@@ -33,12 +33,12 @@ object `package` extends RootModule {
 
   object testng extends JavaModule with ContribModule {
 
-    def testTransitiveDeps =
-      super.testTransitiveDeps() ++
+    def transitiveLocalTestOverrides =
+      super.transitiveLocalTestOverrides() ++
         Seq(
-          build.libs.scalalib.testDep(),
-          build.libs.scalalib.worker.testDep(),
-          build.libs.testrunner.entrypoint.testDep()
+          build.libs.scalalib.localTestOverride(),
+          build.libs.scalalib.worker.localTestOverride(),
+          build.libs.testrunner.entrypoint.localTestOverride()
         )
 
     // pure Java implementation
@@ -60,8 +60,10 @@ object `package` extends RootModule {
     def moduleDeps = Seq(twirllib, playlib.api)
     def compileModuleDeps = Seq(build.libs.scalalib)
 
-    def testTransitiveDeps =
-      super.testTransitiveDeps() ++ Task.traverse(build.Deps.play.keys.toSeq)(worker(_).testDep)()
+    def transitiveLocalTestOverrides =
+      super.transitiveLocalTestOverrides() ++ Task.traverse(
+        build.Deps.play.keys.toSeq
+      )(worker(_).localTestOverride)()
 
     def testArgs = Task {
       super.testArgs() ++
@@ -108,8 +110,8 @@ object `package` extends RootModule {
     def moduleDeps = Seq(scoverage.api)
     def compileModuleDeps = Seq(build.libs.scalalib)
 
-    def testTransitiveDeps =
-      super.testTransitiveDeps() ++ Seq(worker2.testDep())
+    def transitiveLocalTestOverrides =
+      super.transitiveLocalTestOverrides() ++ Seq(worker2.localTestOverride())
 
     def testArgs = Task {
       super.testArgs() ++
@@ -130,7 +132,7 @@ object `package` extends RootModule {
 
     // Worker for Scoverage 2.0
     object worker2 extends build.MillPublishScalaModule {
-      def testDepPaths = Task { Seq(compile().classes) }
+      def localTestOverridePaths = Task { Seq(compile().classes) }
       def compileModuleDeps = Seq(
         build.core.api,
         scoverage.api

core/util/package.mill

@@ -38,13 +38,6 @@ object `package` extends build.MillStableScalaModule with BuildInfo {
     ),
     BuildInfo.Value("scalaVersion", scalaVersion(), "Scala version used to compile mill core."),
     BuildInfo.Value("millVersion", build.millVersion(), "Mill version."),
-    BuildInfo.Value(
-      "millDistModule", {
-        val selfDep = build.dist.publishSelfDependency()
-        s"${selfDep.group}:${selfDep.id}"
-      },
-      "Coordinates of the module pulling the whole of Mill"
-    ),
     BuildInfo.Value("millBinPlatform", build.millBinPlatform(), "Mill binary platform version."),
     BuildInfo.Value(
       "millAllDistDependencies", {

core/util/src/mill/util/Jvm.scala

@@ -677,62 +677,75 @@ object Jvm {
     val resolutionParams0 = resolutionParams
       .addForceVersion(forceVersions.toSeq*)
 
-    val testOverridesRepo =
-      new TestOverridesRepo(os.resource(getClass.getClassLoader) / "mill/local-test-overrides")
-
-    val resolve = Resolve()
-      .withCache(coursierCache0)
-      .withDependencies(rootDeps)
-      .withRepositories(testOverridesRepo +: repositories)
-      .withResolutionParams(resolutionParams0)
-      .withMapDependenciesOpt(mapDependencies)
-      .withBoms(boms.iterator.toSeq)
-
-    resolve.either() match {
-      case Left(error) =>
-        val cantDownloadErrors = error.errors.collect {
-          case cantDownload: CantDownloadModule => cantDownload
-        }
-        if (error.errors.length == cantDownloadErrors.length) {
-          val extraHeader =
-            if (offlineMode)
-              """
-                |*** Mill is in offline mode (--offline) ***
-                |It can not download new dependencies from remote repositories.
-                |You may need to run Mill without the `--offline` option at least once
-                |to download required remote dependencies.
-                |Run `mill __.prepareOffline` to fetch most remote resources at once.
-                |
-                |""".stripMargin
-            else ""
-
-          val header =
-            s"""|
-                |${extraHeader}Resolution failed for ${cantDownloadErrors.length} modules:
-                |--------------------------------------------
-                |""".stripMargin
-
-          val helpMessage =
-            s"""|
-                |--------------------------------------------
-                |
-                |For additional information on library dependencies, see the docs at
-                |${mill.api.BuildInfo.millDocUrl}/mill/Library_Dependencies.html""".stripMargin
-
-          val errLines = cantDownloadErrors
-            .map { err =>
-              s"  ${err.module.trim}:${err.versionConstraint.asString} \n\t" +
-                err.perRepositoryErrors.mkString("\n\t")
-            }
-            .mkString("\n")
-          val msg = header + errLines + "\n" + helpMessage + "\n"
-          Result.Failure(msg)
-        } else {
-          throw error
-        }
-      case Right(resolution0) =>
-        val resolution = customizer.fold(resolution0)(_.apply(resolution0))
-        Result.Success(resolution)
+    val testOverridesClassloaders = System.getenv("MILL_LOCAL_TEST_OVERRIDE_CLASSPATH") match {
+      case null => Nil
+      case cp => Seq(new URLClassLoader(Array(os.Path(cp).toNIO.toUri.toURL)))
+    }
+
+    try {
+      val envTestOverridesRepo = testOverridesClassloaders.map(cl =>
+        new TestOverridesRepo(os.resource(cl) / "mill/local-test-overrides")
+      )
+
+      val resourceTestOverridesRepo =
+        new TestOverridesRepo(os.resource(getClass.getClassLoader) / "mill/local-test-overrides")
+
+      val resolve = Resolve()
+        .withCache(coursierCache0)
+        .withDependencies(rootDeps)
+        .withRepositories(Seq(resourceTestOverridesRepo) ++ envTestOverridesRepo ++ repositories)
+        .withResolutionParams(resolutionParams0)
+        .withMapDependenciesOpt(mapDependencies)
+        .withBoms(boms.iterator.toSeq)
+
+      resolve.either() match {
+        case Left(error) =>
+          val cantDownloadErrors = error.errors.collect {
+            case cantDownload: CantDownloadModule => cantDownload
+          }
+          if (error.errors.length == cantDownloadErrors.length) {
+            val extraHeader =
+              if (offlineMode)
+                """
+                  |*** Mill is in offline mode (--offline) ***
+                  |It can not download new dependencies from remote repositories.
+                  |You may need to run Mill without the `--offline` option at least once
+                  |to download required remote dependencies.
+                  |Run `mill __.prepareOffline` to fetch most remote resources at once.
+                  |
+                  |""".stripMargin
+              else ""
+
+            val header =
+              s"""|
+                  |${extraHeader}Resolution failed for ${cantDownloadErrors.length} modules:
+                  |--------------------------------------------
+                  |""".stripMargin
+
+            val helpMessage =
+              s"""|
+                  |--------------------------------------------
+                  |
+                  |For additional information on library dependencies, see the docs at
+                  |${mill.api.BuildInfo.millDocUrl}/mill/Library_Dependencies.html""".stripMargin
+
+            val errLines = cantDownloadErrors
+              .map { err =>
+                s"  ${err.module.trim}:${err.versionConstraint.asString} \n\t" +
+                  err.perRepositoryErrors.mkString("\n\t")
+              }
+              .mkString("\n")
+            val msg = header + errLines + "\n" + helpMessage + "\n"
+            Result.Failure(msg)
+          } else {
+            throw error
+          }
+        case Right(resolution0) =>
+          val resolution = customizer.fold(resolution0)(_.apply(resolution0))
+          Result.Success(resolution)
+      }
+    } finally {
+      testOverridesClassloaders.foreach(_.close())
     }
   }