Move the semanticdb generation logic to compile and make sure it's shared where possible

contrib/playlib/test/src/mill/playlib/PlayModuleTests.scala

@@ -14,6 +14,7 @@ object PlayModuleTests extends TestSuite with PlayTestSuite {
       val (crossScalaVersion, crossPlayVersion) = (crossValue, crossValue2)
       override def playVersion = crossPlayVersion
       override def scalaVersion = crossScalaVersion
+
       object test extends PlayTests
       override def mvnDeps = Task { super.mvnDeps() ++ Seq(ws()) }
     }

contrib/playlib/test/src/mill/playlib/PlaySingleApiModuleTests.scala

@@ -10,7 +10,7 @@ object PlaySingleApiModuleTests extends TestSuite with PlayTestSuite {
   object playsingleapi extends TestRootModule with PlayApiModule {
     override val moduleDir = os.temp() // workaround problem in `SingleModule`
     override def playVersion = Task { testPlay28 }
-    override def scalaVersion = Task { "2.13.12" }
+    override def scalaVersion = Task { "2.13.17" }
     object test extends PlayTests
 
     lazy val millDiscover = Discover[this.type]

core/api/daemon/src/mill/api/daemon/internal/SemanticDbJavaModuleApi.scala

@@ -9,17 +9,17 @@ trait SemanticDbJavaModuleApi {
   private[mill] def bspCompiledClassesAndSemanticDbFiles: TaskApi[UnresolvedPathApi[?]]
 }
 object SemanticDbJavaModuleApi {
-  val buildTimeJavaSemanticDbVersion = BuildInfo.semanticDbJavaVersion
-  val buildTimeSemanticDbVersion = BuildInfo.semanticDBVersion
+  val buildTimeJavaSemanticDbVersion: String = BuildInfo.semanticDbJavaVersion
+  val buildTimeSemanticDbVersion: String = BuildInfo.semanticDBVersion
 
   private[mill] val contextSemanticDbVersion: InheritableThreadLocal[Option[String]] =
     new InheritableThreadLocal[Option[String]] {
-      protected override def initialValue(): Option[String] = None.asInstanceOf[Option[String]]
+      protected override def initialValue(): Option[String] = None
     }
 
   private[mill] val contextJavaSemanticDbVersion: InheritableThreadLocal[Option[String]] =
     new InheritableThreadLocal[Option[String]] {
-      protected override def initialValue(): Option[String] = None.asInstanceOf[Option[String]]
+      protected override def initialValue(): Option[String] = None
     }
 
   private[mill] def resetContext(): Unit = {

core/api/src/mill/api/BuildCtx.scala

@@ -1,7 +1,8 @@
 package mill.api
 import collection.mutable
 import mill.api.Watchable
-import mill.constants.EnvVars
+import mill.constants.{EnvVars, OutFiles, OutFolderMode}
+
 import scala.util.DynamicVariable
 
 /**
@@ -50,6 +51,7 @@ object BuildCtx {
     }
   }
 
+  /** As [[watchValue]] but watches a file path. */
   def watch(p: os.Path): os.Path = {
     val watchable = Watchable.Path(p.toNIO, false, PathRef(p).sig)
     watchedValues.append(watchable)
@@ -59,4 +61,16 @@ object BuildCtx {
   def watch0(w: Watchable): Unit = watchedValues.append(w)
 
   def evalWatch0(w: Watchable): Unit = evalWatchedValues.append(w)
+
+  /**
+   * Folder in the filesystem where Mill's BSP sessions that require semanticdb store an indicator file (name =
+   * process PID, contents are irrelevant) to communicate to main Mill daemon and other BSP sessions that there is at
+   * least one Mill session that will need the semanticdb.
+   *
+   * The reasoning is that if at least one of Mill's clients requests semanticdb, then there is no point in running
+   * regular `compile` without semanticdb, as eventually we will have to rerun it with semanticdb, and thus we should
+   * compile with semanticdb upfront to avoid paying the price of compling twice (without semanticdb and then with it).
+   */
+  private[mill] def bspSemanticDbSessionsFolder: os.Path =
+    workspaceRoot / os.SubPath(OutFiles.outFor(OutFolderMode.BSP)) / "semanticdb-sessions"
 }

core/constants/src/mill/constants/EnvVars.java

@@ -27,6 +27,12 @@ public class EnvVars {
    */
   public static final String MILL_OUTPUT_DIR = "MILL_OUTPUT_DIR";
 
+  /**
+   * Output directory where Mill workers' state and Mill tasks output should be
+   * written to for the Mill instances running in BSP mode.
+   */
+  public static final String MILL_BSP_OUTPUT_DIR = "MILL_BSP_OUTPUT_DIR";
+
   /**
    * If set to "1", Mill will re-use the regular @{Link OutFiles#out} folder instead of
    * using a separate one for BSP output.

core/constants/src/mill/constants/OutFiles.java

@@ -12,6 +12,12 @@ public class OutFiles {
    */
   private static final String envOutOrNull = System.getenv(EnvVars.MILL_OUTPUT_DIR);
 
+  /**
+   * Allows us to override the `out/mill-bsp-out` folder from the environment via the
+   * {@link EnvVars#MILL_BSP_OUTPUT_DIR} variable.
+   */
+  private static final String envBspOutOrNull = System.getenv(EnvVars.MILL_BSP_OUTPUT_DIR);
+
   /** @see EnvVars#MILL_NO_SEPARATE_BSP_OUTPUT_DIR */
   public static final boolean mergeBspOut =
       "1".equals(System.getenv(EnvVars.MILL_NO_SEPARATE_BSP_OUTPUT_DIR));
@@ -28,24 +34,30 @@ public class OutFiles {
    */
   public static final String out = envOutOrNull == null ? defaultOut : envOutOrNull;
 
+  /**
+   * Default hard-coded value for the Mill `out/` folder path when Mill is running in BSP mode. Unless you know
+   * what you are doing, you should favor using {@link #outFor} instead.
+   */
+  public static final String defaultBspOut = "out/mill-bsp-out";
+
   /**
    * Path of the Mill `out/` folder when Mill is running in BSP mode. Unless you know
    * what you are doing, you should favor using {@link #outFor} instead.
    */
-  public static final String bspOut = "out/mill-bsp-out";
+  public static final String bspOut = envBspOutOrNull == null ? defaultBspOut : envBspOutOrNull;
 
   /**
    * Path of the Mill {@link #out} folder.
    *
    * @param outMode If {@link #envOutOrNull} is set, this parameter is ignored.
    */
   public static String outFor(OutFolderMode outMode) {
-    if (envOutOrNull != null) return envOutOrNull;
     switch (outMode) {
       case REGULAR:
-        return out;
+        return envOutOrNull != null ? envOutOrNull : out;
       case BSP:
-        return mergeBspOut ? out : bspOut;
+        if (envBspOutOrNull != null) return envBspOutOrNull;
+        return mergeBspOut ? outFor(OutFolderMode.REGULAR) : bspOut;
       default:
         throw new IllegalArgumentException("Unknown out folder mode: " + outMode);
     }

core/exec/src/mill/exec/GroupExecution.scala

@@ -553,7 +553,7 @@ object GroupExecution {
           if (path.startsWith(workspace) && !validReadDests.exists(path.startsWith(_))) {
             sys.error(
               s"Reading from ${path.relativeTo(workspace)} not allowed during execution of `$terminal`.\n" +
-                "You can only read files referenced by `Task.Source` or `Task.Sources`, or within a `Task.Input"
+                "You can only read files referenced by `Task.Source` or `Task.Sources`, or within a `Task.Input`"
             )
           }
         }

example/scalalib/spark/1-hello-spark/build.mill

@@ -3,6 +3,8 @@ import mill.*, scalalib.*
 
 object foo extends ScalaModule {
   def scalaVersion = "2.12.20"
+  def semanticDbVersion = "4.9.0" // last version to support this Scala
+
   def mvnDeps = Seq(
     mvn"org.apache.spark::spark-core:3.5.4",
     mvn"org.apache.spark::spark-sql:3.5.4"

example/scalalib/spark/3-semi-realistic/build.mill

@@ -3,6 +3,8 @@ import mill.*, scalalib.*
 
 object `package` extends ScalaModule {
   def scalaVersion = "2.12.20"
+  def semanticDbVersion = "4.9.0" // last version to support this Scala
+
   def mvnDeps = Seq(
     mvn"org.apache.spark::spark-core:3.5.6",
     mvn"org.apache.spark::spark-sql:3.5.6"

integration/bsp-util/src/BspServerTestUtil.scala

@@ -11,6 +11,7 @@ import org.eclipse.lsp4j.jsonrpc.services.JsonRequest
 import java.io.ByteArrayOutputStream
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.concurrent.{CompletableFuture, ExecutorService, Executors, ThreadFactory}
+import scala.annotation.unused
 import scala.jdk.CollectionConverters.*
 import scala.reflect.ClassTag
 
@@ -226,7 +227,7 @@ object BspServerTestUtil {
       workspacePath: os.Path,
       coursierCache: os.Path = os.Path(CacheDefaults.location),
       javaHome: os.Path = os.Path(sys.props("java.home")),
-      javaVersion: String = sys.props("java.version")
+      @unused javaVersion: String = sys.props("java.version")
   ): Seq[(String, String)] =
     Seq(
       workspacePath.toURI.toASCIIString.stripSuffix("/") -> "file:///workspace",

integration/feature/scoverage/resources/build.mill

@@ -17,16 +17,27 @@ object Deps {
 object core extends Cross[CoreCross]("2.13.11")
 trait CoreCross extends CrossScalaModule with ScoverageModule {
   override def scoverageVersion = "2.0.11"
+  override def semanticDbVersion = "4.9.9" // last version to support this Scala
+
+  // customized scoverage data
+  override lazy val scoverage: ScoverageData = new ScoverageData {
+    override def semanticDbVersion = "4.9.9" // last version to support this Scala
+  }
+
   object test extends ScoverageTests with TestModule.ScalaTest {
+    override def semanticDbVersion = "4.9.9" // last version to support this Scala
     override def mvnDeps = Seq(Deps.scalaTest, Deps.millMain)
   }
 }
 
 object extra extends ScalaModule with ScoverageModule {
   override def scoverageVersion = "2.0.11"
   override def scalaVersion = "2.13.11"
+  override def semanticDbVersion = "4.9.9" // last version to support this Scala
+
   // customized scoverage data
   override lazy val scoverage: ScoverageData = new ScoverageData {
     // some customizations
+    override def semanticDbVersion = "4.9.9" // last version to support this Scala
   }
 }

integration/feature/scoverage/src/ScoverageTests.scala

@@ -9,8 +9,9 @@ object ScoverageTests extends UtestIntegrationTestSuite {
     test("test") - retry(3) {
       integrationTest { tester =>
         import tester._
-        assert(eval("__.compile").isSuccess)
-        assert(eval("core[2.13.11].scoverage.xmlReport").isSuccess)
+
+        prepEval("__.compile").runWithClues(r => assert(r.isSuccess))
+        prepEval("core[2.13.11].scoverage.xmlReport").runWithClues(r => assert(r.isSuccess))
       }
     }
   }

integration/ide/bsp-server/resources/snapshots/logging

@@ -3,12 +3,12 @@
 [1-buildInitialize] Entered buildInitialize
 [1-buildInitialize] Got client semanticdbVersion: * Enabling SemanticDB support.
 [1-buildInitialize] buildInitialize took * msec
-[bsp-init-mill-build/build.mill-59] [info] compiling * Scala sources to * ...
-[bsp-init-mill-build/build.mill-59] [info] done compiling
-[bsp-init-build.mill-59] [info] compiling * Scala sources to * ...
-[bsp-init-build.mill-59] [info] done compiling
-[bsp-init-098] [info] compiling * Scala source to * ...
-[bsp-init-098] [info] done compiling
+[bsp-init-mill-build/build.mill-70] [info] compiling * Scala sources to * ...
+[bsp-init-mill-build/build.mill-70] [info] done compiling
+[bsp-init-build.mill-70] [info] compiling * Scala sources to * ...
+[bsp-init-build.mill-70] [info] done compiling
+[bsp-init-109] [info] compiling * Scala source to * ...
+[bsp-init-109] [info] done compiling
 [bsp-init] SNAPSHOT
 [2-workspaceBuildTargets] Entered workspaceBuildTargets
 [2-workspaceBuildTargets] Evaluating * tasks
@@ -29,16 +29,24 @@
 [3-loggingTest] loggingTest took * msec
 [4-compile] Entered buildTargetCompile
 [4-compile] Evaluating 1 task
-[4-compile] Done
+[4-compile-60] errored.exception.compile task failed
+[4-compile] 1 tasks failed
+[4-compile] errored.exception.compile java.lang.RuntimeException: nope
+[4-compile]     scala.sys.package$.error(package.scala:27)
+[4-compile]     build_.package_.build_$package_$errored$exception$$$_$compile$$anonfun$1$$anonfun$1(build.mill:75)
+[4-compile]     mill.api.Task$Named.evaluate(Task.scala:389)
+[4-compile]     mill.api.Task$Named.evaluate$(Task.scala:374)
+[4-compile]     mill.api.Task$Computed.evaluate(Task.scala:400)
+[4-compile] Failed
 [4-compile] buildTargetCompile took * msec
 [5-compile] Entered buildTargetCompile
 [5-compile] Evaluating 1 task
-[5-compile-55] [info] compiling * Scala source to * ...
-[5-compile-55] [error] *:2:3: not found: value nope
-[5-compile-55] [error]   nope
-[5-compile-55] [error]   ^
-[5-compile-55] [error] one error found
-[5-compile-55] errored.compilation-error.semanticDbDataDetailed task failed
+[5-compile-59] [info] compiling * Scala source to * ...
+[5-compile-59] [error] *:2:3: not found: value nope
+[5-compile-59] [error]   nope
+[5-compile-59] [error]   ^
+[5-compile-59] [error] one error found
+[5-compile-59] errored.compilation-error.compile task failed
 [5-compile] Done
 [5-compile] buildTargetCompile took * msec
 [6-compile] Entered buildTargetCompile

libs/androidlib/src/mill/androidlib/AndroidModule.scala

@@ -521,6 +521,7 @@ trait AndroidModule extends JavaModule { outer =>
       .internalWorker()
       .compileJava(
         ZincCompileJava(
+          compileTo = Task.dest,
           upstreamCompileOutput = upstreamCompileOutput(),
           sources = androidLibsRClasses().map(_.path),
           compileClasspath = Seq.empty,
@@ -734,6 +735,7 @@ trait AndroidModule extends JavaModule { outer =>
       .internalWorker()
       .compileJava(
         ZincCompileJava(
+          compileTo = Task.dest,
           upstreamCompileOutput = upstreamCompileOutput(),
           sources = sources.map(_.path),
           compileClasspath = androidTransitiveLibRClasspath().map(_.path),