Merge pull request #567 from alexarchambault/bsp-wrap-script

Generate semantic DBs for Java sources, pass wrapped sources details over BSP

Author: alexarchambault

build.sc

@@ -46,6 +46,14 @@ object stubs extends JavaModule with ScalaCliPublishModule {
     super.javacOptions() ++ Seq("-target", "8", "-source", "8")
   }
 }
+object `scala-cli-bsp` extends JavaModule with ScalaCliPublishModule {
+  def ivyDeps = super.ivyDeps() ++ Seq(
+    Deps.bsp4j
+  )
+  def javacOptions = T {
+    super.javacOptions() ++ Seq("-target", "8", "-source", "8")
+  }
+}
 object integration extends Module {
   object docker extends CliIntegrationDocker {
     object test extends Tests {
@@ -170,6 +178,7 @@ class Build(val crossScalaVersion: String)
   def moduleDeps = Seq(
     `bloop-rifle`(),
     `build-macros`(),
+    `scala-cli-bsp`,
     `test-runner`(),
     `tasty-lib`()
   )
@@ -255,6 +264,10 @@ class Build(val crossScalaVersion: String)
          |  def semanticDbPluginModuleName = "semanticdb-scalac"
          |  def semanticDbPluginVersion = "${Deps.scalametaTrees.dep.version}"
          |
+         |  def semanticDbJavacPluginOrganization = "${Deps.semanticDbJavac.dep.module.organization.value}"
+         |  def semanticDbJavacPluginModuleName = "${Deps.semanticDbJavac.dep.module.name.value}"
+         |  def semanticDbJavacPluginVersion = "${Deps.semanticDbJavac.dep.version}"
+         |
          |  def localRepoResourcePath = "$localRepoResourcePath"
          |
          |  def jmhVersion = "1.29"

modules/bloop-rifle/src/main/scala/scala/build/blooprifle/BloopRifle.scala

@@ -54,6 +54,7 @@ object BloopRifle {
           bloopJava,
           config.javaOpts,
           cp.map(_.toPath),
+          config.workingDir,
           scheduler,
           config.startCheckPeriod,
           config.startCheckTimeout,

modules/bloop-rifle/src/main/scala/scala/build/blooprifle/BloopRifleConfig.scala

@@ -12,6 +12,7 @@ final case class BloopRifleConfig(
   javaPath: String,
   javaOpts: Seq[String],
   classPath: String => Either[Throwable, Seq[File]],
+  workingDir: File,
   bspSocketOrPort: Option[() => BspConnectionAddress],
   bspStdin: Option[InputStream],
   bspStdout: Option[OutputStream],
@@ -116,13 +117,15 @@ object BloopRifleConfig {
 
   def default(
     address: Address,
-    bloopClassPath: String => Either[Throwable, Seq[File]]
+    bloopClassPath: String => Either[Throwable, Seq[File]],
+    workingDir: File
   ): BloopRifleConfig =
     BloopRifleConfig(
       address = address,
       javaPath = "java",
       javaOpts = defaultJavaOpts,
       classPath = bloopClassPath,
+      workingDir = workingDir,
       bspSocketOrPort = None,
       bspStdin = None,
       bspStdout = None,

modules/bloop-rifle/src/main/scala/scala/build/blooprifle/internal/Operations.scala

@@ -102,6 +102,7 @@ object Operations {
     javaPath: String,
     javaOpts: Seq[String],
     classPath: Seq[Path],
+    workingDir: File,
     scheduler: ScheduledExecutorService,
     waitInterval: FiniteDuration,
     timeout: Duration,
@@ -124,6 +125,7 @@ object Operations {
         ) ++
         addressArgs
     val b = new ProcessBuilder(command: _*)
+    b.directory(workingDir)
     b.redirectInput(ProcessBuilder.Redirect.PIPE)
 
     // https://stackoverflow.com/questions/55628999/java-processbuilder-how-to-suppress-output-instead-of-redirecting-it/55629297#55629297

modules/build/src/main/scala/scala/build/Artifacts.scala

@@ -24,6 +24,8 @@ final case class Artifacts(
   compilerDependencies: Seq[AnyDependency],
   compilerArtifacts: Seq[(String, Path)],
   compilerPlugins: Seq[(AnyDependency, String, Path)],
+  javacPluginDependencies: Seq[(AnyDependency, String, Path)],
+  extraJavacPlugins: Seq[Path],
   dependencies: Seq[AnyDependency],
   scalaNativeCli: Seq[Path],
   detailedArtifacts: Seq[(CsDependency, csCore.Publication, csUtil.Artifact, Path)],
@@ -63,6 +65,8 @@ object Artifacts {
   def apply(
     params: ScalaParameters,
     compilerPlugins: Seq[Positioned[AnyDependency]],
+    javacPluginDependencies: Seq[Positioned[AnyDependency]],
+    extraJavacPlugins: Seq[Path],
     dependencies: Seq[Positioned[AnyDependency]],
     extraClassPath: Seq[Path],
     extraCompileOnlyJars: Seq[Path],
@@ -198,10 +202,23 @@ object Artifacts {
         .map(_.flatten)
     }
 
+    val javacPlugins0 = value {
+      javacPluginDependencies
+        .map { posDep =>
+          artifacts(posDep.map(Seq(_)), allExtraRepositories, params, logger)
+            .map(_.map { case (url, path) => (posDep.value, url, path) })
+        }
+        .sequence
+        .left.map(CompositeBuildException(_))
+        .map(_.flatten)
+    }
+
     Artifacts(
       compilerDependencies,
       compilerArtifacts,
       compilerPlugins0,
+      javacPlugins0,
+      extraJavacPlugins,
       updatedDependencies.map(_.value),
       scalaNativeCli,
       fetchRes.fullDetailedArtifacts.collect { case (d, p, a, Some(f)) => (d, p, a, f.toPath) },

modules/build/src/main/scala/scala/build/BloopBuildClient.scala

@@ -4,9 +4,11 @@ import ch.epfl.scala.bsp4j
 
 import java.io.PrintStream
 
+import scala.build.options.Scope
+
 trait BloopBuildClient extends bsp4j.BuildClient {
   def setProjectParams(newParams: Seq[String]): Unit
-  def setGeneratedSources(newGeneratedSources: Seq[GeneratedSource]): Unit
+  def setGeneratedSources(scope: Scope, newGeneratedSources: Seq[GeneratedSource]): Unit
   def diagnostics: Option[Seq[(Either[String, os.Path], bsp4j.Diagnostic)]]
   def clear(): Unit
 }

modules/build/src/main/scala/scala/build/Build.scala

@@ -559,8 +559,10 @@ object Build {
         s"-Xplugin:${path.toAbsolutePath}"
     }
 
+    val generateSemanticDbs = options.scalaOptions.generateSemanticDbs.getOrElse(false)
+
     val semanticDbScalacOptions =
-      if (options.scalaOptions.generateSemanticDbs.getOrElse(false))
+      if (generateSemanticDbs)
         if (params.scalaVersion.startsWith("2."))
           Seq(
             "-Yrangepos",
@@ -574,6 +576,29 @@ object Build {
           )
       else Nil
 
+    val semanticDbJavacOptions =
+      // FIXME Should this be in scalaOptions, now that we use it for javac stuff too?
+      if (generateSemanticDbs) {
+        // from https://github.com/scalameta/metals/blob/04405c0401121b372ea1971c361e05108fb36193/metals/src/main/scala/scala/meta/internal/metals/JavaInteractiveSemanticdb.scala#L137-L146
+        val compilerPackages = Seq(
+          "com.sun.tools.javac.api",
+          "com.sun.tools.javac.code",
+          "com.sun.tools.javac.model",
+          "com.sun.tools.javac.tree",
+          "com.sun.tools.javac.util"
+        )
+        val exports = compilerPackages.flatMap { pkg =>
+          Seq("-J--add-exports", s"-Jjdk.compiler/$pkg=ALL-UNNAMED")
+        }
+
+        Seq(
+          // does the path need to be escaped somehow?
+          s"-Xplugin:semanticdb -sourceroot:${inputs.workspace} -targetroot:javac-classes-directory"
+        ) ++ exports
+      }
+      else
+        Nil
+
     val sourceRootScalacOptions =
       if (params.scalaVersion.startsWith("2.")) Nil
       else Seq("-sourceroot", inputs.workspace.toString)
@@ -601,6 +626,8 @@ object Build {
       compilerClassPath = artifacts.compilerClassPath
     )
 
+    val javacOptions = javacReleaseV ++ semanticDbJavacOptions ++ options.javaOptions.javacOptions
+
     // `test` scope should contains class path to main scope
     val mainClassesPath =
       if (scope == Scope.Test)
@@ -609,6 +636,11 @@ object Build {
 
     value(validate(logger, options))
 
+    val fullClassPath = artifacts.compileClassPath ++
+      mainClassesPath ++
+      artifacts.javacPluginDependencies.map(_._3) ++
+      artifacts.extraJavacPlugins
+
     val project = Project(
       directory = inputs.workspace / Constants.workspaceDirName,
       workspace = inputs.workspace,
@@ -622,13 +654,13 @@ object Build {
           Some(options.scalaNativeOptions.bloopConfig())
         else None,
       projectName = inputs.scopeProjectName(scope),
-      classPath = artifacts.compileClassPath ++ mainClassesPath,
+      classPath = fullClassPath,
       resolution = Some(Project.resolution(artifacts.detailedArtifacts)),
       sources = allSources,
       resourceDirs = sources.resourceDirs,
       scope = scope,
       javaHomeOpt = Option(options.javaHomeLocation().value),
-      javacOptions = javacReleaseV
+      javacOptions = javacOptions
     )
     project
   }
@@ -668,7 +700,7 @@ object Build {
     }
 
     buildClient.clear()
-    buildClient.setGeneratedSources(generatedSources)
+    buildClient.setGeneratedSources(scope, generatedSources)
 
     (classesDir0, params, artifacts, project, updatedBloopConfig)
   }
@@ -723,7 +755,7 @@ object Build {
     }
 
     buildClient.clear()
-    buildClient.setGeneratedSources(generatedSources)
+    buildClient.setGeneratedSources(scope, generatedSources)
     val success = Bloop.compile(
       inputs.scopeProjectName(scope),
       bloopServer,

modules/build/src/main/scala/scala/build/ConsoleBloopBuildClient.scala

@@ -7,14 +7,15 @@ import java.net.URI
 import java.nio.file.Paths
 
 import scala.build.errors.Severity
+import scala.build.options.Scope
 import scala.collection.mutable
 import scala.jdk.CollectionConverters._
 
 class ConsoleBloopBuildClient(
   logger: Logger,
   out: PrintStream,
   keepDiagnostics: Boolean = false,
-  var generatedSources: Seq[GeneratedSource] = Nil
+  generatedSources: mutable.Map[Scope, Seq[GeneratedSource]] = mutable.Map()
 ) extends BloopBuildClient {
   import ConsoleBloopBuildClient._
   private var projectParams = Seq.empty[String]
@@ -29,8 +30,8 @@ class ConsoleBloopBuildClient(
 
   private val diagnostics0 = new mutable.ListBuffer[(Either[String, os.Path], bsp4j.Diagnostic)]
 
-  def setGeneratedSources(newGeneratedSources: Seq[GeneratedSource]) =
-    generatedSources = newGeneratedSources
+  def setGeneratedSources(scope: Scope, newGeneratedSources: Seq[GeneratedSource]) =
+    generatedSources(scope) = newGeneratedSources
   def setProjectParams(newParams: Seq[String]): Unit = {
     projectParams = newParams
   }
@@ -70,7 +71,8 @@ class ConsoleBloopBuildClient(
     logger.debug("Received onBuildPublishDiagnostics from bloop: " + params)
     for (diag <- params.getDiagnostics.asScala) {
 
-      val diagnosticMappings = generatedSources
+      val diagnosticMappings = generatedSources.valuesIterator
+        .flatMap(_.iterator)
         .map { source =>
           val lineShift = -os.read(source.generated)
             .take(source.topWrapperLen)
@@ -139,7 +141,7 @@ class ConsoleBloopBuildClient(
   override def onConnectWithServer(server: bsp4j.BuildServer): Unit = {}
 
   def clear(): Unit = {
-    generatedSources = Nil
+    generatedSources.clear()
     diagnostics0.clear()
     printedStart = false
   }

modules/build/src/main/scala/scala/build/Directories.scala

@@ -11,6 +11,7 @@ trait Directories {
   def virtualProjectsDir: os.Path
   def bspSocketDir: os.Path
   def bloopDaemonDir: os.Path
+  def bloopWorkingDir: os.Path
 }
 
 object Directories {
@@ -28,11 +29,13 @@ object Directories {
       // FIXME I would have preferred to use projDirs.dataLocalDir, but it seems named socket
       // support, or name sockets in general, aren't fine with it.
       os.Path(projDirs.cacheDir, Os.pwd) / "bsp-sockets"
-    lazy val bloopDaemonDir: os.Path = {
+    lazy val bloopDaemonDir: os.Path =
+      bloopWorkingDir / "daemon"
+    lazy val bloopWorkingDir: os.Path = {
       val baseDir =
         if (Properties.isMac) projDirs.cacheDir
         else projDirs.dataLocalDir
-      os.Path(baseDir, Os.pwd) / "bloop" / "daemon"
+      os.Path(baseDir, Os.pwd) / "bloop"
     }
   }
 
@@ -48,7 +51,9 @@ object Directories {
     lazy val bspSocketDir: os.Path =
       dir / "data-local" / "bsp-sockets"
     lazy val bloopDaemonDir: os.Path =
-      dir / "data-local" / "bloop" / "daemon"
+      bloopWorkingDir / "daemon"
+    lazy val bloopWorkingDir: os.Path =
+      dir / "data-local" / "bloop"
   }
 
   def default(): Directories = {

modules/build/src/main/scala/scala/build/ReplArtifacts.scala

@@ -67,7 +67,6 @@ object ReplArtifacts {
     dependencies: Seq[AnyDependency],
     extraClassPath: Seq[Path],
     logger: Logger,
-    directories: Directories,
     repositories: Seq[String]
   ): Either[BuildException, ReplArtifacts] = either {
     val isScala2 = scalaParams.scalaVersion.startsWith("2.")