Support Scala 3's Best Effort compilation

backend/src/main/scala/bloop/BloopClassFileManager.scala

@@ -41,7 +41,7 @@ final class BloopClassFileManager(
   private[this] val generatedFiles = new mutable.HashSet[File]
 
   // Supported compile products by the class file manager
-  private[this] val supportedCompileProducts = List(".sjsir", ".nir", ".tasty")
+  private[this] val supportedCompileProducts = List(".sjsir", ".nir", ".tasty", ".betasty")
   // Files backed up during compilation
   private[this] val movedFiles = new mutable.HashMap[File, File]
 
@@ -215,7 +215,8 @@ final class BloopClassFileManager(
             clientTracer: BraveTracer
         ) => {
           clientTracer.traceTaskVerbose("copy new products to external classes dir") { _ =>
-            val config = ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty)
+            val config =
+              ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty, Set.empty)
             val clientExternalBestEffortDir =
               clientExternalClassesDir.underlying.resolve("META-INF/best-effort")
 
@@ -291,7 +292,8 @@ final class BloopClassFileManager(
             else
               clientTracer.traceTask("populate empty classes dir") { _ =>
                 // Prepopulate external classes dir even though compilation failed
-                val config = ParallelOps.CopyConfiguration(1, CopyMode.NoReplace, Set.empty)
+                val config =
+                  ParallelOps.CopyConfiguration(1, CopyMode.NoReplace, Set.empty, Set.empty)
                 ParallelOps
                   .copyDirectories(config)(
                     Paths.get(readOnlyClassesDirPath),

backend/src/main/scala/bloop/Compiler.scala

@@ -402,15 +402,19 @@ object Compiler {
               val clientClassesDir = clientClassesObserver.classesDir
               clientLogger.debug(s"Triggering background tasks for $clientClassesDir")
               val firstTask = Task { BloopPaths.delete(AbsolutePath(newClassesDir)) }
-              val secondTask = updateExternalClassesDirWithReadOnly(
-                clientClassesDir,
-                clientTracer,
-                clientLogger,
-                compileInputs,
+              val config =
+                ParallelOps.CopyConfiguration(
+                  parallelUnits = 5,
+                  CopyMode.ReplaceIfMetadataMismatch,
+                  denylist = Set.empty,
+                  denyDirs = Set.empty
+                )
+              val secondTask = ParallelOps.copyDirectories(config)(
                 compileProducts.newClassesDir,
-                readOnlyCopyDenylist = mutable.HashSet.empty,
-                allInvalidatedClassFilesForProject,
-                allInvalidatedExtraCompileProducts
+                clientClassesDir.underlying,
+                compileInputs.ioScheduler,
+                enableCancellation = false,
+                compileInputs.logger
               )
               Task
                 .gatherUnordered(List(firstTask, secondTask))
@@ -716,8 +720,11 @@ object Compiler {
           allInvalidatedExtraCompileProducts.iterator.map(_.toPath).toSet
         val invalidatedInThisProject = invalidatedClassFiles ++ invalidatedExtraProducts
         val denyList = invalidatedInThisProject ++ readOnlyCopyDenylist.iterator
+        // Let's not copy outdated betasty from readOnly, since we do not have a mechanism
+        // for tracking that otherwise
+        val denyDir = Set(readOnlyClassesDir.resolve("META-INF/best-effort"))
         val config =
-          ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, denyList)
+          ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, denyList, denyDir)
         val lastCopy = ParallelOps.copyDirectories(config)(
           readOnlyClassesDir,
           clientClassesDir.underlying,
@@ -934,6 +941,8 @@ object Compiler {
                   syntax.replace(readOnlyClassesDirPath, clientClassesDirPath)
                 )
                 if (rebasedFile.exists) {
+                  // In practice this deletes previous successful compilation artifacts
+                  // (like '.tasty' and ".class"), which we do not need in clientDir
                   Files.delete(rebasedFile.underlying)
                 }
               }

backend/src/main/scala/bloop/io/ParallelOps.scala

@@ -44,7 +44,8 @@ object ParallelOps {
   case class CopyConfiguration private (
       parallelUnits: Int,
       mode: CopyMode,
-      denylist: Set[Path]
+      denylist: Set[Path],
+      denyDirs: Set[Path]
   )
 
   case class FileWalk(visited: List[Path], target: List[Path])
@@ -87,7 +88,11 @@ object ParallelOps {
       def visitFile(file: Path, attributes: BasicFileAttributes): FileVisitResult = {
         if (isCancelled.get) FileVisitResult.TERMINATE
         else {
-          if (attributes.isDirectory || configuration.denylist.contains(file)) ()
+          if (
+            attributes.isDirectory || configuration.denylist.contains(
+              file
+            ) || configuration.denyDirs.find(file.startsWith(_)).isDefined
+          ) ()
           else {
             val rebasedFile = currentTargetDirectory.resolve(file.getFileName)
             if (configuration.denylist.contains(rebasedFile)) ()

frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala

@@ -365,7 +365,7 @@ object CompileTask {
     } else {
       // Denylist ensure final dir doesn't contain class files that don't map to source files
       val denylist = products.invalidatedCompileProducts.iterator.map(_.toPath).toSet
-      val config = ParallelOps.CopyConfiguration(5, CopyMode.NoReplace, denylist)
+      val config = ParallelOps.CopyConfiguration(5, CopyMode.NoReplace, denylist, Set.empty)
       val task = tracer.traceTaskVerbose("preparing new read-only classes directory") { _ =>
         ParallelOps.copyDirectories(config)(
           products.readOnlyClassesDir,

frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala

@@ -23,6 +23,7 @@ import bloop.task.Task
 import bloop.util.TestProject
 import bloop.util.TestUtil
 import bloop.Compiler
+import java.nio.file.Files
 
 object LocalBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Local)
 object TcpBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Tcp)
@@ -607,6 +608,74 @@ class BspMetalsClientSpec(
     }
   }
 
+  test("best-effort: correctly manage betasty files when compiling correct and failing projects") {
+    val initFile =
+      """/ErrorFile.scala
+        |object A
+        |object B
+        |""".stripMargin
+    val updatedFile1WithError =
+      """|object A
+         |//object B
+         |error
+         |object C
+         |""".stripMargin
+    val updatedFile2WithoutError =
+      """|//object A
+         |object B
+         |//error
+         |object C
+         |""".stripMargin
+    val updatedFile3WithError =
+      """|//object A
+         |object B
+         |error
+         |//object C
+         |""".stripMargin
+
+    TestUtil.withinWorkspace { workspace =>
+      val `A` = TestProject(
+        workspace,
+        "A",
+        List(initFile),
+        scalaVersion = Some(bestEffortScalaVersion)
+      )
+      def updateProject(content: String) =
+        Files.write(`A`.config.sources.head.resolve("ErrorFile.scala"), content.getBytes())
+      val projects = List(`A`)
+      TestProject.populateWorkspace(workspace, projects)
+      val logger = new RecordingLogger(ansiCodesSupported = false)
+      val extraParams = BloopExtraBuildParams(
+        ownsBuildFiles = None,
+        clientClassesRootDir = None,
+        semanticdbVersion = Some(semanticdbVersion),
+        supportedScalaVersions = Some(List(bestEffortScalaVersion)),
+        javaSemanticdbVersion = None,
+        enableBestEffortMode = Some(true)
+      )
+      loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state =>
+        val compiledState = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState
+        assertBetastyFile("A.betasty", compiledState, "A")
+        assertBetastyFile("B.betasty", compiledState, "A")
+        updateProject(updatedFile1WithError)
+        val compiledState2 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState
+        assertBetastyFile("A.betasty", compiledState2, "A")
+        assertNoBetastyFile("B.betasty", compiledState2, "A")
+        assertBetastyFile("C.betasty", compiledState2, "A")
+        updateProject(updatedFile2WithoutError)
+        val compiledState3 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState
+        assertNoBetastyFile("A.betasty", compiledState3, "A")
+        assertBetastyFile("B.betasty", compiledState3, "A")
+        assertBetastyFile("C.betasty", compiledState3, "A")
+        updateProject(updatedFile3WithError)
+        val compiledState4 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState
+        assertNoBetastyFile("A.betasty", compiledState4, "A")
+        assertBetastyFile("B.betasty", compiledState4, "A")
+        assertNoBetastyFile("C.betasty", compiledState4, "A")
+      }
+    }
+  }
+
   test("compile is successful with semanticDB and javac processorpath") {
     TestUtil.withinWorkspace { workspace =>
       val logger = new RecordingLogger(ansiCodesSupported = false)
@@ -733,6 +802,17 @@ class BspMetalsClientSpec(
     assertIsFile(beTastyFile)
   }
 
+  private def assertNoBetastyFile(
+      expectedBetastyRelativePath: String,
+      state: TestState,
+      projectName: String
+  ): Unit = {
+    val project = state.build.getProjectFor(projectName).get
+    val classesDir = state.client.getUniqueClassesDirFor(project, forceGeneration = true)
+    val beTastyFile = classesDir.resolve(s"META-INF/best-effort/$expectedBetastyRelativePath")
+    assertNotFile(beTastyFile)
+  }
+
   private def assertSemanticdbFileFor(
       sourceFileName: String,
       state: TestState,

frontend/src/test/scala/bloop/testing/BloopHelpers.scala

@@ -83,7 +83,7 @@ trait BloopHelpers {
     val baseDir = sourceConfigDir.getParent
     val relativeConfigDir = RelativePath(sourceConfigDir.getFileName)
 
-    val config = ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty)
+    val config = ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty, Set.empty)
     val copyToNewWorkspace = ParallelOps.copyDirectories(config)(
       baseDir,
       workspace.underlying,
@@ -302,7 +302,8 @@ trait BloopHelpers {
             }
 
             val backupDir = ParallelOps.copyDirectories(
-              ParallelOps.CopyConfiguration(2, ParallelOps.CopyMode.ReplaceExisting, Set.empty)
+              ParallelOps
+                .CopyConfiguration(2, ParallelOps.CopyMode.ReplaceExisting, Set.empty, Set.empty)
             )(
               classesDir,
               newClassesDir,