[SPARK-54851][BUILD] Support generating bloop files via sbt

### What changes were proposed in this pull request?

Some IDEs such as cursor/VSCode works better with bloop as the build server. This PR supports generating the bloop config files via sbt.

### Why are the changes needed?

better IDE support

### Does this PR introduce _any_ user-facing change?

no, dev only

### How was this patch tested?

manual test

### Was this patch authored or co-authored using generative AI tooling?

cursor 2.2.43

Closes #53621 from cloud-fan/bloop.

Lead-authored-by: Wenchen Fan <[email protected]>
Co-authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>

Author: cloud-fan

project/SparkBuild.scala

@@ -1228,8 +1228,15 @@ object ExcludedDependencies {
  * client dependencies.
  */
 object ExcludeShims {
+  import bloop.integrations.sbt.BloopKeys
+
   val shimmedProjects = Set("spark-sql-api", "spark-connect-common", "spark-connect-client-jdbc", "spark-connect-client-jvm")
   val classPathFilter = TaskKey[Classpath => Classpath]("filter for classpath")
+
+  // Filter for bloopInternalClasspath which is Seq[(File, File)]
+  type BloopClasspath = Seq[(java.io.File, java.io.File)]
+  val bloopClasspathFilter = TaskKey[BloopClasspath => BloopClasspath]("filter for bloop classpath")
+
   lazy val settings = Seq(
     classPathFilter := {
       if (!shimmedProjects(moduleName.value)) {
@@ -1238,6 +1245,16 @@ object ExcludeShims {
         identity _
       }
     },
+    bloopClasspathFilter := {
+      if (!shimmedProjects(moduleName.value)) {
+        // Note: bloop output directories use "connect-shims" (without "spark-" prefix)
+        cp => cp.filterNot { case (f1, f2) =>
+          f1.getPath.contains("connect-shims") || f2.getPath.contains("connect-shims")
+        }
+      } else {
+        identity _
+      }
+    },
     Compile / internalDependencyClasspath :=
       classPathFilter.value((Compile / internalDependencyClasspath).value),
     Compile / internalDependencyAsJars :=
@@ -1250,6 +1267,13 @@ object ExcludeShims {
       classPathFilter.value((Test / internalDependencyClasspath).value),
     Test / internalDependencyAsJars :=
       classPathFilter.value((Test / internalDependencyAsJars).value),
+    // Filter bloop's internal classpath for correct IDE integration
+    Compile / BloopKeys.bloopInternalClasspath :=
+      bloopClasspathFilter.value((Compile / BloopKeys.bloopInternalClasspath).value),
+    Runtime / BloopKeys.bloopInternalClasspath :=
+      bloopClasspathFilter.value((Runtime / BloopKeys.bloopInternalClasspath).value),
+    Test / BloopKeys.bloopInternalClasspath :=
+      bloopClasspathFilter.value((Test / BloopKeys.bloopInternalClasspath).value),
   )
 }
 

project/plugins.sbt

@@ -46,3 +46,5 @@ addSbtPlugin("com.github.sbt.junit" % "sbt-jupiter-interface" % "0.17.0")
 addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.7")
 
 addSbtPlugin("com.here.platform" % "sbt-bom" % "1.0.29")
+
+addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "2.0.17")