Add experimental Build Tools API (BTAPI) for compilation (#1325)

erikkerber · restingbull · web-flow · commit 9734aa6f309b · 2025-06-07T11:51:04.000-07:00
* Add experimental Build Tools API (BTAPI) for compilation

* Provision build tools API with http_archive over Maven

* Fixup maven_install

* Add build tools api to the integration tests

---------

Co-authored-by: Corbin McNeely-Smith &lt;restingbull@mcneely-smith.com&gt;
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -29,6 +29,7 @@ use_repo(
     "com_github_jetbrains_kotlin",
     "com_github_jetbrains_kotlin_git",
     "com_github_pinterest_ktlint",
+    "kotlin_build_tools_impl",
     "kotlinx_serialization_core_jvm",
     "kotlinx_serialization_json",
     "kotlinx_serialization_json_jvm",
diff --git a/MODULE.release.bazel b/MODULE.release.bazel
@@ -30,6 +30,7 @@ use_repo(
     "kotlinx_serialization_core_jvm",
     "kotlinx_serialization_json",
     "kotlinx_serialization_json_jvm",
+    "kotlin_build_tools_impl",
 )
 
 register_toolchains("//kotlin/internal:default_toolchain")
diff --git a/docs/kotlin.md b/docs/kotlin.md
@@ -515,9 +515,9 @@ define_kt_toolchain(<a href="#define_kt_toolchain-name">name</a>, <a href="#defi
                     <a href="#define_kt_toolchain-experimental_treat_internal_as_private_in_abi_jars">experimental_treat_internal_as_private_in_abi_jars</a>,
                     <a href="#define_kt_toolchain-experimental_remove_private_classes_in_abi_jars">experimental_remove_private_classes_in_abi_jars</a>, <a href="#define_kt_toolchain-experimental_strict_kotlin_deps">experimental_strict_kotlin_deps</a>,
                     <a href="#define_kt_toolchain-experimental_report_unused_deps">experimental_report_unused_deps</a>, <a href="#define_kt_toolchain-experimental_reduce_classpath_mode">experimental_reduce_classpath_mode</a>,
-                    <a href="#define_kt_toolchain-experimental_multiplex_workers">experimental_multiplex_workers</a>, <a href="#define_kt_toolchain-javac_options">javac_options</a>, <a href="#define_kt_toolchain-kotlinc_options">kotlinc_options</a>, <a href="#define_kt_toolchain-jvm_stdlibs">jvm_stdlibs</a>,
-                    <a href="#define_kt_toolchain-jvm_runtime">jvm_runtime</a>, <a href="#define_kt_toolchain-jacocorunner">jacocorunner</a>, <a href="#define_kt_toolchain-exec_compatible_with">exec_compatible_with</a>, <a href="#define_kt_toolchain-target_compatible_with">target_compatible_with</a>,
-                    <a href="#define_kt_toolchain-target_settings">target_settings</a>)
+                    <a href="#define_kt_toolchain-experimental_multiplex_workers">experimental_multiplex_workers</a>, <a href="#define_kt_toolchain-experimental_build_tools_api">experimental_build_tools_api</a>, <a href="#define_kt_toolchain-javac_options">javac_options</a>,
+                    <a href="#define_kt_toolchain-kotlinc_options">kotlinc_options</a>, <a href="#define_kt_toolchain-jvm_stdlibs">jvm_stdlibs</a>, <a href="#define_kt_toolchain-jvm_runtime">jvm_runtime</a>, <a href="#define_kt_toolchain-jacocorunner">jacocorunner</a>, <a href="#define_kt_toolchain-exec_compatible_with">exec_compatible_with</a>,
+                    <a href="#define_kt_toolchain-target_compatible_with">target_compatible_with</a>, <a href="#define_kt_toolchain-target_settings">target_settings</a>)
 </pre>
 
 Define the Kotlin toolchain.
@@ -538,6 +538,7 @@ Define the Kotlin toolchain.
 | <a id="define_kt_toolchain-experimental_report_unused_deps"></a>experimental_report_unused_deps |  <p align="center"> - </p>   |  `None` |
 | <a id="define_kt_toolchain-experimental_reduce_classpath_mode"></a>experimental_reduce_classpath_mode |  <p align="center"> - </p>   |  `None` |
 | <a id="define_kt_toolchain-experimental_multiplex_workers"></a>experimental_multiplex_workers |  <p align="center"> - </p>   |  `None` |
+| <a id="define_kt_toolchain-experimental_build_tools_api"></a>experimental_build_tools_api |  <p align="center"> - </p>   |  `None` |
 | <a id="define_kt_toolchain-javac_options"></a>javac_options |  <p align="center"> - </p>   |  `Label("@rules_kotlin//kotlin/internal:default_javac_options")` |
 | <a id="define_kt_toolchain-kotlinc_options"></a>kotlinc_options |  <p align="center"> - </p>   |  `Label("@rules_kotlin//kotlin/internal:default_kotlinc_options")` |
 | <a id="define_kt_toolchain-jvm_stdlibs"></a>jvm_stdlibs |  <p align="center"> - </p>   |  `None` |
diff --git a/kotlin/internal/jvm/compile.bzl b/kotlin/internal/jvm/compile.bzl
@@ -462,6 +462,7 @@ def _run_kt_builder_action(
     args.add("--strict_kotlin_deps", toolchains.kt.experimental_strict_kotlin_deps)
     args.add_all("--classpath", compile_deps.compile_jars)
     args.add("--reduced_classpath_mode", toolchains.kt.experimental_reduce_classpath_mode)
+    args.add("--build_tools_api", toolchains.kt.experimental_build_tools_api)
     args.add_all("--sources", srcs.all_srcs, omit_if_empty = True)
     args.add_all("--source_jars", srcs.src_jars + generated_src_jars, omit_if_empty = True)
     args.add_all("--deps_artifacts", deps_artifacts, omit_if_empty = True)
diff --git a/kotlin/internal/toolchains.bzl b/kotlin/internal/toolchains.bzl
@@ -91,6 +91,7 @@ def _kotlin_toolchain_impl(ctx):
         experimental_strict_kotlin_deps = ctx.attr.experimental_strict_kotlin_deps,
         experimental_report_unused_deps = ctx.attr.experimental_report_unused_deps,
         experimental_reduce_classpath_mode = ctx.attr.experimental_reduce_classpath_mode,
+        experimental_build_tools_api = ctx.attr.experimental_build_tools_api,
         javac_options = ctx.attr.javac_options[JavacOptions] if ctx.attr.javac_options else None,
         kotlinc_options = ctx.attr.kotlinc_options[KotlincOptions] if ctx.attr.kotlinc_options else None,
         empty_jar = ctx.file._empty_jar,
@@ -247,6 +248,10 @@ _kt_toolchain = rule(
                 "KOTLINBUILDER_REDUCED",
             ],
         ),
+        "experimental_build_tools_api": attr.bool(
+            doc = "Enables experimental support for Build Tools API integration",
+            default = False,
+        ),
         "javac_options": attr.label(
             doc = "Compiler options for javac",
             providers = [JavacOptions],
@@ -321,6 +326,7 @@ def define_kt_toolchain(
         experimental_report_unused_deps = None,
         experimental_reduce_classpath_mode = None,
         experimental_multiplex_workers = None,
+        experimental_build_tools_api = None,
         javac_options = Label("//kotlin/internal:default_javac_options"),
         kotlinc_options = Label("//kotlin/internal:default_kotlinc_options"),
         jvm_stdlibs = None,
@@ -349,6 +355,7 @@ def define_kt_toolchain(
         experimental_strict_kotlin_deps = experimental_strict_kotlin_deps,
         experimental_report_unused_deps = experimental_report_unused_deps,
         experimental_reduce_classpath_mode = experimental_reduce_classpath_mode,
+        experimental_build_tools_api = experimental_build_tools_api,
         javac_options = javac_options,
         kotlinc_options = kotlinc_options,
         visibility = ["//visibility:public"],
diff --git a/kotlin/settings/BUILD.bazel b/kotlin/settings/BUILD.bazel
@@ -42,3 +42,9 @@ bool_flag(
     build_setting_default = False,
     visibility = ["//visibility:public"],
 )
+
+bool_flag(
+    name = "experimental_build_tools_api",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
diff --git a/kotlin/settings/BUILD.release.bazel b/kotlin/settings/BUILD.release.bazel
@@ -33,3 +33,10 @@ bool_flag(
     build_setting_default = False,
     visibility = ["//visibility:public"],
 )
+
+# --@rules_kotlin//kotlin/settings:experimental_build_tools_api=True
+bool_flag(
+    name = "experimental_build_tools_api",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
diff --git a/src/main/kotlin/BUILD.release.bazel b/src/main/kotlin/BUILD.release.bazel
@@ -47,6 +47,7 @@ java_binary(
         "//kotlin/compiler:symbol-processing-cmdline",
         "//src/main/kotlin/io/bazel/kotlin/compiler",
         "@com_github_jetbrains_kotlin//:home",
+        "@kotlin_build_tools_impl//jar",
         "@kotlinx_serialization_core_jvm//jar",
         "@kotlinx_serialization_json//jar",
         "@kotlinx_serialization_json_jvm//jar",
@@ -55,6 +56,7 @@ java_binary(
         "-D@com_github_jetbrains_kotlinx...serialization-core-jvm=$(rlocationpath @kotlinx_serialization_core_jvm//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json=$(rlocationpath @kotlinx_serialization_json//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json-jvm=$(rlocationpath @kotlinx_serialization_json_jvm//jar)",
+        "-D@com_github_jetbrains_kotlin...build-tools-impl=$(rlocationpath @kotlin_build_tools_impl//jar)",
         "-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rlocationpath //kotlin/compiler:jvm-abi-gen)",
         "-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rlocationpath //kotlin/compiler:kotlin-compiler)",
         "-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath //kotlin/compiler:kotlin-annotation-processing)",
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/cmd/BUILD.bazel b/src/main/kotlin/io/bazel/kotlin/builder/cmd/BUILD.bazel
@@ -26,6 +26,7 @@ kt_bootstrap_binary(
         "//src/main/kotlin:skip-code-gen",
         "//src/main/kotlin/io/bazel/kotlin/compiler:compiler.jar",
         "@com_github_jetbrains_kotlin//:home",
+        "@kotlin_build_tools_impl//jar",
         "@kotlinx_serialization_core_jvm//jar",
         "@kotlinx_serialization_json//jar",
         "@kotlinx_serialization_json_jvm//jar",
@@ -34,6 +35,7 @@ kt_bootstrap_binary(
         "-D@com_github_jetbrains_kotlinx...serialization-core-jvm=$(rlocationpath @kotlinx_serialization_core_jvm//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json=$(rlocationpath @kotlinx_serialization_json//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json-jvm=$(rlocationpath @kotlinx_serialization_json_jvm//jar)",
+        "-D@com_github_jetbrains_kotlin...build-tools-impl=$(rlocationpath @kotlin_build_tools_impl//jar)",
         "-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rlocationpath //kotlin/compiler:jvm-abi-gen)",
         "-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rlocationpath //kotlin/compiler:kotlin-compiler)",
         "-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath //kotlin/compiler:kotlin-annotation-processing)",
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt b/src/main/kotlin/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt
@@ -87,6 +87,7 @@ class KotlinBuilder
         REDUCED_CLASSPATH_MODE("--reduced_classpath_mode"),
         INSTRUMENT_COVERAGE("--instrument_coverage"),
         KSP_GENERATED_JAVA_SRCJAR("--ksp_generated_java_srcjar"),
+        BUILD_TOOLS_API("--build_tools_api"),
       }
     }
 
@@ -169,6 +170,9 @@ class KotlinBuilder
         argMap.optionalSingle(KotlinBuilderFlags.ABI_JAR_REMOVE_PRIVATE_CLASSES)?.let {
           removePrivateClassesInAbiJar = it == "true"
         }
+        argMap.optionalSingle(KotlinBuilderFlags.BUILD_TOOLS_API)?.let {
+          buildToolsApi = it == "true"
+        }
         this
       }
 
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt b/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt
@@ -33,7 +33,7 @@ const val X_FRIENDS_PATH_SEPARATOR = ","
 class KotlinJvmTaskExecutor
   @Inject
   internal constructor(
-    private val compiler: KotlinToolchain.KotlincInvoker,
+    private val compilerBuilder: KotlinToolchain.KotlincInvokerBuilder,
     private val plugins: InternalCompilerPlugins,
   ) {
     private fun combine(
@@ -54,6 +54,8 @@ class KotlinJvmTaskExecutor
       context: CompilationTaskContext,
       task: JvmCompilationTask,
     ) {
+      val compiler = compilerBuilder.build(context.info.buildToolsApi)
+
       val preprocessedTask =
         task
           .preProcessingSteps(context)
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/toolchain/KotlinToolchain.kt b/src/main/kotlin/io/bazel/kotlin/builder/toolchain/KotlinToolchain.kt
@@ -125,6 +125,13 @@ class KotlinToolchain private constructor(
         ).toPath()
     }
 
+    private val BUILD_TOOLS_API by lazy {
+      BazelRunFiles
+        .resolveVerifiedFromProperty(
+          "@com_github_jetbrains_kotlin...build-tools-impl",
+        ).toPath()
+    }
+
     private val JAVA_HOME by lazy {
       FileSystems
         .getDefault()
@@ -144,6 +151,7 @@ class KotlinToolchain private constructor(
         JAVA_HOME,
         KOTLINC.verified().absoluteFile,
         COMPILER.verified().absoluteFile,
+        BUILD_TOOLS_API.verified().absoluteFile,
         JVM_ABI_PLUGIN.verified().absoluteFile,
         SKIP_CODE_GEN_PLUGIN.verified().absoluteFile,
         JDEPS_GEN_PLUGIN.verified().absoluteFile,
@@ -159,6 +167,7 @@ class KotlinToolchain private constructor(
     fun createToolchain(
       javaHome: Path,
       kotlinc: File,
+      buildTools: File,
       compiler: File,
       jvmAbiGenFile: File,
       skipCodeGenFile: File,
@@ -174,6 +183,7 @@ class KotlinToolchain private constructor(
         listOf(
           kotlinc,
           compiler,
+          buildTools,
           // plugins *must* be preloaded. Not doing so causes class conflicts
           // (and a NoClassDef err) in the compiler extension interfaces.
           // This may cause issues in accepting user defined compiler plugins.
@@ -263,7 +273,7 @@ class KotlinToolchain private constructor(
     val id: String,
   )
 
-  open class KotlinCliToolInvoker internal constructor(
+  open class KotlincInvoker internal constructor(
     toolchain: KotlinToolchain,
     clazz: String,
   ) {
@@ -296,12 +306,19 @@ class KotlinToolchain private constructor(
   }
 
   @Singleton
-  class KotlincInvoker
+  class KotlincInvokerBuilder
     @Inject
     constructor(
-      toolchain: KotlinToolchain,
-    ) : KotlinCliToolInvoker(
-        toolchain.toolchainWithReflect(),
-        "io.bazel.kotlin.compiler.BazelK2JVMCompiler",
-      )
+      private val toolchain: KotlinToolchain,
+    ) {
+      fun build(useExperimentalBuildToolsAPI: Boolean): KotlincInvoker {
+        val clazz =
+          if (useExperimentalBuildToolsAPI) {
+            "io.bazel.kotlin.compiler.BuildToolsAPICompiler"
+          } else {
+            "io.bazel.kotlin.compiler.BazelK2JVMCompiler"
+          }
+        return KotlincInvoker(toolchain = toolchain, clazz = clazz)
+      }
+    }
 }
diff --git a/src/main/kotlin/io/bazel/kotlin/compiler/BuildToolsAPICompiler.kt b/src/main/kotlin/io/bazel/kotlin/compiler/BuildToolsAPICompiler.kt
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2025 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.bazel.kotlin.compiler
+
+import org.jetbrains.kotlin.buildtools.api.CompilationResult
+import org.jetbrains.kotlin.buildtools.api.CompilationService
+import org.jetbrains.kotlin.buildtools.api.ExperimentalBuildToolsApi
+import org.jetbrains.kotlin.buildtools.api.ProjectId
+import org.jetbrains.kotlin.cli.common.ExitCode
+import java.util.UUID
+
+@Suppress("unused")
+class BuildToolsAPICompiler {
+  @OptIn(ExperimentalBuildToolsApi::class)
+  fun exec(
+    errStream: java.io.PrintStream,
+    vararg args: String,
+  ): ExitCode {
+    System.setProperty("zip.handler.uses.crc.instead.of.timestamp", "true")
+
+    val kotlinService = CompilationService.loadImplementation(this.javaClass.classLoader!!)
+    // The execution configuration. Controls in-process vs daemon execution strategies. Default is in-process.
+    val executionConfig = kotlinService.makeCompilerExecutionStrategyConfiguration()
+    // The compilation configuration. Controls everything related to incremental compilation.
+    val compilationConfig =
+      kotlinService.makeJvmCompilationConfiguration().apply {
+        // useLogger(BasicKotlinLogger(true, "/tmp/kotlin_log/$label.log"))
+      }
+
+    val result =
+      kotlinService.compileJvm(
+        ProjectId.ProjectUUID(UUID.randomUUID()),
+        executionConfig,
+        compilationConfig,
+        emptyList(),
+        args.toList(),
+      )
+
+    // BTAPI returns a different type than K2JVMCompiler (CompilationResult vs ExitCode).
+    return when (result) {
+      CompilationResult.COMPILATION_SUCCESS -> ExitCode.OK
+      CompilationResult.COMPILATION_ERROR -> ExitCode.COMPILATION_ERROR
+      CompilationResult.COMPILATION_OOM_ERROR -> ExitCode.OOM_ERROR
+      CompilationResult.COMPILER_INTERNAL_ERROR -> ExitCode.INTERNAL_ERROR
+    }
+  }
+}
diff --git a/src/main/kotlin/io/bazel/kotlin/test/BazelIntegrationTestRunner.kt b/src/main/kotlin/io/bazel/kotlin/test/BazelIntegrationTestRunner.kt
diff --git a/src/main/protobuf/kotlin_model.proto b/src/main/protobuf/kotlin_model.proto
diff --git a/src/main/starlark/core/repositories/initialize.release.bzl b/src/main/starlark/core/repositories/initialize.release.bzl
diff --git a/src/main/starlark/core/repositories/versions.bzl b/src/main/starlark/core/repositories/versions.bzl
diff --git a/src/test/kotlin/io/bazel/kotlin/builder/KotlinAbstractTestBuilder.java b/src/test/kotlin/io/bazel/kotlin/builder/KotlinAbstractTestBuilder.java
diff --git a/src/test/kotlin/io/bazel/kotlin/defs.bzl b/src/test/kotlin/io/bazel/kotlin/defs.bzl

Original file line number	Diff line number	Diff line change
`@@ -30,6 +30,7 @@ use_repo(`
`30`	`30`	`"kotlinx_serialization_core_jvm",`
`31`	`31`	`"kotlinx_serialization_json",`
`32`	`32`	`"kotlinx_serialization_json_jvm",`
	`33`	`+ "kotlin_build_tools_impl",`
`33`	`34`	`)`
`34`	`35`
`35`	`36`	`register_toolchains("//kotlin/internal:default_toolchain")`
Original file line number	Diff line number	Diff line change
`@@ -87,6 +87,7 @@ class KotlinBuilder`
`87`	`87`	`REDUCED_CLASSPATH_MODE("--reduced_classpath_mode"),`
`88`	`88`	`INSTRUMENT_COVERAGE("--instrument_coverage"),`
`89`	`89`	`KSP_GENERATED_JAVA_SRCJAR("--ksp_generated_java_srcjar"),`
	`90`	`+ BUILD_TOOLS_API("--build_tools_api"),`
`90`	`91`	`}`
`91`	`92`	`}`
`92`	`93`
`@@ -169,6 +170,9 @@ class KotlinBuilder`
`169`	`170`	`argMap.optionalSingle(KotlinBuilderFlags.ABI_JAR_REMOVE_PRIVATE_CLASSES)?.let {`
`170`	`171`	`removePrivateClassesInAbiJar = it == "true"`
`171`	`172`	`}`
	`173`	`+ argMap.optionalSingle(KotlinBuilderFlags.BUILD_TOOLS_API)?.let {`
	`174`	`+ buildToolsApi = it == "true"`
	`175`	`+ }`
`172`	`176`	`this`
`173`	`177`	`}`
`174`	`178`