Add initial kotlin native toolchain and kt_library rule
#1351
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,9 +12,9 @@ | |
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| load("@com_github_jetbrains_kotlin//:artifacts.bzl", "KOTLINC_ARTIFACTS", "KOTLIN_NATIVE_ARTIFACTS") | ||
| load("//kotlin:jvm.bzl", "kt_jvm_import") | ||
| load("//kotlin/internal:defs.bzl", _KT_COMPILER_REPO = "KT_COMPILER_REPO", _KT_NATIVE_COMPILER_REPO_PREFIX = "KT_NATIVE_COMPILER_REPO_PREFIX") | ||
|
|
||
| KOTLIN_STDLIBS = [ | ||
| "//kotlin/compiler:annotations", | ||
|
|
@@ -25,18 +25,18 @@ KOTLIN_STDLIBS = [ | |
| "//kotlin/compiler:trove4j", | ||
| ] | ||
|
|
||
| def _import_artifacts(artifacts, rule_kind, compiler_repo = _KT_COMPILER_REPO): | ||
| _import_labels(artifacts.plugin, rule_kind, compiler_repo) | ||
| _import_labels(artifacts.runtime, rule_kind, compiler_repo) | ||
| _import_labels(artifacts.compile, rule_kind, compiler_repo, neverlink = 1) | ||
|
|
||
| def _import_labels(labels, rule_kind, compiler_repo, **rule_args): | ||
| for (label, file) in labels.items(): | ||
| if not file.endswith(".jar"): | ||
| native.filegroup( | ||
| name = label, | ||
| srcs = [ | ||
| "@%s//:%s" % (compiler_repo, label), | ||
| ], | ||
| ) | ||
| return | ||
|
|
@@ -46,10 +46,10 @@ def _import_labels(labels, rule_kind, **rule_args): | |
| args = dict(rule_args.items()) | ||
| args["visibility"] = ["//visibility:public"] | ||
| args["name"] = label | ||
| args["jars"] = ["@%s//:%s" % (compiler_repo, label)] | ||
| sources = label + "-sources" | ||
| if sources in labels: | ||
| args["srcjar"] = "@%s//:%s" % (compiler_repo, sources) | ||
| rule_kind(**args) | ||
|
|
||
| def kt_configure_compiler(): | ||
|
|
@@ -63,3 +63,18 @@ def kt_configure_compiler(): | |
|
|
||
| _import_artifacts(KOTLINC_ARTIFACTS.jvm, kt_jvm_import) | ||
| _import_artifacts(KOTLINC_ARTIFACTS.core, kt_jvm_import) | ||
| _import_artifacts(KOTLIN_NATIVE_ARTIFACTS.linux_x86_64, kt_jvm_import, compiler_repo = _KT_NATIVE_COMPILER_REPO_PREFIX + "_linux_x86_64") | ||
| _import_artifacts(KOTLIN_NATIVE_ARTIFACTS.macos_x86_64, kt_jvm_import, compiler_repo = _KT_NATIVE_COMPILER_REPO_PREFIX + "_macos_x86_64") | ||
| _import_artifacts(KOTLIN_NATIVE_ARTIFACTS.macos_aarch64, kt_jvm_import, compiler_repo = _KT_NATIVE_COMPILER_REPO_PREFIX + "_macos_aarch64") | ||
| _import_artifacts(KOTLIN_NATIVE_ARTIFACTS.windows_x86_64, kt_jvm_import, compiler_repo = _KT_NATIVE_COMPILER_REPO_PREFIX + "_windows_x86_64") | ||
|
|
||
| # a convenience alias for kotlin-native to be referenced in other places | ||
| native.alias( | ||
|
||
| actual = select({ | ||
| "@bazel_tools//src/conditions:linux_x86_64": "//kotlin/compiler:kotlin-native-linux-x86_64", | ||
| "@bazel_tools//src/conditions:darwin": "//kotlin/compiler:kotlin-native-macos-x86_64", | ||
| "@bazel_tools//src/conditions:windows": "//kotlin/compiler:kotlin-native-windows_x86_64", | ||
| "@bazel_tools//src/conditions:darwin_arm64": "//kotlin/compiler:kotlin-native-macos_aarch64", | ||
| }), | ||
| name = "kotlin-native", | ||
| ) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| load("//kotlin/internal:defs.bzl", _KtKlibInfo = "KtKlibInfo", _TOOLCHAIN_TYPE = "TOOLCHAIN_TYPE") | ||
| load("//kotlin/internal/utils:utils.bzl", "utils") | ||
|
|
||
| def _kt_klib_library(ctx): | ||
| module_name = utils.derive_module_name(ctx) | ||
| builder_args = utils.init_args(ctx, "kt_klib_library", module_name) | ||
|
|
||
| klib = ctx.actions.declare_file("{}.klib".format(ctx.label.name)) | ||
| outputs = [klib] | ||
|
|
||
| toolchains = ctx.toolchains[_TOOLCHAIN_TYPE] | ||
| deps_klibs = [] | ||
| for dep in ctx.attr.deps: | ||
| deps_klibs.append(dep[_KtKlibInfo].klibs) | ||
| libraries = depset(transitive = deps_klibs) | ||
| builder_args.add_all("--sources", ctx.files.srcs) | ||
| builder_inputs, _, input_manifests = ctx.resolve_command(tools = [toolchains.kotlinbuilder, toolchains.konan_home]) | ||
|
|
||
| builder_args.add("--strict_kotlin_deps", "off") | ||
| builder_args.add("--reduced_classpath_mode", "off") | ||
| builder_args.add("--output_klib", klib.path) | ||
|
|
||
| deps_klibs = [] | ||
| for dep in ctx.attr.deps: | ||
|
||
| deps_klibs.append(dep[_KtKlibInfo].klibs) | ||
| libraries = depset(transitive = deps_klibs) | ||
| builder_args.add_all("--klibs", libraries, omit_if_empty = False) | ||
|
|
||
| # This will be a directory we need to propagate to the compiler | ||
| konan_home = toolchains.konan_home[DefaultInfo].files.to_list()[0] | ||
|
||
| if not konan_home.is_directory: | ||
| fail("konan home must be a directory!") | ||
|
|
||
| ctx.actions.run( | ||
| mnemonic = "KotlinKlibCompile", | ||
| inputs = depset(builder_inputs + ctx.files.srcs, transitive = [libraries]), | ||
| outputs = outputs, | ||
| executable = toolchains.kotlinbuilder.files_to_run.executable, | ||
| tools = [ | ||
| toolchains.kotlinbuilder.files_to_run, | ||
| toolchains.konan_home[DefaultInfo].files_to_run, | ||
| ], | ||
| execution_requirements = {"supports-workers": "1"}, | ||
| arguments = [ctx.actions.args().add_all(toolchains.builder_args), builder_args], | ||
| progress_message = "Compiling Kotlin to Klib %%{label} { kt: %d }" % len(ctx.files.srcs), | ||
| input_manifests = input_manifests, | ||
| env = { | ||
| "REPOSITORY_NAME": utils.builder_workspace_name(ctx), | ||
| "KONAN_HOME": konan_home.path, | ||
| }, | ||
| ) | ||
|
|
||
| return [ | ||
| DefaultInfo(files = depset(outputs)), | ||
| _KtKlibInfo( | ||
| klibs = depset(outputs), | ||
|
There was a problem hiding this comment. You'll want to include the transitive outputs here. |
||
| ), | ||
| ] | ||
|
|
||
| kt_klib_library = rule( | ||
|
||
| implementation = _kt_klib_library, | ||
| doc = """ | ||
| This rule is intended to leverage the new Kotlin IR backend to allow for compiling platform-independent Kotlin code | ||
| to be shared between Kotlin code for different platforms (JS/JVM/WASM etc.). It produces a klib file as the output. | ||
| """, | ||
| attrs = { | ||
| "srcs": attr.label_list( | ||
| doc = "A list of source files to be compiled to klib", | ||
| allow_files = [".kt"], | ||
| ), | ||
| "deps": attr.label_list( | ||
| doc = "A list of other kt_klib_library targets that this library depends on for compilation", | ||
| providers = [_KtKlibInfo], | ||
| ), | ||
| }, | ||
| toolchains = [_TOOLCHAIN_TYPE], | ||
| provides = [_KtKlibInfo], | ||
| ) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I added this to leverage
copy_to_directoryto expose the Kotlin native compiler distribution as a directory in the toolchain here (as it's massive and has a ton of files)Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please do not use
copy_to_directoryit's basically antagonistic to RBE.copy_to_directory uses uses a tree artifact (via declare_directory which is, under the hood, a tarball.) It doesn't save time or number of files. Why? Because bazel only works with files, rather than directories. The intent of that feature is handle actions where the outputs are unknown before execution.
copy_to_directorydoes know the inputs (all the files in the glob), and then applies filters etc to the them via an action... Which means we take a well known set of inputs, execute an action, place them in a tarball, and then unpack them everytime we need them (give or take -- there are some optimizations under the hood.)For RBE, this is ugly: first, transfer all the files into a remote action; second, tar all the outputs and place them in CAS. Given that the files have not changed, this bloats the CAS size, increases transfer costs, adds additional actions... for no good reason, other than laziness on the rule writers part. After all,
copy_to_directorycould be written in starlark to do the filtering... or the rules could handle the filtering during globbing, or...You get the idea.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Updated and removed
copy_to_directory. I've instead useddirectoryfrom bazel-skylib https://github.com/bazelbuild/bazel-skylib/blob/main/rules/directory/directory.bzl which seems to work as well.