Fix associates compile classpath (#1271)

* Fix assoicate compile classpath

* Maybe invalidate build cache

* Add associate jars as input

* Use class not compile jar in associates

* Collect strict class jar only

* Remove unused classpath entry

* Add test for classpath order extracting fun under test

* Flush and yield in test

* Tidy ABI classpath test

* Filter any abi jar from associates from classpath

Author: rbeazleyspot

kotlin/internal/jvm/associates.bzl

@@ -29,35 +29,56 @@ def _collect_associates(ctx, toolchains, associate):
     """Collects the associate jars from the provided dependency and returns
     them as a depset.
 
-    There are two outcomes for this marco:
-    1. When `experimental_strict_associate_dependencies` is enabled and the tag override has not been provided, only the
-        direct java_output compile jars will be collected for each associate target.
-    2. When `experimental_strict_associate_dependencies` is disabled, the complete transitive set of compile jars will
+    There are three outcomes for this marco:
+    1. When `experimental_remove_private_classes_in_abi_jars` is enabled and the tag override has not been provided, only the
+        direct java_output CLASS jars will be collected for each associate target. Due to the stripping of internal and private
+        symbols from the compile jar, class jar is the only one that will contain the internal symbols an associate has access to.
+    2. When `experimental_strict_associate_dependencies` is enabled and the tag override has not been provided, only the
+        direct java_output COMPILE jars will be collected for each associate target.
+    3. When `experimental_strict_associate_dependencies` is disabled, the complete transitive set of compile jars will
         be collected for each assoicate target.
     """
     jars_depset = None
-    if (toolchains.kt.experimental_strict_associate_dependencies and
-        "kt_experimental_strict_associate_dependencies_incompatible" not in ctx.attr.tags):
+    abi_jars_set = _sets.new()
+    if (not "kt_remove_private_classes_in_abi_plugin_incompatible" in ctx.attr.tags and
+        toolchains.kt.experimental_remove_private_classes_in_abi_jars):
+        jars_depset = depset(direct = [a.class_jar for a in associate[JavaInfo].java_outputs])
+        _sets.add_all(abi_jars_set, [a.compile_jar for a in associate[JavaInfo].java_outputs])
+    elif (toolchains.kt.experimental_strict_associate_dependencies and
+          "kt_experimental_strict_associate_dependencies_incompatible" not in ctx.attr.tags):
         jars_depset = depset(direct = [a.compile_jar for a in associate[JavaInfo].java_outputs])
     else:
         jars_depset = depset(transitive = [associate[JavaInfo].compile_jars])
-    return jars_depset
+    return struct(
+        jars = jars_depset,
+        abi_jars_set = abi_jars_set,
+    )
+
+def _java_info(target):
+    return target[JavaInfo] if JavaInfo in target else None
 
 def _get_associates(ctx, toolchains, associates):
     """Creates a struct of associates meta data"""
     if not associates:
         return struct(
             module_name = _utils.derive_module_name(ctx),
             jars = depset(),
+            abi_jar_set = _sets.new(),
+            dep_infos = [],
         )
     elif ctx.attr.module_name:
         fail("If associates have been set then module_name cannot be provided")
     else:
         jars = []
+        abi_jar_set = {}
         module_names = []
+        java_infos = []
         for a in associates:
-            jars.append(_collect_associates(ctx = ctx, toolchains = toolchains, associate = a))
+            jar_bundle = _collect_associates(ctx = ctx, toolchains = toolchains, associate = a)
+            jars.append(jar_bundle.jars)
+            abi_jar_set = jar_bundle.abi_jars_set
             module_names.append(a[_KtJvmInfo].module_name)
+            java_infos.append(_java_info(a))
         module_names = list(_sets.copy_of(module_names))
 
         if len(module_names) > 1:
@@ -69,7 +90,9 @@ def _get_associates(ctx, toolchains, associates):
             fail("Error in rules - a KtJvmInfo was found which did not have a module_name")
         return struct(
             jars = depset(transitive = jars),
+            abi_jar_set = abi_jar_set,
             module_name = module_names[0],
+            dep_infos = java_infos,
         )
 
 associate_utils = struct(

kotlin/internal/jvm/compile.bzl

@@ -33,8 +33,8 @@ load(
     "kotlinc_options_to_flags",
 )
 load(
-    "//kotlin/internal/jvm:associates.bzl",
-    _associate_utils = "associate_utils",
+    "//kotlin/internal/jvm:jvm_deps.bzl",
+    _jvm_deps_utils = "jvm_deps_utils",
 )
 load(
     "//kotlin/internal/jvm:plugins.bzl",
@@ -102,8 +102,8 @@ def _compiler_toolchains(ctx):
         java_runtime = find_java_runtime_toolchain(ctx, ctx.attr._host_javabase),
     )
 
-def _jvm_deps(ctx, toolchains, associate_deps, deps, exports = [], runtime_deps = []):
-    """Encapsulates jvm dependency metadata."""
+def _fail_if_invalid_associate_deps(associate_deps, deps):
+    """Verifies associates not included in target deps."""
     diff = _sets.intersection(
         _sets.copy_of([x.label for x in associate_deps]),
         _sets.copy_of([x.label for x in deps]),
@@ -113,34 +113,6 @@ def _jvm_deps(ctx, toolchains, associate_deps, deps, exports = [], runtime_deps
             "\n------\nTargets should only be put in associates= or deps=, not both:\n%s" %
             ",\n ".join(["    %s" % x for x in list(diff)]),
         )
-    dep_infos = [_java_info(d) for d in associate_deps + deps] + [toolchains.kt.jvm_stdlibs]
-
-    associates = _associate_utils.get_associates(ctx, toolchains = toolchains, associates = associate_deps)
-
-    # Reduced classpath, exclude transitive deps from compilation
-    if (toolchains.kt.experimental_prune_transitive_deps and
-        not "kt_experimental_prune_transitive_deps_incompatible" in ctx.attr.tags):
-        transitive = [
-            d.compile_jars
-            for d in dep_infos
-        ]
-    else:
-        transitive = [
-            d.compile_jars
-            for d in dep_infos
-        ] + [
-            d.transitive_compile_time_jars
-            for d in dep_infos
-        ]
-
-    return struct(
-        module_name = associates.module_name,
-        deps = dep_infos,
-        exports = [_java_info(d) for d in exports],
-        associate_jars = associates.jars,
-        compile_jars = depset(transitive = transitive),
-        runtime_deps = [_java_info(d) for d in runtime_deps],
-    )
 
 def _java_infos_to_compile_jars(java_infos):
     return depset(transitive = [j.compile_jars for j in java_infos])
@@ -568,6 +540,7 @@ def _run_kt_builder_action(
         inputs = depset(
             srcs.all_srcs + srcs.src_jars + generated_src_jars,
             transitive = [
+                compile_deps.associate_jars,
                 compile_deps.compile_jars,
                 transitive_runtime_jars,
                 deps_artifacts,
@@ -608,7 +581,8 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind):
     """
     toolchains = _compiler_toolchains(ctx)
     srcs = _partitioned_srcs(ctx.files.srcs)
-    compile_deps = _jvm_deps(
+    _fail_if_invalid_associate_deps(ctx.attr.associates, ctx.attr.deps)
+    compile_deps = _jvm_deps_utils.jvm_deps(
         ctx,
         toolchains = toolchains,
         associate_deps = ctx.attr.associates,

kotlin/internal/jvm/jvm_deps.bzl

@@ -0,0 +1,60 @@
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+load(
+    "@rules_java//java:defs.bzl",
+    "JavaInfo",
+)
+load("//kotlin/internal/jvm:associates.bzl", _associate_utils = "associate_utils")
+load("//kotlin/internal/utils:sets.bzl", _sets = "sets")
+
+def _java_info(target):
+    return target[JavaInfo] if JavaInfo in target else None
+
+def _jvm_deps(ctx, toolchains, associate_deps, deps, exports = [], runtime_deps = []):
+    """Encapsulates jvm dependency metadata."""
+    dep_infos = [_java_info(d) for d in deps] + [toolchains.kt.jvm_stdlibs]
+
+    associates = _associate_utils.get_associates(ctx, toolchains = toolchains, associates = associate_deps)
+
+    # Reduced classpath, exclude transitive deps from compilation
+    if (toolchains.kt.experimental_prune_transitive_deps and
+        not "kt_experimental_prune_transitive_deps_incompatible" in ctx.attr.tags):
+        transitive = [
+            d.compile_jars
+            for d in dep_infos
+        ]
+    else:
+        transitive = [
+            d.compile_jars
+            for d in dep_infos
+        ] + [
+            d.transitive_compile_time_jars
+            for d in dep_infos
+        ]
+
+    compile_depset_list = depset(transitive = transitive + [associates.jars]).to_list()
+    compile_depset_list_filtered = [jar for jar in compile_depset_list if not _sets.contains(associates.abi_jar_set, jar)]
+
+    return struct(
+        module_name = associates.module_name,
+        deps = dep_infos + associates.dep_infos,
+        exports = [_java_info(d) for d in exports],
+        associate_jars = associates.jars,
+        compile_jars = depset(direct = compile_depset_list_filtered),
+        runtime_deps = [_java_info(d) for d in runtime_deps],
+    )
+
+jvm_deps_utils = struct(
+    jvm_deps = _jvm_deps,
+)

src/test/kotlin/io/bazel/worker/WorkerEnvironmentTest.kt

@@ -35,10 +35,12 @@ class WorkerEnvironmentTest {
           WorkerProtocol.WorkResponse.newBuilder().setRequestId(req.requestId).build()
             .also { println("sent $it") }
             .writeDelimitedTo(stdOut)
+          stdOut.flush()
         }
       }
 
       writeStdIn(give)
+      Thread.sleep(10)
       closeStdIn()
 
       return@inProcess readStdOut()

src/test/starlark/internal/jvm/BUILD.bazel

@@ -0,0 +1,3 @@
+load(":jvm_deps_tests.bzl", "jvm_deps_test_suite")
+
+jvm_deps_test_suite(name = "jvm_tests")