Implement an aspect to automatically index Bazel/Java codebases

Previously, the Bazel example required monkeypatching the `java_library`
rule in order to inject the SemanticDB compiler plugin. This PR fixes
this problem by adding support to automatically index Bazel codebases
without modifying the build configuration (WORKSPACE or BUILD files).

See scip_java.bzl in the diff for more details about how this works.

To make this functionality work, this commit extends the functionality
of the SCIP build tool to optionally allow users to specify a custom
classpath, processorpath, and javac options.

Author: olafurpg

.github/workflows/ci.yml

@@ -55,7 +55,7 @@ jobs:
             docker run -v $PWD/.repos/$REPO:/sources -w /sources sourcegraph/scip-java:latest scip-java index
             file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1) 
           }
-          
+
           sudo apt install parallel
           export -f check_repo
 
@@ -76,6 +76,22 @@ jobs:
       - run: du -h index.scip
         working-directory: examples/bazel-example
 
+  bazel_aspect:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: yarn global add @bazel/bazelisk
+      - run: sbt cli/pack
+      - run: echo "$PWD/scip-java/target/pack/bin" >> $GITHUB_PATH
+      - run: |
+          scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
+      - run: du -h index.scip
+      - run: |
+          scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
+        working-directory: examples/bazel-example
+      - run: du -h index.scip
+        working-directory: examples/bazel-example
+
   check:
     runs-on: ubuntu-latest
     steps:

BUILD

@@ -163,7 +163,14 @@ lazy val javacPlugin = project
       Seq(
         ShadeRule
           .rename(
+            // Don't rename SemanticdbPlugin since the fully-qualified name is
+            // referenced from META-INF/services/com.sun.source.util.Plugin
+            "com.sourcegraph.semanticdb_javac.SemanticdbPlugin" ->
+              "com.sourcegraph.semanticdb_javac.SemanticdbPlugin",
             "com.google.**" -> "com.sourcegraph.shaded.com.google.@1",
+            // Need to shade the semanticdb-javac compiler plugin in order to be
+            // able to index the plugin code itself.
+            "com.sourcegraph.**" -> "com.sourcegraph.shaded.com.sourcegraph.@1",
             "google.**" -> "com.sourcegraph.shaded.google.@1",
             "org.relaxng.**" -> "com.sourcegraph.shaded.relaxng.@1"
           )
@@ -250,7 +257,7 @@ lazy val cli = project
           }
 
           addJar(
-            (javacPlugin / Compile / Keys.`package`).value,
+            (javacPlugin / Compile / assembly).value,
             "semanticdb-plugin.jar"
           )
           addJar(
@@ -259,7 +266,12 @@ lazy val cli = project
           )
           addJar((gradlePlugin / Compile / assembly).value, "gradle-plugin.jar")
 
-          IO.copy(outs)
+          IO.copy(
+            outs,
+            overwrite = true,
+            preserveLastModified = false,
+            preserveExecutable = true
+          )
           val props = new Properties()
           val propsFile = out.resolve("scip-java.properties").toFile
           val copiedJars = outs.collect { case (_, out) =>

build.sbt

@@ -1 +1,2 @@
+aspects/scip_java.bzl
 bazel-bazel-example

examples/bazel-example/.gitignore

@@ -1,5 +1,6 @@
 # This is an end-to-end example of how to consume scip-java from an external repository.
 workspace(name = "scip_java_example")
+
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 ##############
@@ -20,11 +21,11 @@ http_archive(
 ##############
 local_repository(
     name = "scip_java",
-    path = "../.."
+    path = "../..",
 )
 
 # Copy and paste this, not the local_repository:
-# 
+#
 # SCIP_JAVA_VERSION="0.8.20"
 # http_archive(
 #   name = "scip_java",
@@ -44,36 +45,47 @@ http_archive(
         "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0-3.20.0.tar.gz",
     ],
 )
+
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
 rules_proto_dependencies()
+
 rules_proto_toolchains()
 
 ##############
 # JVM External
 ##############
 # To update this version, copy-paste instructions from https://github.com/bazelbuild/rules_jvm_external/releases
 RULES_JVM_EXTERNAL_TAG = "4.2"
+
 RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca"
+
 http_archive(
     name = "rules_jvm_external",
-    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
     sha256 = RULES_JVM_EXTERNAL_SHA,
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
     url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
 )
+
 load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
+
 rules_jvm_external_deps()
+
 load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
+
 rules_jvm_external_setup()
+
 load("@rules_jvm_external//:defs.bzl", "maven_install")
+
 maven_install(
     artifacts = [
-        "com.google.protobuf:protobuf-java:3.15.6", # Required dependency by scip-java.
-        "com.google.protobuf:protobuf-java-util:3.15.6", # Required dependency by scip-java.
+        "com.google.protobuf:protobuf-java:3.15.6",  # Required dependency by scip-java.
+        "com.google.protobuf:protobuf-java-util:3.15.6",  # Required dependency by scip-java.
         # These dependencies are only required for the tests
-        "com.google.guava:guava:31.0-jre", 
+        "com.google.guava:guava:31.0-jre",
         "com.google.auto.value:auto-value:1.5.3",
     ],
     repositories = [
-         "https://repo1.maven.org/maven2",
+        "https://repo1.maven.org/maven2",
     ],
 )

examples/bazel-example/WORKSPACE

@@ -0,0 +1,123 @@
+"""
+Bazel aspect to run scip-java against a Java Bazel codebase.
+
+You can optionally commit this file into your git repository, gitignore it, or
+just delete it. When you run `scip-java index` in a Bazel codebase, this file
+will get re-created and the command will error if the file already exists but with
+different contents.
+
+This aspect is needed for scip-java to inspect the structure of the Bazel build
+and register actions to index all java_library/java_test/java_binary targets.
+The result of running this aspect is that your bazel-bin/ directory will contain
+many *.scip (https://github.com/sourcegraph/scip) and 
+*.semanticdb (https://scalameta.org/docs/semanticdb/specification.html) files.
+These files encode information about which symbols are referenced from which
+locations in your source code.
+
+Use the command below to merge all of these SCIP files into a single index:
+
+    find bazel-bin/ -type f -name '*.scip' | xargs cat > index.scip
+
+Use `src code-intel upload` to upload the unified SCIP file to Sourcegraph:
+
+    npm install -g @sourcegraph/src
+    export SRC_ENDPOINT=SOURCEGRAPH_URL
+    export SRC_ACCESS_TOKEN=TOKEN_VALUE
+    src login # confirm you are correctly authenticated
+    src code-intel upload -file=index.scip
+
+Example command to run this aspect directly:
+
+    bazel build //...  --aspects path/to/scip_java.bzl%scip_java_aspect --output_groups=scip --define=sourceroot=$(pwd) --define=scip_java_binary=$(which scip-java) --define=java_home=$JAVA_HOME
+
+To learn more about aspects: https://bazel.build/extending/aspects
+"""
+
+def _scip_java(target, ctx):
+    if JavaInfo not in target or not hasattr(ctx.rule.attr, "srcs"):
+        return None
+
+    javac_action = None
+    for a in target.actions:
+        if a.mnemonic == "Javac":
+            javac_action = a
+            break
+
+    if not javac_action:
+        return None
+
+    info = target[JavaInfo]
+    compilation = info.compilation_info
+    annotations = info.annotation_processing
+
+    source_files = []
+    for src in ctx.rule.files.srcs:
+        source_files.append(src.path)
+    if len(source_files) == 0:
+        return None
+
+    classpath = [j.path for j in compilation.compilation_classpath.to_list()]
+    bootclasspath = [j.path for j in compilation.boot_classpath]
+
+    processorpath = []
+    processors = []
+    if annotations and annotations.enabled:
+        processorpath += [j.path for j in annotations.processor_classpath.to_list()]
+        processors = annotations.processor_classnames
+
+    build_config = struct(**{
+        "javaHome": ctx.var["java_home"],
+        "classpath": classpath,
+        "sourceFiles": source_files,
+        "javacOptions": compilation.javac_options,
+        "processors": processors,
+        "processorpath": processorpath,
+        "bootclasspath": bootclasspath,
+    })
+    build_config_path = ctx.actions.declare_file(ctx.label.name + ".scip.json")
+
+    scip_output = ctx.actions.declare_file(ctx.label.name + ".scip")
+    targetroot = ctx.actions.declare_directory(ctx.label.name + ".semanticdb")
+    ctx.actions.write(
+        output = build_config_path,
+        content = build_config.to_json(),
+    )
+    ctx.actions.run_shell(
+        command = "\"{}\" index --no-cleanup --index-semanticdb.allow-empty-index --cwd \"{}\" --targetroot {} --scip-config \"{}\" --output \"{}\"".format(
+            ctx.var["scip_java_binary"],
+            ctx.var["sourceroot"],
+            targetroot.path,
+            build_config_path.path,
+            scip_output.path,
+        ),
+        env = {
+            "JAVA_HOME": ctx.var["java_home"],
+            "NO_PROGRESS_BAR": "true",
+        },
+        inputs = [build_config_path],
+        outputs = [scip_output, targetroot],
+    )
+
+    return scip_output
+
+def _scip_java_aspect(target, ctx):
+    scip = _scip_java(target, ctx)
+    if not scip:
+        return struct()
+    return [OutputGroupInfo(scip = [scip])]
+
+scip_java_aspect = aspect(
+    _scip_java_aspect,
+)
+
+def _scip_java_impl(ctx):
+    output = ctx.attr.compilation[OutputGroupInfo]
+    return [
+        OutputGroupInfo(scip = output.scip),
+        DefaultInfo(files = output.scip),
+    ]
+
+scip_java = rule(
+    implementation = _scip_java_impl,
+    attrs = {"compilation": attr.label(aspects = [scip_java_aspect])},
+)

examples/bazel-example/aspects/BUILD

@@ -112,6 +112,14 @@ object Embedded {
     }
   }
 
+  def bazelAspectFile(tmpDir: Path): String = {
+    val tmpFile = copyFile(tmpDir, "scip-java/scip_java.bzl")
+    val contents =
+      new String(Files.readAllBytes(tmpFile), StandardCharsets.UTF_8)
+    Files.deleteIfExists(tmpFile)
+    contents
+  }
+
   private def copyFile(tmpDir: Path, filename: String): Path = {
     val in = this.getClass.getResourceAsStream(s"/$filename")
     val out = tmpDir.resolve(filename)