Skip to content

Commit 9dbad49

Browse files
authored
Merge pull request #610 from sourcegraph/olafurpg/aspect
Add support to auto-index Bazel builds with an aspect
2 parents 00c8ed5 + 72ff816 commit 9dbad49

File tree

21 files changed

+759
-101
lines changed

21 files changed

+759
-101
lines changed

.github/workflows/ci.yml

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,9 @@ jobs:
5353
git clone https://github.com/$REPO.git .repos/$REPO
5454
5555
docker run -v $PWD/.repos/$REPO:/sources -w /sources sourcegraph/scip-java:latest scip-java index
56-
file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1)
56+
file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1)
5757
}
58-
58+
5959
sudo apt install parallel
6060
export -f check_repo
6161
@@ -76,6 +76,24 @@ jobs:
7676
- run: du -h index.scip
7777
working-directory: examples/bazel-example
7878

79+
bazel_aspect:
80+
runs-on: ubuntu-latest
81+
steps:
82+
- uses: actions/checkout@v2
83+
- run: yarn global add @bazel/bazelisk
84+
- run: sbt cli/pack
85+
- run: echo "$PWD/scip-java/target/pack/bin" >> $GITHUB_PATH
86+
- name: Auto-index scip-java codebase
87+
run: |
88+
scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
89+
- run: du -h index.scip
90+
- name: Auto-index example/bazel-workspace
91+
run: |
92+
scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
93+
working-directory: examples/bazel-example
94+
- run: du -h index.scip
95+
working-directory: examples/bazel-example
96+
7997
check:
8098
runs-on: ubuntu-latest
8199
steps:

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,4 @@ bazel-lsif-java
6565
VERSION
6666

6767
semanticdb-gradle-plugin/gradle
68+
aspects/scip_java.bzl

BUILD

Whitespace-only changes.

build.sbt

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,24 @@ lazy val javacPlugin = project
163163
Seq(
164164
ShadeRule
165165
.rename(
166+
// Don't rename SemanticdbPlugin since the fully-qualified name is
167+
// referenced from META-INF/services/com.sun.source.util.Plugin
168+
"com.sourcegraph.semanticdb_javac.SemanticdbPlugin" ->
169+
"com.sourcegraph.semanticdb_javac.SemanticdbPlugin",
170+
// Don't rename PrintJavaVersion because we load it via FQN to
171+
// detect the Java of a JVM installation.
172+
"com.sourcegraph.semanticdb_javac.PrintJavaVersion" ->
173+
"com.sourcegraph.semanticdb_javac.PrintJavaVersion",
174+
// Don't rename InjectSemanticdbOptions because we load it via FQN to
175+
// process a list of Java compiler options.
176+
"com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions" ->
177+
"com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions",
166178
"com.google.**" -> "com.sourcegraph.shaded.com.google.@1",
179+
// Shade everything else in the semanticdb-javac compiler plugin in
180+
// order to be able to index the plugin code itself. Without this step,
181+
// we can't add the plugin to the classpath while compiling the source
182+
// code of the plugin itself because it results in cryptic compile errors.
183+
"com.sourcegraph.**" -> "com.sourcegraph.shaded.com.sourcegraph.@1",
167184
"google.**" -> "com.sourcegraph.shaded.google.@1",
168185
"org.relaxng.**" -> "com.sourcegraph.shaded.relaxng.@1"
169186
)
@@ -259,7 +276,12 @@ lazy val cli = project
259276
)
260277
addJar((gradlePlugin / Compile / assembly).value, "gradle-plugin.jar")
261278

262-
IO.copy(outs)
279+
IO.copy(
280+
outs,
281+
overwrite = true,
282+
preserveLastModified = false,
283+
preserveExecutable = true
284+
)
263285
val props = new Properties()
264286
val propsFile = out.resolve("scip-java.properties").toFile
265287
val copiedJars = outs.collect { case (_, out) =>

docs/getting-started.md

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ title: Getting started
44
---
55

66
By following the instructions on this page, you should be able to generate a
7-
[SCIP](https://github.com/sourcegraph/scip)
8-
index of your Java codebase using Gradle or Maven. See
7+
[SCIP](https://github.com/sourcegraph/scip) index of your Java codebase using
8+
Gradle, Maven, sbt, or Bazel. See
99
[Supported build tools](#supported-build-tools) for an overview of other build
1010
tools that we're planning to support in the future.
1111

@@ -328,13 +328,41 @@ projects, with the following caveats:
328328

329329
### Bazel
330330

331-
Bazel is supported by scip-java, but it requires custom configuration to work
332-
correctly. The `scip-java index` command does not automatically index Bazel builds.
331+
There are two approaches to index Bazel/Java codebases: automatic and manual.
333332

334-
The Bazel integration for scip-java is specifically designed to be compatible
335-
with the Bazel build cache to enable incremental indexing. To achieve this,
336-
scip-java must be configured in `WORKSPACE` and `BUILD` files. The scip-java
337-
repository contains an example for how to configure everything.
333+
Don't hesitate to open an issue in the
334+
[scip-java repository](https://github.com/sourcegraph/scip-java) if you have any
335+
questions about using scip-java with Bazel builds.
336+
337+
#### Automatic - `aspect`
338+
339+
Since scip-java v0.8.24, it's possible to automatically index Bazel/Java
340+
codebases via `scip-java index`.
341+
342+
```sh
343+
scip-java index "--bazel-scip-java-binary=$(which scip-java)"
344+
```
345+
346+
When using this approach, indexing happens mostly inside the Bazel action graph,
347+
benefitting from parallel compilation and the Bazel build cache.
348+
349+
The `--bazel-scip-java-binary` argument is required due to implementation
350+
details, scip-java runs an [aspect](https://bazel.build/extending/aspects) that
351+
requires the absolute path to the `scip-java` binary.
352+
353+
> The current solution for automatic indexing step is not yet 100% hermetic and,
354+
> therefore, relies on `--spawn_strategy=local` under the hood. Depending on
355+
> your use-case, this might be OK or not. If there is demand for it, it's should
356+
> be possible to make the indexing fully hermetic and compatible with Bazel's
357+
> sandbox with some extra work.
358+
359+
#### Manual - `select`
360+
361+
It's possible to index Bazel codebases by integrating scip-java directly into
362+
the build configuration. To achieve this, scip-java must be configured in
363+
`WORKSPACE` and `BUILD` files. The scip-java repository contains an example for
364+
how to configure everything, including how to build scip-java itself from
365+
source.
338366

339367
- [WORKSPACE](https://github.com/sourcegraph/scip-java/blob/main/examples/bazel-example/WORKSPACE):
340368
adds the required dependencies to be able to run scip-java itself.
@@ -343,6 +371,7 @@ repository contains an example for how to configure everything.
343371
scip-java.
344372

345373
Once configured, build the codebase with the SemanticDB compiler plugin.
374+
346375
```sh
347376
bazel build //... --@scip_java//semanticdb-javac:enabled=true
348377
```
@@ -356,7 +385,7 @@ bazel run @scip_java//scip-semanticdb:bazel -- --sourceroot $PWD
356385
# The command below works for the `examples/bazel-example` directory in the sourcegraph/scip-java repository.
357386
❯ jar tf bazel-bin/src/main/java/example/libexample.jar | grep semanticdb$
358387
META-INF/semanticdb/src/main/java/example/Example.java.semanticdb
359-
```
388+
```
360389

361390
Finally, run the following commands to upload the SCIP index to Sourcegraph.
362391

@@ -372,8 +401,3 @@ src login # validate the token authenticates correctly
372401
# 3. Upload SCIP index to Sourcegraph
373402
src code-intel upload # requires index.scip file to exist
374403
```
375-
376-
377-
Don't hesitate to open an issue in the
378-
[scip-java repository](https://github.com/sourcegraph/scip-java) if you have any
379-
questions about using scip-java with Bazel builds.

examples/bazel-example/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
bazel-bazel-example
2+
aspects/scip_java.bzl

examples/bazel-example/WORKSPACE

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# This is an end-to-end example of how to consume scip-java from an external repository.
22
workspace(name = "scip_java_example")
3+
34
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
45

56
##############
@@ -20,11 +21,11 @@ http_archive(
2021
##############
2122
local_repository(
2223
name = "scip_java",
23-
path = "../.."
24+
path = "../..",
2425
)
2526

2627
# Copy and paste this, not the local_repository:
27-
#
28+
#
2829
# SCIP_JAVA_VERSION="0.8.20"
2930
# http_archive(
3031
# name = "scip_java",
@@ -44,36 +45,47 @@ http_archive(
4445
"https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0-3.20.0.tar.gz",
4546
],
4647
)
48+
4749
load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
50+
4851
rules_proto_dependencies()
52+
4953
rules_proto_toolchains()
5054

5155
##############
5256
# JVM External
5357
##############
5458
# To update this version, copy-paste instructions from https://github.com/bazelbuild/rules_jvm_external/releases
5559
RULES_JVM_EXTERNAL_TAG = "4.2"
60+
5661
RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca"
62+
5763
http_archive(
5864
name = "rules_jvm_external",
59-
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
6065
sha256 = RULES_JVM_EXTERNAL_SHA,
66+
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
6167
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
6268
)
69+
6370
load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
71+
6472
rules_jvm_external_deps()
73+
6574
load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
75+
6676
rules_jvm_external_setup()
77+
6778
load("@rules_jvm_external//:defs.bzl", "maven_install")
79+
6880
maven_install(
6981
artifacts = [
70-
"com.google.protobuf:protobuf-java:3.15.6", # Required dependency by scip-java.
71-
"com.google.protobuf:protobuf-java-util:3.15.6", # Required dependency by scip-java.
82+
"com.google.protobuf:protobuf-java:3.15.6", # Required dependency by scip-java.
83+
"com.google.protobuf:protobuf-java-util:3.15.6", # Required dependency by scip-java.
7284
# These dependencies are only required for the tests
73-
"com.google.guava:guava:31.0-jre",
85+
"com.google.guava:guava:31.0-jre",
7486
"com.google.auto.value:auto-value:1.5.3",
7587
],
7688
repositories = [
77-
"https://repo1.maven.org/maven2",
89+
"https://repo1.maven.org/maven2",
7890
],
7991
)

examples/bazel-example/aspects/BUILD

Whitespace-only changes.
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
"""
2+
Bazel aspect to run scip-java against a Java Bazel codebase.
3+
4+
You can optionally commit this file into your git repository, gitignore it, or
5+
just delete it. When you run `scip-java index` in a Bazel codebase, this file
6+
will get re-created and the command will error if the file already exists but with
7+
different contents.
8+
9+
This aspect is needed for scip-java to inspect the structure of the Bazel build
10+
and register actions to index all java_library/java_test/java_binary targets.
11+
The result of running this aspect is that your bazel-bin/ directory will contain
12+
many *.scip (https://github.com/sourcegraph/scip) and
13+
*.semanticdb (https://scalameta.org/docs/semanticdb/specification.html) files.
14+
These files encode information about which symbols are referenced from which
15+
locations in your source code.
16+
17+
This aspect only works on Linux when using the `local` spawn strategy because
18+
the `run_shell` action writes SemanticDB and SCIP files to the provided
19+
--targetroot argument. It should be possible to avoid this requirement
20+
in the future if there's a strong desire to make the aspect work with the
21+
default (sandboxed) spawn strategy.
22+
23+
Use the command below to merge all of these SCIP files into a single index:
24+
25+
find bazel-bin/ -type f -name '*.scip' | xargs cat > index.scip
26+
27+
Use `src code-intel upload` to upload the unified SCIP file to Sourcegraph:
28+
29+
npm install -g @sourcegraph/src
30+
export SRC_ENDPOINT=SOURCEGRAPH_URL
31+
export SRC_ACCESS_TOKEN=TOKEN_VALUE
32+
src login # confirm you are correctly authenticated
33+
src code-intel upload -file=index.scip
34+
35+
Example command to run this aspect directly:
36+
37+
bazel build //... --spawn_strategy=local --aspects path/to/scip_java.bzl%scip_java_aspect --output_groups=scip --define=sourceroot=$(pwd) --define=scip_java_binary=$(which scip-java) --define=java_home=$JAVA_HOME
38+
39+
To learn more about aspects: https://bazel.build/extending/aspects
40+
"""
41+
42+
def _scip_java(target, ctx):
43+
if JavaInfo not in target or not hasattr(ctx.rule.attr, "srcs"):
44+
return None
45+
46+
javac_action = None
47+
for a in target.actions:
48+
if a.mnemonic == "Javac":
49+
javac_action = a
50+
break
51+
52+
if not javac_action:
53+
return None
54+
55+
info = target[JavaInfo]
56+
compilation = info.compilation_info
57+
annotations = info.annotation_processing
58+
59+
source_files = []
60+
for src in ctx.rule.files.srcs:
61+
source_files.append(src.path)
62+
if len(source_files) == 0:
63+
return None
64+
65+
classpath = [j.path for j in compilation.compilation_classpath.to_list()]
66+
bootclasspath = [j.path for j in compilation.boot_classpath]
67+
68+
processorpath = []
69+
processors = []
70+
if annotations and annotations.enabled:
71+
processorpath += [j.path for j in annotations.processor_classpath.to_list()]
72+
processors = annotations.processor_classnames
73+
74+
build_config = struct(**{
75+
"javaHome": ctx.var["java_home"],
76+
"classpath": classpath,
77+
"sourceFiles": source_files,
78+
"javacOptions": compilation.javac_options,
79+
"processors": processors,
80+
"processorpath": processorpath,
81+
"bootclasspath": bootclasspath,
82+
"reportWarningOnEmptyIndex": False,
83+
})
84+
build_config_path = ctx.actions.declare_file(ctx.label.name + ".scip.json")
85+
86+
scip_output = ctx.actions.declare_file(ctx.label.name + ".scip")
87+
targetroot = ctx.actions.declare_directory(ctx.label.name + ".semanticdb")
88+
ctx.actions.write(
89+
output = build_config_path,
90+
content = build_config.to_json(),
91+
)
92+
93+
deps = [javac_action.inputs, annotations.processor_classpath]
94+
ctx.actions.run_shell(
95+
command = "\"{}\" index --no-cleanup --index-semanticdb.allow-empty-index --cwd \"{}\" --targetroot {} --scip-config \"{}\" --output \"{}\"".format(
96+
ctx.var["scip_java_binary"],
97+
ctx.var["sourceroot"],
98+
targetroot.path,
99+
build_config_path.path,
100+
scip_output.path,
101+
),
102+
env = {
103+
"JAVA_HOME": ctx.var["java_home"],
104+
"NO_PROGRESS_BAR": "true",
105+
},
106+
inputs = depset([build_config_path], transitive = deps),
107+
outputs = [scip_output, targetroot],
108+
)
109+
110+
return scip_output
111+
112+
def _scip_java_aspect(target, ctx):
113+
scip = _scip_java(target, ctx)
114+
if not scip:
115+
return struct()
116+
return [OutputGroupInfo(scip = [scip])]
117+
118+
scip_java_aspect = aspect(
119+
_scip_java_aspect,
120+
)
121+
122+
def _scip_java_impl(ctx):
123+
output = ctx.attr.compilation[OutputGroupInfo]
124+
return [
125+
OutputGroupInfo(scip = output.scip),
126+
DefaultInfo(files = output.scip),
127+
]
128+
129+
scip_java = rule(
130+
implementation = _scip_java_impl,
131+
attrs = {"compilation": attr.label(aspects = [scip_java_aspect])},
132+
)

scip-java/src/main/scala/com/sourcegraph/scip_java/Embedded.scala

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,19 @@ object Embedded {
112112
}
113113
}
114114

115+
/**
116+
* Returns the string contents of the scip_java.bzl file on disk.
117+
*/
118+
def bazelAspectFile(tmpDir: Path): String = {
119+
// We could in theory load the resource straight into a string but it was
120+
// easier to copy it to a file and read it from there.
121+
val tmpFile = copyFile(tmpDir, "scip-java/scip_java.bzl")
122+
val contents =
123+
new String(Files.readAllBytes(tmpFile), StandardCharsets.UTF_8)
124+
Files.deleteIfExists(tmpFile)
125+
contents
126+
}
127+
115128
private def copyFile(tmpDir: Path, filename: String): Path = {
116129
val in = this.getClass.getResourceAsStream(s"/$filename")
117130
val out = tmpDir.resolve(filename)

0 commit comments

Comments
 (0)