Debug CI failure

olafurpg · olafurpg · commit ab73d3b4d5b2 · 2023-07-13T11:21:01.000+02:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,76 +5,76 @@ on:
       - main
   pull_request:
 jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    name: Tests
-    strategy:
-      fail-fast: false
-      matrix:
-        # NOTE(olafurpg) Windows is not enabled because it times out due to reasons I don't understand.
-        # os: [windows-latest, ubuntu-latest]
-        os: [ubuntu-latest]
-        java: [8, 11, 17]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          cache: "sbt"
-          java-version: ${{ matrix.java }}
-      - name: Main project tests
-        run: sbt test
+  # test:
+  #   runs-on: ${{ matrix.os }}
+  #   name: Tests
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       # NOTE(olafurpg) Windows is not enabled because it times out due to reasons I don't understand.
+  #       # os: [windows-latest, ubuntu-latest]
+  #       os: [ubuntu-latest]
+  #       java: [8, 11, 17]
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - uses: actions/setup-java@v3
+  #       with:
+  #         distribution: "temurin"
+  #         cache: "sbt"
+  #         java-version: ${{ matrix.java }}
+  #     - name: Main project tests
+  #       run: sbt test
 
-  docker_test:
-    runs-on: ${{ matrix.os }}
-    name: Docker CLI tests
-    strategy:
-      fail-fast: true
-      matrix:
-        os: [ubuntu-latest]
-    steps:
-      - uses: actions/checkout@v2
+  # docker_test:
+  #   runs-on: ${{ matrix.os }}
+  #   name: Docker CLI tests
+  #   strategy:
+  #     fail-fast: true
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #   steps:
+  #     - uses: actions/checkout@v2
 
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          cache: "sbt"
-          java-version: 17
+  #     - uses: actions/setup-java@v3
+  #       with:
+  #         distribution: "temurin"
+  #         cache: "sbt"
+  #         java-version: 17
 
-      - name: Build Dockerised CLI
-        run: sbt cli/docker
+  #     - name: Build Dockerised CLI
+  #       run: sbt cli/docker
 
-      - run: |
+  #     - run: |
 
-          set -eu
-          check_repo() {
-            REPO=$1
-            mkdir -p .repos/$REPO
-            git clone https://github.com/$REPO.git .repos/$REPO
+  #         set -eu
+  #         check_repo() {
+  #           REPO=$1
+  #           mkdir -p .repos/$REPO
+  #           git clone https://github.com/$REPO.git .repos/$REPO
 
-            docker run -v $PWD/.repos/$REPO:/sources -w /sources sourcegraph/scip-java:latest scip-java index
-            file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1) 
-          }
+  #           docker run -v $PWD/.repos/$REPO:/sources -w /sources sourcegraph/scip-java:latest scip-java index
+  #           file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1)
+  #         }
 
-          sudo apt install parallel
-          export -f check_repo
+  #         sudo apt install parallel
+  #         export -f check_repo
 
-          parallel -j4 check_repo ::: circe/circe indeedeng/iwf-java-sdk
+  #         parallel -j4 check_repo ::: circe/circe indeedeng/iwf-java-sdk
 
-  bazel:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - run: yarn global add @bazel/bazelisk
-      - run: bazel build //... --//semanticdb-javac:enabled=true
-      - run: bazel run scip-semanticdb:bazel -- --sourceroot "$PWD"
-      - run: du -h index.scip
-      - run: bazel build //... --@scip_java//semanticdb-javac:enabled=true
-        working-directory: examples/bazel-example
-      - run: bazel run @scip_java//scip-semanticdb:bazel -- --sourceroot "$PWD"
-        working-directory: examples/bazel-example
-      - run: du -h index.scip
-        working-directory: examples/bazel-example
+  # bazel:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - run: yarn global add @bazel/bazelisk
+  #     - run: bazel build //... --//semanticdb-javac:enabled=true
+  #     - run: bazel run scip-semanticdb:bazel -- --sourceroot "$PWD"
+  #     - run: du -h index.scip
+  #     - run: bazel build //... --@scip_java//semanticdb-javac:enabled=true
+  #       working-directory: examples/bazel-example
+  #     - run: bazel run @scip_java//scip-semanticdb:bazel -- --sourceroot "$PWD"
+  #       working-directory: examples/bazel-example
+  #     - run: du -h index.scip
+  #       working-directory: examples/bazel-example
 
   bazel_aspect:
     runs-on: ubuntu-latest
@@ -83,22 +83,19 @@ jobs:
       - run: yarn global add @bazel/bazelisk
       - run: sbt cli/pack
       - run: echo "$PWD/scip-java/target/pack/bin" >> $GITHUB_PATH
-      - run: |
-          scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
-      - run: du -h index.scip
       - run: |
           scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
         working-directory: examples/bazel-example
       - run: du -h index.scip
         working-directory: examples/bazel-example
 
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          java-version: 17
-          cache: "sbt"
-      - run: sbt checkAll
+  # check:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - uses: actions/setup-java@v3
+  #       with:
+  #         distribution: "temurin"
+  #         java-version: 17
+  #         cache: "sbt"
+  #     - run: sbt checkAll
diff --git a/.gitignore b/.gitignore
@@ -65,3 +65,4 @@ bazel-lsif-java
 VERSION
 
 semanticdb-gradle-plugin/gradle
+aspects/scip_java.bzl
diff --git a/build.sbt b/build.sbt
@@ -167,6 +167,10 @@ lazy val javacPlugin = project
             // referenced from META-INF/services/com.sun.source.util.Plugin
             "com.sourcegraph.semanticdb_javac.SemanticdbPlugin" ->
               "com.sourcegraph.semanticdb_javac.SemanticdbPlugin",
+            "com.sourcegraph.semanticdb_javac.PrintJavaVersion" ->
+              "com.sourcegraph.semanticdb_javac.PrintJavaVersion",
+            "com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions" ->
+              "com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions",
             "com.google.**" -> "com.sourcegraph.shaded.com.google.@1",
             // Need to shade the semanticdb-javac compiler plugin in order to be
             // able to index the plugin code itself.
@@ -257,7 +261,7 @@ lazy val cli = project
           }
 
           addJar(
-            (javacPlugin / Compile / assembly).value,
+            (javacPlugin / Compile / Keys.`package`).value,
             "semanticdb-plugin.jar"
           )
           addJar(
diff --git a/examples/bazel-example/.gitignore b/examples/bazel-example/.gitignore
@@ -1,2 +1,2 @@
-aspects/scip_java.bzl
 bazel-bazel-example
+aspects/scip_java.bzl
diff --git a/scip-java/src/main/resources/scip-java/scip_java.bzl b/scip-java/src/main/resources/scip-java/scip_java.bzl
@@ -82,6 +82,8 @@ def _scip_java(target, ctx):
         output = build_config_path,
         content = build_config.to_json(),
     )
+
+    deps = [javac_action.inputs, annotations.processor_classpath]
     ctx.actions.run_shell(
         command = "\"{}\" index --no-cleanup --index-semanticdb.allow-empty-index --cwd \"{}\" --targetroot {} --scip-config \"{}\" --output \"{}\"".format(
             ctx.var["scip_java_binary"],
@@ -94,7 +96,7 @@ def _scip_java(target, ctx):
             "JAVA_HOME": ctx.var["java_home"],
             "NO_PROGRESS_BAR": "true",
         },
-        inputs = [build_config_path],
+        inputs = depset([build_config_path], transitive = deps),
         outputs = [scip_output, targetroot],
     )
 
diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala
@@ -8,12 +8,12 @@ import java.nio.file.Path
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.StandardOpenOption
 import java.nio.file.attribute.BasicFileAttributes
-
 import scala.collection.mutable.ListBuffer
-
 import com.sourcegraph.io.AbsolutePath
 import com.sourcegraph.scip_java.Embedded
 import com.sourcegraph.scip_java.commands.IndexCommand
+import moped.cli.Application
+import os.ProcessOutput.Readlines
 
 class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
 
@@ -31,6 +31,8 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
         val buildCommand = ListBuffer.empty[String]
         buildCommand += "bazel"
         buildCommand += "build"
+        buildCommand += "--noshow_progress"
+        buildCommand += "--sandbox_debug"
 
         val targetSpecs = ListBuffer.empty[String]
         if (index.buildCommand.isEmpty) {
@@ -72,12 +74,18 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
         }
         buildCommand += s"--define=scip_java_binary=$scipJavaBinary"
 
+        val sandbox = new SandboxCommandExtractor(index.app)
         index.app.info(buildCommand.mkString(" "))
         val commandResult = index
           .app
           .process(buildCommand.toList)
-          .call(check = false)
+          .call(check = false, stderr = Readlines(sandbox))
         if (commandResult.exitCode != 0) {
+          // Automatically run the sandbox command to help the user debug the problem.
+          index
+            .app
+            .process("bash", "-c", sandbox.commandLines().mkString("\n"))
+            .call(check = false, stdout = os.Inherit, stderr = os.Inherit)
           index
             .app
             .error(
@@ -127,6 +135,35 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
     }
   }
 
+  /**
+   * Callback to `os.ProcessOutput.Readlines` that processes the stderr output
+   * of a running Bazel process and extract the sandbox command. When the
+   * --sandbox_debug flag is enabled, Bazel prints out the sandbox command that
+   * failed but it doesn't show the stdout/stderr of that command. This
+   * extractor captures the command so that we can automatically re-run the
+   * command to print out the stdout/stderr.
+   */
+  private class SandboxCommandExtractor(app: Application)
+      extends (String => Unit) {
+    private var isSandboxCommandPrinting = false
+    private val lines = ListBuffer.empty[String]
+    def commandLines(): List[String] = lines.toList
+    def command(): List[String] = List("bash", "-c", lines.mkString("\n"))
+    override def apply(line: String): Unit = {
+      if (
+        !isSandboxCommandPrinting && line.startsWith("ERROR:") &&
+        line.endsWith("error executing command ")
+      ) {
+        isSandboxCommandPrinting = true
+      } else if (isSandboxCommandPrinting && !line.startsWith("  ")) {
+        isSandboxCommandPrinting = false
+      } else if (isSandboxCommandPrinting) {
+        lines += line
+      }
+      app.env.standardError.println(line)
+    }
+  }
+
   /**
    * Reads the scip_java.bzl file from resources and writes it to the
    * aspect/scip_java.bzl file inside the Bazel workspace.
diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala
@@ -186,7 +186,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
 
     val compileAttempts = ListBuffer.empty[Try[Unit]]
     compileAttempts += compileJavaFiles(tmp, deps, config, javaFiles)
-    compileAttempts += compileScalaFiles(deps, scalaFiles)
+    compileAttempts += compileScalaFiles(deps, scalaFiles, tmp)
     compileAttempts += compileKotlinFiles(deps, config, kotlinFiles)
     val errors = compileAttempts.collect { case Failure(exception) =>
       exception
@@ -347,11 +347,12 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
 
   private def compileScalaFiles(
       deps: Dependencies,
-      allScalaFiles: List[Path]
+      allScalaFiles: List[Path],
+      tmp: Path
   ): Try[Unit] =
     Try {
       if (deps.dependencies.nonEmpty && allScalaFiles.nonEmpty)
-        withScalaPresentationCompiler(deps) { compiler =>
+        withScalaPresentationCompiler(deps, tmp) { compiler =>
           allScalaFiles.foreach { path =>
             try compileScalaFile(compiler, path)
             catch {
@@ -393,9 +394,9 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
     )
   }
 
-  private def withScalaPresentationCompiler[T](
-      deps: Dependencies
-  )(fn: PresentationCompiler => T): T = {
+  private def withScalaPresentationCompiler[T](deps: Dependencies, tmp: Path)(
+      fn: PresentationCompiler => T
+  ): T = {
     val scalaVersion = deps
       .classpath
       .iterator
@@ -423,7 +424,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
       .iterator()
     if (compilers.hasNext) {
       val classpath = deps.classpath ++ scalaLibrary
-      val argsfile = targetroot.resolve("javacopts.txt")
+      val argsfile = tmp.resolve("javacopts.txt")
       Files.createDirectories(argsfile.getParent)
       Files.write(
         argsfile,
@@ -491,7 +492,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
           AbsolutePath.of(Paths.get(path), index.workingDirectory).toString
         )
     actualClasspath ++= deps.classpath.map(_.toString)
-    val argsfile = targetroot.resolve("javacopts.txt")
+    val argsfile = tmp.resolve("javacopts.txt")
     val arguments = ListBuffer.empty[String]
     arguments += "-encoding"
     arguments += "utf8"
@@ -677,7 +678,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
       !pathString.startsWith("bazel-bin") && !pathString.startsWith("bazel-out")
     ) {
       path = AbsolutePath
-        .of(Paths.get("bazel-out", pathString), workingDirectory)
+        .of(Paths.get("bazel-bin", pathString), workingDirectory)
 
       if (Files.isRegularFile(path))
         return Some(path)
@@ -758,7 +759,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
       processorpath: List[String] = Nil,
       processors: List[String] = Nil,
       javacOptions: List[String] = Nil,
-      jvm: String = "8",
+      jvm: String = "17",
       kind: String = ""
   )
   private object Config {

Original file line number	Diff line number	Diff line change
`@@ -65,3 +65,4 @@ bazel-lsif-java`
`65`	`65`	`VERSION`
`66`	`66`
`67`	`67`	`semanticdb-gradle-plugin/gradle`
	`68`	`+aspects/scip_java.bzl`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-aspects/scip_java.bzl`
`2`	`1`	`bazel-bazel-example`
	`2`	`+aspects/scip_java.bzl`