Fix bug when indexing JDK 11 sources

Previously, PackageHub was unable to auto-index the JDK 11 sources
because it was missing appropriate `--module` flags that are not needed
for JDK 8.

Author: olafurpg

Dockerfile

@@ -1,11 +1,11 @@
-FROM openjdk:8-jdk-alpine@sha256:94792824df2df33402f201713f932b58cb9de94a0cd524164a0f2283343547b3
-COPY bin/coursier coursier
-RUN apk add --no-cache git curl \
-    && git config --global user.email "[email protected]" \
-    && git config --global user.name "Your Name" \
-    && git config --global http.postBuffer 1048576000 \
-    && curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src \
-    && chmod +x /src \
-    && /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:0.5.0-12-69905fcb-SNAPSHOT -o /packagehub
+FROM sourcegraph/lsif-java
+COPY bin/packagehub.sh /packagehub.sh
+RUN chmod +x /packagehub.sh
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "Your Name"
+RUN git config --global http.postBuffer 1048576000
+RUN curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src
+RUN chmod +x /src
+RUN /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:0.5.1-26-2d4609cc-SNAPSHOT -o /packagehub
 ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
-ENTRYPOINT /packagehub --host 0.0.0.0 --port $PORT --src /src --coursier /coursier --postgres.username=$DB_USER --postgres.password=$DB_PASS --postgres.url=$DB_URL --auto-index-delay=PT1M
+CMD ["/packagehub.sh"]

Dockerfile.template

@@ -1,11 +1,11 @@
-FROM openjdk:8-jdk-alpine@sha256:94792824df2df33402f201713f932b58cb9de94a0cd524164a0f2283343547b3
-COPY bin/coursier coursier
-RUN apk add --no-cache git curl \
-    && git config --global user.email "[email protected]" \
-    && git config --global user.name "Your Name" \
-    && git config --global http.postBuffer 1048576000 \
-    && curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src \
-    && chmod +x /src \
-    && /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:VERSION -o /packagehub
+FROM sourcegraph/lsif-java
+COPY bin/packagehub.sh /packagehub.sh
+RUN chmod +x /packagehub.sh
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "Your Name"
+RUN git config --global http.postBuffer 1048576000
+RUN curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src
+RUN chmod +x /src
+RUN /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:VERSION -o /packagehub
 ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
-ENTRYPOINT /packagehub --host 0.0.0.0 --port $PORT --src /src --coursier /coursier --postgres.username=$DB_USER --postgres.password=$DB_PASS --postgres.url=$DB_URL --auto-index-delay=PT1M
+CMD ["/packagehub.sh"]

bin/packagehub.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+/packagehub --host 0.0.0.0 --port $PORT --src /src --coursier /coursier --postgres.username=$DB_USER --postgres.password=$DB_PASS --postgres.url=$DB_URL --auto-index-delay=PT1M

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala

@@ -10,9 +10,11 @@ import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.attribute.BasicFileAttributes
+
 import scala.collection.mutable.ListBuffer
 import scala.jdk.CollectionConverters._
 import scala.util.control.NonFatal
+
 import com.sourcegraph.io.DeleteVisitor
 import com.sourcegraph.lsif_java.Dependencies
 import com.sourcegraph.lsif_java.Embedded
@@ -28,8 +30,8 @@ import moped.macros.ClassShape
 import moped.parsers.JsonParser
 import moped.reporters.Diagnostic
 import moped.reporters.Input
+import os.CommandResult
 import os.ProcessOutput.Readlines
-import os.{CommandResult, ProcessOutput}
 
 /**
  * A custom build tool that is specifically made for lsif-java.
@@ -100,7 +102,11 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
     if (!Files.isDirectory(sourceroot)) {
       throw new NoSuchFileException(sourceroot.toString())
     }
-    val javaFiles = collectAllJavaFiles(sourceroot).map(_.toString())
+    val allJavaFiles = collectAllJavaFiles(sourceroot)
+    val javaFiles = allJavaFiles
+      .filterNot(_.endsWith(moduleInfo))
+      .map(_.toString())
+    val moduleInfos = allJavaFiles.filter(_.endsWith(moduleInfo))
     if (javaFiles.isEmpty) {
       index
         .app
@@ -126,7 +132,16 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
     arguments += actualClasspath.mkString(File.pathSeparator)
     arguments +=
       s"-Xplugin:semanticdb -targetroot:$targetroot -sourceroot:$sourceroot"
-    arguments ++= javaFiles
+    if (!config.indexJdk && moduleInfos.nonEmpty) {
+      moduleInfos.foreach { module =>
+        arguments += "--module"
+        arguments += module.getParent.getFileName.toString
+      }
+      arguments += "--module-source-path"
+      arguments += sourceroot.toString
+    } else {
+      arguments ++= javaFiles
+    }
     val quotedArguments = arguments.map(a => "\"" + a + "\"")
     Files.write(argsfile, quotedArguments.asJava)
     if (javaFiles.size > 1) {
@@ -135,8 +150,17 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
     val pipe = Readlines(line => {
       index.app.reporter.info(line)
     })
+    val javac = Paths.get(
+      os.proc(coursier.toString, "java-home", "--jvm", config.jvm)
+        .call()
+        .out
+        .trim(),
+      "bin",
+      "javac"
+    )
+    index.app.reporter.info(s"$$ $javac @$argsfile")
     val result = os
-      .proc("javac", s"@$argsfile")
+      .proc(javac.toString, s"@$argsfile")
       .call(
         stdout = pipe,
         stderr = pipe,
@@ -168,11 +192,11 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
   private def clean(): Unit = {
     Files.walkFileTree(targetroot, new DeleteVisitor)
   }
+  val moduleInfo = Paths.get("module-info.java")
 
   /** Recursively collects all Java files in the working directory */
   private def collectAllJavaFiles(dir: Path): List[Path] = {
     val javaPattern = FileSystems.getDefault.getPathMatcher("glob:**.java")
-    val moduleInfo = Paths.get("module-info.java")
     val buf = ListBuffer.empty[Path]
     Files.walkFileTree(
       dir,
@@ -190,7 +214,7 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
             file: Path,
             attrs: BasicFileAttributes
         ): FileVisitResult = {
-          if (javaPattern.matches(file) && !file.endsWith(moduleInfo)) {
+          if (javaPattern.matches(file)) {
             buf += file
           }
           FileVisitResult.CONTINUE

packagehub/src/main/scala/com/sourcegraph/packagehub/PackageActor.scala

@@ -30,6 +30,7 @@ import os.SubProcess
 import ujson.Arr
 import ujson.Bool
 import ujson.Obj
+import ujson.Str
 
 /**
  * Actor that creates git repos from package sources and (optionally LSIF
@@ -237,13 +238,13 @@ class PackageActor(
       List("dump.lsif", packagehubCached).asJava
     )
     val build = Obj()
-    val dependencies =
-      dep match {
-        case MavenPackage(dep) =>
-          build("dependencies") = Arr(packageId(dep))
-        case _ =>
-          build("indexJdk") = Bool(false)
-      }
+    dep match {
+      case MavenPackage(dep) =>
+        build("dependencies") = Arr(packageId(dep))
+      case JdkPackage(version) =>
+        build("indexJdk") = Bool(false)
+        build("jvm") = Str(version)
+    }
     Files.write(
       repo.resolve("lsif-java.json"),
       List(ujson.write(build, indent = 2)).asJava