Use the correct JDK version when indexing Java packages.

Previously, PackageHub was not able to index Java 11 libraries because
it always ran the `index-semanticdb` step on Java 8. Now, we shell out
to `cs launch lsif-java --jvm JVM_VERSION` to ensure that the
`index-semanticdb` command always uses the correct Java version.

Author: olafurpg

lsif-java/src/main/scala/com/sourcegraph/lsif_java/Dependencies.scala

@@ -28,6 +28,7 @@ case class Dependencies(
 }
 
 object Dependencies {
+  val empty = Dependencies(Nil, Fetch.Result(), Fetch.Result())
   private val cache: FileCache[Task] = FileCache[Task]()
     .noCredentials
     .withCachePolicies(List(CachePolicy.LocalOnly, CachePolicy.Update))

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala

@@ -14,8 +14,6 @@ abstract class BuildTool(val name: String, index: IndexCommand) {
 
   def isHidden: Boolean = false
 
-  def indexJdk(): Boolean = true
-
   final def sourceroot: Path = index.workingDirectory
   final def targetroot: Path =
     AbsolutePath

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala

@@ -50,9 +50,9 @@ import os.ProcessOutput.Readlines
 class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
 
   protected def defaultTargetroot: Path = Paths.get("target")
-  private def configFile = index.workingDirectory.resolve("lsif-java.json")
+  private def configFile =
+    index.workingDirectory.resolve(LsifBuildTool.ConfigFileName)
   def usedInCurrentDirectory(): Boolean = Files.isRegularFile(configFile)
-  override def indexJdk(): Boolean = parsedConfig.getOrElse(Config()).indexJdk
   override def isHidden: Boolean = true
   def generateSemanticdb(): CommandResult = {
     parsedConfig match {
@@ -132,7 +132,7 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
     arguments += actualClasspath.mkString(File.pathSeparator)
     arguments +=
       s"-Xplugin:semanticdb -targetroot:$targetroot -sourceroot:$sourceroot"
-    if (!config.indexJdk && moduleInfos.nonEmpty) {
+    if (config.kind == "jdk" && moduleInfos.nonEmpty) {
       moduleInfos.foreach { module =>
         arguments += "--module"
         arguments += module.getParent.getFileName.toString
@@ -192,7 +192,8 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
   private def clean(): Unit = {
     Files.walkFileTree(targetroot, new DeleteVisitor)
   }
-  val moduleInfo = Paths.get("module-info.java")
+
+  private val moduleInfo = Paths.get("module-info.java")
 
   /** Recursively collects all Java files in the working directory */
   private def collectAllJavaFiles(dir: Path): List[Path] = {
@@ -280,10 +281,14 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
   private case class Config(
       dependencies: List[Dependency] = Nil,
       jvm: String = "8",
-      indexJdk: Boolean = true
+      kind: String = ""
   )
   private object Config {
     implicit lazy val codec = moped.macros.deriveCodec(Config())
   }
 
 }
+
+object LsifBuildTool {
+  val ConfigFileName = "lsif-java.json"
+}

lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala

@@ -157,7 +157,6 @@ case class IndexCommand(
           generateSemanticdbResult.exitCode
         } else {
           IndexSemanticdbCommand(
-            indexJdk = tool.indexJdk(),
             output = finalOutput,
             targetroot = List(tool.targetroot),
             packagehub = packagehub,

lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala

@@ -33,10 +33,6 @@ final case class IndexSemanticdbCommand(
     @Description(
       "Whether to process the SemanticDB files in parallel"
     ) parallel: Boolean = true,
-    @Description(
-      "Whether to index against symbols in the JDK codebase. " +
-        "If false, no 'packageInformation' LSIF edges will be emitted for JDK symbols."
-    ) indexJdk: Boolean = true,
     @Description("URL to a PackageHub instance")
     @Hidden
     packagehub: Option[String] = None,
@@ -76,7 +72,6 @@ final case class IndexSemanticdbCommand(
         "java",
         format,
         parallel,
-        indexJdk,
         packages.map(_.toPackageInformation).asJava
       )
     LsifSemanticdb.run(options)

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/JavaVersion.java

@@ -1,8 +1,28 @@
 package com.sourcegraph.lsif_semanticdb;
 
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.util.Enumeration;
+import java.util.Optional;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
 public class JavaVersion {
   public final boolean isJava8;
   public final JdkPackage pkg;
+  private static PathMatcher CLASS_PATTERN =
+      FileSystems.getDefault().getPathMatcher("glob:**.class");
+  private static PathMatcher JAR_PATTERN = FileSystems.getDefault().getPathMatcher("glob:**.jar");
+
+  public static int JAVA8_VERSION = 8;
+  public static int JAVA11_VERSION = 11;
+  public static int DEFAULT_JAVA_VERSION = JAVA8_VERSION;
+  private static int JAVA0_MAJOR_VERSION = 44;
 
   public JavaVersion() {
     this(System.getProperty("java.version"));
@@ -19,4 +39,55 @@ private String javaVersion(String version) {
     if (parts.length > 0) return parts[0];
     else return version;
   }
+
+  public static int roundToNearestStableRelease(int version) {
+    if (version <= JAVA8_VERSION) return JAVA8_VERSION;
+    if (version <= JAVA11_VERSION) return JAVA11_VERSION;
+    return version;
+  }
+
+  /**
+   * Return the JVM version of the given jar/class file.
+   *
+   * <p>The JVM version is determined by reading the 5-8th bytes of classfiles, according to the
+   * Java Language spec. See
+   * https://docs.oracle.com/javase/specs/jvms/se16/html/jvms-4.html#jvms-4.1
+   *
+   * @return the JVM version such as <code>8</code> for Java 8 and <code>11</code> for Java 11.
+   */
+  public static Optional<Integer> classfileJvmVersion(Path file) {
+    try {
+      int major = classfileMajorVersion(file);
+      return major < 0 ? Optional.empty() : Optional.of(major - JAVA0_MAJOR_VERSION);
+    } catch (IOException e) {
+      return Optional.empty();
+    }
+  }
+
+  private static int classfileMajorVersion(Path file) throws IOException {
+    if (CLASS_PATTERN.matches(file)) {
+      return classfileMajorVersion(Files.newInputStream(file));
+    } else if (JAR_PATTERN.matches(file)) {
+      try (JarFile jar = new JarFile(file.toFile())) {
+        Enumeration<JarEntry> entries = jar.entries();
+        while (entries.hasMoreElements()) {
+          JarEntry entry = entries.nextElement();
+          if (entry.getName().endsWith(".class")) {
+            return classfileMajorVersion(jar.getInputStream(entry));
+          }
+        }
+      }
+    }
+
+    return -1;
+  }
+
+  private static int classfileMajorVersion(InputStream classfileBytes) throws IOException {
+    DataInputStream in = new DataInputStream(classfileBytes);
+    // See https://docs.oracle.com/javase/specs/jvms/se16/html/jvms-4.html#jvms-4.1
+    int magic = in.readInt(); // u4 magic
+    if (magic != 0xCAFEBABE) return -1;
+    in.readUnsignedShort(); // u2 minor_version
+    return in.readUnsignedShort(); // u2 major_version
+  }
 }

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbOptions.java

@@ -15,7 +15,6 @@ public class LsifSemanticdbOptions {
   public final String language;
   public final LsifOutputFormat format;
   public final boolean parallel;
-  public final boolean indexJdk;
   public final List<MavenPackage> packages;
 
   public LsifSemanticdbOptions(
@@ -27,7 +26,6 @@ public LsifSemanticdbOptions(
       String language,
       LsifOutputFormat format,
       boolean parallel,
-      boolean indexJdk,
       List<MavenPackage> packages) {
     this.targetroots = targetroots;
     this.output = output;
@@ -37,7 +35,6 @@ public LsifSemanticdbOptions(
     this.language = language;
     this.format = format;
     this.parallel = parallel;
-    this.indexJdk = indexJdk;
     this.packages = packages;
   }
 }

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/PackageTable.java

@@ -26,7 +26,6 @@ public class PackageTable implements Function<Package, Integer> {
   private final Set<String> cachedJdkSymbols = new HashSet<>();
   private final Map<Package, Integer> lsif = new ConcurrentHashMap<>();
   private final JavaVersion javaVersion;
-  private final boolean indexJdk;
   private final LsifWriter writer;
 
   private static final PathMatcher JAR_PATTERN =
@@ -37,13 +36,10 @@ public class PackageTable implements Function<Package, Integer> {
   public PackageTable(LsifSemanticdbOptions options, LsifWriter writer) throws IOException {
     this.writer = writer;
     this.javaVersion = new JavaVersion();
-    this.indexJdk = options.indexJdk;
     for (MavenPackage pkg : options.packages) {
       indexPackage(pkg);
     }
-    if (indexJdk) {
-      indexJdk();
-    }
+    indexJdk();
   }
 
   public void writeImportedSymbol(String symbol, int monikerId) {
@@ -106,7 +102,6 @@ private void indexJdk() throws IOException {
    * under the URL "jrt:/".
    */
   private boolean isJrtClassfile(String classfile) {
-    if (!indexJdk) return false;
     if (cachedJdkSymbols.contains(classfile)) return true;
     URL resource = getClass().getResource("/" + classfile);
     boolean isJrt = resource != null && "jrt".equals(resource.getProtocol());
@@ -116,26 +111,6 @@ private boolean isJrtClassfile(String classfile) {
     return isJrt;
   }
 
-  /**
-   * Returns the equivalent of `Path.toString()` but uses forward slashes on all operating systems
-   * (including Windows).
-   */
-  private String relativePathToString(Path path) {
-    StringBuilder out = new StringBuilder();
-    Iterator<Path> it = path.iterator();
-    boolean first = true;
-    while (it.hasNext()) {
-      if (first) {
-        first = false;
-      } else {
-        out.append('/');
-      }
-      String filename = it.next().toString();
-      out.append(filename);
-    }
-    return out.toString();
-  }
-
   /**
    * The boot classpath contains jar files for the JDK in Java 8.
    *