Add support for LSIF Typed

This commit adds the ability to emit LSIF Typed using the following
command:

```
lsif-java index --output dump.lsif-typed
```

LSIF Typed is used as the output format as long as the output filename
ends with `*.lsif-typed`.

Author: olafurpg

lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala

@@ -44,17 +44,19 @@ final case class IndexSemanticdbCommand(
     @Inline() app: Application = Application.default
 ) extends Command {
   def sourceroot: Path = AbsolutePath.of(app.env.workingDirectory)
-  def isProtobufFormat: Boolean =
-    IndexSemanticdbCommand.isProtobufFormat(output)
   def absoluteTargetroots: List[Path] =
     targetroot.map(AbsolutePath.of(_, app.env.workingDirectory))
   def run(): Int = {
     val reporter = new ConsoleLsifSemanticdbReporter(app)
-    val format =
-      if (isProtobufFormat)
-        LsifOutputFormat.PROTOBUF
-      else
-        LsifOutputFormat.JSON
+    val outputFilename = output.getFileName.toString
+    val format = LsifOutputFormat.fromFilename(outputFilename)
+    if (format == LsifOutputFormat.UNKNOWN) {
+      app.error(
+        s"unknown output format for filename '$outputFilename'. " +
+          s"Supported file extension are `*.lsif`, `*.lsif-typed'"
+      )
+      return 1
+    }
     val packages =
       absoluteTargetroots
         .iterator

lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala

@@ -42,6 +42,7 @@ import moped.cli.Command
 import moped.cli.CommandParser
 import moped.reporters.Input
 import moped.reporters.Position
+import moped.reporters.Reporter
 import org.scalameta.ascii.layout.prefs.LayoutPrefsImpl
 
 @CommandName("snapshot-lsif")
@@ -65,7 +66,11 @@ case class SnapshotLsifCommand(
     for {
       inputPath <- input
       in = AbsolutePath.of(inputPath, sourceroot)
-      doc <- SnapshotLsifCommand.parseTextDocument(in, sourceroot)
+      if Files.isRegularFile(in) || {
+        app.error(s"no such file: $in")
+        false
+      }
+      doc <- SnapshotLsifCommand.parseTextDocument(in, sourceroot, app.reporter)
     } {
       val docPath = AbsolutePath
         .of(Paths.get(doc.getUri), sourceroot)
@@ -86,16 +91,21 @@ case class SnapshotLsifCommand(
 
 object SnapshotLsifCommand {
   private val jsonParser = JsonFormat.parser().ignoringUnknownFields()
-  def parseTextDocument(input: Path, sourceroot: Path): List[TextDocument] = {
-    parseSemanticdb(input, parseInput(input), sourceroot)
+  def parseTextDocument(
+      input: Path,
+      sourceroot: Path,
+      reporter: Reporter
+  ): List[TextDocument] = {
+    parseSemanticdb(input, parseInput(input), sourceroot, reporter)
   }
 
   def parseSemanticdb(
       input: Path,
       objects: mutable.Buffer[LsifObject],
-      sourceroot: Path
+      sourceroot: Path,
+      reporter: Reporter
   ): List[TextDocument] = {
-    val lsif = new IndexedLsif(input, objects, sourceroot)
+    val lsif = new IndexedLsif(input, objects, sourceroot, reporter)
     lsif
       .ranges
       .iterator
@@ -169,7 +179,8 @@ object SnapshotLsifCommand {
   class IndexedLsif(
       val path: Path,
       val objects: mutable.Buffer[LsifObject],
-      val sourceroot: Path
+      val sourceroot: Path,
+      val reporter: Reporter
   ) {
     val documents = mutable.Map.empty[Int, TextDocument.Builder]
     val next = mutable.Map.empty[Int, Int]
@@ -432,24 +443,28 @@ object SnapshotLsifCommand {
             case "document" =>
               val relativeFile = Paths.get(URI.create(o.getUri))
               val absoluteFile = sourceroot.resolve(relativeFile)
-              val text =
-                new String(
-                  Files.readAllBytes(absoluteFile),
-                  StandardCharsets.UTF_8
-                )
-              val relativeUri = sourceroot
-                .relativize(absoluteFile)
-                .iterator()
-                .asScala
-                .mkString("/")
-              val language = Language
-                .values()
-                .find(_.name().compareToIgnoreCase(o.getLanguage) == 0)
-                .getOrElse(Language.UNKNOWN_LANGUAGE)
-              textDocument(o.getId)
-                .setUri(relativeUri)
-                .setLanguage(language)
-                .setText(text)
+              if (!Files.isRegularFile(absoluteFile)) {
+                reporter.warning(s"no such file: $absoluteFile")
+              } else {
+                val text =
+                  new String(
+                    Files.readAllBytes(absoluteFile),
+                    StandardCharsets.UTF_8
+                  )
+                val relativeUri = sourceroot
+                  .relativize(absoluteFile)
+                  .iterator()
+                  .asScala
+                  .mkString("/")
+                val language = Language
+                  .values()
+                  .find(_.name().compareToIgnoreCase(o.getLanguage) == 0)
+                  .getOrElse(Language.UNKNOWN_LANGUAGE)
+                textDocument(o.getId)
+                  .setUri(relativeUri)
+                  .setLanguage(language)
+                  .setText(text)
+              }
             case "definitionResult" =>
               isDefinitionResult += o.getId()
             case "hoverResult" =>

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputFormat.java

@@ -6,6 +6,25 @@
  * <p>The Protobuf format is experimental and currently only exists as a proof-of-concept.
  */
 public enum LsifOutputFormat {
-  JSON,
-  PROTOBUF
+  GRAPH_NDJSON,
+  GRAPH_PROTOBUF,
+  TYPED_PROTOBUF,
+  TYPED_NDJSON,
+  UNKNOWN;
+
+  public boolean isTyped() {
+    return this == TYPED_NDJSON || this == TYPED_PROTOBUF;
+  }
+
+  public boolean isNewlineDelimitedJSON() {
+    return this == GRAPH_NDJSON || this == TYPED_NDJSON;
+  }
+
+  public static LsifOutputFormat fromFilename(String name) {
+    if (name.endsWith(".lsif")) return GRAPH_NDJSON;
+    if (name.endsWith(".lsif-protobuf")) return GRAPH_PROTOBUF;
+    if (name.endsWith(".lsif-typed")) return TYPED_PROTOBUF;
+    if (name.endsWith(".lsif-typed.ndjson")) return TYPED_NDJSON;
+    return UNKNOWN;
+  }
 }

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputStream.java

@@ -49,10 +49,10 @@ public void writeLsifObject(LsifObject.Builder object) {
     b.output.reset();
     try {
       switch (options.format) {
-        case PROTOBUF:
+        case GRAPH_PROTOBUF:
           object.buildPartial().writeTo(b.output);
           break;
-        case JSON:
+        case GRAPH_NDJSON:
         default:
           jsonPrinter.appendTo(object, b.writer);
           b.writer.flush();
@@ -69,7 +69,9 @@ public void flush() throws IOException {
       byte[] bytes = buffer.poll();
       while (bytes != null) {
         out.write(bytes);
-        out.write(NEWLINE);
+        if (options.format.isNewlineDelimitedJSON()) {
+          out.write(NEWLINE);
+        }
         bytes = buffer.poll();
       }
       out.flush();

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java

@@ -7,14 +7,18 @@
 import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence;
 import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence.Role;
 import com.sourcegraph.semanticdb_javac.SemanticdbSymbols;
+import lib.codeintel.lsif_typed.LsifTyped;
 
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 /** The core logic that converts SemanticDB into LSIF. */
 public class LsifSemanticdb {
@@ -49,6 +53,116 @@ private void run() throws IOException {
       return;
     }
     options.reporter.startProcessing(files.size());
+    if (options.format.isTyped()) {
+      runTyped(files, packages);
+    } else {
+      runGraph(files, packages);
+    }
+    writer.build();
+    options.reporter.endProcessing();
+  }
+
+  private void runTyped(List<Path> files, PackageTable packages) {
+    writer.emitTyped(typedMetadata());
+    filesStream(files).forEach(document -> processTypedDocument(document, packages));
+  }
+
+  private String typedSymbol(String symbol, Package pkg) {
+    if (symbol.startsWith("local")) {
+      return "local " + symbol.substring("local".length());
+    }
+    return "semanticdb maven " + pkg.repoName() + " " + pkg.version() + " " + symbol;
+  }
+
+  private void processTypedDocument(Path path, PackageTable packages) {
+    for (LsifTextDocument doc : parseTextDocument(path).collect(Collectors.toList())) {
+      if (doc.semanticdb.getOccurrencesCount() == 0) {
+        continue;
+      }
+
+      Path absolutePath = Paths.get(URI.create(doc.semanticdb.getUri()));
+      String relativePath =
+          StreamSupport.stream(options.sourceroot.relativize(absolutePath).spliterator(), false)
+              .map(p -> p.getFileName().toString())
+              .collect(Collectors.joining("/"));
+      LsifTyped.Document.Builder tdoc =
+          LsifTyped.Document.newBuilder().setRelativePath(relativePath);
+      for (SymbolOccurrence occ : doc.sortedSymbolOccurrences()) {
+        int role = 0;
+        if (occ.getRole() == Role.DEFINITION) {
+          role |= LsifTyped.SymbolRole.Definition_VALUE;
+        }
+        boolean isSingleLineRange = occ.getRange().getStartLine() == occ.getRange().getEndLine();
+        Iterable<Integer> range =
+            isSingleLineRange
+                ? Arrays.asList(
+                    occ.getRange().getStartLine(),
+                    occ.getRange().getStartCharacter(),
+                    occ.getRange().getEndCharacter())
+                : Arrays.asList(
+                    occ.getRange().getStartLine(),
+                    occ.getRange().getStartCharacter(),
+                    occ.getRange().getEndLine(),
+                    occ.getRange().getEndCharacter());
+        Package pkg = packages.packageForSymbol(occ.getSymbol()).orElse(Package.EMPTY);
+        tdoc.addOccurrences(
+            LsifTyped.Occurrence.newBuilder()
+                .addAllRange(range)
+                .setSymbol(typedSymbol(occ.getSymbol(), pkg))
+                .setSymbolRoles(role));
+      }
+      Symtab symtab = new Symtab(doc.semanticdb);
+      for (SymbolInformation info : doc.semanticdb.getSymbolsList()) {
+        Package pkg = packages.packageForSymbol(info.getSymbol()).orElse(Package.EMPTY);
+        LsifTyped.SymbolInformation.Builder tinfo =
+            LsifTyped.SymbolInformation.newBuilder().setSymbol(typedSymbol(info.getSymbol(), pkg));
+
+        for (String overriddenSymbol : info.getOverriddenSymbolsList()) {
+          if (overriddenSymbol.equals("java/lang/Object#")) {
+            // Skip java/lang/Object# since it's the parent of all classes
+            // making it noisy for "find implementations" results.
+            continue;
+          }
+          Package overriddenSymbolPkg =
+              packages.packageForSymbol(overriddenSymbol).orElse(Package.EMPTY);
+          tinfo.addRelationships(
+              LsifTyped.Relationship.newBuilder()
+                  .setSymbol(typedSymbol(overriddenSymbol, overriddenSymbolPkg))
+                  .setIsImplementation(true)
+                  .setIsReference(SemanticdbSymbols.isMethod(info.getSymbol())));
+        }
+        if (info.hasSignature()) {
+          String language =
+              doc.semanticdb.getLanguage().toString().toLowerCase(Locale.ROOT).intern();
+          String signature = new SignatureFormatter(info, symtab).formatSymbol();
+          tinfo.addDocumentation("```" + language + "\n" + signature + "\n```");
+        }
+        String documentation = info.getDocumentation().getMessage();
+        if (!documentation.isEmpty()) {
+          tinfo.addDocumentation(documentation);
+        }
+        tdoc.addSymbols(tinfo);
+      }
+      writer.emitTyped(LsifTyped.Index.newBuilder().addDocuments(tdoc).build());
+    }
+  }
+
+  private LsifTyped.Index typedMetadata() {
+    return LsifTyped.Index.newBuilder()
+        .setMetadata(
+            LsifTyped.Metadata.newBuilder()
+                .setVersion(LsifTyped.ProtocolVersion.UnspecifiedProtocolVersion)
+                .setProjectRoot(options.sourceroot.toUri().toString())
+                .setTextDocumentEncoding(LsifTyped.TextEncoding.UTF8)
+                .setToolInfo(
+                    LsifTyped.ToolInfo.newBuilder()
+                        .setName(options.toolInfo.getName())
+                        .setVersion(options.toolInfo.getVersion())
+                        .addAllArguments(options.toolInfo.getArgsList())))
+        .build();
+  }
+
+  private void runGraph(List<Path> files, PackageTable packages) {
     writer.emitMetaData();
     int projectId = writer.emitProject(options.language);
 
@@ -57,11 +171,7 @@ private void run() throws IOException {
         filesStream(files)
             .flatMap(d -> processPath(d, isExportedSymbol, packages))
             .collect(Collectors.toList());
-
     writer.emitContains(projectId, documentIds);
-
-    writer.build();
-    options.reporter.endProcessing();
   }
 
   private Stream<Path> filesStream(List<Path> files) {

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifWriter.java

@@ -6,6 +6,8 @@
 import com.sourcegraph.lsif_protocol.LsifPosition;
 import com.sourcegraph.semanticdb_javac.Semanticdb;
 import com.sourcegraph.semanticdb_javac.SemanticdbSymbols;
+import lib.codeintel.lsif_typed.LsifTyped;
+
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -15,6 +17,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /** High-level utility methods to write LSIF vertex/edge objects into the LSIF output stream. */
@@ -36,6 +39,10 @@ public LsifWriter(LsifSemanticdbOptions options) throws IOException {
     this.options = options;
   }
 
+  public void emitTyped(LsifTyped.Index index) {
+    this.output.write(index.toByteArray());
+  }
+
   public void emitMetaData() {
     emitObject(
         lsifVertex("metaData")

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/Package.java

@@ -1,8 +1,31 @@
 package com.sourcegraph.lsif_semanticdb;
 
 public abstract class Package {
+  public static final Package EMPTY =
+      new Package() {
+        @Override
+        public String repoName() {
+          return ".";
+        }
+
+        @Override
+        public String version() {
+          return ".";
+        }
+      };
 
   public abstract String repoName();
 
   public abstract String version();
+
+  public final String lsifTypedEncoding() {
+    return "maven " + encode(repoName()) + " " + encode(version());
+  }
+
+  private String encode(String value) {
+    if (value.contains(" ")) {
+      return "`" + value + "`";
+    }
+    return value;
+  }
 }