Index SemanticDB synthetics

Previously, lsif-java only indexed the "Occurrences" section of a
SemanticDB `TextDocument`. Now, we additionall index the "Synthetics"
section, which includes inferred expression from Scala implicits, for
comprehensions, `.apply` methods and more. The benefit of this fix is
that "find references" on an implicit or an `apply` method now shows
usages where those symbols got inferred (the user didn't explicitly
write a reference to them).

It could be that this fix alone is not sufficient to enable "find
references" to show results on Sourcegraph. The Sourcegraph backend
will also need to accept offset ranges with zero length.

Author: olafurpg

build.sbt

@@ -313,7 +313,8 @@ lazy val minimizedScala = project
   .in(file("tests/minimized-scala"))
   .settings(
     (publish / skip) := true,
-    semanticdbOptions ++= List("-P:semanticdb:text:on")
+    semanticdbOptions ++=
+      List("-P:semanticdb:text:on", "-P:semanticdb:synthetics:on")
   )
   .dependsOn(minimized)
 

lsif-java/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala

@@ -3,6 +3,7 @@ package com.sourcegraph.lsif_java
 import scala.jdk.CollectionConverters._
 
 import com.sourcegraph.lsif_java.commands.CommentSyntax
+import com.sourcegraph.lsif_semanticdb.LsifTextDocument
 import com.sourcegraph.lsif_semanticdb.SignatureFormatter
 import com.sourcegraph.lsif_semanticdb.Symtab
 import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence
@@ -14,8 +15,8 @@ object SemanticdbPrinters {
       doc: TextDocument,
       comments: CommentSyntax = CommentSyntax.default
   ): String = {
-    val occurrencesByLine = doc
-      .getOccurrencesList
+    val occurrencesByLine = LsifTextDocument
+      .sortedSymbolOccurrences(doc)
       .asScala
       .groupBy(_.getRange.getStartLine)
     val out = new StringBuilder()
@@ -80,13 +81,7 @@ object SemanticdbPrinters {
       .append(occ.getSymbol)
       .append(
         if (isMultiline)
-          " "
-        else
-          ""
-      )
-      .append(
-        if (isMultiline)
-          s"${r.getEndLine - r.getStartLine}:${r.getEndCharacter}"
+          s" ${r.getEndLine - r.getStartLine}:${r.getEndCharacter}"
         else
           ""
       )
@@ -102,7 +97,10 @@ object SemanticdbPrinters {
             ""
         }
       )
-      .append("\n")
+    while (out.last == ' ') { // Trim trailing whitespace
+      out.setLength(out.length() - 1)
+    }
+    out.append("\n")
   }
 
 }

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java

@@ -7,6 +7,7 @@
 import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence;
 import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence.Role;
 import com.sourcegraph.semanticdb_javac.SemanticdbSymbols;
+
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -189,9 +190,7 @@ private Integer processDocumentUnsafe(
 
   private Stream<LsifTextDocument> parseTextDocument(Path semanticdbPath) {
     try {
-      CodedInputStream in = CodedInputStream.newInstance(Files.readAllBytes(semanticdbPath));
-      in.setRecursionLimit(1000);
-      return Semanticdb.TextDocuments.parseFrom(in).getDocumentsList().stream()
+      return textDocumentsParseFrom(semanticdbPath).getDocumentsList().stream()
           .filter(sdb -> !sdb.getOccurrencesList().isEmpty())
           .map(sdb -> new LsifTextDocument(semanticdbPath, sdb, options.sourceroot));
     } catch (IOException e) {
@@ -200,4 +199,19 @@ private Stream<LsifTextDocument> parseTextDocument(Path semanticdbPath) {
       return Stream.empty();
     }
   }
+
+  private Semanticdb.TextDocuments textDocumentsParseFrom(Path semanticdbPath) throws IOException {
+    byte[] bytes = Files.readAllBytes(semanticdbPath);
+    try {
+      CodedInputStream in = CodedInputStream.newInstance(bytes);
+      in.setRecursionLimit(1000);
+      return Semanticdb.TextDocuments.parseFrom(in);
+    } catch (NoSuchMethodError ignored) {
+      // NOTE(olafur): For some reason, NoSuchMethodError gets thrown when running `snapshots/run`
+      // in the sbt build. I'm unable to reproduce the error in `snapshots/test` or when running the
+      // published version
+      // of `lsif-java index`.
+      return Semanticdb.TextDocuments.parseFrom(bytes);
+    }
+  }
 }

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifTextDocument.java

@@ -37,13 +37,42 @@ public String toString() {
   }
 
   public List<Semanticdb.SymbolOccurrence> sortedSymbolOccurrences() {
+    return LsifTextDocument.sortedSymbolOccurrences(semanticdb);
+  }
+
+  public static List<Semanticdb.SymbolOccurrence> sortedSymbolOccurrences(
+      Semanticdb.TextDocument semanticdb) {
     ArrayList<Semanticdb.SymbolOccurrence> result =
         new ArrayList<>(semanticdb.getOccurrencesList().size());
     result.addAll(semanticdb.getOccurrencesList());
+    for (Semanticdb.Synthetic synthetic : semanticdb.getSyntheticsList()) {
+      addAllSyntheticOccurrences(synthetic, result);
+    }
     result.sort((o1, o2) -> new RangeComparator().compare(o1.getRange(), o2.getRange()));
     return result;
   }
 
+  private static void addAllSyntheticOccurrences(
+      Semanticdb.Synthetic synthetic, ArrayList<Semanticdb.SymbolOccurrence> buffer) {
+    Semanticdb.Range offsetRange =
+        Semanticdb.Range.newBuilder(synthetic.getRange())
+            .setStartLine(synthetic.getRange().getEndLine())
+            .setStartCharacter(synthetic.getRange().getEndCharacter())
+            .build();
+    new SemanticdbTreeVisitor() {
+      @Override
+      void visitIdTree(Semanticdb.IdTree tree) {
+        Semanticdb.SymbolOccurrence syntheticOccurrence =
+            Semanticdb.SymbolOccurrence.newBuilder()
+                .setRange(offsetRange)
+                .setSymbol(tree.getSymbol())
+                .setRole(Semanticdb.SymbolOccurrence.Role.REFERENCE)
+                .build();
+        buffer.add(syntheticOccurrence);
+      }
+    }.visitTree(synthetic.getTree());
+  }
+
   private void setSemanticdb(Semanticdb.TextDocument semanticdb) {
     this.semanticdb = semanticdb;
     for (Semanticdb.SymbolInformation info : semanticdb.getSymbolsList()) {

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbTreeVisitor.java

@@ -0,0 +1,80 @@
+package com.sourcegraph.lsif_semanticdb;
+
+import com.sourcegraph.semanticdb_javac.Semanticdb.*;
+
+public abstract class SemanticdbTreeVisitor {
+  public void visitTree(Tree tree) {
+    if (tree.hasApplyTree()) {
+      this.visitApplyTree(tree.getApplyTree());
+    } else if (tree.hasFunctionTree()) {
+      this.visitFunctionTree(tree.getFunctionTree());
+    } else if (tree.hasIdTree()) {
+      this.visitIdTree(tree.getIdTree());
+    } else if (tree.hasLiteralTree()) {
+      this.visitLiteralTree(tree.getLiteralTree());
+    } else if (tree.hasMacroExpansionTree()) {
+      this.visitMacroExpansionTree(tree.getMacroExpansionTree());
+    } else if (tree.hasOriginalTree()) {
+      this.visitOriginalTree(tree.getOriginalTree());
+    } else if (tree.hasSelectTree()) {
+      this.visitSelectTree(tree.getSelectTree());
+    } else if (tree.hasTypeApplyTree()) {
+      this.visitTypeApplyTree(tree.getTypeApplyTree());
+    } else if (tree.hasAnnotationTree()) {
+      this.visitAnnotationTree(tree.getAnnotationTree());
+    } else if (tree.hasAssignTree()) {
+      this.visitAssignTree(tree.getAssignTree());
+    } else if (tree.hasBinopTree()) {
+      this.visitBinaryOperatorTree(tree.getBinopTree());
+    }
+  }
+
+  void visitApplyTree(ApplyTree tree) {
+    visitTree(tree.getFunction());
+    for (Tree argument : tree.getArgumentsList()) {
+      visitTree(argument);
+    }
+  }
+
+  void visitFunctionTree(FunctionTree tree) {
+    for (IdTree parameter : tree.getParametersList()) {
+      visitIdTree(parameter);
+    }
+    visitTree(tree.getBody());
+  }
+
+  void visitIdTree(IdTree tree) {}
+
+  void visitLiteralTree(LiteralTree tree) {}
+
+  void visitMacroExpansionTree(MacroExpansionTree tree) {
+    visitTree(tree.getBeforeExpansion());
+  }
+
+  void visitOriginalTree(OriginalTree tree) {}
+
+  void visitSelectTree(SelectTree tree) {
+    visitTree(tree.getQualifier());
+    visitIdTree(tree.getId());
+  }
+
+  void visitTypeApplyTree(TypeApplyTree tree) {
+    visitTree(tree.getFunction());
+  }
+
+  void visitAnnotationTree(AnnotationTree tree) {
+    for (Tree parameter : tree.getParametersList()) {
+      visitTree(parameter);
+    }
+  }
+
+  void visitAssignTree(AssignTree tree) {
+    visitTree(tree.getLhs());
+    visitTree(tree.getRhs());
+  }
+
+  void visitBinaryOperatorTree(BinaryOperatorTree tree) {
+    visitTree(tree.getLhs());
+    visitTree(tree.getRhs());
+  }
+}

semanticdb-java/src/main/protobuf/semanticdb.proto

@@ -26,6 +26,7 @@ message TextDocument {
   Language language = 10;
   repeated SymbolInformation symbols = 5;
   repeated SymbolOccurrence occurrences = 6;
+  repeated Synthetic synthetics = 12;
 }
 
 enum Language {
@@ -267,13 +268,21 @@ message RepeatedType {
   Type tpe = 1;
 }
 
+message Synthetic {
+  Range range = 1;
+  Tree tree = 2;
+}
+
 message Tree {
   oneof sealed_value {
     ApplyTree apply_tree = 1;
+    FunctionTree function_tree = 2;
     IdTree id_tree = 3;
     LiteralTree literal_tree = 4;
+    MacroExpansionTree macro_expansion_tree = 5;
     OriginalTree original_tree = 6;
     SelectTree select_tree = 7;
+    TypeApplyTree type_apply_tree = 8;
     // -- OUT OF SPEC -- //
     AnnotationTree annotation_tree = 9;
     AssignTree assign_tree = 10;
@@ -287,23 +296,40 @@ message ApplyTree {
   repeated Tree arguments = 2;
 }
 
+
+message FunctionTree {
+  repeated IdTree parameters = 1;
+  Tree body = 2;
+}
+
 message IdTree {
   string symbol = 1;
 }
 
-message OriginalTree {
-  Range range = 1;
-}
 
 message LiteralTree {
   Constant constant = 1;
 }
 
+message MacroExpansionTree {
+  Tree before_expansion = 1;
+  Type tpe = 2;
+}
+
+message OriginalTree {
+  Range range = 1;
+}
+
 message SelectTree {
   Tree qualifier = 1;
   IdTree id = 2;
 }
 
+message TypeApplyTree {
+  Tree function = 1;
+  repeated Type type_arguments = 2;
+}
+
 // -- OUT OF SPEC -- //
 message AnnotationTree {
   Type tpe = 1;

tests/minimized-scala/src/main/scala/minimized/MinimizedScalaSynthetic.scala

@@ -0,0 +1,19 @@
+package minimized
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+
+class MinimizedScalaSynthetic {
+  def everything(): Unit = Future(1)
+  def applyTree(): Unit = Future.apply[Int](1)
+  def applyTree2(): Unit = List.apply[Int](1).sorted
+  def selectTree(): Unit = Future[Int](1)
+  def typeApplyTree(): Unit = Future.apply(1)
+  def forComprehensions(): Unit =
+    for {
+      x <- Future(1)
+      y <- Future.successful(1)
+      if y < 2
+      z <- Future.apply[Int](1)
+    } yield x + y + z
+}