Make semanticdb cli able to generate a semanticdb file for a given scala file

Author: poechsel

semanticdb/src/dotty/semanticdb/Main.scala

@@ -5,59 +5,99 @@ import scala.tasty.file._
 import scala.NotImplementedError
 
 import dotty.tools.dotc.Driver
+import dotty.tools.dotc.reporting.Reporter
+
+import java.io.File
+import java.nio.file._
 
 object Main {
-  def parseArguments(args: Array[String]): (Map[String, String], Boolean) = {
-    def nextArgument(optionMap: Map[String, String], args: List[String]): Map[String, String] = args match {
-      case "--out" :: file :: tail => nextArgument(optionMap + ("outfile" -> file), tail)
-      case "-o" :: file :: tail => nextArgument(optionMap + ("outfile" -> file), tail)
+  val userHome = System.getProperty("user.home")
+  val classpaths =
+  userHome + "/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.12.8.jar" ::
+  "out/bootstrap/dotty-library-bootstrapped/scala-0.12/dotty-library_0.12-0.12.0-bin-SNAPSHOT.jar" :: Nil
+
+  val help = """Usage semanticdb [options] [file]
+   |Generate semanticdb's information related to the source file [file]
+   |Options are:
+   | -h,--help                    Show help
+   | -o <file>, --out <file>      Place the output into <file> (default: out.semanticdb)
+   | -t <folder>, --temp <folder> Use <folder> as the temp directory to store build artifacts
+   """.stripMargin
+
+  type CliArgs = Map[String, String]
+
+  def parseArguments(args: Array[String]): Option[CliArgs] = {
+    val optRegex = "$-.*".r
+    def nextArgument(optionMap: CliArgs, args: List[String]): Option[CliArgs] = args match {
+      case "--out" :: file :: tail => nextArgument(optionMap + ("out" -> file), tail)
+      case "-o" :: file :: tail => nextArgument(optionMap + ("out" -> file), tail)
       case "--help" :: tail => nextArgument(optionMap + ("help" -> ""), tail)
       case "-h" :: tail => nextArgument(optionMap + ("help" -> ""), tail)
-      case "--temp" :: folder :: tail => nextArgument(optionMap + ("temp" -> folder), tail)
-      case "-t" :: folder :: tail => nextArgument(optionMap + ("temp" -> folder), tail)
+      case "--classpath" :: folder :: tail => nextArgument(optionMap + ("classpath" -> folder), tail)
+      case "-c" :: folder :: tail => nextArgument(optionMap + ("classpath" -> folder), tail)
+      case optRegex(_) :: _=> None
       case file :: tail => nextArgument(optionMap + ("input" -> file), tail)
-      case Nil => optionMap
+      case Nil => Some(optionMap)
     }
 
-    val help = """Usage semanticdb [options] [file]
-    |Generate semanticdb's information related to the source file [file]
-    |Options are:
-    | -h,--help                    Show help
-    | -o <file>, --out <file>      Place the output into <file>
-    | -t <folder>, --temp <folder> Use <folder> as the temp directory to store build artifacts
-    """.stripMargin
-
-    var optionMap = nextArgument(Map(), args.toList)
-    if (optionMap.contains("help") || !optionMap.contains("input")) {
-      println(help)
-      return (optionMap, false)
-    } else {
-      if (!optionMap.contains("temp")) {
-        optionMap += ("temp" -> "/home/pierre/epfl/semester/dotty/semanticdb/testClass/")
+    nextArgument(Map(), args.toList) match {
+    case Some(args : CliArgs) => {
+      var cleanedArgs = args
+      cleanedArgs += "out" -> cleanedArgs.getOrElse("out", "out.semanticdb")
+      if (cleanedArgs.contains("help") || !cleanedArgs.contains("input")) {
+        return None
+      } else {
+        cleanedArgs += "classpath" -> cleanedArgs.getOrElse("classpath", Files.createTempDirectory("semanticdb").toString)
+        val tempFolder = new File(cleanedArgs("classpath"));
+        if (!tempFolder.exists()){
+            tempFolder.mkdir();
+        }
+        return Some(cleanedArgs)
       }
-      val driver = new Driver
-      val compilerParams : List[String] =
-      optionMap("input") ::
-      "-Yno-inline" ::
-      "-d" :: optionMap("temp") ::
-      "-classpath" :: "/home/pierre/epfl/semester/dotty/library/../out/bootstrap/dotty-library-bootstrapped/scala-0.12/dotty-library_0.12-0.12.0-bin-SNAPSHOT.jar" ::
-      Nil
-
-      val foo = driver.process(compilerParams.toArray)
-      return (optionMap, true)
     }
+    case None => None
+    }
+  }
+
+  def compile(cliArgs : CliArgs) : Reporter
+ = {
+    val driver = new Driver
+    val compilerParams : List[String] =
+     "-classpath" :: classpaths.mkString(":") ::
+     "-Yno-inline" ::
+     "-d" :: cliArgs("classpath") ::
+     cliArgs("input") ::
+    Nil
+
+    val reporter = driver.process(compilerParams.toArray)
+    return reporter
   }
 
 
   def main(args: Array[String]): Unit = {
     val extraClasspath = "." // TODO allow to set it from the args with -classpath XYZ
     val classes = args.toList
-    val (optionMap, canTreat) = parseArguments(args)
-    if (args.isEmpty) {
-      println("Dotty Semantic DB: No classes where passed as argument")
-    } else {
-      println("Running Dotty Semantic DB on: " + args.mkString(" "))
-      //ConsumeTasty(extraClasspath, classes, new SemanticdbConsumer())
+
+
+    parseArguments(args) match {
+      case None => println(help)
+      case Some(cliArgs) => {
+        val reporter = compile(cliArgs)
+
+        if (reporter.hasErrors) {
+          println("Compile error:")
+          println(reporter)
+        } else {
+          val classNames = Utils.getClassNames(cliArgs("classpath"), cliArgs("input"))
+          val scalaFile = Paths.get(cliArgs("input")).toAbsolutePath
+          val sdbconsumer = new SemanticdbConsumer(scalaFile)
+          val _ = ConsumeTasty(cliArgs("classpath"), classNames, sdbconsumer)
+          val textDocument = sdbconsumer.toSemanticdb()
+          val os = Files.newOutputStream(Paths.get(cliArgs("out")))
+          try textDocument.writeTo(os)
+          finally os.close()
+        }
+      }
     }
   }
 }

semanticdb/src/dotty/semanticdb/SemanticdbConsumer.scala

@@ -19,8 +19,8 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
   val semantic: s.TextDocument = s.TextDocument()
   var occurrences: Seq[s.SymbolOccurrence] = Seq()
 
-  def toSemanticdb(text: String): s.TextDocument = {
-    s.TextDocument(text = text, occurrences = occurrences)
+  def toSemanticdb(): s.TextDocument = {
+    s.TextDocument(text = sourceCode, occurrences = occurrences)
   }
   val package_definitions: Set[Tuple2[String, Int]] = Set()
   val symbolsCache: HashMap[(String, s.Range), String] = HashMap()

semanticdb/src/dotty/semanticdb/TastyScalaFileInferrer.scala

@@ -0,0 +1,49 @@
+package dotty.semanticdb
+
+import scala.tasty.Reflection
+
+import scala.meta.internal.{semanticdb => s}
+import scala.tasty.Reflection
+import scala.tasty.file.TastyConsumer
+import java.lang.reflect.InvocationTargetException
+
+class TastyScalaFileInferrer extends TastyConsumer {
+  /* Visitor over a tasty tree.
+  Aims at finding the scala file from where this tree originated.
+  */
+
+  /* If a scala file was found sourcePath is Some(scalaFile),
+  Otherwise None */
+  var sourcePath: Option[String] = None
+  final def apply(reflect: Reflection)(root: reflect.Tree): Unit = {
+    import reflect._
+    object ChildTraverser extends TreeTraverser {
+      override def traverseTree(tree: Tree)(implicit ctx: Context): Unit =
+        tree match {
+          case IsClassDef(cdef) => {
+            cdef.symbol.annots.foreach { annot =>
+              annot match {
+                case Term.Apply(Term.Select(Term.New(t), _),
+                                List(Term.Literal(Constant.String(path))))
+                    if t.symbol.name == "SourceFile" =>
+                  // we found the path to a file. In this case, we do not need to
+                  // continue traversing the tree
+                  sourcePath = Some(path)
+                case x => super.traverseTree(tree)
+              }
+              true
+            }
+          }
+          case _ => {
+            // If we already found a sourcePath in this tasty file,
+            // we abort our search here to avoid spending too much time here
+            if (sourcePath == None)
+              super.traverseTree(tree)
+            else
+              ()
+          }
+        }
+    }
+    ChildTraverser.traverseTree(root)(reflect.rootContext)
+  }
+}

semanticdb/src/dotty/semanticdb/Utils.scala

@@ -0,0 +1,80 @@
+package dotty.semanticdb
+
+
+import scala.tasty.Reflection
+import scala.tasty.file._
+import scala.collection.mutable.HashMap
+
+import org.junit.Test
+import org.junit.Assert._
+import java.nio.file._
+import scala.meta.internal.{semanticdb => s}
+import scala.collection.JavaConverters._
+import java.io.File
+import scala.tasty.Reflection
+import scala.tasty.file.TastyConsumer
+import java.lang.reflect.InvocationTargetException
+
+object Utils {
+  /** Infers a tuple (class path, class name) from a given path */
+  def getClasspathClassname(file: Path): (String, String) = {
+    val pat = """(.*)\..*""".r
+    val classpath = file.getParent().getParent().toString()
+    val modulename = file.getParent().getFileName().toString()
+    val sourcename =
+      file.toFile().getName().toString() match {
+        case pat(name) => name
+        case _         => ""
+      }
+    return (classpath, modulename + "." + sourcename)
+  }
+
+  /** List all tasty files occuring in the folder f or one of its subfolders */
+  def recursiveListFiles(f: File): Array[File] = {
+    val pattern = ".*\\.tasty".r
+    val files = f.listFiles
+    val folders = files.filter(_.isDirectory)
+    val tastyfiles = files.filter(_.toString match {
+      case pattern(x: _*) => true
+      case _              => false
+    })
+    tastyfiles ++ folders.flatMap(recursiveListFiles)
+  }
+
+  /** Returns a mapping from *.scala file to a list of tasty files. */
+  def getTastyFiles(classPath: Path): HashMap[String, List[Path]] = {
+    val source_to_tasty: HashMap[String, List[Path]] = HashMap()
+    val tastyfiles = recursiveListFiles(classPath.toFile())
+    recursiveListFiles(classPath.toFile()).map(tasty_path => {
+      val (classpath, classname) = getClasspathClassname(tasty_path.toPath())
+      // We add an exception here to avoid crashing if we encountered
+      // a bad tasty file
+      try {
+        val inspecter = new TastyScalaFileInferrer
+        ConsumeTasty(classpath, classname :: Nil, inspecter)
+        inspecter.sourcePath.foreach(
+          source =>
+            source_to_tasty +=
+              (source -> (tasty_path
+                .toPath() :: source_to_tasty.getOrElse(source, Nil))))
+      } catch {
+        case _: InvocationTargetException => println(tasty_path)
+      }
+    })
+    source_to_tasty
+  }
+
+  /*
+  Returns the list of names of class defined inside the scala file [scalaFile]
+  extracted from the compilation artifacts found in [classPath].
+  */
+  def getClassNames(classPath: String, scalaFile: String): List[String] = {
+    val tastyFiles =
+      getTastyFiles(Paths.get(classPath).toAbsolutePath)
+      .getOrElse(Paths.get(scalaFile).toAbsolutePath.toString, Nil)
+
+    val tastyClasses = tastyFiles.map(getClasspathClassname)
+    val (_, classnames) = tastyClasses.unzip
+    return classnames
+  }
+}

semanticdb/test/dotty/semanticdb/Tests.scala

@@ -132,7 +132,7 @@ class Tests {
       val sdbconsumer = new SemanticdbConsumer(scalaFile)
 
       val _ = ConsumeTasty(classpaths.head, classnames, sdbconsumer)
-      sdbconsumer.toSemanticdb(scalac.text)
+      sdbconsumer.toSemanticdb()
     }
   }