Temporarily parse using directives with a custom parser

Author: alexarchambault

build.sc

@@ -140,8 +140,7 @@ class Build(val crossScalaVersion: String)
     Deps.scalametaTrees,
     Deps.scalaparse,
     Deps.shapeless,
-    Deps.swoval,
-    Deps.usingDirectives
+    Deps.swoval
   )
 
   private def vcsState = {

examples/utest/MyTests.scala

@@ -1,4 +1,4 @@
-@using lib "com.lihaoyi::utest::0.7.10"
+// using "com.lihaoyi::utest::0.7.10"
 
 import utest._
 

modules/build/src/main/scala/scala/build/preprocessing/Directive.scala

@@ -0,0 +1,12 @@
+package scala.build.preprocessing
+
+final case class Directive(
+  tpe: Directive.Type,
+  values: Seq[String]
+)
+
+object Directive {
+  sealed abstract class Type(val name: String) extends Product with Serializable
+  case object Using                            extends Type("using")
+  case object Require                          extends Type("require")
+}

modules/build/src/main/scala/scala/build/preprocessing/DirectivesOutputStreamReporter.scala

@@ -1,17 +1,13 @@
 package scala.build.preprocessing
 
-import com.virtuslab.using_directives.{Context, UsingDirectivesProcessor}
-import com.virtuslab.using_directives.config.Settings
-import com.virtuslab.using_directives.custom.model.Path
-import com.virtuslab.using_directives.reporter.ConsoleReporter
 import dependency.AnyDependency
 import dependency.parser.DependencyParser
 
 import java.nio.charset.StandardCharsets
 
 import scala.build.{Inputs, Os, Sources}
 import scala.build.internal.AmmUtil
-import scala.build.options.{BuildOptions, ClassPathOptions}
+import scala.build.options.{BuildOptions, ClassPathOptions, ScalaOptions}
 import scala.collection.JavaConverters._
 
 case object ScalaPreprocessor extends Preprocessor {
@@ -82,41 +78,44 @@ case object ScalaPreprocessor extends Preprocessor {
     }
   }
 
-  private def processUsing(
-    content: String,
-    printablePath: String
-  ): Option[(BuildOptions, String)] = {
-
-    val processor = {
-      val reporter = new DirectivesOutputStreamReporter(System.err) // TODO Get that via a logger
-      val settings = new Settings
-      val context  = new Context(reporter, settings)
-      new UsingDirectivesProcessor(context)
-    }
-
-    val contentChars = content.toCharArray
-    val directives   = processor.extract(contentChars)
-
-    val updatedOptions = DirectivesProcessor.process(directives.getFlattenedMap.asScala.toMap)
+  private def directivesBuildOptions(directives: Seq[Directive]): BuildOptions =
+    directives
+      .filter(_.tpe == Directive.Using)
+      .map { dir =>
+        dir.values match {
+          case Seq(depStr) if depStr.split(":").count(_.trim.nonEmpty) == 3 =>
+            DependencyParser.parse(depStr) match {
+              case Left(err) => sys.error(s"Error parsing dependency '$depStr': $err")
+              case Right(dep) =>
+                BuildOptions(
+                  classPathOptions = ClassPathOptions(
+                    extraDependencies = Seq(dep)
+                  )
+                )
+            }
+          case Seq("scala", scalaVer) if scalaVer.nonEmpty =>
+            BuildOptions(
+              scalaOptions = ScalaOptions(
+                scalaVersion = Some(scalaVer)
+              )
+            )
+          case _ =>
+            sys.error(s"Unrecognized using directive: ${dir.values.mkString(" ")}")
+        }
+      }
+      .foldLeft(BuildOptions())(_ orElse _)
 
-    val codeOffset = directives.getCodeOffset()
+  private def processUsing(content: String, printablePath: String): Option[(BuildOptions, String)] =
+    TemporaryDirectivesParser.parseDirectives(content).flatMap {
+      case (directives, updatedContent) =>
+        // TODO Warn about unrecognized directives
+        // TODO Report via some diagnostics malformed directives
 
-    val updatedContentOpt =
-      if (codeOffset > 0)
-        Some {
-          val headerBytes = contentChars
-            .iterator
-            .take(codeOffset)
-            .map(c => if (c.isControl) c else ' ')
-            .toArray
-          val mainBytes = contentChars.drop(codeOffset)
-          new String(headerBytes ++ mainBytes)
+        TemporaryDirectivesParser.parseDirectives(content).map {
+          case (directives, updatedContent) =>
+            (directivesBuildOptions(directives), updatedContent)
         }
-      else None
-
-    if (updatedContentOpt.isEmpty) None
-    else Some((updatedOptions, updatedContentOpt.getOrElse(content)))
-  }
+    }
 
   private def processSpecialImports(
     content: String,

modules/build/src/main/scala/scala/build/preprocessing/DirectivesProcessor.scala

@@ -0,0 +1,68 @@
+package scala.build.preprocessing
+
+import fastparse._
+import fastparse.NoWhitespace.noWhitespaceImplicit
+
+object TemporaryDirectivesParser {
+
+  private def directive[_: P] = {
+    def ws = P(" ".rep(1))
+    def sc = P(";")
+    def nl = P(("\r".? ~ "\n").rep(1))
+    def tpe = {
+      def usingTpe = P(ws.? ~ ("using" | "@using" | "// using"))
+        .map(_ => (Directive.Using: Directive.Type))
+      def requireTpe = P(ws.? ~ ("require" | "@require" | "// require"))
+        .map(_ => (Directive.Require: Directive.Type))
+      P(usingTpe | requireTpe)
+    }
+
+    def simpleElem = P(CharPred(c => !c.isWhitespace && c != '"').rep(1).!)
+    def charInQuote = P(
+      CharPred(c => c != '"' && c != '\\').! |
+        P("\\\\").map(_ => "\\") |
+        P("\\\"").map(_ => "\"")
+    )
+    def quotedElem = P("\"" ~ charInQuote.rep ~ "\"").map(_.mkString)
+    def elem       = P(simpleElem | quotedElem)
+
+    P(tpe ~ ws ~ elem ~ (ws ~ elem).rep(0) ~ nl.? ~ sc.? ~ nl.?).map {
+      case (tpe0, firstElem, otherElems) =>
+        Directive(tpe0, firstElem +: otherElems)
+    }
+  }
+
+  private def maybeDirective[_: P] = {
+    // TODO Use some cuts above to also catch malformed directives?
+    P(directive.?)
+  }
+
+  private def parseDirective(content: String, fromIndex: Int): Option[(Directive, Int)] = {
+    // TODO Don't create a new String here
+    val res = parse(content.drop(fromIndex), maybeDirective(_))
+    res.fold((err, idx, _) => sys.error(err), (dirOpt, idx) => dirOpt.map((_, idx + fromIndex)))
+  }
+
+  def parseDirectives(content: String): Option[(List[Directive], String)] = {
+
+    def helper(fromIndex: Int, acc: List[Directive]): (List[Directive], Int) =
+      parseDirective(content, fromIndex) match {
+        case None                  => (acc.reverse, fromIndex)
+        case Some((dir, newIndex)) => helper(newIndex, dir :: acc)
+      }
+
+    val (directives, codeStartsAt) = helper(0, Nil)
+
+    if (codeStartsAt == 0) {
+      assert(directives.isEmpty)
+      None
+    }
+    else
+      Some((
+        directives,
+        content.take(codeStartsAt).map(c => if (c.isControl) c else ' ') ++
+          content.drop(codeStartsAt)
+      ))
+  }
+
+}

modules/build/src/main/scala/scala/build/preprocessing/ScalaPreprocessor.scala

@@ -217,7 +217,7 @@ class BuildTests extends munit.FunSuite {
   test("dependencies - using") {
     val testInputs = TestInputs(
       os.rel / "simple.sc" ->
-        """@using lib "com.lihaoyi::geny:0.6.5"
+        """using "com.lihaoyi::geny:0.6.5"
           |import geny.Generator
           |val g = Generator("Hel", "lo")
           |println(g.mkString)
@@ -252,8 +252,8 @@ class BuildTests extends munit.FunSuite {
   test("several dependencies - using") {
     val testInputs = TestInputs(
       os.rel / "simple.sc" ->
-        """@using lib "com.lihaoyi::geny:0.6.5"
-          |@using lib "com.lihaoyi::pprint:0.6.6"
+        """using "com.lihaoyi::geny:0.6.5"
+          |using "com.lihaoyi::pprint:0.6.6"
           |import geny.Generator
           |val g = Generator("Hel", "lo")
           |pprint.log(g)

modules/build/src/main/scala/scala/build/preprocessing/TemporaryDirectivesParser.scala

@@ -45,9 +45,9 @@ class SourcesTests extends munit.FunSuite {
   test("dependencies in .scala - using") {
     val testInputs = TestInputs(
       os.rel / "something.scala" ->
-        """@using lib "org1:name1:1.1"
-          |@using lib "org2::name2:2.2"
-          |@using lib "org3:::name3:3.3"
+        """using "org1:name1:1.1"
+          |using "org2::name2:2.2"
+          |using "org3:::name3:3.3"
           |import scala.collection.mutable
           |
           |object Something {
@@ -154,9 +154,9 @@ class SourcesTests extends munit.FunSuite {
   test("dependencies in .sc - using") {
     val testInputs = TestInputs(
       os.rel / "something.sc" ->
-        """@using lib "org1:name1:1.1"
-          |@using lib "org2::name2:2.2"
-          |@using lib "org3:::name3:3.3"
+        """using "org1:name1:1.1"
+          |using "org2::name2:2.2"
+          |using "org3:::name3:3.3"
           |import scala.collection.mutable
           |
           |def a = 1

modules/build/src/test/scala/scala/build/tests/BuildTests.scala

@@ -21,7 +21,7 @@ object TestUtil {
       val generated0 = generated()
       assert(
         generated0.map(_.toString).toSet == expected.toSet, {
-          pprint.log(generated0)
+          pprint.log(generated0.map(_.toString))
           pprint.log(expected)
           ""
         }