Accept resources in Github Gists (#415)

lwronski · web-flow · commit eba1c6675333 · 2021-11-29T17:35:31.000+01:00
* Accept resources in Github Gists
diff --git a/modules/build/src/main/scala/scala/build/Build.scala b/modules/build/src/main/scala/scala/build/Build.scala
@@ -159,7 +159,9 @@ object Build {
 
       val baseOptions = overrideOptions.orElse(sharedOptions)
 
-      val sources = value(crossSources.scopedSources(baseOptions))
+      val crossSources0 = crossSources.withVirtualDir(inputs0, scope, baseOptions)
+
+      val sources = value(crossSources0.scopedSources(baseOptions))
         .sources(scope, baseOptions)
 
       val generatedSources = sources.generateSources(inputs0.generatedSrcRoot(scope))
diff --git a/modules/build/src/main/scala/scala/build/CrossSources.scala b/modules/build/src/main/scala/scala/build/CrossSources.scala
@@ -14,6 +14,18 @@ final case class CrossSources(
   buildOptions: Seq[HasBuildRequirements[BuildOptions]]
 ) {
 
+  def withVirtualDir(inputs: Inputs, scope: Scope, options: BuildOptions): CrossSources = {
+
+    val srcRootPath = inputs.generatedSrcRoot(scope)
+    val resourceDirs0 = options.classPathOptions.resourceVirtualDir.map { path =>
+      HasBuildRequirements(BuildRequirements(), srcRootPath / path)
+    }
+
+    copy(
+      resourceDirs = resourceDirs ++ resourceDirs0
+    )
+  }
+
   def sharedOptions(baseOptions: BuildOptions): BuildOptions =
     buildOptions
       .filter(_.requirements.isEmpty)
diff --git a/modules/build/src/main/scala/scala/build/Inputs.scala b/modules/build/src/main/scala/scala/build/Inputs.scala
@@ -195,6 +195,8 @@ object Inputs {
       extends Virtual with AnyScalaFile
   final case class VirtualJavaFile(content: Array[Byte], source: String)
       extends Virtual with Compiled
+  final case class VirtualData(content: Array[Byte], source: String)
+      extends Virtual
 
   private def inputsHash(elements: Seq[Element]): String = {
     def bytes(s: String): Array[Byte] = s.getBytes(StandardCharsets.UTF_8)
@@ -226,7 +228,7 @@ object Inputs {
   }
 
   private def forValidatedElems(
-    validElems: Seq[Compiled],
+    validElems: Seq[Element],
     baseProjectName: String,
     directories: Directories
   ): Inputs = {
@@ -262,7 +264,7 @@ object Inputs {
   }
 
   private val githubGistsArchiveRegex: Regex =
-    s""":\\/\\/gist\\.github\\.com\\/[^\\/]*?\\/[^\\/]*$$""".r
+    s"""gist\\.github\\.com\\/[^\\/]*?\\/[^\\/]*$$""".r
 
   private def forNonEmptyArgs(
     args: Seq[String],
@@ -288,18 +290,19 @@ object Inputs {
           val url =
             if (githubGistsArchiveRegex.findFirstMatchIn(arg).nonEmpty) s"$arg/download" else arg
           download(url).map { content =>
-            def resolve(path: String, content: Array[Byte]): Compiled = {
+            def resolve(path: String, content: Array[Byte]): Element = {
               val wrapperPath =
                 os.sub / path.split("/").last
 
               if (path.endsWith(".scala")) VirtualScalaFile(content, path)
               else if (path.endsWith(".java")) VirtualJavaFile(content, path)
-              else VirtualScript(content, path, wrapperPath)
+              else if (path.endsWith(".sc")) VirtualScript(content, path, wrapperPath)
+              else VirtualData(content, path)
             }
             if (githubGistsArchiveRegex.findFirstMatchIn(arg).nonEmpty) {
               val zipInputStream = new ZipInputStream(new ByteArrayInputStream(content))
               @tailrec
-              def readArchive(acc: Seq[Compiled]): Seq[Compiled] =
+              def readArchive(acc: Seq[Element]): Seq[Element] =
                 zipInputStream.getNextEntry() match {
                   case entry: ZipEntry if !entry.isDirectory =>
                     val content = {
diff --git a/modules/build/src/main/scala/scala/build/Sources.scala b/modules/build/src/main/scala/scala/build/Sources.scala
@@ -39,6 +39,7 @@ object Sources {
     Seq(
       ScriptPreprocessor(codeWrapper),
       JavaPreprocessor,
-      ScalaPreprocessor
+      ScalaPreprocessor,
+      DataPreprocessor
     )
 }
diff --git a/modules/build/src/main/scala/scala/build/options/ClassPathOptions.scala b/modules/build/src/main/scala/scala/build/options/ClassPathOptions.scala
@@ -10,7 +10,8 @@ final case class ClassPathOptions(
   extraCompileOnlyJars: Seq[os.Path] = Nil,
   extraSourceJars: Seq[os.Path] = Nil,
   fetchSources: Option[Boolean] = None,
-  extraDependencies: Seq[Positioned[AnyDependency]] = Nil
+  extraDependencies: Seq[Positioned[AnyDependency]] = Nil,
+  resourceVirtualDir: Seq[os.SubPath] = Nil
 )
 
 object ClassPathOptions {
diff --git a/modules/build/src/main/scala/scala/build/options/HashedType.scala b/modules/build/src/main/scala/scala/build/options/HashedType.scala
@@ -19,6 +19,10 @@ object HashedType {
     path => path.toString
   }
 
+  implicit val subPath: HashedType[os.SubPath] = {
+    path => path.toString
+  }
+
   implicit val boolean: HashedType[Boolean] = {
     bool => bool.toString
   }
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/DataPreprocessor.scala b/modules/build/src/main/scala/scala/build/preprocessing/DataPreprocessor.scala
@@ -0,0 +1,33 @@
+package scala.build.preprocessing
+
+import java.nio.charset.StandardCharsets
+
+import scala.build.Inputs
+import scala.build.errors.BuildException
+
+case object DataPreprocessor extends Preprocessor {
+  def preprocess(input: Inputs.SingleElement)
+    : Option[Either[BuildException, Seq[PreprocessedSource]]] =
+    input match {
+      case file: Inputs.VirtualData =>
+        val content = new String(file.content, StandardCharsets.UTF_8)
+
+        val inMemory = Seq(
+          PreprocessedSource.InMemory(
+            Left(file.source),
+            file.subPath,
+            content,
+            0,
+            None,
+            None,
+            Nil,
+            None,
+            file.scopePath
+          )
+        )
+
+        Some(Right(inMemory))
+      case _ =>
+        None
+    }
+}
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/directives/Directive.scala b/modules/build/src/main/scala/scala/build/preprocessing/directives/Directive.scala
@@ -17,6 +17,12 @@ object Directive {
   case object Using                            extends Type("using")
   case object Require                          extends Type("require")
 
+  def osRootResource(cwd: ScopePath): (Option[os.SubPath], Option[os.Path]) =
+    cwd.root match {
+      case Left(_)     => (Some(cwd.path), None)
+      case Right(root) => (None, Some(root / cwd.path))
+    }
+
   def osRoot(cwd: ScopePath, pos: Option[Position]): Either[BuildException, os.Path] =
     cwd.root match {
       case Left(virtualRoot) =>
diff --git a/modules/build/src/main/scala/scala/build/preprocessing/directives/UsingResourcesDirectiveHandler.scala b/modules/build/src/main/scala/scala/build/preprocessing/directives/UsingResourcesDirectiveHandler.scala
@@ -1,6 +1,6 @@
 package scala.build.preprocessing.directives
 
-import scala.build.EitherCps.{either, value}
+import scala.build.EitherCps.either
 import scala.build.Position
 import scala.build.errors.BuildException
 import scala.build.options.{BuildOptions, ClassPathOptions}
@@ -24,11 +24,15 @@ case object UsingResourcesDirectiveHandler extends UsingDirectiveHandler {
     directive.values match {
       case Seq("resourceDir" | "resourceDirs", paths @ _*) =>
         val res = either {
-          val root   = value(Directive.osRoot(cwd, Some(directive.position)))
-          val paths0 = paths.map(os.Path(_, root))
+          val (virtualRootOpt, rootOpt) = Directive.osRootResource(cwd)
+          val paths0                    = rootOpt.map(root => paths.map(os.Path(_, root)))
+          val virtualPaths = virtualRootOpt.map(virtualRoot =>
+            paths.map(path => virtualRoot / os.SubPath(path))
+          )
           BuildOptions(
             classPathOptions = ClassPathOptions(
-              extraClassPath = paths0
+              extraClassPath = paths0.toList.flatten,
+              resourceVirtualDir = virtualPaths.toList.flatten
             )
           )
         }
@@ -43,12 +47,16 @@ case object UsingResourcesDirectiveHandler extends UsingDirectiveHandler {
     cwd: ScopePath,
     positionOpt: Option[Position]
   ): Either[BuildException, BuildOptions] = either {
-    val root   = value(Directive.osRoot(cwd, positionOpt))
-    val paths  = DirectiveUtil.stringValues(values)
-    val paths0 = paths.map(os.Path(_, root))
+    val (virtualRootOpt, rootOpt) = Directive.osRootResource(cwd)
+    val paths                     = DirectiveUtil.stringValues(values)
+    val paths0                    = rootOpt.map(root => paths.map(os.Path(_, root)))
+    val virtualPaths = virtualRootOpt.map(virtualRoot =>
+      paths.map(path => virtualRoot / os.SubPath(path))
+    )
     BuildOptions(
       classPathOptions = ClassPathOptions(
-        extraClassPath = paths0
+        extraClassPath = paths0.toList.flatten,
+        resourceVirtualDir = virtualPaths.toList.flatten
       )
     )
   }
diff --git a/modules/integration/src/main/scala/scala/cli/integration/TestInputs.scala b/modules/integration/src/main/scala/scala/cli/integration/TestInputs.scala
@@ -1,10 +1,12 @@
 package scala.cli.integration
 
-import java.io.IOException
+import java.io.{FileOutputStream, IOException}
 import java.nio.charset.StandardCharsets
 import java.security.SecureRandom
 import java.util.concurrent.atomic.AtomicInteger
+import java.util.zip.{ZipEntry, ZipOutputStream}
 
+import scala.cli.integration.TestInputs.compress
 import scala.util.control.NonFatal
 
 final case class TestInputs(
@@ -22,6 +24,12 @@ final case class TestInputs(
     writeIn(tmpDir)
     tmpDir
   }
+  def asZip[T](f: (os.Path, os.Path) => T): T =
+    TestInputs.withTmpDir { tmpDir =>
+      val zipArchivePath = tmpDir / s"${tmpDir.last}.zip"
+      compress(zipArchivePath, files)
+      f(tmpDir, zipArchivePath)
+    }
   def fromRoot[T](f: os.Path => T): T =
     TestInputs.withTmpDir { tmpDir =>
       writeIn(tmpDir)
@@ -31,6 +39,17 @@ final case class TestInputs(
 
 object TestInputs {
 
+  def compress(zipFilepath: os.Path, files: Seq[(os.RelPath, String)]) = {
+    val zip = new ZipOutputStream(new FileOutputStream(zipFilepath.toString()))
+    try for ((relPath, content) <- files) {
+      zip.putNextEntry(new ZipEntry(relPath.toString()))
+      val in: Array[Byte] = content.getBytes
+      zip.write(in)
+      zip.closeEntry()
+    }
+    finally zip.close()
+  }
+
   private lazy val baseTmpDir = {
     Option(System.getenv("SCALA_CLI_TMP")).getOrElse {
       sys.error("SCALA_CLI_TMP not set")
diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala
@@ -5,7 +5,7 @@ import com.eed3si9n.expecty.Expecty.expect
 import java.io.{ByteArrayOutputStream, File}
 import java.nio.charset.Charset
 
-import scala.util.Properties
+import scala.util.{Properties, Random}
 
 abstract class RunTestDefinitions(val scalaVersionOpt: Option[String])
     extends munit.FunSuite with TestScalaVersionArgs {
@@ -692,6 +692,71 @@ abstract class RunTestDefinitions(val scalaVersionOpt: Option[String])
     }
   }
 
+  private def generateDummyGistUrl(root: os.Path) =
+    root / "gist.github.com" /
+      Random.alphanumeric.take(10).mkString("") /
+      Random.alphanumeric.take(10).mkString("")
+
+  test("Github Gists Scala URL with resource in directive") {
+    val inputs = TestInputs(
+      Seq(
+        os.rel / "Hello.scala" ->
+          s"""// using resourceDir "./"
+             |import scala.io.Source
+             |
+             |object Hello extends App {
+             |    val inputs = Source.fromResource("input").getLines.map(_.toInt).toSeq
+             |    println(inputs.mkString(","))
+             |}
+             |""".stripMargin,
+        os.rel / "input" ->
+          s"""1
+             |2
+             |""".stripMargin
+      )
+    )
+    inputs.asZip { (root, zipPath) =>
+      val message = "1,2"
+      val gistUrl = generateDummyGistUrl(root)
+      val gistUri = gistUrl.toNIO.toUri.toString
+
+      os.copy(zipPath, gistUrl / "download", createFolders = true)
+      val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(gistUri))
+        .call(cwd = root)
+        .out.text().trim
+      expect(output == message)
+    }
+  }
+
+  test("Github Gists Scala Script URL with resource in directive") {
+    val inputs = TestInputs(
+      Seq(
+        os.rel / "hello.sc" ->
+          s"""// using resourceDir "./"
+             |import scala.io.Source
+             |
+             |val inputs = Source.fromResource("input").getLines.map(_.toInt).toSeq
+             |println(inputs.mkString(","))
+             |""".stripMargin,
+        os.rel / "input" ->
+          s"""1
+             |2
+             |""".stripMargin
+      )
+    )
+    inputs.asZip { (root, zipPath) =>
+      val message = "1,2"
+      val gistUrl = generateDummyGistUrl(root)
+      val gistUri = gistUrl.toNIO.toUri.toString
+
+      os.copy(zipPath, gistUrl / "download", createFolders = true)
+      val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(gistUri))
+        .call(cwd = root)
+        .out.text().trim
+      expect(output == message)
+    }
+  }
+
   def compileTimeOnlyJars(): Unit = {
     // format: off
     val cmd = Seq[os.Shellable](
diff --git a/website/docs/cookbooks/gists.md b/website/docs/cookbooks/gists.md
@@ -45,3 +45,37 @@ gh gist create file.scala
 ```
 
 Then you (and others) can run it quickly, using the `scala-cli` approach shown above.
+
+
+## Resources from gists 
+
+You can also use resources from gists archive. This is done by passing `resourceDir` in using directives.
+
+For example, given the gist `https://gist.github.com/lwronski/7ee12fa4b8b8bac3211841273df82080` which containing Scala code and text file:
+
+```scala title=Hello.scala
+// using resourceDir "./"
+import scala.io.Source
+
+object Hello extends App {
+    val inputs = Source.fromResource("input").getLines.map(_.toInt).toSeq
+    println(inputs.mkString(","))
+}
+```
+```scala title=input
+1
+2
+3
+4
+```
+
+and run them:
+```bash
+scala-cli https://gist.github.com/lwronski/7ee12fa4b8b8bac3211841273df82080
+# 1,2,3,4
+```
+<!-- Expected:
+1,2,3,4
+-->
+
+it will print `1,2,3,4` to the standard output.

Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,7 @@ object Sources {`
`39`	`39`	`Seq(`
`40`	`40`	`ScriptPreprocessor(codeWrapper),`
`41`	`41`	`JavaPreprocessor,`
`42`		`- ScalaPreprocessor`
	`42`	`+ ScalaPreprocessor,`
	`43`	`+ DataPreprocessor`
`43`	`44`	`)`
`44`	`45`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,8 @@ final case class ClassPathOptions(`
`10`	`10`	`extraCompileOnlyJars: Seq[os.Path] = Nil,`
`11`	`11`	`extraSourceJars: Seq[os.Path] = Nil,`
`12`	`12`	`fetchSources: Option[Boolean] = None,`
`13`		`- extraDependencies: Seq[Positioned[AnyDependency]] = Nil`
	`13`	`+ extraDependencies: Seq[Positioned[AnyDependency]] = Nil,`
	`14`	`+ resourceVirtualDir: Seq[os.SubPath] = Nil`
`14`	`15`	`)`
`15`	`16`
`16`	`17`	`object ClassPathOptions {`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,10 @@ object HashedType {`
`19`	`19`	`path => path.toString`
`20`	`20`	`}`
`21`	`21`
	`22`	`+ implicit val subPath: HashedType[os.SubPath] = {`
	`23`	`+ path => path.toString`
	`24`	`+ }`
	`25`	`+`
`22`	`26`	`implicit val boolean: HashedType[Boolean] = {`
`23`	`27`	`bool => bool.toString`
`24`	`28`	`}`