Merge pull request #11379 from philwalk/minimal-scripting-support

Minimal scripting support

Author: anatoliykmetyuk

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -274,7 +274,7 @@ object Contexts {
     /** Sourcefile corresponding to given abstract file, memoized */
     def getSource(file: AbstractFile, codec: => Codec = Codec(settings.encoding.value)) = {
       util.Stats.record("Context.getSource")
-      base.sources.getOrElseUpdate(file, new SourceFile(file, codec))
+      base.sources.getOrElseUpdate(file, SourceFile(file, codec))
     }
 
     /** SourceFile with given path name, memoized */

compiler/src/dotty/tools/dotc/util/SourceFile.scala

@@ -22,19 +22,34 @@ object ScriptSourceFile {
   @sharable private val headerPattern = Pattern.compile("""^(::)?!#.*(\r|\n|\r\n)""", Pattern.MULTILINE)
   private val headerStarts  = List("#!", "::#!")
 
+  /** Return true if has a script header */
+  def hasScriptHeader(content: Array[Char]): Boolean =
+    headerStarts.exists(content.startsWith(_))
+
   def apply(file: AbstractFile, content: Array[Char]): SourceFile = {
     /** Length of the script header from the given content, if there is one.
-     *  The header begins with "#!" or "::#!" and ends with a line starting
-     *  with "!#" or "::!#".
+     *  The header begins with "#!" or "::#!" and is either a single line,
+     *  or it ends with a line starting with "!#" or "::!#", if present.
      */
     val headerLength =
       if (headerStarts exists (content startsWith _)) {
         val matcher = headerPattern matcher content.mkString
         if (matcher.find) matcher.end
-        else throw new IOException("script file does not close its header with !# or ::!#")
+        else content.indexOf('\n') // end of first line
       }
       else 0
-    new SourceFile(file, content drop headerLength) {
+
+    // overwrite hash-bang lines with all spaces to preserve line numbers
+    val hashBangLines = content.take(headerLength).mkString.split("\\r?\\n")
+    if hashBangLines.nonEmpty then
+      for i <- 0 until headerLength do
+        content(i) match {
+          case '\r' | '\n' =>
+          case _ =>
+            content(i) = ' '
+        }
+
+    new SourceFile(file, content) {
       override val underlying = new SourceFile(this.file, this.content)
     }
   }
@@ -245,6 +260,25 @@ object SourceFile {
       else
         sourcePath.toString
   }
+
+  /** Return true if file is a script:
+   *  if filename extension is not .scala and has a script header.
+   */
+  def isScript(file: AbstractFile, content: Array[Char]): Boolean =
+    ScriptSourceFile.hasScriptHeader(content)
+
+  def apply(file: AbstractFile, codec: Codec): SourceFile =
+    // see note above re: Files.exists is remarkably slow
+    val chars =
+    try
+      new String(file.toByteArray, codec.charSet).toCharArray
+    catch
+      case _: java.nio.file.NoSuchFileException => Array[Char]()
+
+    if isScript(file, chars) then
+      ScriptSourceFile(file, chars)
+    else
+      new SourceFile(file, chars)
 }
 
 @sharable object NoSource extends SourceFile(NoAbstractFile, Array[Char]()) {

compiler/src/dotty/tools/scripting/Main.scala

@@ -1,22 +1,108 @@
 package dotty.tools.scripting
 
 import java.io.File
+import java.nio.file.Path
+import dotty.tools.dotc.config.Properties.isWin 
 
 /** Main entry point to the Scripting execution engine */
 object Main:
   /** All arguments before -script <target_script> are compiler arguments.
       All arguments afterwards are script arguments.*/
-  def distinguishArgs(args: Array[String]): (Array[String], File, Array[String]) =
-    val (compilerArgs, rest) = args.splitAt(args.indexOf("-script"))
+  private def distinguishArgs(args: Array[String]): (Array[String], File, Array[String], Boolean, Boolean) =
+    val (leftArgs, rest) = args.splitAt(args.indexOf("-script"))
+    assert(rest.size >= 2,s"internal error: rest == Array(${rest.mkString(",")})")
+
     val file = File(rest(1))
     val scriptArgs = rest.drop(2)
-    (compilerArgs, file, scriptArgs)
+    var saveJar = false
+    var invokeFlag = true // by default, script main method is invoked
+    val compilerArgs = leftArgs.filter {
+      case "-save" | "-savecompiled" =>
+        saveJar = true
+        false
+      case "-compile-only" =>
+        invokeFlag = false // no call to script main method
+        false
+      case _ =>
+        true
+    }
+    (compilerArgs, file, scriptArgs, saveJar, invokeFlag)
   end distinguishArgs
 
   def main(args: Array[String]): Unit =
-    val (compilerArgs, scriptFile, scriptArgs) = distinguishArgs(args)
-    try ScriptingDriver(compilerArgs, scriptFile, scriptArgs).compileAndRun()
+    val (compilerArgs, scriptFile, scriptArgs, saveJar, invokeFlag) = distinguishArgs(args)
+    val driver = ScriptingDriver(compilerArgs, scriptFile, scriptArgs)
+    try driver.compileAndRun { (outDir:Path, classpath:String, mainClass: String) =>
+      if saveJar then
+        // write a standalone jar to the script parent directory
+        writeJarfile(outDir, scriptFile, scriptArgs, classpath, mainClass)
+      invokeFlag
+    }
     catch
       case ScriptingException(msg) =>
         println(s"Error: $msg")
         sys.exit(1)
+
+      case e: java.lang.reflect.InvocationTargetException =>
+        throw e.getCause
+
+  private def writeJarfile(outDir: Path, scriptFile: File, scriptArgs:Array[String],
+      classpath:String, mainClassName: String): Unit =
+
+    val javaClasspath = sys.props("java.class.path")
+    val runtimeClasspath = s"${classpath}$pathsep$javaClasspath"
+
+    val jarTargetDir: Path = Option(scriptFile.toPath.getParent) match {
+      case None => sys.error(s"no parent directory for script file [$scriptFile]")
+      case Some(parent) => parent
+    }
+
+    def scriptBasename = scriptFile.getName.takeWhile(_!='.')
+    val jarPath = s"$jarTargetDir/$scriptBasename.jar"
+
+    val cpPaths = runtimeClasspath.split(pathsep).map(_.absPath)
+
+    import java.util.jar.Attributes.Name
+    val cpString:String = cpPaths.distinct.mkString(" ")
+    val manifestAttributes:Seq[(Name, String)] = Seq(
+      (Name.MANIFEST_VERSION, "1.0"),
+      (Name.MAIN_CLASS, mainClassName),
+      (Name.CLASS_PATH, cpString),
+    )
+    import dotty.tools.io.{Jar, Directory}
+    val jar = new Jar(jarPath)
+    val writer = jar.jarWriter(manifestAttributes:_*)
+    try
+      writer.writeAllFrom(Directory(outDir))
+    finally
+      writer.close()
+  end writeJarfile
+
+  def pathsep = sys.props("path.separator")
+
+
+  extension(file: File){
+    def norm: String = file.toString.norm
+  }
+
+  extension(path: String) {
+    // Normalize path separator, convert relative path to absolute
+    def norm: String =
+      path.replace('\\', '/') match {
+        case s if s.secondChar == ":" => s.drop(2)
+        case s if s.startsWith("./") => s.drop(2)
+        case s => s
+      }
+   
+    // convert to absolute path relative to cwd.
+    def absPath: String = norm match
+      case str if str.isAbsolute => norm
+      case _ => s"/${sys.props("user.dir").norm}/$norm"
+
+    def absFile: File = File(path.absPath)
+
+    // Treat norm paths with a leading '/' as absolute.
+    // Windows java.io.File#isAbsolute treats them as relative.
+    def isAbsolute = path.norm.startsWith("/") || (isWin && path.secondChar == ":")
+    def secondChar: String = path.take(2).drop(1).mkString("")
+  }

compiler/src/dotty/tools/scripting/ScriptingDriver.scala

@@ -17,7 +17,7 @@ import dotty.tools.dotc.config.Settings.Setting._
 import sys.process._
 
 class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs: Array[String]) extends Driver:
-  def compileAndRun(): Unit =
+  def compileAndRun(pack:(Path, String, String) => Boolean = null): Unit =
     val outDir = Files.createTempDirectory("scala3-scripting")
     val (toCompile, rootCtx) = setup(compilerArgs :+ scriptFile.getAbsolutePath, initCtx.fresh)
     given Context = rootCtx.fresh.setSetting(rootCtx.settings.outputDir,
@@ -26,7 +26,16 @@ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs:
     if doCompile(newCompiler, toCompile).hasErrors then
       throw ScriptingException("Errors encountered during compilation")
 
-    try detectMainMethod(outDir, ctx.settings.classpath.value).invoke(null, scriptArgs)
+    try
+      val (mainClass, mainMethod) = detectMainClassAndMethod(outDir, ctx.settings.classpath.value, scriptFile)
+      val invokeMain: Boolean =
+        Option(pack) match
+          case Some(func) =>
+            func(outDir, ctx.settings.classpath.value, mainClass)
+          case None =>
+            true
+        end match
+      if invokeMain then mainMethod.invoke(null, scriptArgs)
     catch
       case e: java.lang.reflect.InvocationTargetException =>
         throw e.getCause
@@ -41,12 +50,13 @@ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs:
     target.delete()
   end deleteFile
 
-  private def detectMainMethod(outDir: Path, classpath: String): Method =
+  private def detectMainClassAndMethod(outDir: Path, classpath: String,
+      scriptFile: File): (String, Method) =
     val outDirURL = outDir.toUri.toURL
-    val classpathUrls = classpath.split(":").map(File(_).toURI.toURL)
+    val classpathUrls = classpath.split(pathsep).map(File(_).toURI.toURL)
     val cl = URLClassLoader(classpathUrls :+ outDirURL)
 
-    def collectMainMethods(target: File, path: String): List[Method] =
+    def collectMainMethods(target: File, path: String): List[(String, Method)] =
       val nameWithoutExtension = target.getName.takeWhile(_ != '.')
       val targetPath =
         if path.nonEmpty then s"${path}.${nameWithoutExtension}"
@@ -61,7 +71,7 @@ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs:
         val cls = cl.loadClass(targetPath)
         try
           val method = cls.getMethod("main", classOf[Array[String]])
-          if Modifier.isStatic(method.getModifiers) then List(method) else Nil
+          if Modifier.isStatic(method.getModifiers) then List((cls.getName, method)) else Nil
         catch
           case _: java.lang.NoSuchMethodException => Nil
       else Nil
@@ -74,13 +84,16 @@ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs:
 
     candidates match
       case Nil =>
-        throw ScriptingException("No main methods detected in your script")
+        throw ScriptingException(s"No main methods detected in script ${scriptFile}")
       case _ :: _ :: _ =>
         throw ScriptingException("A script must contain only one main method. " +
           s"Detected the following main methods:\n${candidates.mkString("\n")}")
       case m :: Nil => m
     end match
-  end detectMainMethod
+  end detectMainClassAndMethod
+
+  def pathsep = sys.props("path.separator")
+
 end ScriptingDriver
 
 case class ScriptingException(msg: String) extends RuntimeException(msg)

compiler/test-resources/scripting/hashBang.sc

@@ -0,0 +1,25 @@
+#!/usr/bin/env scala
+# comment
+STUFF=nada
+!#
+// everything above this point should be ignored by the compiler
+def main(args: Array[String]): Unit =
+  args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) }
+  System.err.printf("mainClassFromStack: %s\n",mainFromStack)
+  assert(mainFromStack.contains("hashBang"),s"fromStack[$mainFromStack]")
+
+  lazy val mainFromStack:String = {
+    val result = new java.io.StringWriter()
+    new RuntimeException("stack").printStackTrace(new java.io.PrintWriter(result))
+    val stack = result.toString.split("[\r\n]+").toList
+    if verbose then for( s <- stack ){ System.err.printf("[%s]\n",s) }
+    stack.filter { str => str.contains(".main(") }.map {
+      // derive main class name from stack when main object is NOT declared in source
+      _.replaceAll("[.].*","").
+      replaceAll("\\s+at\\s+","")
+    }.distinct.take(1).mkString("")
+  }
+
+  lazy val verbose = Option(System.getenv("DOTC_VERBOSE")) match
+    case None => false
+    case _ => true

compiler/test-resources/scripting/mainClassOnStack.sc

@@ -0,0 +1,29 @@
+#!/usr/bin/env scala
+export STUFF=nada
+#lots of other stuff that isn't valid scala
+!#
+// everything above this point should be ignored by the compiler
+object Zoo {
+  def main(args: Array[String]): Unit =
+    args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) }
+    printf("mainClassFromStack: %s\n",mainClassFromStack)
+    assert(mainClassFromStack == "Zoo",s"fromStack[$mainClassFromStack]")
+
+  lazy val mainClassFromStack:String = {
+    val result = new java.io.StringWriter()
+    new RuntimeException("stack").printStackTrace(new java.io.PrintWriter(result))
+    val stack = result.toString.split("[\r\n]+").toList
+    if verbose then for( s <- stack ){ System.err.printf("[%s]\n",s) }
+    val shortStack = stack.filter { str => str.contains(".main(") && ! str.contains("$") }.map {
+      // derive main class name from stack when main object is declared in source
+      _.replaceAll("[.].*","").
+      replaceAll("\\s+at\\s+","")
+    }
+    // for( s <- shortStack ){ System.err.printf("[%s]\n",s) }
+    shortStack.take(1).mkString("|")
+  }
+
+  lazy val verbose = Option(System.getenv("DOTC_VERBOSE")) match
+    case None => false
+    case _ => true
+}

compiler/test-resources/scripting/scriptParent.scala

@@ -0,0 +1,14 @@
+import java.nio.file.Paths
+
+object ScriptParent {
+  def main(args: Array[String]): Unit = {
+    args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) }
+    val scriptName = Option(sys.props("script.path")) match {
+      case  None =>
+        printf("no script.path property\n")
+      case Some(script) =>
+        val p = Paths.get(script).toAbsolutePath.toFile.getParent
+        printf("parentDir: [%s]\n",p)
+    }
+  }
+}

compiler/test-resources/scripting/scriptPath.sc

@@ -0,0 +1,10 @@
+#!/usr/bin/env scala
+
+  def main(args: Array[String]): Unit =
+    args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) }
+    val path = Option(sys.props("script.path")) match {
+    case None => printf("no script.path property is defined\n")
+    case Some(path) =>
+      printf("script.path: %s\n",path)
+      assert(path.endsWith("scriptPath.sc"),s"actual path [$path]")
+    }

compiler/test-resources/scripting/touchFile.sc

@@ -0,0 +1,8 @@
+#!/usr/bin/env scala
+
+import java.io.File
+
+// create an empty file
+def main(args: Array[String]): Unit =
+  val file = File("touchedFile.out")
+  file.createNewFile();