More Script Polish (#6029)

TODO:

- [x] Example of script depending on programmatic module
- [x] Packaging scripts into assemblies and Graal native images
- [x] More details on how testing works, and list of supported classes
- [x] Fix tab completion to work for script files as well as module
selectors
- [x] Script REPL and Console instructions
- [ ] Script autoformatting
- [x] Script pinned java version
- [x] Error message tests
- [x] Fleshed out use cases for scripts

Author: lihaoyi

.github/actions/post-build-selective/action.yml

@@ -4,6 +4,10 @@ inputs:
     default: ''
     type: string
 
+  extra:
+    default: ''
+    type: string
+
   shell:
     required: true
     type: string
@@ -43,7 +47,7 @@ runs:
     #- run: ./mill -i -k selective.resolveChanged ${{ inputs.millargs }}
     #  shell: ${{ inputs.shell }}
 
-    - run: ./mill -i -k selective.run ${{ inputs.millargs }}
+    - run: ./mill -i -k ${{ inputs.extra }} selective.run ${{ inputs.millargs }}
       shell: ${{ inputs.shell }}
 
     - name: Clean-up Test Reports

.github/workflows/post-build-selective.yml

@@ -16,6 +16,9 @@ on:
       os:
         default: 'ubuntu-latest'
         type: string
+      extra:
+        default: ''
+        type: string
       timeout-minutes:
         default: 60
         type: number
@@ -63,4 +66,5 @@ jobs:
 
         with:
           millargs: ${{ inputs.millargs }}
-          shell: ${{ inputs.shell }}
+          shell: ${{ inputs.shell }}
+          extra: ${{ inputs.extra }}

.github/workflows/run-tests.yml

@@ -204,8 +204,9 @@ jobs:
             millargs: "'integration.invalidation.__.packaged.daemon'"
 
           - java-version: 11
+            # turn of parallelism since it sometimes causes issues with concurrent migrations
             millargs: '"example.migrating.javalib.__.packaged.daemon"'
-
+            extra: "--jobs=1"
           - java-version: 11
             millargs: '"libs.{util,javalib,androidlib,graphviz}.__.test"'
 
@@ -223,6 +224,7 @@ jobs:
       os: windows-latest
       java-version: ${{ matrix.java-version }}
       millargs: ${{ matrix.millargs }}
+      extra: ${{ matrix.extra || '' }}
       shell: powershell
 
   itest:

core/api/daemon/src/mill/api/daemon/ExecResult.scala

@@ -130,7 +130,7 @@ object ExecResult {
     catch {
       case e: InvocationTargetException =>
         Result.Failure(makeResultException(e.getCause, new java.lang.Exception()).left.get)
-      case e: Exception =>
+      case e: java.lang.Exception =>
         Result.Failure(makeResultException(e, new java.lang.Exception()).left.get)
     }
   }

core/api/daemon/src/mill/api/daemon/Result.scala

@@ -6,7 +6,7 @@ import scala.util.boundary
 /**
  * Represents a computation that either succeeds with a value [[T]] or
  * fails. Basically equivalent to `Either[String, T]`, with converters
- * back and forther via [[Result.toEither]] or [[Result.fromEither]]
+ * back and forth via [[Result.toEither]] or [[Result.fromEither]]
  */
 sealed trait Result[+T] {
   def map[V](f: T => V): Result[V]

core/api/src/mill/api/Discover.scala

@@ -16,6 +16,7 @@ import scala.collection.mutable
  * can then be used later to look up the `MainData` for any module.
  */
 final class Discover(val classInfo: Map[Class[?], Discover.ClassInfo]) {
+  private[mill] val allTaskNames = classInfo.values.flatMap(_.declaredTaskNameSet).toSet
   private[mill] def resolveEntrypoint(cls: Class[?], name: String) = {
     val res = for {
       (cls2, node) <- classInfo

core/exec/src/mill/exec/GroupExecution.scala

@@ -111,15 +111,29 @@ trait GroupExecution {
               case ujson.Obj(kvs) => ujson.Obj.from(kvs.map((k, v) => (k, rec(v))))
               case v => v
             }
-            val (resultData, serializedPaths) = PathRef.withSerializedPaths {
-              PathRef.currentOverrideModulePath.withValue(labelled.ctx.millSourcePath) {
-                upickle.read[Any](rec(jsonData))(
-                  using labelled.readWriterOpt.get.asInstanceOf[upickle.Reader[Any]]
-                )
+
+            val (execRes, serializedPaths) =
+              try {
+                val (resultData, serializedPaths) = PathRef.withSerializedPaths {
+                  PathRef.currentOverrideModulePath.withValue(labelled.ctx.millSourcePath) {
+                    upickle.read[Any](rec(jsonData))(
+                      using labelled.readWriterOpt.get.asInstanceOf[upickle.Reader[Any]]
+                    )
+                  }
+                }
+                (ExecResult.Success(Val(resultData), resultData.##), serializedPaths)
+              } catch {
+                case e: upickle.core.TraceVisitor.TraceException =>
+                  (
+                    ExecResult.Failure(
+                      s"Failed de-serializing config override: ${e.getCause.getMessage}"
+                    ),
+                    Nil
+                  )
               }
-            }
+
             GroupExecution.Results(
-              Map(labelled -> ExecResult.Success(Val(resultData), resultData.##)),
+              Map(labelled -> execRes),
               Nil,
               cached = true,
               inputsHash,

core/resolve/src/mill/resolve/Resolve.scala

@@ -193,7 +193,7 @@ private[mill] object Resolve {
       )
       .head
 
-    ResolveCore.catchWrapException(
+    mill.api.ExecResult.catchWrapException(
       definition.invoke(p).asInstanceOf[Task.Named[?]]
     )
   }
@@ -206,7 +206,7 @@ private[mill] object Resolve {
       nullCommandDefaults: Boolean,
       allowPositionalCommandArgs: Boolean
   ) = {
-    ResolveCore.catchWrapException {
+    mill.api.ExecResult.catchWrapException {
       val invoked = invokeCommand0(
         p,
         r.segments.last.value,

core/resolve/src/mill/resolve/ResolveCore.scala

@@ -3,7 +3,6 @@ package mill.resolve
 import mill.api.*
 import mill.api.internal.{Reflect, Resolved, RootModule0}
 
-import java.lang.reflect.InvocationTargetException
 import java.lang.reflect.Method
 
 /**
@@ -79,16 +78,6 @@ private object ResolveCore {
     }
   }
 
-  def catchWrapException[T](t: => T): mill.api.Result[T] = {
-    try mill.api.Result.Success(t)
-    catch {
-      case e: InvocationTargetException =>
-        mill.api.Result.Failure(makeResultException(e.getCause, new java.lang.Exception()).left.get)
-      case e: Exception =>
-        mill.api.Result.Failure(makeResultException(e, new java.lang.Exception()).left.get)
-    }
-  }
-
   def makeResultException(e: Throwable, base: Exception): Left[String, Nothing] =
     mill.api.ExecResult.makeResultException(e, base)
 
@@ -231,7 +220,7 @@ private object ResolveCore {
             if (classOf[Cross[?]].isAssignableFrom(m.cls)) {
               instantiateModule(rootModule, current.segments, cache).flatMap {
                 case c: Cross[_] =>
-                  catchWrapException(
+                  mill.api.ExecResult.catchWrapException(
                     if (cross == Seq("__")) for ((_, v) <- c.valuesToModules.toSeq) yield v
                     else if (cross.contains("_")) {
                       for {
@@ -291,7 +280,7 @@ private object ResolveCore {
         case (mill.api.Result.Success(current), Segment.Cross(vs)) =>
           assert(!vs.contains("_"), vs)
 
-          catchWrapException(
+          mill.api.ExecResult.catchWrapException(
             current
               .asInstanceOf[Cross[?]]
               .segmentsToModules(vs.toList)
@@ -445,7 +434,8 @@ private object ResolveCore {
           .collect {
             case (name, memberCls, getter) =>
               val resolved = Resolved.Module(Segments.labels(cache.decode(name)), memberCls)
-              val getter2 = Some((mod: Module) => catchWrapException(getter(mod)))
+              val getter2 =
+                Some((mod: Module) => mill.api.ExecResult.catchWrapException(getter(mod)))
               (resolved, getter2)
           }
           .toSeq

example/javalib/basic/1-script/build.mill

@@ -1,5 +1,7 @@
 //// SNIPPET:FILE
 /** See Also: Foo.java */
+//// SNIPPET:END
+//// SNIPPET:CMD
 /** Usage
 > ./mill Foo.java --text hello
 compiling 1 Java source to...