Convert ./mill init to generate .mill.yaml format, add !append support (#6441)

* We generate `.mill.yaml` files when importing Java projects using
`./mill init`.

* We still generate Scala code in `mill-build/src/`, which lets us avoid
duplication in the `.mill.yaml` files since the yaml files can `extend`
the shared `trait` containing the common config

* Scala project import makes use of cross values which are currently not
supported in `.mill.yaml` so we leave those as generating `.mill` files
for now.

* You can now tag top-level YAML values with `!append`, which
concatenates the Seq the overridden value rather than completely
replacing it. This is necessary for `.mill.yaml` Java project import to
work, as we make use of `super ++` in some places. This is implemented
via `parseYaml0` converting the `!append` tags into a synthetic
`{"$millAppend": ...}` wrapper, which is then deserialized into an
`Appendable[T]` that lets us decide later on whether to append something
or not

The `!append` stuff is mentioned in some updated doc examples which
demonstrate and exercise it

Author: lihaoyi

.scalafix.conf

@@ -2,4 +2,9 @@ rules = [
     RemoveUnused,
     NoAutoTupling
 ]
-OrganizeImports.targetDialect = Scala3
+OrganizeImports.targetDialect = Scala3
+project {
+  excludePaths = [
+    "glob:**/out/**",
+  ]
+}

core/api/src/mill/api/Evaluator.scala

@@ -33,7 +33,8 @@ trait Evaluator extends AutoCloseable with EvaluatorApi {
   private[mill] def effectiveThreadCount: Int
   private[mill] def offline: Boolean
   private[mill] def useFileLocks: Boolean = false
-  private[mill] def staticBuildOverrides: Map[String, Located[BufferedValue]] = Map()
+  private[mill] def staticBuildOverrides: Map[String, Located[internal.Appendable[BufferedValue]]] =
+    Map()
   def withBaseLogger(newBaseLogger: Logger): Evaluator
 
   def resolveSegments(

core/api/src/mill/api/Module.scala

@@ -45,7 +45,8 @@ trait Module extends Module.BaseClass with ModuleCtx.Wrapper with ModuleApi {
   private[mill] val moduleLinearized: Seq[Class[?]] =
     OverrideMapping.computeLinearization(this.getClass)
 
-  private[mill] def moduleDynamicBuildOverrides: Map[String, internal.Located[BufferedValue]] =
+  private[mill] def moduleDynamicBuildOverrides
+      : Map[String, internal.Located[internal.Appendable[BufferedValue]]] =
     Map()
 }
 

core/api/src/mill/api/ModuleCtx.scala

@@ -46,7 +46,8 @@ object ModuleCtx extends LowPriCtx {
     def moduleSegments: Segments = moduleCtx.segments
     def moduleCtx: ModuleCtx
     private[mill] def moduleLinearized: Seq[Class[?]]
-    private[mill] def moduleDynamicBuildOverrides: Map[String, internal.Located[BufferedValue]] =
+    private[mill] def moduleDynamicBuildOverrides
+        : Map[String, internal.Located[internal.Appendable[BufferedValue]]] =
       Map()
   }
 

core/api/src/mill/api/ScriptModule.scala

@@ -25,7 +25,11 @@ trait ScriptModule extends ExternalModule {
     .rest
     .map { case (k, v) =>
       val newKey = (moduleSegments ++ mill.api.Segment.Label(k.value)).render
-      (newKey, internal.Located(scriptConfig.scriptFile, k.index, v))
+      val (actualValue, append) = internal.Appendable.unwrapAppendMarker(v)
+      (
+        newKey,
+        internal.Located(scriptConfig.scriptFile, k.index, internal.Appendable(actualValue, append))
+      )
     }
 }
 @experimental

core/api/src/mill/api/internal/HeaderData.scala

@@ -2,9 +2,14 @@ package mill.api.internal
 
 private[mill] case class HeaderData(
     `extends`: Located[OneOrMore[Located[String]]] = Located(null, -1, OneOrMore(Nil)),
-    moduleDeps: Located[Seq[Located[String]]] = Located(null, -1, Nil),
-    compileModuleDeps: Located[Seq[Located[String]]] = Located(null, -1, Nil),
-    runModuleDeps: Located[Seq[Located[String]]] = Located(null, -1, Nil),
+    moduleDeps: Located[Appendable[Seq[Located[String]]]] =
+      Located(null, -1, Appendable(Nil)),
+    compileModuleDeps: Located[Appendable[Seq[Located[String]]]] =
+      Located(null, -1, Appendable(Nil)),
+    runModuleDeps: Located[Appendable[Seq[Located[String]]]] =
+      Located(null, -1, Appendable(Nil)),
+    bomModuleDeps: Located[Appendable[Seq[Located[String]]]] =
+      Located(null, -1, Appendable(Nil)),
     @upickle.implicits.flatten rest: Map[Located[String], upickle.core.BufferedValue]
 )
 private[mill] object HeaderData {
@@ -17,6 +22,8 @@ private[mill] object HeaderData {
   def headerDataReader(path: os.Path) = {
     implicit def locatedReader[T: upickle.Reader]: Located.UpickleReader[T] =
       new Located.UpickleReader[T](path)
+    implicit def appendableReader[T: upickle.Reader]: Appendable.UpickleReader[T] =
+      new Appendable.UpickleReader[T]
     upickle.macroR[HeaderData]
   }
 }

core/api/src/mill/api/internal/Located.scala

@@ -1,10 +1,56 @@
 package mill.api.internal
 
+/** Tracks the source location (file path and character index) of a parsed value */
 case class Located[T](path: os.Path, index: Int, value: T)
 
+/** Wraps a value with append semantics (whether to replace or append to existing value) */
+case class Appendable[T](value: T, append: Boolean = false)
+
+object Appendable {
+  import upickle.core.BufferedValue
+
+  /** Marker key for append mode: `{$millAppend: <actual-value>}` */
+  val AppendMarkerKey = "$millAppend"
+
+  /**
+   * Extract append flag and actual value from a BufferedValue that may contain
+   * the marker object `{$millAppend: <value>}` (produced by YAML `!append` tag parsing).
+   * Returns (actualValue, appendFlag).
+   */
+  def unwrapAppendMarker(v: BufferedValue): (BufferedValue, Boolean) = {
+    v match {
+      case obj: BufferedValue.Obj if obj.value0.size == 1 =>
+        obj.value0.head match {
+          case (BufferedValue.Str(k, _), value) if k.toString == AppendMarkerKey =>
+            (value, true)
+          case _ => (v, false)
+        }
+      case _ => (v, false)
+    }
+  }
+
+  /**
+   * Upickle reader for Appendable[T] that buffers the value, checks for the
+   * `!append` marker object, and deserializes accordingly.
+   */
+  class UpickleReader[T](implicit r: upickle.Reader[T])
+      extends upickle.Reader.MapReader[BufferedValue, BufferedValue, Appendable[T]](
+        BufferedValue.Builder
+      ) {
+    def mapNonNullsFunction(buffered: BufferedValue): Appendable[T] = {
+      val (actualValue, append) = unwrapAppendMarker(buffered)
+      val deserialized = BufferedValue.transform(actualValue, r)
+      Appendable(deserialized, append)
+    }
+  }
+}
+
 object Located {
+
+  /** Upickle reader for Located[T] that wraps values with their source location */
   class UpickleReader[T](path: os.Path)(implicit r: upickle.Reader[T])
       extends upickle.Reader[Located[T]] {
+
     private def wrap(index: Int, v: T): Located[T] = Located(path, index, v)
 
     def visitArray(length: Int, index: Int): upickle.core.ArrVisitor[Any, Located[T]] = {
@@ -16,8 +62,11 @@ object Located {
       }
     }
 
-    def visitObject(length: Int, jsonableKeys: Boolean, index: Int)
-        : upickle.core.ObjVisitor[Any, Located[T]] = {
+    def visitObject(
+        length: Int,
+        jsonableKeys: Boolean,
+        index: Int
+    ): upickle.core.ObjVisitor[Any, Located[T]] = {
       val delegate = r.visitObject(length, jsonableKeys, index)
       new upickle.core.ObjVisitor[Any, Located[T]] {
         def subVisitor = delegate.subVisitor

core/eval/src/mill/eval/ScriptModuleInit.scala

@@ -157,9 +157,9 @@ class ScriptModuleInit extends ((String, Evaluator) => Seq[Result[ExternalModule
             moduleFor(
               scriptFile,
               parsedHeaderData.`extends`.value.value.headOption,
-              parsedHeaderData.moduleDeps.value,
-              parsedHeaderData.compileModuleDeps.value,
-              parsedHeaderData.runModuleDeps.value,
+              parsedHeaderData.moduleDeps.value.value,
+              parsedHeaderData.compileModuleDeps.value.value,
+              parsedHeaderData.runModuleDeps.value.value,
               eval,
               parsedHeaderData
             )