Improve example zips (#6122)

- Make `ExampleParser` return proper `enum` cases, rather than
stringly-typed tuples
- For examples where `build.mill` doesn't contain any actual code or
YAML header config, e.g. the script examples where it's just for docs
and usage blocks, remove the `build.mill` and render a `readme.adoc`
instead
- For examples where the `build.mill` does contain code or header
config, use the `testRepoSourceRoot` containing any `build.mill`
templates expanded, rather than the raw `moduleDir`.

Tested manually via ` ./mill show dist.exampleZips`

Author: lihaoyi

dist/package.mill

@@ -5,6 +5,7 @@ import scalalib.*
 import publish.*
 import mill.api.ModuleRef
 import mill.util.Jvm
+import mill.testkit.Chunk
 //import de.tobiasroeser.mill.vcs.version.VcsVersion
 import millbuild.*
 import mill.api.BuildCtx
@@ -268,9 +269,14 @@ object `package` extends MillJavaModule with DistModule {
     build.example.exampleModules.map(_.moduleDir)
   }
 
-  def examplePathRefs: Task[Seq[PathRef]] = Task.Sources(
-    build.example.exampleModules.map(_.moduleDir)*
-  )
+  def examplePathRefs: Task[Seq[PathRef]] =
+    Task.traverse(build.example.exampleModules)(_.testRepoSourceRoot)
+
+  def exampleRendered: Task[Seq[PathRef]] =
+    Task.traverse(build.example.exampleModules)(_.rendered)
+
+  def exampleParsed: Task[Seq[Seq[Chunk]]] =
+    Task.traverse(build.example.exampleModules)(_.parsed)
 
   def exampleArtifactNames = Task {
     for (path <- examplePaths()) yield {
@@ -281,9 +287,17 @@ object `package` extends MillJavaModule with DistModule {
   }
 
   def exampleZips: T[Seq[PathRef]] = Task {
-    examplePathRefs().zip(exampleArtifactNames()).map {
-      case (pr, exampleStr) =>
+    examplePathRefs().zip(exampleArtifactNames()).zip(exampleRendered()).zip(exampleParsed()).map {
+      case (((pr, exampleStr), rendered), parsed) =>
         os.copy(pr.path, Task.dest / exampleStr, createFolders = true)
+
+        // When the `build.mill` contains no Scala or Yaml chunks, it purely serves
+        // as the doc-site content and isn't used in the build. So we don't need to include it
+        // and cna instead just render the readme.adoc file for people to read
+        if (!parsed.exists(c => c.isInstanceOf[Chunk.Scala] || c.isInstanceOf[Chunk.Yaml])) {
+          os.remove.all(Task.dest / exampleStr / "build.mill")
+          os.copy(rendered.path, Task.dest / exampleStr / "readme.adoc")
+        }
         val ignoreErrorsOnCI = Task.dest / exampleStr / "ignoreErrorsOnCI"
         if (os.exists(ignoreErrorsOnCI)) os.remove(ignoreErrorsOnCI)
         val buildMill = Task.dest / exampleStr / "build.mill"

example/package.mill

@@ -13,6 +13,7 @@ import mill.contrib.buildinfo.BuildInfo
 import mill.T
 import mill.api.Cross
 import mill.api.BuildCtx
+import mill.testkit.Chunk
 
 object `package` extends Module {
   def exampleModules: Seq[ExampleCrossModule] = moduleInternal
@@ -246,25 +247,26 @@ object `package` extends Module {
     /**
      * Parses a `build.mill` for specific comments and return the split-by-type content
      */
-    def parsed: T[Seq[(String, String)]] = Task {
+    def parsed: T[Seq[mill.testkit.Chunk]] = Task {
       mill.testkit.ExampleParser(testRepoSourceRoot().path)
     }
 
     def rendered = Task {
       var seenCode = false
       var seenFrontMatter = false
       val examplePath = moduleDir.subRelativeTo(BuildCtx.workspaceRoot)
-      val frontMatter = parsed().takeWhile(_._1 == "yaml").map(_._2).mkString("\n")
-      val withoutFrontMatter = parsed().dropWhile(_._1 == "yaml")
+      val frontMatter = parsed().takeWhile(_.isInstanceOf[Chunk.Yaml]).map {
+        case Chunk.Yaml(lines) => lines.mkString("\n")
+      }.mkString("\n")
+      val withoutFrontMatter = parsed().dropWhile(_.isInstanceOf[Chunk.Yaml])
       val exampleDashed = examplePath.segments.mkString("-")
       val download =
         s"{mill-download-url}/mill-dist-${build.millVersion()}-$exampleDashed.zip[download]"
       os.write(
         Task.dest / "example.adoc",
         withoutFrontMatter
-          .filter(_._2.nonEmpty)
           .map {
-            case (s"see:$path", txt) =>
+            case Chunk.See(path, lines) =>
               // avoid .stripMargin, as the embedded content may contain the margin symbol
               val lang = os.FilePath(path).ext match {
                 case "mill" => "scala"
@@ -274,9 +276,9 @@ object `package` extends Module {
 .$path ($download, {mill-example-url}/$examplePath/$path[browse])
 [source,$lang,subs="attributes,verbatim"]
 ----
-$txt
+${lines.mkString("\n")}
 ----"""
-            case ("scala", txt) =>
+            case Chunk.Scala(lines) =>
               val title =
                 if (seenCode) ""
                 else {
@@ -297,19 +299,18 @@ ${
                   }
                 }
 
-$txt
+${lines.mkString("\n")}
 ----
 """
-            case ("comment", txt) => txt + "\n"
-            case ("example", txt) =>
+            case Chunk.Comment(lines) => lines.mkString("\n") + "\n"
+            case Chunk.Example(lines) =>
               // avoid .stripMargin, as the embedded content may contain the margin symbol
               s"""
 [source,console,subs="attributes,verbatim"]
 ----
-$txt
+${lines.mkString("\n")}
 ----"""
-            case (kind, txt) =>
-              throw Exception(s"Unknown kind: $kind\n\nText: $txt")
+            case Chunk.Yaml(_) => ""
           }
           .mkString("\n")
       )

mill-build/src-testkit/mill/testkit/ExampleParser.scala

@@ -1,10 +1,30 @@
 package mill.testkit
 
+enum Chunk derives upickle.default.ReadWriter {
+  case Yaml(lines: Seq[String])
+  case Example(lines: Seq[String])
+  case See(path: String, lines: Seq[String])
+  case Scala(lines: Seq[String])
+  case Comment(lines: Seq[String])
+}
+
 object ExampleParser {
-  def apply(testRepoRoot: os.Path): Seq[(String, String)] = {
+  def apply(testRepoRoot: os.Path): Seq[Chunk] = {
+    val result = collection.mutable.Buffer[Chunk]()
 
-    val states = collection.mutable.Buffer("yaml")
-    val chunks = collection.mutable.Buffer(collection.mutable.Buffer.empty[String])
+    def appendLine(line: String): Unit = {
+      if (result.nonEmpty) {
+        val last = result.last
+        val updated = last match {
+          case Chunk.Yaml(lines) => Chunk.Yaml(lines :+ line)
+          case Chunk.Example(lines) => Chunk.Example(lines :+ line)
+          case Chunk.See(path, lines) => Chunk.See(path, lines :+ line)
+          case Chunk.Scala(lines) => Chunk.Scala(lines :+ line)
+          case Chunk.Comment(lines) => Chunk.Comment(lines :+ line)
+        }
+        result(result.length - 1) = updated
+      }
+    }
 
     val rootBuildFileNames = Seq("build.mill")
     val buildFile = rootBuildFileNames.map(testRepoRoot / _)
@@ -14,26 +34,47 @@ object ExampleParser {
           s"No build file named ${rootBuildFileNames.mkString("/")} found in $testRepoRoot"
         )
       )
+
     for (line <- os.read.lines(buildFile)) {
-      val (newState, restOpt) = line match {
-        case s"/** Usage" => ("example", None)
+      val (newChunkType, lineToAdd) = line match {
+        case s"/** Usage" => (Chunk.Example(Vector()), None)
         case s"/** See Also: $path */" =>
-          (s"see:$path", Some(os.read(os.Path(path, testRepoRoot))))
-        case s"*/" => ("scala", None)
-        case line @ s"//|$_" if states.last == "yaml" => ("yaml", Some(line))
-        case s"//$rest" if !rest.startsWith("|") => ("comment", Some(rest.stripPrefix(" ")))
+          (Chunk.See(path, Vector()), Some(os.read(os.Path(path, testRepoRoot))))
+        case s"*/" => (Chunk.Scala(Vector()), None)
+        case line @ s"//|$_" if result.nonEmpty && result.last.isInstanceOf[Chunk.Yaml] =>
+          (Chunk.Yaml(Vector()), Some(line))
+        case s"//$rest" if !rest.startsWith("|") =>
+          (Chunk.Comment(Vector()), Some(rest.stripPrefix(" ")))
         case l =>
-          (if (Seq("comment", "yaml").contains(states.last)) "scala" else states.last, Some(l))
+          val chunkType = if (result.nonEmpty) {
+            result.last match {
+              case _: Chunk.Comment | _: Chunk.Yaml => Chunk.Scala(Vector())
+              case other => other
+            }
+          } else Chunk.Yaml(Vector()) // initial state
+          (chunkType, Some(l))
       }
 
-      if (newState != states.last) {
-        states.append(newState)
-        chunks.append(collection.mutable.Buffer.empty[String])
+      if (result.isEmpty || newChunkType.getClass != result.last.getClass) {
+        result.append(newChunkType)
       }
 
-      restOpt.foreach(r => chunks.last.append(r))
+      lineToAdd.foreach(appendLine)
     }
 
-    states.zip(chunks.map(_.mkString("\n").trim)).filter(_._2.nonEmpty).toSeq
+    result.filter {
+      case Chunk.Yaml(lines) => lines.mkString("\n").trim.nonEmpty
+      case Chunk.Example(lines) => lines.mkString("\n").trim.nonEmpty
+      case Chunk.See(_, lines) => lines.mkString("\n").trim.nonEmpty
+      case Chunk.Scala(lines) => lines.mkString("\n").trim.nonEmpty
+      case Chunk.Comment(lines) => lines.mkString("\n").trim.nonEmpty
+    }.map {
+      case Chunk.Yaml(lines) => Chunk.Yaml(lines.mkString("\n").trim.linesIterator.toVector)
+      case Chunk.Example(lines) => Chunk.Example(lines.mkString("\n").trim.linesIterator.toVector)
+      case Chunk.See(path, lines) =>
+        Chunk.See(path, lines.mkString("\n").trim.linesIterator.toVector)
+      case Chunk.Scala(lines) => Chunk.Scala(lines.mkString("\n").trim.linesIterator.toVector)
+      case Chunk.Comment(lines) => Chunk.Comment(lines.mkString("\n").trim.linesIterator.toVector)
+    }.toSeq
   }
 }

testkit/src/mill/testkit/ExampleTester.scala

@@ -1,6 +1,7 @@
 package mill.testkit
 
 import mill.constants.Util.isWindows
+import mill.testkit.Chunk
 import utest.*
 
 import scala.util.control.NonFatal
@@ -236,7 +237,9 @@ class ExampleTester(
     val parsed = ExampleParser(workspaceSourcePath)
     val ignoreErrors = System.getenv("CI") != null &&
       os.exists(workspaceSourcePath / "ignoreErrorsOnCI")
-    val usageComment = parsed.collect { case ("example", txt) => txt }.mkString("\n\n")
+    val usageComment = parsed.collect { case Chunk.Example(lines) =>
+      lines.mkString("\n")
+    }.mkString("\n\n")
     val commandBlocks = ("\n" + usageComment.trim).split("\n> ").filter(_.nonEmpty)
 
     try {