Add support for -doc-root-content

Author: romanowski

project/Build.scala

@@ -1528,13 +1528,7 @@ object Build {
       def generateDocumentation(targets: String, name: String, outDir: String, ref: String, params: String = "") = Def.taskDyn {
           val projectVersion = version.value
           IO.createDirectory(file(outDir))
-          val managedSources =
-            (`stdlib-bootstrapped`/Compile/sourceManaged).value / "scala-library-src"
-          val projectRoot = (ThisBuild/baseDirectory).value.toPath
-          val stdLibRoot = projectRoot.relativize(managedSources.toPath.normalize())
-          val scalaSourceLink =
-            s"$stdLibRoot=github://scala/scala/v${stdlibVersion(Bootstrapped)}#src/library"
-          val sourceLinks = s"-source-links:$scalaSourceLink,github://lampepfl/dotty "
+          val sourceLinks = "-source-links:github://lampepfl/dotty "
           val revision = s"-revision $ref -project-version $projectVersion"
           val cmd = s""" -d $outDir -project "$name" $sourceLinks $revision $params $targets"""
           run.in(Compile).toTask(cmd)
@@ -1595,6 +1589,12 @@ object Build {
 
             val roots = joinProducts(dottyJars)
 
+            val managedSources =
+              (`stdlib-bootstrapped`/Compile/sourceManaged).value / "scala-library-src"
+            val projectRoot = (ThisBuild/baseDirectory).value.toPath
+            val stdLibRoot = projectRoot.relativize(managedSources.toPath.normalize())
+            val docRootFile = stdLibRoot.resolve("rootdoc.txt")
+
             if (dottyJars.isEmpty) Def.task { streams.value.log.error("Dotty lib wasn't found") }
             else Def.task{
               IO.write(dest / "versions" / "latest-nightly-base", majorVersion)
@@ -1613,7 +1613,9 @@ object Build {
               "-skip-by-regex:.+\\.internal($|\\..+) " +
               "-skip-by-regex:.+\\.impl($|\\..+) " +
               "-comment-syntax wiki -siteroot scala3doc/scala3-docs -project-logo scala3doc/scala3-docs/logo.svg " +
-              "-external-mappings:.*java.*::javadoc::https://docs.oracle.com/javase/8/docs/api/"
+              "-external-mappings:.*java.*::javadoc::https://docs.oracle.com/javase/8/docs/api/ " +
+              s"-source-links:$stdLibRoot=github://scala/scala/v${stdlibVersion(Bootstrapped)}#src/library " +
+              s"-doc-root-content $docRootFile"
               ))
           }.evaluated,
 

scala3doc/src/dotty/dokka/ExternalDocLink.scala

@@ -25,7 +25,7 @@ object ExternalDocLink:
 
     def tryParse[T](descr: String)(op: => T): Option[T] = try Some(op) catch
       case e: RuntimeException =>
-        report.warn("Unable to parse $descr", e)
+        report.warn(s"Unable to parse $descr", e)
         None
 
     def parsePackageList(elements: List[String]) = elements match

scala3doc/src/dotty/dokka/IO.java

@@ -43,4 +43,8 @@ public FileVisitResult visitFile(
   public static String read(Path path) throws IOException {
     return new String(Files.readAllBytes(path), Charset.defaultCharset());
   }
+
+  public static String read(String path) throws IOException {
+    return read(Paths.get(path));
+  }
 }

scala3doc/src/dotty/dokka/Scala3doc.scala

@@ -68,6 +68,7 @@ object Scala3doc:
     externalMappings: List[ExternalDocLink] = Nil,
     identifiersToSkip: List[String] = Nil,
     regexesToSkip: List[String] = Nil,
+    rootDocPath: Option[String] = None
   )
 
   def run(args: Array[String], rootContext: CompilerContext): Reporter =

scala3doc/src/dotty/dokka/Scala3docArgs.scala

@@ -48,9 +48,11 @@ class Scala3docArgs extends SettingGroup with CommonScalaSettings:
   val skipByRegex: Setting[List[String]] =
     MultiStringSetting("-skip-by-regex", "regex", "Regexes that match fully qualified names of packages or top-level classes to skip when generating documentation")
 
+  val docRootContent: Setting[String] =
+    StringSetting("-doc-root-content", "path", "The file from which the root package documentation should be imported.", "")
 
   def scala3docSpecificSettings: Set[Setting[_]] =
-    Set(sourceLinks, syntax, revision, externalDocumentationMappings, skipById, skipByRegex, deprecatedSkipPackages)
+    Set(sourceLinks, syntax, revision, externalDocumentationMappings, skipById, skipByRegex, deprecatedSkipPackages, docRootContent)
 
 object Scala3docArgs:
   def extract(args: List[String], rootCtx: CompilerContext):(Scala3doc.Args, CompilerContext) =
@@ -139,5 +141,6 @@ object Scala3docArgs:
       externalMappings,
       skipById.get ++ deprecatedSkipPackages.get,
       skipByRegex.get,
+      docRootContent.nonDefault
     )
     (docArgs, newContext)

scala3doc/src/dotty/dokka/ScalaModuleCreator.scala

@@ -16,7 +16,7 @@ import kotlin.coroutines.Continuation
 
 class ScalaModuleProvider(using ctx: DocContext) extends SourceToDocumentableTranslator:
    override def invoke(sourceSet: DokkaSourceSet, cxt: DokkaContext, unused: Continuation[? >: DModule]) =
-    val result = DokkaTastyInspector(new MarkdownParser(_ => null)).result()
+    val (result, rootDoc) = DokkaTastyInspector(new MarkdownParser(_ => null)).result()
     val (rootPck, rest) = result.partition(_.name == "<empty>")
     val packageMembers = (rest ++ rootPck.flatMap(_.allMembers)).sortBy(_.name)
 
@@ -32,7 +32,7 @@ class ScalaModuleProvider(using ctx: DocContext) extends SourceToDocumentableTra
       null,
       JSet(ctx.sourceSet),
       PropertyContainer.Companion.empty()
-    ).withNewMembers(packageMembers).withKind(Kind.RootPackage).asInstanceOf[DPackage]
+    ).withNewMembers(packageMembers).withKind(Kind.RootPackage).withDocs(rootDoc).asInstanceOf[DPackage]
 
     new DModule(
       sourceSet.getDisplayName,

scala3doc/src/dotty/dokka/model/api/internalExtensions.scala

@@ -79,6 +79,10 @@ extension (member: Member)
     val ext = MemberExtension.getFrom(member).getOrElse(MemberExtension.empty).copy(kind = kind)
     putInMember(ext)
 
+  def withDocs(docs: Option[Comment]): Member =
+    val ext = MemberExtension.getFrom(member).getOrElse(MemberExtension.empty).copy(rawDoc = docs)
+    putInMember(ext)
+
   def withMembers(newMembers: Seq[Member]): Member =
     val original = member.compositeMemberExt.getOrElse(CompositeMemberExtension())
     val newExt = original.copy(members = newMembers)

scala3doc/src/dotty/dokka/tasty/ScalaDocSupport.scala

@@ -12,39 +12,43 @@ import dotty.dokka.tasty.comments.Comment
 trait ScaladocSupport { self: TastyParser =>
   import qctx.reflect._
 
-  def parseComment(docstring: String,  tree: Tree): Comment =
-    val commentString: String =
-      if tree.symbol.isClassDef || tree.symbol.owner.isClassDef then
-        import dotty.tools.dotc
-        given ctx: dotc.core.Contexts.Context = qctx.asInstanceOf[scala.quoted.runtime.impl.QuotesImpl].ctx
-
-        val sym = tree.symbol.asInstanceOf[dotc.core.Symbols.Symbol]
-
-        comments.CommentExpander.cookComment(sym)(using ctx)
-          .get.expanded.get
-      else
-        docstring
-
-    val preparsed =
-      comments.Preparser.preparse(comments.Cleaner.clean(commentString))
+  def parseCommentString(comment: String, sym: Symbol, pos: Option[Position]): Comment =
+    val preparsed = comments.Preparser.preparse(comments.Cleaner.clean(comment))
 
     val commentSyntax =
       preparsed.syntax.headOption match {
         case Some(commentSetting) =>
           CommentSyntax.parse(commentSetting).getOrElse {
             val msg = s"not a valid comment syntax: $commentSetting, defaulting to Markdown syntax."
             // we should update pos with span from documentation
-            report.warning(msg, tree.pos)
+            pos.fold(report.warning(msg))(report.warning(msg, _))
+
             CommentSyntax.default
           }
         case None => ctx.args.defaultSyntax
       }
 
     val parser = commentSyntax match {
       case CommentSyntax.Wiki =>
-        comments.WikiCommentParser(comments.Repr(qctx)(tree.symbol))
+        comments.WikiCommentParser(comments.Repr(qctx)(sym))
       case CommentSyntax.Markdown =>
-        comments.MarkdownCommentParser(comments.Repr(qctx)(tree.symbol))
+        comments.MarkdownCommentParser(comments.Repr(qctx)(sym))
     }
     parser.parse(preparsed)
+
+  def parseComment(docstring: String,  tree: Tree): Comment =
+    val commentString: String =
+      if tree.symbol.isClassDef || tree.symbol.owner.isClassDef then
+        import dotty.tools.dotc
+        given ctx: dotc.core.Contexts.Context = qctx.asInstanceOf[scala.quoted.runtime.impl.QuotesImpl].ctx
+
+        val sym = tree.symbol.asInstanceOf[dotc.core.Symbols.Symbol]
+
+        comments.CommentExpander.cookComment(sym)(using ctx)
+          .get.expanded.get
+      else
+        docstring
+
+    parseCommentString(commentString, tree.symbol, Some(tree.pos))
+
 }

scala3doc/src/dotty/dokka/tasty/TastyParser.scala

@@ -24,15 +24,20 @@ import dotty.tools.dotc
 
 import dotty.dokka.tasty.comments.MemberLookup
 import dotty.dokka.tasty.comments.QueryParser
+import dotty.dokka.tasty.comments.Comment
 import dotty.dokka.model.api._
 
+import java.nio.file.Paths
+import java.nio.file.Files
+
 /** Responsible for collectively inspecting all the Tasty files we're interested in.
   *
   * Delegates most of the work to [[TastyParser]] [[dotty.dokka.tasty.TastyParser]].
   */
 case class DokkaTastyInspector(parser: Parser)(using ctx: DocContext) extends DocTastyInspector:
 
   private val topLevels = Seq.newBuilder[(String, Member)]
+  private var rootDoc: Option[Comment] = None
 
   def processCompilationUnit(using quotes: Quotes)(root: quotes.reflect.Tree): Unit = ()
 
@@ -84,24 +89,49 @@ case class DokkaTastyInspector(parser: Parser)(using ctx: DocContext) extends Do
 
       isSkippedById(sym) || isSkippedByRx(sym)
 
-    hackForeachTree { root =>
-      if !isSkipped(root.symbol) then
-        val parser = new TastyParser(q, this)(isSkipped)
+    val parser = new TastyParser(q, this)(isSkipped)
+    def driFor(link: String): Option[DRI] =
+      val symOps = new SymOps[q.type](q)
+      import symOps._
+      Try(QueryParser(link).readQuery()).toOption.flatMap(q =>
+        MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri}
+    )
+    ctx.staticSiteContext.foreach(_.memberLinkResolver = driFor)
+
+    var alreadyProcessed = false
+    def processRootDocIfNeeded(tree: parser.qctx.reflect.Tree) =
+      def readFile(pathStr: String)(using CompilerContext): Option[String] =
+        try
+          val path = Paths.get(pathStr)
+          if Files.exists(path) then Some(IO.read(path))
+          else
+            report.inform("Rootdoc at $pathStr does not exisits")
+            None
+        catch
+          case e: RuntimeException =>
+            report.warning(s"Unable to read root package doc from $pathStr: ${throwableToString(e)}")
+            None
 
-        def driFor(link: String): Option[DRI] =
-          val symOps = new SymOps[q.type](q)
-          import symOps._
-          Try(QueryParser(link).readQuery()).toOption.flatMap(q =>
-            MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri}
-          )
+      if !alreadyProcessed then
+        alreadyProcessed = true
+        ctx.args.rootDocPath.flatMap(readFile).map { content =>
+          import parser.qctx.reflect._
+          def root(s: Symbol): Symbol = if s.owner.isNoSymbol then s else root(s.owner)
+          val topLevelPck = root(tree.symbol)
+          rootDoc = Some(parser.parseCommentString(content, topLevelPck, None))
+        }
 
-        ctx.staticSiteContext.foreach(_.memberLinkResolver = driFor)
-        topLevels ++= parser.parseRootTree(root.asInstanceOf[parser.qctx.reflect.Tree])
+    hackForeachTree { root =>
+      if !isSkipped(root.symbol) then
+        val treeRoot = root.asInstanceOf[parser.qctx.reflect.Tree]
+        processRootDocIfNeeded(treeRoot)
+        topLevels ++= parser.parseRootTree(treeRoot)
     }
 
 
-  def result(): List[Member] =
+  def result(): (List[Member], Option[Comment]) =
     topLevels.clear()
+    rootDoc = None
     val filePaths = ctx.args.tastyFiles.map(_.getAbsolutePath).toList
     val classpath = ctx.args.classpath.split(java.io.File.pathSeparator).toList
 
@@ -114,7 +144,7 @@ case class DokkaTastyInspector(parser: Parser)(using ctx: DocContext) extends Do
         p1.withNewMembers(p2.allMembers) // TODO add doc
       )
       basePck.withMembers((basePck.allMembers ++ rest).sortBy(_.name))
-    }.toList
+    }.toList -> rootDoc
 
 /** Parses a single Tasty compilation unit. */
 case class TastyParser(