Run snippet compiler on static sites

Author: pikinier20

scaladoc-testcases/docs/docs/index.md

@@ -0,0 +1,9 @@
+---
+
+
+---
+
+```scala sc:compile
+2 + List(0)
+```
+

scaladoc-testcases/src/tests/snippetCompilerTest.scala

@@ -27,13 +27,13 @@ package snippetCompiler
   * val d: Long = "asd"
   * ```
   *
-  * ```scala sc:failing
+  * ```scala sc:fail
   * def a = 2
   * val x = 1 + List()
   * a
   * ```
   *
-  * ```scala sc:failing
+  * ```scala sc:fail
   * def a = 2
   * ```
   *
@@ -45,7 +45,6 @@ package snippetCompiler
 class A { }
 
 /**
- *
  * ```scala sc:compile
  * val c: Int = 4.5
  * ```

scaladoc/src/dotty/tools/scaladoc/DocContext.scala

@@ -76,10 +76,15 @@ case class DocContext(args: Scaladoc.Args, compilerContext: CompilerContext):
 
   lazy val snippetCompilerArgs = snippets.SnippetCompilerArgs.load(args.snippetCompiler, args.snippetCompilerDebug)(using compilerContext)
 
+  lazy val snippetChecker = snippets.SnippetChecker(args.classpath, args.tastyDirs)
+
   lazy val staticSiteContext = args.docsRoot.map(path => StaticSiteContext(
       File(path).getAbsoluteFile(),
       args,
-      sourceLinks
+      sourceLinks,
+      snippetCompilerArgs,
+      snippetChecker
     )(using compilerContext))
 
+
   val externalDocumentationLinks = args.externalMappings

scaladoc/src/dotty/tools/scaladoc/site/LoadedTemplate.scala

@@ -51,4 +51,4 @@ case class LoadedTemplate(
       ("site" -> (getMap("site") + ("posts" -> posts))) + ("urls" -> sourceLinks.toMap) +
       ("page" -> (getMap("page") + ("title" -> templateFile.title)))
 
-    templateFile.resolveInner(RenderingContext(updatedSettings, ctx.layouts))
+    templateFile.resolveInner(RenderingContext(updatedSettings, ctx.layouts))(using ctx)

scaladoc/src/dotty/tools/scaladoc/site/StaticSiteContext.scala

@@ -13,7 +13,9 @@ import collection.JavaConverters._
 class StaticSiteContext(
   val root: File,
   val args: Scaladoc.Args,
-  val sourceLinks: SourceLinks)(using val outerCtx: CompilerContext):
+  val sourceLinks: SourceLinks,
+  val snippetCompilerArgs: snippets.SnippetCompilerArgs,
+  val snippetChecker: snippets.SnippetChecker)(using val outerCtx: CompilerContext):
 
   var memberLinkResolver: String => Option[DRI] = _ => None
 

scaladoc/src/dotty/tools/scaladoc/site/common.scala

@@ -36,7 +36,8 @@ val defaultMarkdownOptions: DataHolder =
       EmojiExtension.create(),
       YamlFrontMatterExtension.create(),
       StrikethroughExtension.create(),
-      WikiLinkExtension.create()
+      WikiLinkExtension.create(),
+      tasty.comments.markdown.SnippetRenderingExtension
     ))
 
 def emptyTemplate(file: File, title: String): TemplateFile = TemplateFile(

scaladoc/src/dotty/tools/scaladoc/site/templates.scala

@@ -2,7 +2,7 @@ package dotty.tools.scaladoc
 package site
 
 import java.io.File
-import java.nio.file.Files
+import java.nio.file.{Files, Paths}
 
 import com.vladsch.flexmark.ext.anchorlink.AnchorLinkExtension
 import com.vladsch.flexmark.ext.autolink.AutolinkExtension
@@ -18,6 +18,7 @@ import liqp.Template
 import scala.collection.JavaConverters._
 
 import scala.io.Source
+import dotty.tools.scaladoc.snippets._
 
 case class RenderingContext(
   properties: Map[String, Object],
@@ -55,7 +56,22 @@ case class TemplateFile(
 ):
   def isIndexPage() = file.isFile && (file.getName == "index.md" || file.getName == "index.html")
 
-  private[site] def resolveInner(ctx: RenderingContext): ResolvedPage =
+  private[site] def resolveInner(ctx: RenderingContext)(using ssctx: StaticSiteContext): ResolvedPage =
+
+    lazy val snippetCheckingFunc: SnippetChecker.SnippetCheckingFunc =
+      val path = Some(Paths.get(file.getAbsolutePath))
+      val pathBasedArg = ssctx.snippetCompilerArgs.get(path)
+      (str: String, lineOffset: SnippetChecker.LineOffset, argOverride: Option[SCFlags]) => {
+          val arg = argOverride.fold(pathBasedArg)(pathBasedArg.overrideFlag(_))
+          ssctx.snippetChecker.checkSnippet(str, None, arg, lineOffset).collect {
+              case r: SnippetCompilationResult if !r.isSuccessful =>
+                val msg = s"In static site (${file.getAbsolutePath}):\n${r.getSummary}"
+                report.error(msg)(using ssctx.outerCtx)
+                r
+              case r => r
+          }
+      }
+
     if (ctx.resolving.contains(file.getAbsolutePath))
       throw new RuntimeException(s"Cycle in templates involving $file: ${ctx.resolving}")
 
@@ -74,9 +90,13 @@ case class TemplateFile(
     val rendered = Template.parse(this.rawCode).render(mutableProperties)
     // We want to render markdown only if next template is html
     val code = if (isHtml || layoutTemplate.exists(!_.isHtml)) rendered else
+      // Snippet compiler currently supports markdown only
       val parser: Parser = Parser.builder(defaultMarkdownOptions).build()
-      HtmlRenderer.builder(defaultMarkdownOptions).build().render(parser.parse(rendered))
+      val parsedMd = parser.parse(rendered)
+      val processed = FlexmarkSnippetProcessor.processSnippets(parsedMd, ssctx.snippetCompilerArgs.debug, snippetCheckingFunc)
+      HtmlRenderer.builder(defaultMarkdownOptions).build().render(processed)
+
     layoutTemplate match
       case None => ResolvedPage(code, resources ++ ctx.resources)
       case Some(layoutTemplate) =>
-        layoutTemplate.resolveInner(ctx.nest(code, file, resources))
+        layoutTemplate.resolveInner(ctx.nest(code, file, resources))

scaladoc/src/dotty/tools/scaladoc/snippets/FlexmarkSnippetProcessor.scala

@@ -0,0 +1,47 @@
+package dotty.tools.scaladoc
+package snippets
+
+import com.vladsch.flexmark.util.{ast => mdu, sequence}
+import com.vladsch.flexmark.{ast => mda}
+import com.vladsch.flexmark.formatter.Formatter
+import com.vladsch.flexmark.util.options.MutableDataSet
+import collection.JavaConverters._
+
+import dotty.tools.scaladoc.tasty.comments.markdown.ExtendedFencedCodeBlock
+
+object FlexmarkSnippetProcessor:
+  def processSnippets(root: mdu.Node, debug: Boolean, checkingFunc: => SnippetChecker.SnippetCheckingFunc): mdu.Node = {
+    lazy val cf: SnippetChecker.SnippetCheckingFunc = checkingFunc
+
+    val nodes = root.getDescendants().asScala.collect {
+      case fcb: mda.FencedCodeBlock => fcb
+    }.toList
+
+    nodes.foreach { node =>
+      val snippet = node.getContentChars.toString
+      val lineOffset = node.getStartLineNumber
+      val info = node.getInfo.toString
+      val argOverride =
+        info.split(" ")
+          .find(_.startsWith("sc:"))
+          .map(_.stripPrefix("sc:"))
+          .map(SCFlagsParser.parse)
+          .flatMap(_.toOption)
+      val snippetCompilationResult = cf(snippet, lineOffset, argOverride) match {
+        case result@Some(SnippetCompilationResult(wrapped, _, _, _)) if debug =>
+          val s = sequence.BasedSequence.EmptyBasedSequence()
+            .append(wrapped)
+            .append(sequence.BasedSequence.EOL)
+          val content = mdu.BlockContent()
+          content.add(s, 0)
+          node.setContent(content)
+          result
+        case result => result
+      }
+
+      node.insertBefore(new ExtendedFencedCodeBlock(node, snippetCompilationResult))
+      node.unlink()
+    }
+
+    root
+  }

scaladoc/src/dotty/tools/scaladoc/snippets/SnippetChecker.scala

@@ -3,12 +3,16 @@ package snippets
 
 import dotty.tools.scaladoc.DocContext
 import java.nio.file.Paths
+import java.io.File
 
-class SnippetChecker()(using ctx: DocContext):
+class SnippetChecker(val classpath: String, val tastyDirs: Seq[File]):
   private val sep = System.getProperty("path.separator")
-  private val cp = System.getProperty("java.class.path") + sep +
-      Paths.get(ctx.args.classpath).toAbsolutePath + sep +
-      ctx.args.tastyDirs.map(_.getAbsolutePath()).mkString(sep)
+  private val cp = List(
+    System.getProperty("java.class.path"),
+    Paths.get(classpath).toAbsolutePath,
+    tastyDirs.map(_.getAbsolutePath()).mkString(sep)
+  ).mkString(sep)
+
   private val compiler: SnippetCompiler = SnippetCompiler(classpath = cp)
 
   def checkSnippet(

scaladoc/src/dotty/tools/scaladoc/snippets/SnippetCompiler.scala

@@ -42,13 +42,16 @@ class SnippetCompiler(
   private def nullableMessage(msgOrNull: String): String =
     if (msgOrNull == null) "" else msgOrNull
 
-  private def createReportMessage(diagnostics: Seq[Diagnostic], line: Int, column: Int): Seq[SnippetCompilerMessage] = {
+  private def createReportMessage(wrappedSnippet: WrappedSnippet, arg: SnippetCompilerArg, diagnostics: Seq[Diagnostic]): Seq[SnippetCompilerMessage] = {
+    val line = wrappedSnippet.lineOffset
+    val column = wrappedSnippet.columnOffset
+    val lineBoilerplate = wrappedSnippet.lineBoilerplate
     val infos = diagnostics.toSeq.sortBy(_.pos.source.path)
     val errorMessages = infos.map {
       case diagnostic if diagnostic.position.isPresent =>
         val diagPos = diagnostic.position.get
         val pos = Some(
-          Position(diagPos.line + line, diagPos.column + column, diagPos.lineContent, diagPos.line)
+          Position(diagPos.line + line, diagPos.column + column, diagPos.lineContent, if arg.debug then diagPos.line else diagPos.line - lineBoilerplate)
         )
         val msg = nullableMessage(diagnostic.message)
         val level = MessageLevel.fromOrdinal(diagnostic.level)
@@ -85,7 +88,7 @@ class SnippetCompiler(
     run.compileFromStrings(List(wrappedSnippet.snippet))
 
     val messages =
-      createReportMessage(context.reporter.pendingMessages(using context), wrappedSnippet.lineOffset, wrappedSnippet.columnOffset) ++
+      createReportMessage(wrappedSnippet, arg, context.reporter.pendingMessages(using context)) ++
       additionalMessages(wrappedSnippet, arg, context)
 
     val t = Option.when(!context.reporter.hasErrors)(target)