Markdown preprocessor for static site

Author: BarkingBad

project/Build.scala

@@ -329,6 +329,7 @@ object Build {
         ".*java.*::javadoc::https://docs.oracle.com/javase/8/docs/api/",
       "-skip-by-regex:.+\\.internal($|\\..+)",
       "-skip-by-regex:.+\\.impl($|\\..+)",
+      "-format", "md",
       "-project-logo", "docs/logo.svg",
       "-social-links:" +
         "github::https://github.com/lampepfl/dotty," +

scaladoc/src/dotty/tools/scaladoc/Scaladoc.scala

@@ -58,7 +58,8 @@ object Scaladoc:
     snippetCompilerDebug: Boolean = false,
     noLinkWarnings: Boolean = false,
     versionsDictionaryUrl: Option[String] = None,
-    generateInkuire : Boolean = false
+    generateInkuire : Boolean = false,
+    projectFormat: String = "html",
   )
 
   def run(args: Array[String], rootContext: CompilerContext): Reporter =
@@ -223,7 +224,8 @@ object Scaladoc:
         noLinkWarnings.get,
         snippetCompilerDebug.get,
         versionsDictionaryUrl.nonDefault,
-        generateInkuire.get
+        generateInkuire.get,
+        projectFormat.get
       )
       (Some(docArgs), newContext)
     }

scaladoc/src/dotty/tools/scaladoc/ScaladocSettings.scala

@@ -120,5 +120,9 @@ class ScaladocSettings extends SettingGroup with AllScalaSettings:
   val generateInkuire: Setting[Boolean] =
     BooleanSetting("-Ygenerate-inkuire", "Generates InkuireDB and enables Hoogle-like searches", false)
 
+  val projectFormat: Setting[String] =
+    StringSetting("-format", "format of the static site output", "Format of the static site output", "html")
+
+
   def scaladocSpecificSettings: Set[Setting[_]] =
     Set(sourceLinks, syntax, revision, externalDocumentationMappings, socialLinks, skipById, skipByRegex, deprecatedSkipPackages, docRootContent, snippetCompiler, snippetCompilerDebug, generateInkuire)

scaladoc/src/dotty/tools/scaladoc/renderers/HtmlRenderer.scala

@@ -101,7 +101,15 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
 
   def renderPage(page: Page, parents: Vector[Link]): Seq[String] =
     val newParents = parents :+ page.link
-    val content = renderContent(page)
+    val content = ctx.args.projectFormat match
+      case "html" => html(
+        mkHead(page),
+        body(
+          if !page.hasFrame then renderContent(page)
+          else mkFrame(page.link, newParents, renderContent(page))
+        )
+      )
+      case "md" => renderContent(page)
     write(page.link.dri, content) +: page.children.flatMap(renderPage(_, newParents))
 
   private def specificResources(page: Page): Set[String] =

scaladoc/src/dotty/tools/scaladoc/renderers/SiteRenderer.scala

@@ -56,11 +56,24 @@ trait SiteRenderer(using DocContext) extends Locations:
       else
         processLocalLink(str)
 
-    val document = Jsoup.parse(content.resolved.code)
-    document.select("a").forEach(element =>
-      element.attr("href", processLocalLinkWithGuard(element.attr("href")))
-    )
-    document.select("img").forEach { element =>
-      element.attr("src", processLocalLink(element.attr("src")))
-    } // foreach does not work here. Why?
-    raw(document.outerHtml())
+    summon[DocContext].args.projectFormat match
+      case "html" =>
+        val document = Jsoup.parse(content.resolved.code)
+        document.select("a").forEach(element =>
+          element.attr("href", processLocalLinkWithGuard(element.attr("href")))
+        )
+        document.select("img").forEach { element =>
+          element.attr("src", processLocalLink(element.attr("src")))
+        } // foreach does not work here. Why?
+        raw(document.outerHtml())
+      case "md" =>
+        val links = """(?<!\\!)\[(.*)\\]\\((.*)\\)""".r
+        val imgs = """!\\[(.*)\\]\\((.*)\\)""".r
+
+        raw(links.replaceAllIn(
+          imgs.replaceAllIn(
+            content.resolved.code,
+            m => s"[${m.group(1)}](${processLocalLink(m.group(2))})"
+          ),
+          m => s"[${m.group(1)}](${processLocalLinkWithGuard(m.group(2))})"
+        ))

scaladoc/src/dotty/tools/scaladoc/site/StaticSiteContext.scala

@@ -34,7 +34,7 @@ class StaticSiteContext(
   lazy val layouts: Map[String, TemplateFile] =
     val layoutRoot = new File(root, "_layouts")
     val dirs: Array[File] = Option(layoutRoot.listFiles()).getOrElse(Array())
-    dirs.map { it => loadTemplateFile(it) }.map { it => it.name -> it }.toMap
+    dirs.map { it => loadTemplateFile(it)(using this) }.map { it => it.name -> it }.toMap
 
   lazy val sideBarConfig =
     val sidebarFile = root.toPath.resolve("sidebar.yml")
@@ -90,7 +90,7 @@ class StaticSiteContext(
               val msg = s"ERROR: Multiple index pages for $from found in ${indexes.map(_.file)}"
               throw new java.lang.RuntimeException(msg)
 
-        val templateFile = if (from.isDirectory) loadIndexPage() else loadTemplateFile(from)
+        val templateFile = if (from.isDirectory) loadIndexPage() else loadTemplateFile(from)(using this)
 
         def dateFrom(p: LoadedTemplate, default: String = "1900-01-01"): String =
           val pageSettings = p.templateFile.settings.get("page").collect{ case m: Map[String @unchecked, _] => m }

scaladoc/src/dotty/tools/scaladoc/site/common.scala

@@ -21,7 +21,7 @@ import scala.collection.JavaConverters._
 val docsRootDRI: DRI = DRI(location = "docs/index", symbolUUID = staticFileSymbolUUID)
 val apiPageDRI: DRI = DRI(location = "api/index")
 
-val defaultMarkdownOptions: DataHolder =
+def defaultMarkdownOptions(using ctx: StaticSiteContext): DataHolder =
   new MutableDataSet()
     .setFrom(ParserEmulationProfile.COMMONMARK.getOptions)
     .set(AnchorLinkExtension.ANCHORLINKS_WRAP_TEXT, false)
@@ -37,7 +37,10 @@ val defaultMarkdownOptions: DataHolder =
       YamlFrontMatterExtension.create(),
       StrikethroughExtension.create(),
       WikiLinkExtension.create(),
-      tasty.comments.markdown.SnippetFormattingExtension,
+      (ctx.args.projectFormat match
+        case "html" => tasty.comments.markdown.SnippetRenderingExtension
+        case "md" => tasty.comments.markdown.SnippetFormattingExtension
+      ),
     ))
 
 def emptyTemplate(file: File, title: String): TemplateFile = TemplateFile(
@@ -56,9 +59,9 @@ def emptyTemplate(file: File, title: String): TemplateFile = TemplateFile(
 final val ConfigSeparator = "---"
 final val LineSeparator = "\n"
 
-val yamlParser: Parser = Parser.builder(defaultMarkdownOptions).build()
+def yamlParser(using ctx: StaticSiteContext): Parser = Parser.builder(defaultMarkdownOptions).build()
 
-def loadTemplateFile(file: File): TemplateFile = {
+def loadTemplateFile(file: File)(using ctx: StaticSiteContext): TemplateFile = {
   val lines = Files.readAllLines(file.toPath).asScala.toList
 
   val (config, content) = if (lines.head == ConfigSeparator) {

scaladoc/src/dotty/tools/scaladoc/site/templates.scala

@@ -109,11 +109,12 @@ case class TemplateFile(
       val parsedMd = parser.parse(rendered)
       val processed = FlexmarkSnippetProcessor.processSnippets(parsedMd, ssctx.snippetCompilerArgs.debug, snippetCheckingFunc)(using ssctx.outerCtx)
 
-      Formatter.builder(defaultMarkdownOptions).build().render(processed)
+      ssctx.args.projectFormat match
+        case "html" => HtmlRenderer.builder(defaultMarkdownOptions).build().render(processed)
+        case "md" => Formatter.builder(defaultMarkdownOptions).build().render(processed)
 
-      // HtmlRenderer.builder(defaultMarkdownOptions).build().render(processed)
 
-    layoutTemplate match
-      case None => ResolvedPage(code, resources ++ ctx.resources)
-      case Some(layoutTemplate) =>
-        layoutTemplate.resolveInner(ctx.nest(code, file, resources))
+    if layoutTemplate.isEmpty || ssctx.args.projectFormat == "md" then
+      ResolvedPage(code, resources ++ ctx.resources)
+    else
+      layoutTemplate.get.resolveInner(ctx.nest(code, file, resources))