Skip to content

Commit eb72ec3

Browse files
BarkingBadBarkingBad
committed
Add preprocessing of markdown files for docs.scala-lang
1 parent 01c70a2 commit eb72ec3

21 files changed

+431
-166
lines changed

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,7 @@ compiler/test-coursier/run/*.jar
105105

106106
# Docs
107107
docs-for-dotty-page/*
108+
109+
# docs.scala-lang deplyment temp dir
110+
docsScalaLang/
111+
docs/_site/

‎docs/docsScalaLangResources/scaladoc-assets.html

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<!--- This file is copied from dotty repo located at the github lampepefl/dotty inside docs/docsScalaLangResources/scaladoc-assets.html -->
2+
<script src="{{ site.baseurl }}/scripts/scaladoc-scalajs.js" type="text/javascript"></script>
3+
<link rel="stylesheet" href="{{ site.baseurl }}/resources/css/colors.css" type="text/css" />
4+
<link rel="stylesheet" href="{{ site.baseurl }}/resources/css/code-snippets.css" type="text/css" />
5+
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.9.0/css/all.min.css">

‎project/Build.scala

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import sbtbuildinfo.BuildInfoPlugin.autoImport._
2929
import scala.util.Properties.isJavaAtLeast
3030

3131
import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._
32+
import org.scalajs.linker.interface.ModuleInitializer
3233

3334
object DottyJSPlugin extends AutoPlugin {
3435
import Build._
@@ -1248,6 +1249,7 @@ object Build {
12481249
// Note: the two tasks below should be one, but a bug in Tasty prevents that
12491250
val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
12501251
val generateTestcasesDocumentation = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
1252+
val renderScaladocScalajsToFile = inputKey[Unit]("Copy the output of the scaladoc js files")
12511253

12521254
lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")).
12531255
dependsOn(`scala3-compiler-bootstrapped`).
@@ -1256,8 +1258,12 @@ object Build {
12561258
enablePlugins(DottyJSPlugin).
12571259
dependsOn(`scala3-library-bootstrappedJS`).
12581260
settings(
1261+
Compile / scalaJSMainModuleInitializer := (sys.env.get("scaladoc.projectFormat") match {
1262+
case Some("md") => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "markdownMain"))
1263+
case _ => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "main"))
1264+
}),
12591265
Test / fork := false,
1260-
scalaJSUseMainModuleInitializer := true,
1266+
Compile / scalaJSUseMainModuleInitializer := true,
12611267
libraryDependencies += ("org.scala-js" %%% "scalajs-dom" % "1.1.0").cross(CrossVersion.for3Use2_13)
12621268
)
12631269

@@ -1311,7 +1317,7 @@ object Build {
13111317
).
13121318
settings(
13131319
Compile / resourceGenerators += Def.task {
1314-
val jsDestinationFile = (Compile / resourceManaged).value / "dotty_res" / "scripts" / "searchbar.js"
1320+
val jsDestinationFile = (Compile / resourceManaged).value / "dotty_res" / "scripts" / "scaladoc-scalajs.js"
13151321
sbt.IO.copyFile((`scaladoc-js` / Compile / fullOptJS).value.data, jsDestinationFile)
13161322
Seq(jsDestinationFile)
13171323
}.taskValue,
@@ -1441,6 +1447,23 @@ object Build {
14411447
)
14421448
}.value,
14431449

1450+
renderScaladocScalajsToFile := Def.inputTask {
1451+
val extraArgs = spaceDelimited("<arg>").parsed
1452+
val (destJS, destCSS, csses) = extraArgs match {
1453+
case js :: css :: tail => (js, css, tail)
1454+
case js :: Nil => (js, "", Nil)
1455+
case _ => throw new IllegalArgumentException("No js destination provided")
1456+
}
1457+
val jsDestinationFile: File = Paths.get(destJS).toFile
1458+
sbt.IO.copyFile((`scaladoc-js` / Compile / fullOptJS).value.data, jsDestinationFile)
1459+
csses.map { file =>
1460+
val cssDesitnationFile = Paths.get(destCSS).toFile / file
1461+
val cssSourceFile = (`scaladoc-js` / Compile / resourceDirectory).value / file
1462+
sbt.IO.copyFile(cssSourceFile, cssDesitnationFile)
1463+
cssDesitnationFile
1464+
}
1465+
}.evaluated,
1466+
14441467
Test / buildInfoKeys := Seq[BuildInfoKey](
14451468
(Test / Build.testcasesOutputDir),
14461469
(Test / Build.testcasesSourceRoot),

‎project/scripts/genDocsScalaLang

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
shopt -s extglob # needed for rm everything but x
5+
echo "Working directory: $PWD"
6+
7+
GENDOC_EXTRA_ARGS=$@
8+
GIT_HEAD=$(git rev-parse HEAD) # save current head for commit message in gh-pages
9+
PREVIOUS_SNAPSHOTS_DIR="$PWD/../prev_snapshots"
10+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)"
11+
SITE_OUT_DIR="$PWD/docs/_site"
12+
13+
DOCS_SCALA_LANG_DIR="$PWD/docsScalaLang"
14+
15+
rm -rf $DOCS_SCALA_LANG_DIR
16+
mkdir -pv $DOCS_SCALA_LANG_DIR
17+
git clone "https://github.com/scala/docs.scala-lang.git" $DOCS_SCALA_LANG_DIR
18+
19+
SBT="$SCRIPT_DIR/sbt"
20+
mkdir -pv $SITE_OUT_DIR
21+
env "scaladoc.projectFormat=md" "$SBT" "scaladoc/renderScaladocScalajsToFile $DOCS_SCALA_LANG_DIR/scripts/scaladoc-scalajs.js $DOCS_SCALA_LANG_DIR/resources/css code-snippets.css"
22+
"dist/target/pack/bin/scaladoc" "-d" "$SITE_OUT_DIR" "-format" "md" "-siteroot" "docs" "/dev/null"
23+
24+
if [ ! -d "$SITE_OUT_DIR" ]; then
25+
echo "Output directory did not exist: $SITE_OUT_DIR" 1>&2
26+
exit 1
27+
fi
28+
29+
# Copy reference and scaladoc docs
30+
cp -rf "$SITE_OUT_DIR/docs/reference"/* "$DOCS_SCALA_LANG_DIR/_scala3-reference"
31+
cp -rf "$SITE_OUT_DIR/docs/usage/scaladoc"/* "$DOCS_SCALA_LANG_DIR/_overviews/scala3-scaladoc"
32+
33+
cp -rf "$SITE_OUT_DIR/docs/reference/contextual/motivation.md" "$DOCS_SCALA_LANG_DIR/_scala3-reference/contextual.md"
34+
cp -rf "$SITE_OUT_DIR/docs/reference/metaprogramming/toc.md" "$DOCS_SCALA_LANG_DIR/_scala3-reference/metaprogramming.md"
35+
cp -rf "$SITE_OUT_DIR/docs/resources/talks.md" "$DOCS_SCALA_LANG_DIR/scala3/talks.md"
36+
cp -rf "$SITE_OUT_DIR/docs/usage/getting-started.md" "$DOCS_SCALA_LANG_DIR/scala3/getting-started.md"
37+
cp -rf "$SITE_OUT_DIR/docs/usage/language-versions.md" "$DOCS_SCALA_LANG_DIR/_scala3-reference/language-versions.md"
38+
cp -rf "$SITE_OUT_DIR/docs/usage/worksheet-mode.md" "$DOCS_SCALA_LANG_DIR/_overviews/scala3-book/tools-worksheets.md"
39+
40+
41+
# Copy csses and html importing these assets
42+
cp -f "$SITE_OUT_DIR/styles/colors.css" "$DOCS_SCALA_LANG_DIR/resources/css/colors.css"
43+
cp -f "$PWD/docs/docsScalaLangResources/scaladoc-assets.html" "$DOCS_SCALA_LANG_DIR/_includes/scaladoc-assets.html"
44+
45+
# Hack inclusion of these assests by the docs.scala-lang jekyll builder
46+
echo "{% include scaladoc-assets.html %}" >> "$DOCS_SCALA_LANG_DIR/_layouts/inner-page-parent-dropdown.html"

‎scaladoc-js/src/Main.scala

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,22 @@
11
package dotty.tools.scaladoc
22

3-
object Main extends App {
4-
Searchbar()
5-
SocialLinks()
6-
CodeSnippets()
7-
DropdownHandler()
8-
Ux()
9-
}
3+
object Main:
4+
5+
private def common(): Unit =
6+
CodeSnippets()
7+
8+
def main(): Unit =
9+
Searchbar()
10+
SocialLinks()
11+
DropdownHandler()
12+
Ux()
13+
common()
14+
15+
/**
16+
* This main is conditionally enabled by system env variable `scaladoc.projectFormat=md`
17+
* passed in ./projects/scripts/genDocsScalaLang
18+
* The reason why we have to pass the condition by env variable is because js is build before scaladoc,
19+
* so we cannot access its args
20+
*/
21+
def markdownMain(): Unit =
22+
common()

‎scaladoc/src/dotty/tools/scaladoc/Scaladoc.scala

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ object Scaladoc:
5959
versionsDictionaryUrl: Option[String] = None,
6060
generateInkuire : Boolean = false,
6161
apiSubdirectory : Boolean = false,
62-
scastieConfiguration: String = ""
62+
scastieConfiguration: String = "",
63+
projectFormat: String = "html",
6364
)
6465

6566
def run(args: Array[String], rootContext: CompilerContext): Reporter =
@@ -225,7 +226,8 @@ object Scaladoc:
225226
versionsDictionaryUrl.nonDefault,
226227
generateInkuire.get,
227228
apiSubdirectory.get,
228-
scastieConfiguration.get
229+
scastieConfiguration.get,
230+
projectFormat.get,
229231
)
230232
(Some(docArgs), newContext)
231233
}
@@ -234,6 +236,10 @@ object Scaladoc:
234236
given docContext: DocContext = new DocContext(args, ctx)
235237
val module = ScalaModuleProvider.mkModule()
236238

237-
new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members).render()
239+
val renderer = args.projectFormat match
240+
case "html" => new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members)
241+
case "md" => new dotty.tools.scaladoc.renderers.MarkdownRenderer(module.rootPackage, module.members)
242+
243+
renderer.render()
238244
report.inform("generation completed successfully")
239245
docContext

‎scaladoc/src/dotty/tools/scaladoc/ScaladocSettings.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,16 @@ class ScaladocSettings extends SettingGroup with AllScalaSettings:
122122

123123
val scastieConfiguration: Setting[String] =
124124
StringSetting("-scastie-configuration", "Scastie configuration", "Additional configuration passed to Scastie in code snippets", "")
125+
126+
val projectFormat: Setting[String] =
127+
ChoiceSetting(
128+
"-format",
129+
"format of the static site output",
130+
"Format of the static site output. The default value is html, which converts all static articles into a webpage. " +
131+
"The md format only preprocess markdown files and should not be used as a direct output, but rather as a sources generator for an outer templating engine like Jekyll",
132+
List("html", "md"),
133+
"html"
134+
)
125135

126136
def scaladocSpecificSettings: Set[Setting[_]] =
127137
Set(sourceLinks, syntax, revision, externalDocumentationMappings, socialLinks, skipById, skipByRegex, deprecatedSkipPackages, docRootContent, snippetCompiler, generateInkuire, scastieConfiguration)

‎scaladoc/src/dotty/tools/scaladoc/renderers/HtmlRenderer.scala

Lines changed: 9 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -14,120 +14,21 @@ import java.nio.file.Files
1414
import java.nio.file.FileVisitOption
1515
import java.io.File
1616

17-
case class Page(link: Link, content: Member | ResolvedTemplate | String, children: Seq[Page]):
18-
def withNewChildren(newChildren: Seq[Page]) = copy(children = children ++ newChildren)
17+
class HtmlRenderer(rootPackage: Member, members: Map[DRI, Member])(using ctx: DocContext)
18+
extends Renderer(rootPackage, members, extension = "html"):
1919

20-
def withTitle(newTitle: String) = copy(link = link.copy(name = newTitle))
21-
22-
def hasFrame = content match
23-
case t: ResolvedTemplate => t.hasFrame
24-
case _ => true
25-
26-
class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx: DocContext)
27-
extends SiteRenderer, Resources, Locations, Writer:
28-
private val args = summon[DocContext].args
29-
val staticSite = summon[DocContext].staticSiteContext
30-
31-
val effectiveMembers = members
32-
33-
private def memberPage(member: Member): Page =
34-
val childrenPages = member.members.filter(_.needsOwnPage)
35-
Page(Link(member.name, member.dri), member, childrenPages.map(memberPage))
36-
37-
val navigablePage: Page =
38-
val rootPckPage = memberPage(rootPackage)
39-
staticSite match
40-
case None => rootPckPage.withTitle(args.name)
41-
case Some(siteContext) =>
42-
val (indexes, templates) = siteContext.templates.partition(f =>
43-
f.templateFile.isIndexPage() && f.file.toPath.getParent() == siteContext.docsPath)
44-
if (indexes.size > 1)
45-
val msg = s"ERROR: Multiple index pages for doc found ${indexes.map(_.file)}"
46-
report.error(msg)
47-
48-
val templatePages = templates.map(templateToPage(_, siteContext))
49-
50-
indexes.headOption match
51-
case None if templatePages.isEmpty=>
52-
rootPckPage.withTitle(args.name)
53-
case None =>
54-
Page(Link(args.name, docsRootDRI),"", templatePages :+ rootPckPage.withTitle("API"))
55-
case Some(indexPage) =>
56-
val newChildren = templatePages :+ rootPckPage.withTitle("API")
57-
templateToPage(indexPage, siteContext).withNewChildren(newChildren)
58-
59-
val hiddenPages: Seq[Page] =
60-
staticSite match
61-
case None =>
62-
Seq(navigablePage.copy( // Add index page that is a copy of api/index.html
63-
link = navigablePage.link.copy(dri = docsRootDRI),
64-
children = Nil
65-
))
66-
case Some(siteContext) =>
67-
// In case that we do not have an index page and we do not have any API entries
68-
// we want to create empty index page, so there is one
69-
val actualIndexTemplate = siteContext.indexTemplate() match {
70-
case None if effectiveMembers.isEmpty => Seq(siteContext.emptyIndexTemplate)
71-
case templates => templates.toSeq
72-
}
73-
74-
(siteContext.orphanedTemplates ++ actualIndexTemplate).map(templateToPage(_, siteContext))
75-
76-
/**
77-
* Here we have to retrive index pages from hidden pages and replace fake index pages in navigable page tree.
78-
*/
79-
val allPages: Seq[Page] =
80-
def traversePages(page: Page): (Page, Seq[Page]) =
81-
val (newChildren, newPagesToRemove): (Seq[Page], Seq[Page]) = page.children.map(traversePages(_)).foldLeft((Seq[Page](), Seq[Page]())) {
82-
case ((pAcc, ptrAcc), (p, ptr)) => (pAcc :+ p, ptrAcc ++ ptr)
83-
}
84-
hiddenPages.find(_.link == page.link) match
85-
case None =>
86-
(page.copy(children = newChildren), newPagesToRemove)
87-
case Some(newPage) =>
88-
(newPage.copy(children = newChildren), newPagesToRemove :+ newPage)
89-
90-
val (newNavigablePage, pagesToRemove) = traversePages(navigablePage)
91-
92-
val all = newNavigablePage +: hiddenPages.filterNot(pagesToRemove.contains)
93-
// We need to check for conflicts only if we have top-level member called blog or docs
94-
val hasPotentialConflict =
95-
rootPackage.members.exists(m => m.name.startsWith("docs") || m.name.startsWith("blog"))
96-
97-
if hasPotentialConflict then
98-
def walk(page: Page): Unit =
99-
if page.link.dri.isStaticFile then
100-
val dest = absolutePath(page.link.dri)
101-
if apiPaths.contains(dest) then
102-
report.error(s"Conflict between static page and API member for $dest. $pathsConflictResoultionMsg")
103-
page.children.foreach(walk)
104-
105-
all.foreach (walk)
106-
107-
all
108-
109-
def renderContent(page: Page) = page.content match
110-
case m: Member =>
111-
val signatureRenderer = new SignatureRenderer:
112-
def currentDri: DRI = page.link.dri
113-
def link(dri: DRI): Option[String] =
114-
Some(pathToPage(currentDri, dri)).filter(_ != UnresolvedLocationLink)
115-
116-
MemberRenderer(signatureRenderer).fullMember(m)
117-
case t: ResolvedTemplate => siteContent(page.link.dri, t)
118-
case a: String => raw(a)
119-
120-
121-
def renderPage(page: Page, parents: Vector[Link]): Seq[String] =
122-
val newParents = parents :+ page.link
123-
val content = html(
20+
override def pageContent(page: Page, parents: Vector[Link]): AppliedTag =
21+
html(
12422
mkHead(page),
12523
body(
12624
if !page.hasFrame then renderContent(page)
127-
else mkFrame(page.link, newParents, renderContent(page))
25+
else mkFrame(page.link, parents, renderContent(page))
12826
)
12927
)
130-
write(page.link.dri, content) +: page.children.flatMap(renderPage(_, newParents))
28+
29+
override def render(): Unit =
30+
val renderedResources = renderResources()
31+
super.render()
13132

13233
private def specificResources(page: Page): Set[String] =
13334
page.children.toSet.flatMap(specificResources) ++ (page.content match
@@ -157,10 +58,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
15758
val resources = siteResourcesPaths.toSeq.map(pathToResource) ++ allResources(allPages) ++ onlyRenderedResources
15859
resources.flatMap(renderResource)
15960

160-
def render(): Unit =
161-
val renderedResources = renderResources()
162-
val sites = allPages.map(renderPage(_, Vector.empty))
163-
16461
def mkHead(page: Page): AppliedTag =
16562
val resources = page.content match
16663
case t: ResolvedTemplate =>
@@ -229,14 +126,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
229126

230127
renderNested(navigablePage, toplevel = true)._2
231128

232-
private def canonicalUrl(l: String): AppliedTag | String =
233-
val canon = args.docCanonicalBaseUrl
234-
if !canon.isEmpty then
235-
val canonicalUrl = if canon.endsWith("/") then canon else canon + "/"
236-
link(rel := "canonical", href := canonicalUrl + l)
237-
else
238-
"" // return empty tag
239-
240129
private def hasSocialLinks = !args.socialLinks.isEmpty
241130

242131
private def socialLinks(whiteIcon: Boolean = true) =

‎scaladoc/src/dotty/tools/scaladoc/renderers/Locations.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ trait Locations(using ctx: DocContext):
2121

2222
// We generate this collection only if there may be a conflict with resources.
2323
// Potentially can be quite big.
24-
lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath)
24+
lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath(_))
2525

2626
var cache = new JHashMap[DRI, Seq[String]]()
2727

@@ -80,7 +80,7 @@ trait Locations(using ctx: DocContext):
8080
pathToRaw(from, to.split("/").toList)
8181

8282
def resolveRoot(dri: DRI, path: String): String = resolveRoot(rawLocation(dri), path)
83-
def absolutePath(dri: DRI): String = rawLocation(dri).mkString("", "/", ".html")
83+
def absolutePath(dri: DRI, extension: String = "html"): String = rawLocation(dri).mkString("", "/", s".$extension")
8484

8585
def resolveLink(dri: DRI, url: String): String =
8686
if URI(url).isAbsolute then url else resolveRoot(dri, url)

0 commit comments

Comments
 (0)