From 5062997cbbab1fce701267314cc696a4d4bd1efb Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Sun, 21 Mar 2021 17:53:21 +0100 Subject: [PATCH] Scaladoc: add Wiki code blocks to Md syntax The idea here is that in the wild, the parts of Wiki syntax that are the most used are lists, code blocks, monospace text, and links. With this PR we now more-or-less support all of them in Markdown syntax, which means that one will be able to use it for old doc comments and get acceptable-looking output. --- scaladoc-testcases/src/tests/tests.scala | 15 ++ .../parsers/WikiCodeBlockParser.scala | 155 ++++++++++++++++++ .../tasty/comments/MarkdownParser.scala | 13 +- 3 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 scaladoc/src/dotty/tools/scaladoc/parsers/WikiCodeBlockParser.scala diff --git a/scaladoc-testcases/src/tests/tests.scala b/scaladoc-testcases/src/tests/tests.scala index 77f38de225d5..4bcf4ce522e7 100644 --- a/scaladoc-testcases/src/tests/tests.scala +++ b/scaladoc-testcases/src/tests/tests.scala @@ -20,7 +20,22 @@ package tests * ```scala * is.an("actual code block") * with.multiple("lines") + * """ + * {{{ + * and an embedded Wiki code block for good measure + * }}} + * """ + * ``` + * + * While + * {{{ + * this.is("a Wiki code block") + * """ * ``` + * with an embedded Md code block for good measure + * ``` + * """ + * }}} * * And this * diff --git a/scaladoc/src/dotty/tools/scaladoc/parsers/WikiCodeBlockParser.scala b/scaladoc/src/dotty/tools/scaladoc/parsers/WikiCodeBlockParser.scala new file mode 100644 index 000000000000..865d78193886 --- /dev/null +++ b/scaladoc/src/dotty/tools/scaladoc/parsers/WikiCodeBlockParser.scala @@ -0,0 +1,155 @@ +package dotty.tools.scaladoc.parsers + +import com.vladsch.flexmark.ast._ +import com.vladsch.flexmark.parser.Parser +import com.vladsch.flexmark.parser.core._ +import com.vladsch.flexmark.parser.block._ +import com.vladsch.flexmark.util.ast.Block +import com.vladsch.flexmark.util.ast.BlockContent +import com.vladsch.flexmark.util.options.DataHolder +import com.vladsch.flexmark.util.sequence.BasedSequence +import com.vladsch.flexmark.util.sequence.SegmentedSequence + +import java.{util => ju} +import ju.regex.Matcher +import ju.regex.Pattern + + +/** Copied from FencedCodeBlockParser. */ +object WikiCodeBlockParser { + private val OPENING_FENCE = Pattern.compile("^\\{{3}") + private val CLOSING_FENCE = Pattern.compile("^(\\}{3})(?=[ \t]*$)$") + + class Factory extends CustomBlockParserFactory { + override def getAfterDependents = + new ju.HashSet[Class[_ <: CustomBlockParserFactory]](ju.Arrays.asList( + classOf[BlockQuoteParser.Factory], + classOf[HeadingParser.Factory], + //FencedCodeBlockParser.Factory.class, + //HtmlBlockParser.Factory.class, + //ThematicBreakParser.Factory.class, + //ListBlockParser.Factory.class, + //IndentedCodeBlockParser.Factory.class + )) + + override def getBeforeDependents = + new ju.HashSet[Class[_ <: CustomBlockParserFactory]](ju.Arrays.asList( + //BlockQuoteParser.Factory.class, + //HeadingParser.Factory.class, + //FencedCodeBlockParser.Factory.class, + classOf[HtmlBlockParser.Factory], + classOf[ThematicBreakParser.Factory], + classOf[ListBlockParser.Factory], + classOf[IndentedCodeBlockParser.Factory], + )) + + override def affectsGlobalScope = false + + override def create(options: DataHolder) = + new WikiCodeBlockParser.BlockFactory(options) + } + + private[WikiCodeBlockParser] class BlockFactory (val options: DataHolder) + extends AbstractBlockParserFactory(options) { + def tryStart(state: ParserState, matchedBlockParser: MatchedBlockParser): BlockStart = { + val nextNonSpace = state.getNextNonSpaceIndex + val line = state.getLine + var matcher: Matcher = null + if state.getIndent < 4 then { + val trySequence = line.subSequence(nextNonSpace, line.length) + matcher = OPENING_FENCE.matcher(trySequence) + if matcher.find then { + val fenceLength = matcher.group(0).length + val blockParser = + new WikiCodeBlockParser(state.getProperties, fenceLength, state.getIndent, nextNonSpace) + blockParser.block.setOpeningMarker(trySequence.subSequence(0, fenceLength)) + return BlockStart.of(blockParser).atIndex(nextNonSpace + fenceLength) + } + } + BlockStart.none + } + } +} + +/** Copied from FencedCodeBlockParser. */ +class WikiCodeBlockParser( + options: DataHolder, + var fenceLength: Int, + private var fenceIndent: Int, + private var fenceMarkerIndent: Int +) extends AbstractBlockParser { + + this.fenceMarkerIndent = fenceIndent + fenceMarkerIndent + + final private val block = new FencedCodeBlock() + private var content = new BlockContent + private val codeContentBlock = options.get(Parser.FENCED_CODE_CONTENT_BLOCK) + + def getBlock: Block = block + def getFenceIndent: Int = fenceIndent + def getFenceMarkerIndent: Int = fenceMarkerIndent + def tryContinue(state: ParserState): BlockContinue = { + val nextNonSpace = state.getNextNonSpaceIndex + var newIndex = state.getIndex + val line = state.getLine + var matcher: Matcher = null + val matches = + state.getIndent <= 3 + && nextNonSpace < line.length + + if matches then { + val trySequence = line.subSequence(nextNonSpace, line.length) + matcher = WikiCodeBlockParser.CLOSING_FENCE.matcher(trySequence) + if matcher.find then { + val foundFenceLength = matcher.group(0).length + if (foundFenceLength >= fenceLength) { // closing fence - we're at end of line, so we can finalize now + block.setClosingMarker(trySequence.subSequence(0, foundFenceLength)) + return BlockContinue.finished + } + } + } + // skip optional spaces of fence indent + var i = fenceIndent + while ({ + i > 0 && newIndex < line.length && line.charAt(newIndex) == ' ' + }) do { + newIndex += 1 + i -= 1 + } + BlockContinue.atIndex(newIndex) + } + + override def addLine(state: ParserState, line: BasedSequence): Unit = { + content.add(line, state.getIndent) + } + + override def isPropagatingLastBlankLine(lastMatchedBlockParser: BlockParser) = false + + override def closeBlock(state: ParserState): Unit = { // first line, if not blank, has the info string + val lines = content.getLines + if (lines.size > 0) { + val info = lines.get(0) + if (!info.isBlank) block.setInfo(info.trim) + val chars = content.getSpanningChars + val spanningChars = chars.baseSubSequence(chars.getStartOffset, lines.get(0).getEndOffset) + if (lines.size > 1) { // have more lines + val segments = lines.subList(1, lines.size) + block.setContent(spanningChars, segments) + if this.codeContentBlock then { + val codeBlock = new CodeBlock() + codeBlock.setContent(segments) + codeBlock.setCharsFromContent + block.appendChild(codeBlock) + } else { + val codeBlock = new Text(SegmentedSequence.of(segments)) + block.appendChild(codeBlock) + } + } + else block.setContent(spanningChars, BasedSequence.EMPTY_LIST) + } + else block.setContent(content) + block.setCharsFromContent + content = null + } +} + diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/MarkdownParser.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/MarkdownParser.scala index 11f8c49eebe8..a3ab2d1a5eca 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/MarkdownParser.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/MarkdownParser.scala @@ -44,14 +44,19 @@ object MarkdownParser { val markdownOptions: DataHolder = mkMarkdownOptions(Seq(WikiLinkExtension.create())) - private val parser = Parser.builder(markdownOptions).build() - val RENDERER = Formatter.builder(markdownOptions).build() def parseToMarkdown(text: String, extensions: Extension*): mdu.Document = + val options = + if extensions.isEmpty then + markdownOptions + else + mkMarkdownOptions(extensions) + val thisParser = - if(extensions.isEmpty) parser - else Parser.builder(mkMarkdownOptions(extensions)).build() + Parser.builder(options) + .customBlockParserFactory(parsers.WikiCodeBlockParser.Factory()) + .build() // We need to remove safe tag markers as they break flexmark parsing thisParser.parse(text.replace(safeTagMarker.toString, "")).asInstanceOf[mdu.Document]