feat(core): add option to merge consecutive tokens with same style (#972) (#991)

Co-authored-by: Anthony Fu <[email protected]>

Author: yvbopeng

packages/core/src/highlight/code-to-hast.ts

@@ -47,13 +47,18 @@ export function codeToHast(
 
   const {
     mergeWhitespaces = true,
+    mergeSameStyleTokens = false,
   } = options
 
   if (mergeWhitespaces === true)
     tokens = mergeWhitespaceTokens(tokens)
   else if (mergeWhitespaces === 'never')
     tokens = splitWhitespaceTokens(tokens)
 
+  if (mergeSameStyleTokens) {
+    tokens = mergeAdjacentStyledTokens(tokens)
+  }
+
   const contextSource = {
     ...transformerContext,
     get source() {
@@ -306,3 +311,35 @@ function splitWhitespaceTokens(tokens: ThemedToken[][]): ThemedToken[][] {
     })
   })
 }
+
+function mergeAdjacentStyledTokens(tokens: ThemedToken[][]): ThemedToken[][] {
+  return tokens.map((line) => {
+    const newLine: ThemedToken[] = []
+    for (const token of line) {
+      if (newLine.length === 0) {
+        newLine.push({ ...token })
+        continue
+      }
+
+      const prevToken = newLine[newLine.length - 1]
+      const prevStyle = prevToken.htmlStyle || stringifyTokenStyle(getTokenStyleObject(prevToken))
+      const currentStyle = token.htmlStyle || stringifyTokenStyle(getTokenStyleObject(token))
+      const isPrevDecorated = prevToken.fontStyle && (
+        (prevToken.fontStyle & FontStyle.Underline)
+        || (prevToken.fontStyle & FontStyle.Strikethrough)
+      )
+      const isDecorated = token.fontStyle && (
+        (token.fontStyle & FontStyle.Underline)
+        || (token.fontStyle & FontStyle.Strikethrough)
+      )
+
+      if (!isPrevDecorated && !isDecorated && prevStyle === currentStyle) {
+        prevToken.content += token.content
+      }
+      else {
+        newLine.push({ ...token })
+      }
+    }
+    return newLine
+  })
+}

packages/core/test/hast.test.ts

@@ -94,3 +94,44 @@ it('render whitespace', async () => {
       <span class="line"><span style="color:#59873A">		tab</span><span style="color:#999999">()</span></span></code></pre>"
     `)
 })
+
+describe('merge same style', () => {
+  it('merges adjacent tokens with same style', async () => {
+    using shiki = await createHighlighter({
+      themes: ['min-light'],
+      langs: ['yaml'],
+    })
+
+    const code = 'name: CI'
+    const html = await shiki.codeToHtml(code, {
+      lang: 'yaml',
+      theme: 'min-light',
+      mergeSameStyleTokens: true,
+    })
+
+    expect(html).toMatchInlineSnapshot(`"<pre class="shiki min-light" style="background-color:#ffffff;color:#24292eff" tabindex="0"><code><span class="line"><span style="color:#D32F2F">name:</span><span style="color:#22863A"> CI</span></span></code></pre>"`)
+  })
+
+  it('does not merge tokens with decorations', async () => {
+    using shiki = await createHighlighter({
+      themes: ['min-light'],
+      langs: ['yaml'],
+    })
+
+    const code = 'name: CI'
+    const html = await shiki.codeToHtml(code, {
+      lang: 'yaml',
+      theme: 'min-light',
+      mergeSameStyleTokens: true,
+      decorations: [
+        {
+          start: { line: 0, character: 0 },
+          end: { line: 0, character: 4 },
+          properties: { class: 'highlighted-word' },
+        },
+      ],
+    })
+
+    expect(html).toMatchInlineSnapshot(`"<pre class="shiki min-light" style="background-color:#ffffff;color:#24292eff" tabindex="0"><code><span class="line"><span style="color:#D32F2F" class="highlighted-word">name</span><span style="color:#D32F2F">:</span><span style="color:#22863A"> CI</span></span></code></pre>"`)
+  })
+})

packages/types/src/options.ts

@@ -142,6 +142,14 @@ export interface CodeToHastOptionsCommon<Languages extends string = string>
    */
   mergeWhitespaces?: boolean | 'never'
 
+  /**
+   * Merge consecutive tokens with the same style to reduce the number of DOM nodes.
+   * This can improve rendering performance but may affect the structure of the output.
+   *
+   * @default false
+   */
+  mergeSameStyleTokens?: boolean
+
   /**
    * The structure of the generated HAST and HTML.
    *