Core: Switch to Shiki JavaScript Regex engine by default

Author: fuma-nama

.changeset/lucky-onions-eat.md

@@ -0,0 +1,11 @@
+---
+'fumadocs-core': major
+---
+
+**Switch to Shiki JavaScript Regex engine by default**
+
+This is important for Cloudflare Worker compatibility, JavaScript engine is the new default over Oniguruma (WASM).
+
+- `rehype-code`: replaced the `experimentalJSEngine` option with `engine: js | oniguruma`.
+- `fumadocs-core/highlight`: use JS engine by default, drop custom engine support, use Shiki directly instead.
+ 

apps/docs/source.config.ts

@@ -53,7 +53,6 @@ export default defineConfig({
   mdxOptions: {
     rehypeCodeOptions: {
       lazy: true,
-      experimentalJSEngine: true,
       langs: ['ts', 'js', 'html', 'tsx', 'mdx'],
       inline: 'tailing-curly-colon',
       themes: {

packages/core/src/highlight/client.tsx

@@ -22,10 +22,5 @@ export function useShiki(
     return `${id}:${state}`;
   }, [code, deps, id, options.lang]);
 
-  return use(
-    (promises[key] ??= highlight(code, {
-      ...options,
-      engine: options.engine ?? 'js',
-    })),
-  );
+  return use((promises[key] ??= highlight(code, options)));
 }

packages/core/src/highlight/shiki.ts

@@ -1,12 +1,10 @@
 import {
-  type Awaitable,
   type BundledHighlighterOptions,
   type BundledLanguage,
   type CodeOptionsMeta,
   type CodeOptionsThemes,
   type CodeToHastOptionsCommon,
   type Highlighter,
-  type RegexEngine,
 } from 'shiki';
 import type { BundledTheme } from 'shiki/themes';
 import {
@@ -25,7 +23,12 @@ export const defaultThemes = {
 
 export type HighlightOptionsCommon = CodeToHastOptionsCommon<BundledLanguage> &
   CodeOptionsMeta & {
-    engine?: 'js' | 'oniguruma' | Awaitable<RegexEngine>;
+    /**
+     * The Regex Engine for Shiki
+     *
+     * @defaultValue 'js'
+     */
+    engine?: 'js' | 'oniguruma';
     components?: Partial<Components>;
 
     fallbackLanguage?: BundledLanguage;
@@ -46,7 +49,7 @@ export async function highlightHast(
     lang: initialLang,
     fallbackLanguage,
     components: _,
-    engine = 'oniguruma',
+    engine = 'js',
     ...rest
   } = options;
   let lang = initialLang;
@@ -64,25 +67,10 @@ export async function highlightHast(
     themesToLoad = Object.values(themes.themes).filter((v) => v !== undefined);
   }
 
-  let highlighter;
-  if (typeof engine === 'string') {
-    highlighter = await getHighlighter(engine, {
-      langs: [],
-      themes: themesToLoad,
-    });
-  } else {
-    highlighter = await getHighlighter('custom', {
-      engine,
-      langs: [],
-      themes: themesToLoad,
-    });
-
-    if (process.env.NODE_ENV === 'development') {
-      console.warn(
-        '[Fumadocs `highlight()`] Avoid passing `engine` directly. For custom engines, use `shiki` directly instead.',
-      );
-    }
-  }
+  const highlighter = await getHighlighter(engine, {
+    langs: [],
+    themes: themesToLoad,
+  });
 
   try {
     await highlighter.loadLanguage(lang as BundledLanguage);
@@ -110,14 +98,17 @@ export function hastToJsx(hast: Root, options?: Partial<ToJsxOptions>) {
 }
 
 /**
- * Get Shiki highlighter instance of Fumadocs (mostly for internal use, don't recommend you to use it).
+ * Get Shiki highlighter instance of Fumadocs (mostly for internal use, you should use Shiki directly over this).
  *
- * @param engineType - engine type, the engine specified in `options` will only be effective when this is set to `custom`.
+ * @param engineType - Shiki Regex engine to use.
  * @param options - Shiki options.
  */
 export async function getHighlighter(
-  engineType: 'js' | 'oniguruma' | 'custom',
-  options: BundledHighlighterOptions<BundledLanguage, BundledTheme>,
+  engineType: 'js' | 'oniguruma',
+  options: Omit<
+    BundledHighlighterOptions<BundledLanguage, BundledTheme>,
+    'engine'
+  >,
 ) {
   const { createHighlighter } = await import('shiki');
   let highlighter = highlighters.get(engineType);
@@ -129,12 +120,10 @@ export async function getHighlighter(
       engine = import('shiki/engine/javascript').then((res) =>
         res.createJavaScriptRegexEngine(),
       );
-    } else if (engineType === 'oniguruma' || !options.engine) {
+    } else {
       engine = import('shiki/engine/oniguruma').then((res) =>
         res.createOnigurumaEngine(import('shiki/wasm')),
       );
-    } else {
-      engine = options.engine;
     }
 
     highlighter = createHighlighter({

packages/core/src/mdx-plugins/rehype-code.ts

@@ -24,7 +24,7 @@ export const rehypeCodeDefaultOptions: RehypeCodeOptions = {
   themes: defaultThemes,
   defaultColor: false,
   defaultLanguage: 'plaintext',
-  experimentalJSEngine: false,
+  engine: 'js',
   transformers: [
     transformerNotationHighlight({
       matchAlgorithm: 'v3',
@@ -61,6 +61,13 @@ function parseLineNumber(str: string, data: Record<string, unknown>) {
 }
 
 export type RehypeCodeOptions = RehypeShikiOptions & {
+  /**
+   * The regex engine to use.
+   *
+   * @defaultValue 'js'
+   */
+  engine?: 'js' | 'oniguruma';
+
   /**
    * Filter meta string before processing
    */
@@ -77,13 +84,6 @@ export type RehypeCodeOptions = RehypeShikiOptions & {
    * @defaultValue true
    */
   tab?: boolean;
-
-  /**
-   * Enable Shiki's experimental JS engine
-   *
-   * @defaultValue false
-   */
-  experimentalJSEngine?: boolean;
 };
 
 /**
@@ -124,19 +124,16 @@ export function rehypeCode(
     transformers.push(transformerTab());
   }
 
-  const highlighter = getHighlighter(
-    options.experimentalJSEngine ? 'js' : 'oniguruma',
-    {
-      themes:
-        'themes' in options
-          ? (Object.values(options.themes).filter(Boolean) as BuiltinTheme[])
-          : [options.theme],
-      langs:
-        options.langs ??
-        (options.lazy ? ['ts', 'tsx'] : Object.keys(bundledLanguages)),
-      langAlias: options.langAlias,
-    },
-  );
+  const highlighter = getHighlighter(options.engine ?? 'js', {
+    themes:
+      'themes' in options
+        ? (Object.values(options.themes).filter(Boolean) as BuiltinTheme[])
+        : [options.theme],
+    langs:
+      options.langs ??
+      (options.lazy ? ['ts', 'tsx'] : Object.keys(bundledLanguages)),
+    langAlias: options.langAlias,
+  });
 
   const transformer = highlighter.then((loaded) =>
     rehypeShikiFromHighlighter(loaded, {

packages/mdx-remote/test/index.test.ts

@@ -14,7 +14,6 @@ const files = await glob('./fixtures/*.mdx', {
 const compiler = createCompiler({
   development: true,
   rehypeCodeOptions: {
-    experimentalJSEngine: true,
     lazy: true,
     themes: {
       light: 'github-light',
@@ -26,7 +25,6 @@ const compiler = createCompiler({
 const compilerProduction = createCompiler({
   development: false,
   rehypeCodeOptions: {
-    experimentalJSEngine: true,
     lazy: true,
     themes: {
       light: 'github-light',