feat: preserve vfile data from unified processor

packages/@contentlayer/core/package.json

@@ -42,7 +42,8 @@
     "remark-rehype": "^10.1.0",
     "source-map-support": "^0.5.21",
     "type-fest": "^2.12.2",
-    "unified": "^10.1.2"
+    "unified": "^10.1.2",
+    "vfile": "^5.3.4"
   },
   "devDependencies": {
     "@types/source-map-support": "^0.5.4",

packages/@contentlayer/core/src/data-types.ts

@@ -1,3 +1,5 @@
+import { Data } from 'vfile'
+
 export type Document = Record<string, any> & DocumentMeta
 
 export type NestedDocument = Record<string, any> & Omit<DocumentMeta, '_id'>
@@ -18,6 +20,8 @@ export type Markdown = {
   raw: string
   /** Generated HTML based on Markdown source */
   html: string
+  /** Information generated by Markdown processor */
+  data: Data
 }
 
 export type MDX = {

packages/@contentlayer/core/src/markdown/markdown.ts

@@ -6,6 +6,7 @@ import remarkFrontmatter from 'remark-frontmatter'
 import remarkParse from 'remark-parse'
 import remark2rehype from 'remark-rehype'
 import { unified } from 'unified'
+import { Data } from 'vfile'
 
 import type { RawDocumentData } from '../data-types.js'
 import type { MarkdownOptions, MarkdownUnifiedBuilderCallback } from '../plugin.js'
@@ -19,7 +20,7 @@ export const markdownToHtml = ({
   mdString: string
   options?: MarkdownOptions | MarkdownUnifiedBuilderCallback
   rawDocumentData: RawDocumentData
-}): T.Effect<OT.HasTracer & HasConsole, UnexpectedMarkdownError, string> =>
+}): T.Effect<OT.HasTracer & HasConsole, UnexpectedMarkdownError, { html: string; data: Data }> =>
   pipe(
     T.gen(function* ($) {
       // const matterResult = matter(mdString)
@@ -32,7 +33,7 @@ export const markdownToHtml = ({
         return yield* $(
           T.tryPromise(async () => {
             const { parse: parseWasm } = await import('markdown-wasm/dist/markdown.node.js')
-            return parseWasm(mdString)
+            return { html: parseWasm(mdString), data: {} }
           }),
         )
       }
@@ -67,7 +68,7 @@ export const markdownToHtml = ({
 
       const res = yield* $(T.tryPromise(() => builder.process(mdString)))
 
-      return res.toString()
+      return { html: String(res), data: res.data }
     }),
     T.catchAllDefect(T.fail),
     T.mapError((error) => new UnexpectedMarkdownError({ error })),

packages/@contentlayer/source-contentful/src/fetchData/mapping.ts

@@ -175,10 +175,10 @@ const getDataForFieldDef = ({
       case 'date':
         return new Date(rawFieldData)
       case 'markdown':
-        const html = yield* $(
+        const { html, data } = yield* $(
           core.markdownToHtml({ mdString: rawFieldData, options: options?.markdown, rawDocumentData: {} }),
         )
-        return <core.Markdown>{ raw: rawFieldData, html }
+        return <core.Markdown>{ raw: rawFieldData, html, data }
       // NOTE `mdx` support for Contentful is experimental and not clearly defined
       case 'mdx':
         const code = yield* $(

packages/@contentlayer/source-files/src/fetchData/mapping.ts

@@ -265,19 +265,19 @@ const getDataForFieldDef = ({
           const rawContent = yield* $(getFromDocumentContext('rawContent'))
           if (rawContent.kind !== 'markdown' && rawContent.kind !== 'mdx') return utils.assertNever(rawContent)
 
-          const html = yield* $(
+          const { html, data } = yield* $(
             core.markdownToHtml({
               mdString: rawContent.rawDocumentContent,
               options: options?.markdown,
               rawDocumentData,
             }),
           )
-          return identity<core.Markdown>({ raw: rawFieldData, html })
+          return identity<core.Markdown>({ raw: rawFieldData, html, data })
         } else {
-          const html = yield* $(
+          const { html, data } = yield* $(
             core.markdownToHtml({ mdString: rawFieldData, options: options?.markdown, rawDocumentData }),
           )
-          return identity<core.Markdown>({ raw: rawFieldData, html })
+          return identity<core.Markdown>({ raw: rawFieldData, html, data })
         }
       }
       case 'mdx': {

packages/integration-tests/src/markdown/index.test.ts

@@ -8,6 +8,7 @@ import rehypeStringify from 'rehype-stringify'
 import remarkFrontmatter from 'remark-frontmatter'
 import remarkParse from 'remark-parse'
 import remark2rehype from 'remark-rehype'
+import type { Plugin } from 'unified'
 import { expect, test, vi } from 'vitest'
 
 test('markdown builder pattern', async () => {
@@ -45,3 +46,43 @@ test('markdown builder pattern', async () => {
 
   expect(spyFn).toHaveBeenCalledOnce()
 })
+
+test('pass vfile.data from markdown processor', async () => {
+  const Post = defineDocumentType(() => ({
+    name: 'Post',
+    filePathPattern: '**/*.md',
+    fields: {},
+  }))
+
+  const testDirPath = fileURLToPath(new URL('.', import.meta.url))
+
+  await fs.rm(path.join(testDirPath, '.contentlayer'), { recursive: true, force: true })
+
+  process.env['INIT_CWD'] = testDirPath
+
+  const testMessage = 'Hello, world!'
+
+  const testPlugin: Plugin = () => {
+    return function transformer(tree, file) {
+      file.data.test = testMessage
+    }
+  }
+
+  const source = await makeSource({
+    contentDirPath: path.join(testDirPath, 'posts'),
+    documentTypes: [Post],
+    markdown: {
+      remarkPlugins: [testPlugin],
+    },
+  })
+
+  await core.runMain({ tracingServiceName: 'test', verbose: false })(
+    core.generateDotpkg({ config: { source, esbuildHash: 'STATIC_HASH' }, verbose: true }),
+  )
+
+  const generated = JSON.parse(
+    await fs.readFile(path.join(testDirPath, '.contentlayer', 'generated/Post/post-1.md.json'), { encoding: 'utf-8' }),
+  )
+
+  expect(generated.body.data.test).toBe(testMessage)
+})