Fix 192 - Add RawDocumentData on vFile data field for Markdown/MDX

Implementation for the request in contentlayerdev#192 that also partially solves for contentlayerdev#11.

In `@contentlayer/core` I added a new `data` field to the  options argument of `markdownToHtml` and `bundleMDX` which can be used to pass arbitrary data to the resulting document's `vFile` `data` property.

Not knowing how you want to structure utility functions for this library, in the initial implementation I've inlined the `addRawDocumentMeta` remark plugin used to append the vFile inside of both `markdownToHtml` and `bundleMDX`. Please advise me if you'd like this extracted out to somewhere else instead.

In this PR I've only updated the `mapping.ts` file for the `source-files` package, as I'm unsure whether other sources like `source-contentful` expose the same `RawDocumentData` that the filesystem source does. Other sources can pass whatever document metadata is pertinent to them to the markdown processors using this addition to their APIs.

packages/@contentlayer/core/src/markdown.ts

@@ -5,16 +5,19 @@ import rehypeStringify from 'rehype-stringify'
 import remarkFrontmatter from 'remark-frontmatter'
 import remarkParse from 'remark-parse'
 import remark2rehype from 'remark-rehype'
+import type { Transformer } from 'unified'
 import { unified } from 'unified'
 
 import type { MarkdownOptions } from './plugin.js'
 
 export const markdownToHtml = ({
   mdString,
   options,
+  data = {},
 }: {
   mdString: string
   options?: MarkdownOptions
+  data: any
 }): T.Effect<OT.HasTracer & HasConsole, UnexpectedMarkdownError, string> =>
   pipe(
     T.gen(function* ($) {
@@ -35,6 +38,12 @@ export const markdownToHtml = ({
 
       const builder = unified()
 
+      const addRawDocumentMeta = (): Transformer => (_, vfile) => {
+        Object.assign(vfile.data, data)
+      }
+
+      builder.use(addRawDocumentMeta)
+
       // parses out the frontmatter (which is needed for full-document parsing)
       builder.use(remarkFrontmatter)
 

packages/@contentlayer/core/src/mdx.ts

@@ -3,17 +3,20 @@ import { OT, pipe, T, Tagged } from '@contentlayer/utils/effect'
 import * as mdxBundler from 'mdx-bundler'
 import type { BundleMDXOptions } from 'mdx-bundler/dist/types'
 import * as path from 'path'
+import type { Transformer } from 'unified'
 
 import type { MDXOptions } from './plugin.js'
 
 export const bundleMDX = ({
   mdxString,
   options,
   contentDirPath,
+  data = {},
 }: {
   mdxString: string
   options?: MDXOptions
   contentDirPath: string
+  data: any
 }): T.Effect<OT.HasTracer, UnexpectedMDXError, string> =>
   pipe(
     T.gen(function* ($) {
@@ -28,10 +31,14 @@ export const bundleMDX = ({
         path.isAbsolute(contentDirPath) ? contentDirPath : path.join(process.cwd(), contentDirPath)
       const cwd = cwd_ ?? getCwdFromContentDirPath()
 
+      const addRawDocumentMeta = (): Transformer => (_, vfile) => {
+        Object.assign(vfile.data, data)
+      }
+
       const mdxOptions: BundleMDXOptions<any> = {
         mdxOptions: (opts) => {
           opts.rehypePlugins = [...(opts.rehypePlugins ?? []), ...(rehypePlugins ?? [])]
-          opts.remarkPlugins = [...(opts.remarkPlugins ?? []), ...(remarkPlugins ?? [])]
+          opts.remarkPlugins = [addRawDocumentMeta, ...(opts.remarkPlugins ?? []), ...(remarkPlugins ?? [])]
           return opts
         },
         cwd,

packages/@contentlayer/source-files/src/fetchData/mapping.ts

@@ -273,35 +273,51 @@ const getDataForFieldDef = ({
         return value
       case 'markdown': {
         const isBodyField = fieldDef.name === options.fieldOptions.bodyFieldName
+        const data: RawDocumentData = {
+          sourceFilePath: relativeFilePath,
+          sourceFileName: path.basename(relativeFilePath),
+          sourceFileDir: path.dirname(relativeFilePath),
+          contentType: `markdown`,
+          flattenedPath: getFlattenedPath(relativeFilePath),
+        }
         // NOTE for the body field, we're passing the entire document file contents to MDX (e.g. in case some remark/rehype plugins need access to the frontmatter)
         // TODO we should come up with a better way to do this
         if (isBodyField) {
           const rawContent = yield* $(getFromDocumentContext('rawContent'))
           if (rawContent.kind !== 'markdown' && rawContent.kind !== 'mdx') return utils.assertNever(rawContent)
 
           const html = yield* $(
-            core.markdownToHtml({ mdString: rawContent.rawDocumentContent, options: options?.markdown }),
+            core.markdownToHtml({ mdString: rawContent.rawDocumentContent, options: options?.markdown, data }),
           )
           return identity<core.Markdown>({ raw: rawFieldData, html })
         } else {
-          const html = yield* $(core.markdownToHtml({ mdString: rawFieldData, options: options?.markdown }))
+          const html = yield* $(core.markdownToHtml({ mdString: rawFieldData, options: options?.markdown, data }))
           return identity<core.Markdown>({ raw: rawFieldData, html })
         }
       }
       case 'mdx': {
         const isBodyField = fieldDef.name === options.fieldOptions.bodyFieldName
+        const data: RawDocumentData = {
+          sourceFilePath: relativeFilePath,
+          sourceFileName: path.basename(relativeFilePath),
+          sourceFileDir: path.dirname(relativeFilePath),
+          contentType: `mdx`,
+          flattenedPath: getFlattenedPath(relativeFilePath),
+        }
         // NOTE for the body field, we're passing the entire document file contents to MDX (e.g. in case some remark/rehype plugins need access to the frontmatter)
         // TODO we should come up with a better way to do this
         if (isBodyField) {
           const rawContent = yield* $(getFromDocumentContext('rawContent'))
           if (rawContent.kind !== 'mdx' && rawContent.kind !== 'markdown') return utils.assertNever(rawContent)
 
           const code = yield* $(
-            core.bundleMDX({ mdxString: rawContent.rawDocumentContent, options: options?.mdx, contentDirPath }),
+            core.bundleMDX({ mdxString: rawContent.rawDocumentContent, options: options?.mdx, contentDirPath, data }),
           )
           return identity<core.MDX>({ raw: rawFieldData, code })
         } else {
-          const code = yield* $(core.bundleMDX({ mdxString: rawFieldData, options: options?.mdx, contentDirPath }))
+          const code = yield* $(
+            core.bundleMDX({ mdxString: rawFieldData, options: options?.mdx, contentDirPath, data }),
+          )
           return identity<core.MDX>({ raw: rawFieldData, code })
         }
       }