feat(cli): store markdown file references in docs definition jsFiles

packages/cli/docs-markdown-utils/src/__test__/replaceReferencedMarkdown.test.ts

@@ -31,7 +31,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="test2.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -60,7 +60,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="test2" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -101,7 +101,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="feature.mdx" plan="pro" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -124,7 +124,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="pricing.md" tier="enterprise" price="$999" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -145,7 +145,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="feature.md" name="authentication" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -169,7 +169,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="feature.md" plan="pro" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -196,7 +196,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="test.md" prop1="value1" prop2='value2' prop3={"value3"} />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -219,7 +219,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="test.md" plan="pro" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -242,7 +242,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="feature.md" name="API Keys" level="enterprise" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -271,7 +271,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="plan-tier.mdx" plan="pro" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -300,7 +300,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="details.md" name="API" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -334,7 +334,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="plan-tier.mdx" plan="pro" tier="enterprise" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -358,7 +358,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="static.md" plan="pro" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -380,7 +380,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="outer.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -404,7 +404,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="level1.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -435,7 +435,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="snippetA.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -465,7 +465,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="self-ref.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -489,7 +489,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="outer.md" name="Fern" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -516,7 +516,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="parent.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -550,7 +550,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="snippetA.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,
@@ -584,7 +584,7 @@ describe("replaceReferencedMarkdown", () => {
             <Markdown src="snippetA.md" />
         `;
 
-        const result = await replaceReferencedMarkdown({
+        const { markdown: result } = await replaceReferencedMarkdown({
             markdown,
             absolutePathToFernFolder,
             absolutePathToMarkdownFile,

packages/cli/docs-markdown-utils/src/index.ts

@@ -4,5 +4,9 @@ export { isMdxExpression, isMdxJsxAttribute, isMdxJsxElement, isMdxJsxExpression
 export { getReplacedHref, parseImagePaths, replaceImagePathsAndUrls, trimAnchor } from "./parseImagePaths";
 export { parseMarkdownToTree } from "./parseMarkdownToTree";
 export { replaceReferencedCode } from "./replaceReferencedCode";
-export { replaceReferencedMarkdown } from "./replaceReferencedMarkdown";
+export {
+    type ReferencedMarkdownFile,
+    type ReplaceReferencedMarkdownResult,
+    replaceReferencedMarkdown
+} from "./replaceReferencedMarkdown";
 export { walkEstreeJsxAttributes } from "./walk-estree-jsx-attributes";

packages/cli/docs-markdown-utils/src/replaceReferencedMarkdown.ts

@@ -1,8 +1,19 @@
-import { AbsoluteFilePath, dirname, RelativeFilePath, resolve } from "@fern-api/fs-utils";
+import { AbsoluteFilePath, dirname, RelativeFilePath, relative, resolve } from "@fern-api/fs-utils";
 import { TaskContext } from "@fern-api/task-context";
 import { readFile } from "fs/promises";
 import grayMatter from "gray-matter";
 
+export interface ReferencedMarkdownFile {
+    absoluteFilePath: AbsoluteFilePath;
+    relativeFilePath: RelativeFilePath;
+    content: string;
+}
+
+export interface ReplaceReferencedMarkdownResult {
+    markdown: string;
+    referencedFiles: ReferencedMarkdownFile[];
+}
+
 async function defaultMarkdownLoader(filepath: AbsoluteFilePath) {
     // strip frontmatter from the referenced markdown
     const { content } = grayMatter(await readFile(filepath));
@@ -63,17 +74,20 @@ export async function replaceReferencedMarkdown({
     // allow for custom markdown loader for testing
     markdownLoader = defaultMarkdownLoader,
     // track ancestor files to detect circular references
-    ancestorFiles = new Set<string>()
+    ancestorFiles = new Set<string>(),
+    // collect referenced files for tracking
+    collectedFiles = new Map<AbsoluteFilePath, ReferencedMarkdownFile>()
 }: {
     markdown: string;
     absolutePathToFernFolder: AbsoluteFilePath;
     absolutePathToMarkdownFile: AbsoluteFilePath;
     context: TaskContext;
     markdownLoader?: (filepath: AbsoluteFilePath) => Promise<string>;
     ancestorFiles?: Set<string>;
-}): Promise<string> {
+    collectedFiles?: Map<AbsoluteFilePath, ReferencedMarkdownFile>;
+}): Promise<ReplaceReferencedMarkdownResult> {
     if (!markdown.includes("<Markdown")) {
-        return markdown;
+        return { markdown, referencedFiles: Array.from(collectedFiles.values()) };
     }
 
     const regex = /([ \t]*)<Markdown\s+([^>]+)\/>/g;
@@ -114,7 +128,18 @@ export async function replaceReferencedMarkdown({
         }
 
         try {
-            let replaceString = await markdownLoader(filepath);
+            const rawContent = await markdownLoader(filepath);
+
+            // Store the referenced file with its raw content (before variable substitution)
+            if (!collectedFiles.has(filepath)) {
+                collectedFiles.set(filepath, {
+                    absoluteFilePath: filepath,
+                    relativeFilePath: relative(absolutePathToFernFolder, filepath),
+                    content: rawContent
+                });
+            }
+
+            let replaceString = rawContent;
 
             const { src: _, ...variables } = attributes;
 
@@ -138,14 +163,16 @@ export async function replaceReferencedMarkdown({
             // Recursively replace referenced markdown in the loaded content
             const newAncestorFiles = new Set(ancestorFiles);
             newAncestorFiles.add(filepath);
-            replaceString = await replaceReferencedMarkdown({
+            const result = await replaceReferencedMarkdown({
                 markdown: replaceString,
                 absolutePathToFernFolder,
                 absolutePathToMarkdownFile: filepath,
                 context,
                 markdownLoader,
-                ancestorFiles: newAncestorFiles
+                ancestorFiles: newAncestorFiles,
+                collectedFiles
             });
+            replaceString = result.markdown;
 
             replaceString = replaceString
                 .split("\n")
@@ -158,5 +185,5 @@ export async function replaceReferencedMarkdown({
         }
     }
 
-    return newMarkdown;
+    return { markdown: newMarkdown, referencedFiles: Array.from(collectedFiles.values()) };
 }

packages/cli/docs-preview/src/previewDocs.ts

@@ -173,7 +173,7 @@ export async function getPreviewDocsDefinition({
                 continue;
             }
 
-            const markdownReplacedMd = await replaceReferencedMarkdown({
+            const { markdown: markdownReplacedMd } = await replaceReferencedMarkdown({
                 markdown,
                 absolutePathToFernFolder: docsWorkspace.absoluteFilePath,
                 absolutePathToMarkdownFile: absoluteFilePath,

packages/cli/docs-resolver/src/DocsDefinitionResolver.ts

@@ -3,6 +3,7 @@ import { docsYml, parseAudiences, parseDocsConfiguration, WithoutQuestionMarks }
 import { assertNever, isNonNullish, replaceEnvVariables, visitDiscriminatedUnion } from "@fern-api/core-utils";
 import {
     parseImagePaths,
+    type ReferencedMarkdownFile,
     replaceImagePathsAndUrls,
     replaceReferencedCode,
     replaceReferencedMarkdown
@@ -219,6 +220,7 @@ export class DocsDefinitionResolver {
     private markdownFilesToNoIndex: Map<AbsoluteFilePath, boolean> = new Map();
     private markdownFilesToTags: Map<AbsoluteFilePath, string[]> = new Map();
     private rawMarkdownFiles: Record<RelativeFilePath, string> = {};
+    private referencedMarkdownFiles: ReferencedMarkdownFile[] = [];
     public async resolve(): Promise<DocsV1Write.DocsDefinition> {
         const resolveStartTime = performance.now();
         const startMemory = process.memoryUsage();
@@ -301,18 +303,28 @@ export class DocsDefinitionResolver {
 
         // replaces all instances of <Markdown src="path/to/file.md" /> with the content of the referenced markdown file
         // this should happen before we parse image paths, as the referenced markdown files may contain images.
+        // Also collects all referenced markdown files to store in jsFiles
         this.taskContext.logger.debug("Replacing referenced markdown files...");
         const refMdStart = performance.now();
         for (const [relativePath, markdown] of Object.entries(this.parsedDocsConfig.pages)) {
-            this.parsedDocsConfig.pages[RelativeFilePath.of(relativePath)] = await replaceReferencedMarkdown({
+            const result = await replaceReferencedMarkdown({
                 markdown,
                 absolutePathToFernFolder: this.docsWorkspace.absoluteFilePath,
                 absolutePathToMarkdownFile: this.resolveFilepath(relativePath),
                 context: this.taskContext
             });
+            this.parsedDocsConfig.pages[RelativeFilePath.of(relativePath)] = result.markdown;
+            // Collect referenced markdown files (deduplicated by absolute path)
+            for (const refFile of result.referencedFiles) {
+                if (!this.referencedMarkdownFiles.some((f) => f.absoluteFilePath === refFile.absoluteFilePath)) {
+                    this.referencedMarkdownFiles.push(refFile);
+                }
+            }
         }
         const refMdTime = performance.now() - refMdStart;
-        this.taskContext.logger.debug(`Replaced referenced markdown in ${refMdTime.toFixed(0)}ms`);
+        this.taskContext.logger.debug(
+            `Replaced referenced markdown in ${refMdTime.toFixed(0)}ms, found ${this.referencedMarkdownFiles.length} referenced files`
+        );
 
         // replaces all instances of <Code src="path/to/file.js" /> with the content of the referenced code file
         this.taskContext.logger.debug("Replacing referenced code files...");
@@ -478,6 +490,17 @@ export class DocsDefinitionResolver {
             );
         }
 
+        // Add referenced markdown files to jsFiles so they are preserved in the docs definition
+        if (this.referencedMarkdownFiles.length > 0) {
+            this.taskContext.logger.debug(
+                `Adding ${this.referencedMarkdownFiles.length} referenced markdown files to jsFiles...`
+            );
+            for (const refFile of this.referencedMarkdownFiles) {
+                // Use the relative path as the key, and the raw content as the value
+                jsFiles[refFile.relativeFilePath] = refFile.content;
+            }
+        }
+
         const totalResolveTime = performance.now() - resolveStartTime;
         const endMemory = process.memoryUsage();
         this.taskContext.logger.debug(