diff --git a/packages/cli/docs-markdown-utils/src/__test__/replaceReferencedMarkdown.test.ts b/packages/cli/docs-markdown-utils/src/__test__/replaceReferencedMarkdown.test.ts index eabce6337898..9aa6a020b78a 100644 --- a/packages/cli/docs-markdown-utils/src/__test__/replaceReferencedMarkdown.test.ts +++ b/packages/cli/docs-markdown-utils/src/__test__/replaceReferencedMarkdown.test.ts @@ -31,7 +31,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -60,7 +60,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -101,7 +101,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -124,7 +124,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -145,7 +145,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -169,7 +169,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -196,7 +196,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -219,7 +219,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -242,7 +242,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -271,7 +271,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -300,7 +300,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -334,7 +334,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -358,7 +358,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -380,7 +380,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -404,7 +404,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -435,7 +435,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -465,7 +465,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -489,7 +489,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -516,7 +516,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -550,7 +550,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, @@ -584,7 +584,7 @@ describe("replaceReferencedMarkdown", () => { `; - const result = await replaceReferencedMarkdown({ + const { markdown: result } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder, absolutePathToMarkdownFile, diff --git a/packages/cli/docs-markdown-utils/src/index.ts b/packages/cli/docs-markdown-utils/src/index.ts index 0b9106fd7132..d9a1fe0d6eca 100644 --- a/packages/cli/docs-markdown-utils/src/index.ts +++ b/packages/cli/docs-markdown-utils/src/index.ts @@ -4,5 +4,9 @@ export { isMdxExpression, isMdxJsxAttribute, isMdxJsxElement, isMdxJsxExpression export { getReplacedHref, parseImagePaths, replaceImagePathsAndUrls, trimAnchor } from "./parseImagePaths"; export { parseMarkdownToTree } from "./parseMarkdownToTree"; export { replaceReferencedCode } from "./replaceReferencedCode"; -export { replaceReferencedMarkdown } from "./replaceReferencedMarkdown"; +export { + type ReferencedMarkdownFile, + type ReplaceReferencedMarkdownResult, + replaceReferencedMarkdown +} from "./replaceReferencedMarkdown"; export { walkEstreeJsxAttributes } from "./walk-estree-jsx-attributes"; diff --git a/packages/cli/docs-markdown-utils/src/replaceReferencedMarkdown.ts b/packages/cli/docs-markdown-utils/src/replaceReferencedMarkdown.ts index 38f0d41e2c23..12e3caa2255a 100644 --- a/packages/cli/docs-markdown-utils/src/replaceReferencedMarkdown.ts +++ b/packages/cli/docs-markdown-utils/src/replaceReferencedMarkdown.ts @@ -1,8 +1,19 @@ -import { AbsoluteFilePath, dirname, RelativeFilePath, resolve } from "@fern-api/fs-utils"; +import { AbsoluteFilePath, dirname, RelativeFilePath, relative, resolve } from "@fern-api/fs-utils"; import { TaskContext } from "@fern-api/task-context"; import { readFile } from "fs/promises"; import grayMatter from "gray-matter"; +export interface ReferencedMarkdownFile { + absoluteFilePath: AbsoluteFilePath; + relativeFilePath: RelativeFilePath; + content: string; +} + +export interface ReplaceReferencedMarkdownResult { + markdown: string; + referencedFiles: ReferencedMarkdownFile[]; +} + async function defaultMarkdownLoader(filepath: AbsoluteFilePath) { // strip frontmatter from the referenced markdown const { content } = grayMatter(await readFile(filepath)); @@ -63,7 +74,9 @@ export async function replaceReferencedMarkdown({ // allow for custom markdown loader for testing markdownLoader = defaultMarkdownLoader, // track ancestor files to detect circular references - ancestorFiles = new Set() + ancestorFiles = new Set(), + // collect referenced files for tracking + collectedFiles = new Map() }: { markdown: string; absolutePathToFernFolder: AbsoluteFilePath; @@ -71,9 +84,10 @@ export async function replaceReferencedMarkdown({ context: TaskContext; markdownLoader?: (filepath: AbsoluteFilePath) => Promise; ancestorFiles?: Set; -}): Promise { + collectedFiles?: Map; +}): Promise { if (!markdown.includes("]+)\/>/g; @@ -114,7 +128,18 @@ export async function replaceReferencedMarkdown({ } try { - let replaceString = await markdownLoader(filepath); + const rawContent = await markdownLoader(filepath); + + // Store the referenced file with its raw content (before variable substitution) + if (!collectedFiles.has(filepath)) { + collectedFiles.set(filepath, { + absoluteFilePath: filepath, + relativeFilePath: relative(absolutePathToFernFolder, filepath), + content: rawContent + }); + } + + let replaceString = rawContent; const { src: _, ...variables } = attributes; @@ -138,14 +163,16 @@ export async function replaceReferencedMarkdown({ // Recursively replace referenced markdown in the loaded content const newAncestorFiles = new Set(ancestorFiles); newAncestorFiles.add(filepath); - replaceString = await replaceReferencedMarkdown({ + const result = await replaceReferencedMarkdown({ markdown: replaceString, absolutePathToFernFolder, absolutePathToMarkdownFile: filepath, context, markdownLoader, - ancestorFiles: newAncestorFiles + ancestorFiles: newAncestorFiles, + collectedFiles }); + replaceString = result.markdown; replaceString = replaceString .split("\n") @@ -158,5 +185,5 @@ export async function replaceReferencedMarkdown({ } } - return newMarkdown; + return { markdown: newMarkdown, referencedFiles: Array.from(collectedFiles.values()) }; } diff --git a/packages/cli/docs-preview/src/previewDocs.ts b/packages/cli/docs-preview/src/previewDocs.ts index ac2308feed0f..65feb8682449 100644 --- a/packages/cli/docs-preview/src/previewDocs.ts +++ b/packages/cli/docs-preview/src/previewDocs.ts @@ -173,7 +173,7 @@ export async function getPreviewDocsDefinition({ continue; } - const markdownReplacedMd = await replaceReferencedMarkdown({ + const { markdown: markdownReplacedMd } = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder: docsWorkspace.absoluteFilePath, absolutePathToMarkdownFile: absoluteFilePath, diff --git a/packages/cli/docs-resolver/src/DocsDefinitionResolver.ts b/packages/cli/docs-resolver/src/DocsDefinitionResolver.ts index e011ee6a95d5..3a18418eb895 100644 --- a/packages/cli/docs-resolver/src/DocsDefinitionResolver.ts +++ b/packages/cli/docs-resolver/src/DocsDefinitionResolver.ts @@ -3,6 +3,7 @@ import { docsYml, parseAudiences, parseDocsConfiguration, WithoutQuestionMarks } import { assertNever, isNonNullish, replaceEnvVariables, visitDiscriminatedUnion } from "@fern-api/core-utils"; import { parseImagePaths, + type ReferencedMarkdownFile, replaceImagePathsAndUrls, replaceReferencedCode, replaceReferencedMarkdown @@ -219,6 +220,7 @@ export class DocsDefinitionResolver { private markdownFilesToNoIndex: Map = new Map(); private markdownFilesToTags: Map = new Map(); private rawMarkdownFiles: Record = {}; + private referencedMarkdownFiles: ReferencedMarkdownFile[] = []; public async resolve(): Promise { const resolveStartTime = performance.now(); const startMemory = process.memoryUsage(); @@ -301,18 +303,28 @@ export class DocsDefinitionResolver { // replaces all instances of with the content of the referenced markdown file // this should happen before we parse image paths, as the referenced markdown files may contain images. + // Also collects all referenced markdown files to store in jsFiles this.taskContext.logger.debug("Replacing referenced markdown files..."); const refMdStart = performance.now(); for (const [relativePath, markdown] of Object.entries(this.parsedDocsConfig.pages)) { - this.parsedDocsConfig.pages[RelativeFilePath.of(relativePath)] = await replaceReferencedMarkdown({ + const result = await replaceReferencedMarkdown({ markdown, absolutePathToFernFolder: this.docsWorkspace.absoluteFilePath, absolutePathToMarkdownFile: this.resolveFilepath(relativePath), context: this.taskContext }); + this.parsedDocsConfig.pages[RelativeFilePath.of(relativePath)] = result.markdown; + // Collect referenced markdown files (deduplicated by absolute path) + for (const refFile of result.referencedFiles) { + if (!this.referencedMarkdownFiles.some((f) => f.absoluteFilePath === refFile.absoluteFilePath)) { + this.referencedMarkdownFiles.push(refFile); + } + } } const refMdTime = performance.now() - refMdStart; - this.taskContext.logger.debug(`Replaced referenced markdown in ${refMdTime.toFixed(0)}ms`); + this.taskContext.logger.debug( + `Replaced referenced markdown in ${refMdTime.toFixed(0)}ms, found ${this.referencedMarkdownFiles.length} referenced files` + ); // replaces all instances of with the content of the referenced code file this.taskContext.logger.debug("Replacing referenced code files..."); @@ -478,6 +490,17 @@ export class DocsDefinitionResolver { ); } + // Add referenced markdown files to jsFiles so they are preserved in the docs definition + if (this.referencedMarkdownFiles.length > 0) { + this.taskContext.logger.debug( + `Adding ${this.referencedMarkdownFiles.length} referenced markdown files to jsFiles...` + ); + for (const refFile of this.referencedMarkdownFiles) { + // Use the relative path as the key, and the raw content as the value + jsFiles[refFile.relativeFilePath] = refFile.content; + } + } + const totalResolveTime = performance.now() - resolveStartTime; const endMemory = process.memoryUsage(); this.taskContext.logger.debug(