centralize md instance as part of the normalized configuration (#1034)

* centralize md instance as part of the normalized configuration

* md is mandatory

* mdparser → createMarkdownIt; move hook later

* remove unused root option

* destructure

---------

Co-authored-by: Mike Bostock <[email protected]>

Author: Fil

src/config.ts

@@ -7,7 +7,7 @@ import {pathToFileURL} from "node:url";
 import type MarkdownIt from "markdown-it";
 import {visitMarkdownFiles} from "./files.js";
 import {formatIsoDate, formatLocaleDate} from "./format.js";
-import {parseMarkdown} from "./markdown.js";
+import {createMarkdownIt, parseMarkdown} from "./markdown.js";
 import {resolvePath} from "./path.js";
 import {resolveTheme} from "./theme.js";
 
@@ -53,7 +53,7 @@ export interface Config {
   style: null | Style; // defaults to {theme: ["light", "dark"]}
   deploy: null | {workspace: string; project: string};
   search: boolean; // default to false
-  markdownIt?: (md: MarkdownIt) => MarkdownIt;
+  md: MarkdownIt;
 }
 
 /**
@@ -79,12 +79,12 @@ export async function readDefaultConfig(root?: string): Promise<Config> {
   return normalizeConfig((await import(pathToFileURL(tsPath).href)).default, root);
 }
 
-async function readPages(root: string): Promise<Page[]> {
+async function readPages(root: string, md: MarkdownIt): Promise<Page[]> {
   const pages: Page[] = [];
   for await (const file of visitMarkdownFiles(root)) {
     if (file === "index.md" || file === "404.md") continue;
     const source = await readFile(join(root, file), "utf8");
-    const parsed = parseMarkdown(source, {root, path: file});
+    const parsed = parseMarkdown(source, {path: file, md});
     if (parsed?.data?.draft) continue;
     const name = basename(file, ".md");
     const page = {path: join("/", dirname(file), name), name: parsed.title ?? "Untitled"};
@@ -117,14 +117,14 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
       currentDate
     )}">${formatLocaleDate(currentDate)}</a>.`
   } = spec;
-  const {markdownIt} = spec;
   root = String(root);
   output = String(output);
   base = normalizeBase(base);
   if (style === null) style = null;
   else if (style !== undefined) style = {path: String(style)};
   else style = {theme: (theme = normalizeTheme(theme))};
-  let {title, pages = await readPages(root), pager = true, toc = true} = spec;
+  const md = createMarkdownIt(spec);
+  let {title, pages = await readPages(root, md), pager = true, toc = true} = spec;
   if (title !== undefined) title = String(title);
   pages = Array.from(pages, normalizePageOrSection);
   sidebar = sidebar === undefined ? pages.length > 0 : Boolean(sidebar);
@@ -136,7 +136,6 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
   toc = normalizeToc(toc);
   deploy = deploy ? {workspace: String(deploy.workspace).replace(/^@+/, ""), project: String(deploy.project)} : null;
   search = Boolean(search);
-  if (markdownIt !== undefined && typeof markdownIt !== "function") throw new Error("markdownIt must be a function");
   return {
     root,
     output,
@@ -153,7 +152,7 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
     style,
     deploy,
     search,
-    markdownIt
+    md
   };
 }
 

src/markdown.ts

@@ -35,6 +35,7 @@ export interface ParseContext {
   code: MarkdownCode[];
   startLine: number;
   currentLine: number;
+  path: string;
 }
 
 function uniqueCodeId(context: ParseContext, content: string): string {
@@ -85,8 +86,9 @@ function getLiveSource(content: string, tag: string, attributes: Record<string,
 //   console.error(red(`${error.name}: ${warning}`));
 // }
 
-function makeFenceRenderer(root: string, baseRenderer: RenderRule, sourcePath: string): RenderRule {
+function makeFenceRenderer(baseRenderer: RenderRule): RenderRule {
   return (tokens, idx, options, context: ParseContext, self) => {
+    const {path} = context;
     const token = tokens[idx];
     const {tag, attributes} = parseInfo(token.info);
     token.info = tag;
@@ -97,7 +99,7 @@ function makeFenceRenderer(root: string, baseRenderer: RenderRule, sourcePath: s
       if (source != null) {
         const id = uniqueCodeId(context, source);
         // TODO const sourceLine = context.startLine + context.currentLine;
-        const node = parseJavaScript(source, {path: sourcePath});
+        const node = parseJavaScript(source, {path});
         context.code.push({id, node});
         html += `<div id="cell-${id}" class="observablehq observablehq--block${
           node.expression ? " observablehq--loading" : ""
@@ -177,7 +179,7 @@ function parsePlaceholder(content: string, replacer: (i: number, j: number) => v
 
 function transformPlaceholderBlock(token) {
   const input = token.content;
-  if (/^\s*<script(\s|>)/.test(input)) return [token]; // ignore <script> elements
+  if (/^\s*<script[\s>]/.test(input)) return [token]; // ignore <script> elements
   const output: any[] = [];
   let i = 0;
   parsePlaceholder(input, (j, k) => {
@@ -245,13 +247,14 @@ const transformPlaceholderCore: RuleCore = (state) => {
   state.tokens = output;
 };
 
-function makePlaceholderRenderer(root: string, sourcePath: string): RenderRule {
+function makePlaceholderRenderer(): RenderRule {
   return (tokens, idx, options, context: ParseContext) => {
+    const {path} = context;
     const token = tokens[idx];
     const id = uniqueCodeId(context, token.content);
     try {
       // TODO sourceLine: context.startLine + context.currentLine
-      const node = parseJavaScript(token.content, {path: sourcePath, inline: true});
+      const node = parseJavaScript(token.content, {path, inline: true});
       context.code.push({id, node});
       return `<span id="cell-${id}" class="observablehq--loading"></span>`;
     } catch (error) {
@@ -273,32 +276,34 @@ function makeSoftbreakRenderer(baseRenderer: RenderRule): RenderRule {
 }
 
 export interface ParseOptions {
-  root: string;
+  md: MarkdownIt;
   path: string;
-  markdownIt?: Config["markdownIt"];
   style?: Config["style"];
 }
 
-export function parseMarkdown(input: string, {root, path, markdownIt, style: configStyle}: ParseOptions): MarkdownPage {
-  const parts = matter(input, {});
-  let md = MarkdownIt({html: true, linkify: true});
+export function createMarkdownIt({markdownIt}: {markdownIt?: (md: MarkdownIt) => MarkdownIt} = {}): MarkdownIt {
+  const md = MarkdownIt({html: true, linkify: true});
   md.linkify.set({fuzzyLink: false, fuzzyEmail: false});
-  if (markdownIt !== undefined) md = markdownIt(md);
   md.use(MarkdownItAnchor, {permalink: MarkdownItAnchor.permalink.headerLink({class: "observablehq-header-anchor"})});
   md.inline.ruler.push("placeholder", transformPlaceholderInline);
   md.core.ruler.before("linkify", "placeholder", transformPlaceholderCore);
-  md.renderer.rules.placeholder = makePlaceholderRenderer(root, path);
-  md.renderer.rules.fence = makeFenceRenderer(root, md.renderer.rules.fence!, path);
+  md.renderer.rules.placeholder = makePlaceholderRenderer();
+  md.renderer.rules.fence = makeFenceRenderer(md.renderer.rules.fence!);
   md.renderer.rules.softbreak = makeSoftbreakRenderer(md.renderer.rules.softbreak!);
+  return markdownIt === undefined ? md : markdownIt(md);
+}
+
+export function parseMarkdown(input: string, {md, path, style: configStyle}: ParseOptions): MarkdownPage {
+  const {content, data} = matter(input, {});
   const code: MarkdownCode[] = [];
-  const context: ParseContext = {code, startLine: 0, currentLine: 0};
-  const tokens = md.parse(parts.content, context);
+  const context: ParseContext = {code, startLine: 0, currentLine: 0, path};
+  const tokens = md.parse(content, context);
   const html = md.renderer.render(tokens, md.options, context); // Note: mutates code, assets!
-  const style = getStylesheet(path, parts.data, configStyle);
+  const style = getStylesheet(path, data, configStyle);
   return {
     html,
-    data: isEmpty(parts.data) ? null : parts.data,
-    title: parts.data?.title ?? findTitle(tokens) ?? null,
+    data: isEmpty(data) ? null : data,
+    title: data?.title ?? findTitle(tokens) ?? null,
     style,
     code
   };

src/search.ts

@@ -43,7 +43,7 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro
   for await (const file of visitMarkdownFiles(root)) {
     const path = join(root, file);
     const source = await readFile(path, "utf8");
-    const {html, title, data} = parseMarkdown(source, {root, path: "/" + file.slice(0, -3)});
+    const {html, title, data} = parseMarkdown(source, {...config, path: "/" + file.slice(0, -3)});
 
     // Skip pages that opt-out of indexing, and skip unlisted pages unless
     // opted-in. We only log the first case.

test/config-test.ts

@@ -1,12 +1,15 @@
 import assert from "node:assert";
+import MarkdownIt from "markdown-it";
 import {normalizeConfig as config, mergeToc, readConfig, setCurrentDate} from "../src/config.js";
 
 const root = "test/input/build/config";
 
 describe("readConfig(undefined, root)", () => {
   before(() => setCurrentDate(new Date("2024-01-11T01:02:03")));
   it("imports the config file at the specified root", async () => {
-    assert.deepStrictEqual(await readConfig(undefined, "test/input/build/config"), {
+    const {md, ...config} = await readConfig(undefined, "test/input/build/config");
+    assert(md instanceof MarkdownIt);
+    assert.deepStrictEqual(config, {
       root: "test/input/build/config",
       output: "dist",
       base: "/",
@@ -30,12 +33,13 @@ describe("readConfig(undefined, root)", () => {
         workspace: "acme",
         project: "bi"
       },
-      search: false,
-      markdownIt: undefined
+      search: false
     });
   });
   it("returns the default config if no config file is found", async () => {
-    assert.deepStrictEqual(await readConfig(undefined, "test/input/build/simple"), {
+    const {md, ...config} = await readConfig(undefined, "test/input/build/simple");
+    assert(md instanceof MarkdownIt);
+    assert.deepStrictEqual(config, {
       root: "test/input/build/simple",
       output: "dist",
       base: "/",
@@ -51,8 +55,7 @@ describe("readConfig(undefined, root)", () => {
       footer:
         'Built with <a href="https://observablehq.com/" target="_blank">Observable</a> on <a title="2024-01-11T01:02:03">Jan 11, 2024</a>.',
       deploy: null,
-      search: false,
-      markdownIt: undefined
+      search: false
     });
   });
 });

test/markdown-test.ts

@@ -3,11 +3,13 @@ import {readdirSync, statSync} from "node:fs";
 import {mkdir, readFile, unlink, writeFile} from "node:fs/promises";
 import {basename, join, resolve} from "node:path/posix";
 import deepEqual from "fast-deep-equal";
+import {normalizeConfig} from "../src/config.js";
 import {isEnoent} from "../src/error.js";
 import type {MarkdownPage} from "../src/markdown.js";
 import {parseMarkdown} from "../src/markdown.js";
 
-describe("parseMarkdown(input)", () => {
+describe("parseMarkdown(input)", async () => {
+  const {md} = await normalizeConfig();
   const inputRoot = "test/input";
   const outputRoot = "test/output";
   for (const name of readdirSync(inputRoot)) {
@@ -20,7 +22,7 @@ describe("parseMarkdown(input)", () => {
 
     (only ? it.only : skip ? it.skip : it)(`test/input/${name}`, async () => {
       const source = await readFile(path, "utf8");
-      const snapshot = parseMarkdown(source, {root: "test/input", path: name});
+      const snapshot = parseMarkdown(source, {path: name, md});
       let allequal = true;
       for (const ext of ["html", "json"]) {
         const actual = ext === "json" ? jsonMeta(snapshot) : snapshot[ext];

test/resolvers-test.ts

@@ -1,78 +1,80 @@
 import assert from "node:assert";
+import {normalizeConfig} from "../src/config.js";
 import {parseMarkdown} from "../src/markdown.js";
 import {getResolvers} from "../src/resolvers.js";
 
-describe("getResolvers(page, {root, path})", () => {
+describe("getResolvers(page, {root, path})", async () => {
+  const {md} = await normalizeConfig();
   const builtins = ["npm:@observablehq/runtime", "npm:@observablehq/stdlib", "observablehq:client"];
   it("resolves directly-attached files", async () => {
-    const options = {root: "test/input", path: "attached.md"};
+    const options = {root: "test/input", path: "attached.md", md};
     const page = parseMarkdown("${FileAttachment('foo.csv')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.files, new Set(["./foo.csv"]));
   });
   it("ignores files that are outside of the source root", async () => {
-    const options = {root: "test/input", path: "attached.md"};
+    const options = {root: "test/input", path: "attached.md", md};
     const page = parseMarkdown("${FileAttachment('../foo.csv')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.files, new Set([]));
   });
   it("detects file methods", async () => {
-    const options = {root: "test/input", path: "attached.md"};
+    const options = {root: "test/input", path: "attached.md", md};
     const page = parseMarkdown("${FileAttachment('foo.csv').csv}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["npm:d3-dsv", ...builtins]));
   });
   it("detects local static imports", async () => {
-    const options = {root: "test/input/imports", path: "attached.md"};
+    const options = {root: "test/input/imports", path: "attached.md", md};
     const page = parseMarkdown("```js\nimport './bar.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./bar.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./bar.js"]));
   });
   it("detects local transitive static imports", async () => {
-    const options = {root: "test/input/imports", path: "attached.md"};
+    const options = {root: "test/input/imports", path: "attached.md", md};
     const page = parseMarkdown("```js\nimport './other/foo.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./other/foo.js", "./bar.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./other/foo.js", "./bar.js"]));
   });
   it("detects local transitive static imports (2)", async () => {
-    const options = {root: "test/input/imports", path: "attached.md"};
+    const options = {root: "test/input/imports", path: "attached.md", md};
     const page = parseMarkdown("```js\nimport './transitive-static-import.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js", ...builtins])); // prettier-ignore
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local transitive dynamic imports", async () => {
-    const options = {root: "test/input/imports", path: "attached.md"};
+    const options = {root: "test/input/imports", path: "attached.md", md};
     const page = parseMarkdown("```js\nimport './dynamic-import.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./dynamic-import.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
   });
   it("detects local transitive dynamic imports (2)", async () => {
-    const options = {root: "test/input/imports", path: "attached.md"};
+    const options = {root: "test/input/imports", path: "attached.md", md};
     const page = parseMarkdown("```js\nimport('./dynamic-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
   });
   it("detects local transitive dynamic imports (3)", async () => {
-    const options = {root: "test/input/imports", path: "attached.md"};
+    const options = {root: "test/input/imports", path: "attached.md", md};
     const page = parseMarkdown("```js\nimport('./transitive-dynamic-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-dynamic-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local transitive dynamic imports (4)", async () => {
-    const options = {root: "test/input/imports", path: "attached.md"};
+    const options = {root: "test/input/imports", path: "attached.md", md};
     const page = parseMarkdown("```js\nimport('./transitive-static-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local dynamic imports", async () => {
-    const options = {root: "test/input", path: "attached.md"};
+    const options = {root: "test/input", path: "attached.md", md};
     const page = parseMarkdown("${import('./foo.js')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));