diff --git a/apps/roam/src/utils/exportUtils.ts b/apps/roam/src/utils/exportUtils.ts new file mode 100644 index 000000000..e5dc5a2b2 --- /dev/null +++ b/apps/roam/src/utils/exportUtils.ts @@ -0,0 +1,141 @@ +import type { Result } from "roamjs-components/types/query-builder"; +import { PullBlock, TreeNode, ViewType } from "roamjs-components/types"; +import type { DiscourseNode } from "./getDiscourseNodes"; +import matchDiscourseNode from "./matchDiscourseNode"; + +type DiscourseExportResult = Result & { type: string }; + +export const uniqJsonArray = >(arr: T[]) => + Array.from( + new Set( + arr.map((r) => + JSON.stringify( + Object.entries(r).sort(([k], [k2]) => k.localeCompare(k2)), + ), + ), + ), + ).map((entries) => Object.fromEntries(JSON.parse(entries))) as T[]; + +export const getPageData = ({ + results, + allNodes, + isExportDiscourseGraph, +}: { + results: Result[]; + allNodes: DiscourseNode[]; + isExportDiscourseGraph?: boolean; +}): (Result & { type: string })[] => { + if (isExportDiscourseGraph) return results as DiscourseExportResult[]; + + const matchedTexts = new Set(); + const mappedResults = results.flatMap((r) => + Object.keys(r) + .filter((k) => k.endsWith(`-uid`) && k !== "text-uid") + .map((k) => ({ + ...r, + text: r[k.slice(0, -4)].toString(), + uid: r[k] as string, + })) + .concat({ + text: r.text, + uid: r.uid, + }), + ); + return allNodes.flatMap((n) => + mappedResults + .filter(({ text }) => { + if (!text) return false; + if (matchedTexts.has(text)) return false; + const isMatch = matchDiscourseNode({ title: text, ...n }); + if (isMatch) matchedTexts.add(text); + + return isMatch; + }) + .map((node) => ({ ...node, type: n.text })), + ); +}; + +const getContentFromNodes = ({ + title, + allNodes, +}: { + title: string; + allNodes: DiscourseNode[]; +}) => { + const nodeFormat = allNodes.find((a) => + matchDiscourseNode({ title, ...a }), + )?.format; + if (!nodeFormat) return title; + const regex = new RegExp( + `^${nodeFormat + .replace(/[[\]\\^$.|?*+()]/g, "\\$&") + .replace("{content}", "(.*?)") + .replace(/{[^}]+}/g, "(?:.*?)")}$`, + ); + return regex.exec(title)?.[1] || title; +}; + +export const getFilename = ({ + title = "", + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + extension = ".md", +}: { + title?: string; + maxFilenameLength: number; + simplifiedFilename: boolean; + allNodes: DiscourseNode[]; + removeSpecialCharacters: boolean; + extension?: string; +}) => { + const baseName = simplifiedFilename + ? getContentFromNodes({ title, allNodes }) + : title; + const name = `${ + removeSpecialCharacters + ? baseName.replace(/[<>:"/\\|\?*[\]]/g, "") + : baseName + }${extension}`; + + return name.length > maxFilenameLength + ? `${name.substring( + 0, + Math.ceil((maxFilenameLength - 3) / 2), + )}...${name.slice(-Math.floor((maxFilenameLength - 3) / 2))}` + : name; +}; + +export const toLink = (filename: string, uid: string, linkType: string) => { + const extensionRemoved = filename.replace(/\.\w+$/, ""); + if (linkType === "wikilinks") return `[[${extensionRemoved}]]`; + if (linkType === "alias") return `[${filename}](${filename})`; + if (linkType === "roam url") + return `[${extensionRemoved}](https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/${uid})`; + return filename; +}; + +export const pullBlockToTreeNode = ( + n: PullBlock, + v: `:${ViewType}`, +): TreeNode => ({ + text: n[":block/string"] || n[":node/title"] || "", + open: typeof n[":block/open"] === "undefined" ? true : n[":block/open"], + order: n[":block/order"] || 0, + uid: n[":block/uid"] || "", + heading: n[":block/heading"] || 0, + viewType: (n[":children/view-type"] || v).slice(1) as ViewType, + editTime: new Date(n[":edit/time"] || 0), + props: { imageResize: {}, iframe: {} }, + textAlign: n[":block/text-align"] || "left", + children: (n[":block/children"] || []) + .sort(({ [":block/order"]: a = 0 }, { [":block/order"]: b = 0 }) => a - b) + .map((r) => pullBlockToTreeNode(r, n[":children/view-type"] || v)), + parents: (n[":block/parents"] || []).map((p) => p[":db/id"] || 0), +}); + +export const collectUids = (t: TreeNode): string[] => [ + t.uid, + ...t.children.flatMap(collectUids), +]; diff --git a/apps/roam/src/utils/getExportTypes.ts b/apps/roam/src/utils/getExportTypes.ts index 3287ba820..8670efc70 100644 --- a/apps/roam/src/utils/getExportTypes.ts +++ b/apps/roam/src/utils/getExportTypes.ts @@ -1,25 +1,25 @@ -import { BLOCK_REF_REGEX } from "roamjs-components/dom/constants"; -import getTextByBlockUid from "roamjs-components/queries/getTextByBlockUid"; import normalizePageTitle from "roamjs-components/queries/normalizePageTitle"; import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; -import getPageViewType from "roamjs-components/queries/getPageViewType"; -import { PullBlock, TreeNode, ViewType } from "roamjs-components/types"; -import { Result } from "roamjs-components/types/query-builder"; -import XRegExp from "xregexp"; +import { PullBlock, TreeNode } from "roamjs-components/types"; +import type { Result } from "roamjs-components/types/query-builder"; import getDiscourseNodes from "./getDiscourseNodes"; import isFlagEnabled from "./isFlagEnabled"; -import matchDiscourseNode from "./matchDiscourseNode"; import getDiscourseRelations from "./getDiscourseRelations"; import type { ExportDialogProps } from "~/components/Export"; import getPageMetadata from "./getPageMetadata"; import getDiscourseContextResults from "./getDiscourseContextResults"; import { getRelationDataUtil } from "./getRelationData"; import { ExportTypes } from "./types"; -import { - findReferencedNodeInText, - getReferencedNodeInFormat, -} from "./formatUtils"; import { getExportSettings } from "./getExportSettings"; +import { pageToMarkdown, toMarkdown } from "./pageToMarkdown"; +import { + uniqJsonArray, + getPageData, + getFilename, + toLink, + pullBlockToTreeNode, + collectUids, +} from "./exportUtils"; export const updateExportProgress = (detail: { progress: number; @@ -31,295 +31,12 @@ export const updateExportProgress = (detail: { }), ); -const pullBlockToTreeNode = (n: PullBlock, v: `:${ViewType}`): TreeNode => ({ - text: n[":block/string"] || n[":node/title"] || "", - open: typeof n[":block/open"] === "undefined" ? true : n[":block/open"], - order: n[":block/order"] || 0, - uid: n[":block/uid"] || "", - heading: n[":block/heading"] || 0, - viewType: (n[":children/view-type"] || v).slice(1) as ViewType, - editTime: new Date(n[":edit/time"] || 0), - props: { imageResize: {}, iframe: {} }, - textAlign: n[":block/text-align"] || "left", - children: (n[":block/children"] || []) - .sort(({ [":block/order"]: a = 0 }, { [":block/order"]: b = 0 }) => a - b) - .map((r) => pullBlockToTreeNode(r, n[":children/view-type"] || v)), - parents: (n[":block/parents"] || []).map((p) => p[":db/id"] || 0), -}); - -const getContentFromNodes = ({ - title, - allNodes, -}: { - title: string; - allNodes: ReturnType; -}) => { - const nodeFormat = allNodes.find((a) => - matchDiscourseNode({ title, ...a }), - )?.format; - if (!nodeFormat) return title; - const regex = new RegExp( - `^${nodeFormat - .replace(/\[/g, "\\[") - .replace(/]/g, "\\]") - .replace("{content}", "(.*?)") - .replace(/{[^}]+}/g, "(?:.*?)")}$`, - ); - return regex.exec(title)?.[1] || title; -}; - -const getFilename = ({ - title = "", - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - extension = ".md", -}: { - title?: string; - maxFilenameLength: number; - simplifiedFilename: boolean; - allNodes: ReturnType; - removeSpecialCharacters: boolean; - extension?: string; -}) => { - const baseName = simplifiedFilename - ? getContentFromNodes({ title, allNodes }) - : title; - const name = `${ - removeSpecialCharacters - ? baseName.replace(/[<>:"/\\|\?*[\]]/g, "") - : baseName - }${extension}`; - - return name.length > maxFilenameLength - ? `${name.substring( - 0, - Math.ceil((maxFilenameLength - 3) / 2), - )}...${name.slice(-Math.floor((maxFilenameLength - 3) / 2))}` - : name; -}; - -const uniqJsonArray = >(arr: T[]) => - Array.from( - new Set( - arr.map((r) => - JSON.stringify( - Object.entries(r).sort(([k], [k2]) => k.localeCompare(k2)), - ), - ), - ), - ).map((entries) => Object.fromEntries(JSON.parse(entries))) as T[]; -const viewTypeToPrefix = { - bullet: "- ", - document: "", - numbered: "1. ", -}; - -const collectUids = (t: TreeNode): string[] => [ - t.uid, - ...t.children.flatMap(collectUids), -]; - -const MATCHES_NONE = /$.+^/; - -// Roam embed syntax: {{[[embed]]: ((block-uid)) }} -// Roam embed syntax: {{[[embed-path]]: ((block-uid)) }} -// Also handles multiple parentheses: {{[[embed]]: ((((block-uid)))) }} -const EMBED_REGEX = - /{{\[\[(?:embed|embed-path)\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; - -// Roam embed-children syntax: {{[[embed-children]]: ((block-uid)) }} -const EMBED_CHILDREN_REGEX = - /{{\[\[embed-children\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; - -const toLink = (filename: string, uid: string, linkType: string) => { - const extensionRemoved = filename.replace(/\.\w+$/, ""); - if (linkType === "wikilinks") return `[[${extensionRemoved}]]`; - if (linkType === "alias") return `[${filename}](${filename})`; - if (linkType === "roam url") - return `[${extensionRemoved}](https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/${uid})`; - return filename; -}; - -const toMarkdown = ({ - c, - i = 0, - v = "bullet", - opts, -}: { - c: TreeNode; - i?: number; - v?: ViewType; - opts: { - refs: boolean; - embeds: boolean; - simplifiedFilename: boolean; - maxFilenameLength: number; - allNodes: ReturnType; - removeSpecialCharacters: boolean; - linkType: string; - flatten?: boolean; - }; -}): string => { - const { - refs, - embeds, - simplifiedFilename, - maxFilenameLength, - allNodes, - removeSpecialCharacters, - linkType, - flatten = false, - } = opts; - const processedText = c.text - .replace(embeds ? EMBED_REGEX : MATCHES_NONE, (_, blockUid) => { - const reference = getFullTreeByParentUid(blockUid); - return toMarkdown({ c: reference, i, v, opts }); - }) - .replace(embeds ? EMBED_CHILDREN_REGEX : MATCHES_NONE, (_, blockUid) => { - const reference = getFullTreeByParentUid(blockUid); - return reference.children - .map((child) => toMarkdown({ c: child, i, v, opts })) - .join("\n"); - }) - .replace(refs ? BLOCK_REF_REGEX : MATCHES_NONE, (_, blockUid) => { - const reference = getTextByBlockUid(blockUid); - return reference || blockUid; - }) - .replace(/{{\[\[TODO\]\]}}/g, v === "bullet" ? "[ ]" : "- [ ]") - .replace(/{{\[\[DONE\]\]}}/g, v === "bullet" ? "[x]" : "- [x]") - .replace(/\_\_(.+?)\_\_/g, "_$1_") // convert Roam italics __ to markdown italics _ - .replace(/(? { - if (s.name === "match") { - const name = getFilename({ - title: s.value, - allNodes, - maxFilenameLength, - simplifiedFilename, - removeSpecialCharacters, - }); - return toLink(name, c.uid, linkType); - } else if (s.name === "left" || s.name === "right") { - return ""; - } else { - return s.value; - } - }) - .join("") || processedText - : processedText; - const indentation = flatten ? "" : "".padStart(i * 4, " "); - // If this block contains an embed, treat it as document to avoid extra prefixes - const effectiveViewType = - embeds && (EMBED_REGEX.test(c.text) || EMBED_CHILDREN_REGEX.test(c.text)) - ? "document" - : v; - const viewTypePrefix = viewTypeToPrefix[effectiveViewType]; - const headingPrefix = c.heading ? `${"".padStart(c.heading, "#")} ` : ""; - const childrenMarkdown = (c.children || []) - .filter((nested) => !!nested.text || !!nested.children?.length) - .map((nested) => { - const childViewType = v !== "bullet" ? v : nested.viewType || "bullet"; - const childMarkdown = toMarkdown({ - c: nested, - i: i + 1, - v: childViewType, - opts, - }); - return `\n${childMarkdown}`; - }) - .join(""); - const lineBreak = v === "document" ? "\n" : ""; - - return `${indentation}${viewTypePrefix}${headingPrefix}${finalProcessedText}${lineBreak}${childrenMarkdown}`; -}; - -const handleDiscourseContext = async ({ - includeDiscourseContext, - uid, - pageTitle, - appendRefNodeContext, -}: { - includeDiscourseContext: boolean; - uid: string; - pageTitle: string; - appendRefNodeContext: boolean; -}) => { - if (!includeDiscourseContext) return []; - - const discourseResults = await getDiscourseContextResults({ - uid, - }); - if (!appendRefNodeContext) return discourseResults; - - const referencedDiscourseNode = getReferencedNodeInFormat({ uid }); - if (referencedDiscourseNode) { - const referencedResult = findReferencedNodeInText({ - text: pageTitle, - discourseNode: referencedDiscourseNode, - }); - if (!referencedResult) return discourseResults; - const appendedContext = { - label: referencedDiscourseNode.text, - results: { [referencedResult.uid]: referencedResult }, - }; - return [...discourseResults, appendedContext]; - } - - return discourseResults; -}; - -const handleFrontmatter = ({ - frontmatter, - rest, - result, -}: { - frontmatter: string[]; - rest: Record; - result: Result; -}) => { - const yaml = frontmatter.length - ? frontmatter - : [ - "title: {text}", - `url: https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/{uid}`, - `author: {author}`, - "date: {date}", - ]; - const resultCols = Object.keys(rest).filter((k) => !k.includes("uid")); - const yamlLines = yaml.concat(resultCols.map((k) => `${k}: {${k}}`)); - const content = yamlLines - .map((s) => - s.replace(/{([^}]+)}/g, (_, capt: string) => { - if (capt === "text") { - // Wrap title in quotes and escape additional quotes - const escapedText = result[capt].toString().replace(/"/g, '\\"'); - return `"${escapedText}"`; - } - return result[capt].toString(); - }), - ) - .join("\n"); - const output = `---\n${content}\n---`; - return output; -}; - type getExportTypesProps = { results?: ExportDialogProps["results"]; exportId: string; isExportDiscourseGraph: boolean; }; -export type DiscourseExportResult = Result & { type: string }; - const getExportTypes = ({ results, exportId, @@ -331,59 +48,17 @@ const getExportTypes = ({ allNodes.map((a) => [a.type, a.text]), ); nodeLabelByType["*"] = "Any"; - const getPageData = async ( - isExportDiscourseGraph?: boolean, - ): Promise<(Result & { type: string })[]> => { - const allResults = results || []; - - if (isExportDiscourseGraph) return allResults as DiscourseExportResult[]; - - const matchedTexts = new Set(); - return allNodes.flatMap((n) => - (allResults - ? allResults.flatMap((r) => - Object.keys(r) - .filter((k) => k.endsWith(`-uid`) && k !== "text-uid") - .map((k) => ({ - ...r, - text: r[k.slice(0, -4)].toString(), - uid: r[k] as string, - })) - .concat({ - text: r.text, - uid: r.uid, - }), - ) - : ( - window.roamAlphaAPI.q( - "[:find (pull ?e [:block/uid :node/title]) :where [?e :node/title _]]", - ) as [Record][] - ).map(([{ title, uid }]) => ({ - text: title, - uid, - })) - ) - .filter(({ text }) => { - if (!text) return false; - if (matchedTexts.has(text)) return false; - const isMatch = matchDiscourseNode({ title: text, ...n }); - if (isMatch) matchedTexts.add(text); - return isMatch; - }) - .map((node) => ({ ...node, type: n.text })), - ); - }; const getRelationData = () => getRelationDataUtil({ allRelations, nodeLabelByType, local: true }); - const getJsonData = async () => { + const getJsonData = async (results: Result[]) => { const grammar = allRelations.map(({ label, destination, source }) => ({ label, destination: nodeLabelByType[destination], source: nodeLabelByType[source], })); - const nodes = (await getPageData()).map(({ text, uid }) => { + const nodes = getPageData({ results, allNodes }).map(({ text, uid }) => { const { date, displayName } = getPageMetadata(text); const { children } = getFullTreeByParentUid(uid); return { @@ -407,134 +82,31 @@ const getExportTypes = ({ { name: "Markdown", callback: async ({ includeDiscourseContext = false }) => { + if (!results) return []; + const settings = { + ...getExportSettings(), + includeDiscourseContext, + }; const { - frontmatter, - optsRefs, - optsEmbeds, simplifiedFilename, maxFilenameLength, removeSpecialCharacters, - linkType, - appendRefNodeContext, - } = getExportSettings(); - const allPages = await getPageData(isExportDiscourseGraph); - const gatherings = allPages.map( - ({ text, uid, context: _, type, ...rest }, i, all) => - async () => { - updateExportProgress({ progress: i / all.length, id: exportId }); - // skip a beat to let progress render - await new Promise((resolve) => setTimeout(resolve)); - const v = getPageViewType(text) || "bullet"; - const { date, displayName } = getPageMetadata(text); - const treeNode = getFullTreeByParentUid(uid); - - const discourseResults = await handleDiscourseContext({ - includeDiscourseContext, - pageTitle: text, - uid, - appendRefNodeContext, - }); - - const referenceResults = isFlagEnabled("render references") - ? ( - window.roamAlphaAPI.data.fast.q( - `[:find (pull ?pr [:node/title]) (pull ?r [:block/heading [:block/string :as "text"] [:children/view-type :as "viewType"] {:block/children ...}]) :where [?p :node/title "${normalizePageTitle( - text, - )}"] [?r :block/refs ?p] [?r :block/page ?pr]]`, - ) as [PullBlock, PullBlock][] - ).filter( - ([, { [":block/children"]: children }]) => - Array.isArray(children) && children.length, - ) - : []; - - const result: Result = { - ...rest, - date, - text, - uid, - author: displayName, - type, - }; - const yamlLines = handleFrontmatter({ - frontmatter, - rest, - result, - }); + } = settings; + const allPages = getPageData({ + results, + allNodes, + isExportDiscourseGraph, + }); + const gatherings = allPages.map((result, i, all) => async () => { + updateExportProgress({ progress: i / all.length, id: exportId }); + // skip a beat to let progress render + await new Promise((resolve) => setTimeout(resolve)); + return pageToMarkdown(result, { + ...settings, + allNodes, + }); + }); - const content = `${yamlLines}\n\n${treeNode.children - .map((c) => - toMarkdown({ - c, - v, - i: 0, - opts: { - refs: optsRefs, - embeds: optsEmbeds, - simplifiedFilename, - allNodes, - maxFilenameLength, - removeSpecialCharacters, - linkType, - }, - }), - ) - .join("\n")}\n${ - discourseResults.length - ? `\n###### Discourse Context\n\n${discourseResults - .flatMap((r) => - Object.values(r.results).map( - (t) => - `- **${r.label}::** ${toLink( - getFilename({ - title: t.text, - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - }), - t.uid, - linkType, - )}`, - ), - ) - .join("\n")}\n` - : "" - }${ - referenceResults.length - ? `\n###### References\n\n${referenceResults - .map( - (r_1) => - `${toLink( - getFilename({ - title: r_1[0][":node/title"], - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - }), - r_1[0][":block/uid"] || "", - linkType, - )}\n\n${toMarkdown({ - c: pullBlockToTreeNode(r_1[1], ":bullet"), - opts: { - refs: optsRefs, - embeds: optsEmbeds, - simplifiedFilename, - allNodes, - maxFilenameLength, - removeSpecialCharacters, - linkType, - }, - })}`, - ) - .join("\n")}\n` - : "" - }`; - const uids = new Set(collectUids(treeNode)); - return { title: text, content, uids }; - }, - ); const pages = await gatherings.reduce( (p, c) => p.then((arr) => @@ -562,7 +134,8 @@ const getExportTypes = ({ { name: "JSON", callback: async ({ filename }) => { - const data = await getJsonData(); + if (!results) return []; + const data = await getJsonData(results); return [ { title: `${filename.replace(/\.json$/, "")}.json`, @@ -574,8 +147,9 @@ const getExportTypes = ({ { name: "Neo4j", callback: async ({ filename }) => { + if (!results) return []; const nodeHeader = "uid:ID,label:LABEL,title,author,date\n"; - const nodeData = (await getPageData()) + const nodeData = getPageData({ results, allNodes }) .map(({ text, uid, type }) => { const value = text.replace(new RegExp(`^\\[\\[\\w*\\]\\] - `), ""); const { displayName, date } = getPageMetadata(text); @@ -628,6 +202,7 @@ const getExportTypes = ({ { name: "PDF", callback: async ({ includeDiscourseContext = false }) => { + if (!results) return []; const { optsRefs, optsEmbeds, @@ -636,7 +211,11 @@ const getExportTypes = ({ removeSpecialCharacters, linkType, } = getExportSettings(); - const allPages = await getPageData(isExportDiscourseGraph); + const allPages = getPageData({ + results, + allNodes, + isExportDiscourseGraph, + }); const gatherings = allPages.map(({ text, uid }, i, all) => async () => { updateExportProgress({ progress: i / all.length, id: exportId }); // skip a beat to let progress render diff --git a/apps/roam/src/utils/pageToMarkdown.ts b/apps/roam/src/utils/pageToMarkdown.ts new file mode 100644 index 000000000..4dc760631 --- /dev/null +++ b/apps/roam/src/utils/pageToMarkdown.ts @@ -0,0 +1,347 @@ +import { BLOCK_REF_REGEX } from "roamjs-components/dom/constants"; +import normalizePageTitle from "roamjs-components/queries/normalizePageTitle"; +import getTextByBlockUid from "roamjs-components/queries/getTextByBlockUid"; +import getPageMetadata from "./getPageMetadata"; +import getPageViewType from "roamjs-components/queries/getPageViewType"; +import type { Result } from "roamjs-components/types/query-builder"; +import type { DiscourseNode } from "./getDiscourseNodes"; +import type { PullBlock, TreeNode, ViewType } from "roamjs-components/types"; +import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; +import getDiscourseContextResults from "./getDiscourseContextResults"; +import isFlagEnabled from "./isFlagEnabled"; +import XRegExp from "xregexp"; +import { + findReferencedNodeInText, + getReferencedNodeInFormat, +} from "./formatUtils"; +import { + getFilename, + toLink, + pullBlockToTreeNode, + collectUids, +} from "./exportUtils"; + +const MATCHES_NONE = /$.+^/; + +// Roam embed syntax: {{[[embed]]: ((block-uid)) }} +// Roam embed syntax: {{[[embed-path]]: ((block-uid)) }} +// Also handles multiple parentheses: {{[[embed]]: ((((block-uid)))) }} +const EMBED_REGEX = + /{{\[\[(?:embed|embed-path)\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; + +// Roam embed-children syntax: {{[[embed-children]]: ((block-uid)) }} +const EMBED_CHILDREN_REGEX = + /{{\[\[embed-children\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; + +const viewTypeToPrefix = { + bullet: "- ", + document: "", + numbered: "1. ", +}; + +const handleDiscourseContext = async ({ + includeDiscourseContext, + uid, + pageTitle, + appendRefNodeContext, +}: { + includeDiscourseContext: boolean; + uid: string; + pageTitle: string; + appendRefNodeContext: boolean; +}) => { + if (!includeDiscourseContext) return []; + + const discourseResults = await getDiscourseContextResults({ + uid, + }); + if (!appendRefNodeContext) return discourseResults; + + const referencedDiscourseNode = getReferencedNodeInFormat({ uid }); + if (referencedDiscourseNode) { + const referencedResult = findReferencedNodeInText({ + text: pageTitle, + discourseNode: referencedDiscourseNode, + }); + if (!referencedResult) return discourseResults; + const appendedContext = { + label: referencedDiscourseNode.text, + results: { [referencedResult.uid]: referencedResult }, + }; + return [...discourseResults, appendedContext]; + } + + return discourseResults; +}; + +const handleFrontmatter = ({ + frontmatter, + rest, + result, +}: { + frontmatter: string[]; + rest: Record; + result: Result; +}) => { + const yaml = frontmatter.length + ? frontmatter + : [ + "title: {text}", + `url: https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/{uid}`, + `author: {author}`, + "date: {date}", + ]; + const resultCols = Object.keys(rest).filter((k) => !k.includes("uid")); + const yamlLines = yaml.concat(resultCols.map((k) => `${k}: {${k}}`)); + const content = yamlLines + .map((s) => + s.replace(/{([^}]+)}/g, (_, capt: string) => { + if (capt === "text") { + // Wrap title in quotes and escape additional quotes + const escapedText = result[capt].toString().replace(/"/g, '\\"'); + return `"${escapedText}"`; + } + return result[capt].toString(); + }), + ) + .join("\n"); + const output = `---\n${content}\n---`; + return output; +}; + +export const toMarkdown = ({ + c, + i = 0, + v = "bullet", + opts, +}: { + c: TreeNode; + i?: number; + v?: ViewType; + opts: { + refs: boolean; + embeds: boolean; + simplifiedFilename: boolean; + maxFilenameLength: number; + allNodes: DiscourseNode[]; + removeSpecialCharacters: boolean; + linkType: string; + flatten?: boolean; + }; +}): string => { + const { + refs, + embeds, + simplifiedFilename, + maxFilenameLength, + allNodes, + removeSpecialCharacters, + linkType, + flatten = false, + } = opts; + const processedText = c.text + .replace(embeds ? EMBED_REGEX : MATCHES_NONE, (_, blockUid) => { + const reference = getFullTreeByParentUid(blockUid); + return toMarkdown({ c: reference, i, v, opts }); + }) + .replace(embeds ? EMBED_CHILDREN_REGEX : MATCHES_NONE, (_, blockUid) => { + const reference = getFullTreeByParentUid(blockUid); + return reference.children + .map((child) => toMarkdown({ c: child, i, v, opts })) + .join("\n"); + }) + .replace(refs ? BLOCK_REF_REGEX : MATCHES_NONE, (_, blockUid) => { + const reference = getTextByBlockUid(blockUid); + return reference || blockUid; + }) + .replace(/{{\[\[TODO\]\]}}/g, v === "bullet" ? "[ ]" : "- [ ]") + .replace(/{{\[\[DONE\]\]}}/g, v === "bullet" ? "[x]" : "- [x]") + .replace(/\_\_(.+?)\_\_/g, "_$1_") // convert Roam italics __ to markdown italics _ + .replace(/(? { + if (s.name === "match") { + const name = getFilename({ + title: s.value, + allNodes, + maxFilenameLength, + simplifiedFilename, + removeSpecialCharacters, + }); + return toLink(name, c.uid, linkType); + } else if (s.name === "left" || s.name === "right") { + return ""; + } else { + return s.value; + } + }) + .join("") || processedText + : processedText; + const indentation = flatten ? "" : "".padStart(i * 4, " "); + // If this block contains an embed, treat it as document to avoid extra prefixes + const effectiveViewType = + embeds && (EMBED_REGEX.test(c.text) || EMBED_CHILDREN_REGEX.test(c.text)) + ? "document" + : v; + const viewTypePrefix = viewTypeToPrefix[effectiveViewType]; + const headingPrefix = c.heading ? `${"".padStart(c.heading, "#")} ` : ""; + const childrenMarkdown = (c.children || []) + .filter((nested) => !!nested.text || !!nested.children?.length) + .map((nested) => { + const childViewType = v !== "bullet" ? v : nested.viewType || "bullet"; + const childMarkdown = toMarkdown({ + c: nested, + i: i + 1, + v: childViewType, + opts, + }); + return `\n${childMarkdown}`; + }) + .join(""); + const lineBreak = v === "document" ? "\n" : ""; + + return `${indentation}${viewTypePrefix}${headingPrefix}${finalProcessedText}${lineBreak}${childrenMarkdown}`; +}; + +export const pageToMarkdown = async ( + { text, uid, context: _, type, ...rest }: Result, + { + includeDiscourseContext, + appendRefNodeContext, + frontmatter, + optsRefs, + optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }: { + includeDiscourseContext: boolean; + appendRefNodeContext: boolean; + frontmatter: string[]; + optsRefs: boolean; + optsEmbeds: boolean; + simplifiedFilename: boolean; + allNodes: DiscourseNode[]; + maxFilenameLength: number; + removeSpecialCharacters: boolean; + linkType: string; + }, +): Promise<{ title: string; content: string; uids: Set }> => { + const v = getPageViewType(text) || "bullet"; + const { date, displayName } = getPageMetadata(text); + const treeNode = getFullTreeByParentUid(uid); + + const discourseResults = await handleDiscourseContext({ + includeDiscourseContext, + pageTitle: text, + uid, + appendRefNodeContext, + }); + + const referenceResults = isFlagEnabled("render references") + ? ( + window.roamAlphaAPI.data.fast.q( + `[:find (pull ?pr [:node/title :block/uid]) (pull ?r [:block/heading [:block/string :as "text"] [:children/view-type :as "viewType"] {:block/children ...}]) :where [?p :node/title "${normalizePageTitle( + text, + )}"] [?r :block/refs ?p] [?r :block/page ?pr]]`, + ) as [PullBlock, PullBlock][] + ).filter( + ([, { [":block/children"]: children }]) => + Array.isArray(children) && children.length, + ) + : []; + + const result: Result = { + ...rest, + date, + text, + uid, + author: displayName, + type, + }; + const yamlLines = handleFrontmatter({ + frontmatter, + rest, + result, + }); + + const content = `${yamlLines}\n\n${treeNode.children + .map((c) => + toMarkdown({ + c, + v, + i: 0, + opts: { + refs: optsRefs, + embeds: optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }, + }), + ) + .join("\n")}\n${ + discourseResults.length + ? `\n###### Discourse Context\n\n${discourseResults + .flatMap((r) => + Object.values(r.results).map( + (t) => + `- **${r.label}::** ${toLink( + getFilename({ + title: t.text, + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + }), + t.uid, + linkType, + )}`, + ), + ) + .join("\n")}\n` + : "" + }${ + referenceResults.length + ? `\n###### References\n\n${referenceResults + .map( + (r_1) => + `${toLink( + getFilename({ + title: r_1[0][":node/title"], + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + }), + r_1[0][":block/uid"] || "", + linkType, + )}\n\n${toMarkdown({ + c: pullBlockToTreeNode(r_1[1], ":bullet"), + opts: { + refs: optsRefs, + embeds: optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }, + })}`, + ) + .join("\n")}\n` + : "" + }`; + const uids = new Set(collectUids(treeNode)); + return { title: text, content, uids }; +};