docs: add util to generate clean markdown for a file (medusajs#11303)

Author: shahednasser

www/packages/docs-utils/package.json

@@ -28,7 +28,9 @@
     "watch": "tsc --watch"
   },
   "dependencies": {
+    "@mdx-js/mdx": "^3.1.0",
     "remark-frontmatter": "^5.0.0",
+    "remark-mdx": "^3.1.0",
     "remark-parse": "^11.0.0",
     "remark-stringify": "^11.0.0",
     "to-vfile": "^8.0.0",

…rehype-plugins/src/utils/estree-to-js.ts …/packages/docs-utils/src/estree-to-js.tswww/packages/remark-rehype-plugins/src/utils/estree-to-js.ts renamed to www/packages/docs-utils/src/estree-to-js.ts www/packages/remark-rehype-plugins/src/utils/estree-to-js.ts renamed to www/packages/docs-utils/src/estree-to-js.ts

@@ -7,7 +7,7 @@ import {
   ExpressionJsVarLiteral,
   LiteralExpression,
   ObjectExpression,
-} from "../types/index.js"
+} from "types"
 
 export function estreeToJs(estree: Estree) {
   // TODO improve on this utility. Currently it's implemented to work

…plugins/src/utils/expression-is-utils.ts …es/docs-utils/src/expression-is-utils.tswww/packages/remark-rehype-plugins/src/utils/expression-is-utils.ts renamed to www/packages/docs-utils/src/expression-is-utils.ts www/packages/remark-rehype-plugins/src/utils/expression-is-utils.ts renamed to www/packages/docs-utils/src/expression-is-utils.ts

@@ -1,4 +1,4 @@
-import { ExpressionJsVarLiteral, ExpressionJsVarObj } from "../types/index.js"
+import { ExpressionJsVarLiteral, ExpressionJsVarObj } from "types"
 
 export function isExpressionJsVarLiteral(
   expression: unknown

www/packages/docs-utils/src/get-clean-md.ts

@@ -0,0 +1,136 @@
+import remarkMdx from "remark-mdx"
+import remarkParse from "remark-parse"
+import remarkStringify from "remark-stringify"
+import { read } from "to-vfile"
+import { UnistNode, UnistNodeWithData, UnistTree } from "types"
+import { Plugin, Transformer, unified } from "unified"
+import { SKIP } from "unist-util-visit"
+import type { VFile } from "vfile"
+import {
+  parseCard,
+  parseCardList,
+  parseCodeTabs,
+  parseDetails,
+  parseNote,
+  parsePrerequisites,
+  parseSourceCodeLink,
+  parseTable,
+  parseTabs,
+  parseTypeList,
+  parseWorkflowDiagram,
+} from "./utils/parse-elms.js"
+
+const parseComponentsPlugin = (): Transformer => {
+  return async (tree) => {
+    const { visit } = await import("unist-util-visit")
+
+    let pageTitle = ""
+
+    visit(
+      tree as UnistTree,
+      ["mdxJsxFlowElement", "element", "mdxjsEsm", "heading"],
+      (node: UnistNode, index, parent) => {
+        if (typeof index !== "number" || !parent) {
+          return
+        }
+        if (
+          node.type === "mdxjsEsm" &&
+          node.value?.startsWith("export const metadata = ") &&
+          node.data &&
+          "estree" in node.data
+        ) {
+          const regexMatch = /title: (?<title>.+),?/.exec(node.value)
+          if (regexMatch?.groups?.title) {
+            pageTitle = regexMatch.groups.title
+              .replace(/,$/, "")
+              .replaceAll(/\$\{.+\}/g, "")
+              .replaceAll(/^['"`]/g, "")
+              .replaceAll(/['"`]$/g, "")
+              .trim()
+          }
+        }
+        if (node.type === "heading") {
+          if (
+            node.depth === 1 &&
+            node.children?.length &&
+            node.children[0].value === "metadata.title"
+          ) {
+            node.children[0] = {
+              type: "text",
+              value: pageTitle,
+            }
+          }
+          return
+        }
+        if (
+          node.type === "mdxjsEsm" ||
+          node.name === "Feedback" ||
+          node.name === "ChildDocs" ||
+          node.name === "DetailsList"
+        ) {
+          parent?.children.splice(index, 1)
+          return [SKIP, index]
+        }
+        switch (node.name) {
+          case "Card":
+            return parseCard(node, index, parent)
+          case "CardList":
+            return parseCardList(node as UnistNodeWithData, index, parent)
+          case "CodeTabs":
+            return parseCodeTabs(node as UnistNodeWithData, index, parent)
+          case "Details":
+            return parseDetails(node as UnistNodeWithData, index, parent)
+          case "Note":
+            return parseNote(node, index, parent)
+          case "Prerequisites":
+            return parsePrerequisites(node as UnistNodeWithData, index, parent)
+          case "SourceCodeLink":
+            return parseSourceCodeLink(node as UnistNodeWithData, index, parent)
+          case "Table":
+            return parseTable(node as UnistNodeWithData, index, parent)
+          case "Tabs":
+            return parseTabs(node as UnistNodeWithData, index, parent)
+          case "TypeList":
+            return parseTypeList(node as UnistNodeWithData, index, parent)
+          case "WorkflowDiagram":
+            return parseWorkflowDiagram(
+              node as UnistNodeWithData,
+              index,
+              parent
+            )
+        }
+      }
+    )
+  }
+}
+
+const getParsedAsString = (file: VFile): string => {
+  return file.toString().replaceAll(/^([\s]*)\* /gm, "$1- ")
+}
+
+export const getCleanMd = async (
+  filePath: string,
+  plugins?: {
+    before?: Plugin[]
+    after?: Plugin[]
+  }
+): Promise<string> => {
+  if (!filePath.endsWith(".md") && !filePath.endsWith(".mdx")) {
+    return ""
+  }
+  const unifier = unified().use(remarkParse).use(remarkMdx).use(remarkStringify)
+
+  plugins?.before?.forEach((plugin) => {
+    unifier.use(...(Array.isArray(plugin) ? plugin : [plugin]))
+  })
+
+  unifier.use(parseComponentsPlugin)
+
+  plugins?.after?.forEach((plugin) => {
+    unifier.use(...(Array.isArray(plugin) ? plugin : [plugin]))
+  })
+
+  const parsed = await unifier.process(await read(filePath))
+
+  return getParsedAsString(parsed)
+}

www/packages/docs-utils/src/index.ts

@@ -1,4 +1,7 @@
+export * from "./estree-to-js.js"
+export * from "./expression-is-utils.js"
 export * from "./find-title.js"
+export * from "./get-clean-md.js"
 export * from "./get-file-slug-sync.js"
 export * from "./get-file-slug.js"
 export * from "./get-front-matter.js"