From dd786b91636c8324172c6182696698e825340fff Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Wed, 13 Aug 2025 08:02:16 -0600 Subject: [PATCH] =?UTF-8?q?=E2=84=B9=20Improve=20formatting=20of=20myst-to?= =?UTF-8?q?-md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/cool-ducks-sing.md | 5 + .changeset/tender-pianos-cheat.md | 5 + package-lock.json | 1 + packages/myst-ext-card/tests/card.spec.ts | 16 +-- .../myst-ext-exercise/tests/exercise.spec.ts | 12 +- packages/myst-ext-grid/tests/grid.spec.ts | 2 +- packages/myst-ext-icon/tests/icon.spec.ts | 4 +- packages/myst-ext-proof/tests/proof.spec.ts | 4 +- .../myst-ext-reactive/tests/reactive.spec.ts | 4 +- packages/myst-ext-tabs/tests/tabs.spec.ts | 10 +- packages/myst-parser/src/directives.ts | 4 +- packages/myst-parser/src/roles.ts | 1 + packages/myst-to-md/package.json | 3 +- packages/myst-to-md/src/directives.ts | 55 ++++++- packages/myst-to-md/src/index.ts | 27 +++- packages/myst-to-md/src/misc.ts | 16 ++- packages/myst-to-md/src/roles.ts | 14 +- .../myst-to-md/src/transforms/definitions.ts | 24 ++++ packages/myst-to-md/src/transforms/index.ts | 15 ++ packages/myst-to-md/src/transforms/utils.ts | 16 +++ .../tests/{run.spec.ts => basic.spec.ts} | 42 +----- .../myst-to-md/tests/{ => mdast}/basic.yml | 22 ++- .../tests/{ => mdast}/directives.yml | 136 +++++++++++++----- packages/myst-to-md/tests/mdast/mdast.spec.ts | 40 ++++++ .../myst-to-md/tests/{ => mdast}/misc.yml | 42 +++--- .../tests/{ => mdast}/references.yml | 19 ++- .../myst-to-md/tests/{ => mdast}/roles.yml | 24 ++-- .../tests/roundtrip/roundtrip.spec.ts | 42 ++++++ .../myst-to-md/tests/roundtrip/roundtrip.yml | 67 +++++++++ 29 files changed, 501 insertions(+), 171 deletions(-) create mode 100644 .changeset/cool-ducks-sing.md create mode 100644 .changeset/tender-pianos-cheat.md create mode 100644 packages/myst-to-md/src/transforms/definitions.ts create mode 100644 packages/myst-to-md/src/transforms/index.ts create mode 100644 packages/myst-to-md/src/transforms/utils.ts rename packages/myst-to-md/tests/{run.spec.ts => basic.spec.ts} (64%) rename packages/myst-to-md/tests/{ => mdast}/basic.yml (94%) rename packages/myst-to-md/tests/{ => mdast}/directives.yml (89%) create mode 100644 packages/myst-to-md/tests/mdast/mdast.spec.ts rename packages/myst-to-md/tests/{ => mdast}/misc.yml (93%) rename packages/myst-to-md/tests/{ => mdast}/references.yml (93%) rename packages/myst-to-md/tests/{ => mdast}/roles.yml (93%) create mode 100644 packages/myst-to-md/tests/roundtrip/roundtrip.spec.ts create mode 100644 packages/myst-to-md/tests/roundtrip/roundtrip.yml diff --git a/.changeset/cool-ducks-sing.md b/.changeset/cool-ducks-sing.md new file mode 100644 index 0000000000..9fc2895d23 --- /dev/null +++ b/.changeset/cool-ducks-sing.md @@ -0,0 +1,5 @@ +--- +"myst-parser": patch +--- + +Add body/options to the mystDirective and mystRole nodes diff --git a/.changeset/tender-pianos-cheat.md b/.changeset/tender-pianos-cheat.md new file mode 100644 index 0000000000..ea19179725 --- /dev/null +++ b/.changeset/tender-pianos-cheat.md @@ -0,0 +1,5 @@ +--- +"myst-to-md": patch +--- + +Improve formatting of raw myst-directives diff --git a/package-lock.json b/package-lock.json index 3611042d5d..6f601f8e45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16785,6 +16785,7 @@ "license": "MIT", "dependencies": { "js-yaml": "^4.1.0", + "longest-streak": "^3.1.0", "mdast-util-gfm-footnote": "^1.0.2", "mdast-util-gfm-table": "^1.0.7", "mdast-util-to-markdown": "^1.5.0", diff --git a/packages/myst-ext-card/tests/card.spec.ts b/packages/myst-ext-card/tests/card.spec.ts index 0571248d4d..3172693e55 100644 --- a/packages/myst-ext-card/tests/card.spec.ts +++ b/packages/myst-ext-card/tests/card.spec.ts @@ -11,7 +11,7 @@ describe('card directive', () => { { type: 'mystDirective', name: 'card', - args: 'Card Title', + args: [{ type: 'text', value: 'Card Title' }], value: 'Header\n^^^\n\nCard content\n+++\nFooter', position: { start: { @@ -142,7 +142,7 @@ describe('card directive', () => { const output = mystParse(content, { directives: [cardDirective], }); - expect(output).toEqual(expected); + expect(output).toMatchObject(expected); }); it('card directive parses with options', async () => { const content = @@ -153,11 +153,11 @@ describe('card directive', () => { { type: 'mystDirective', name: 'card', - args: 'Card Title', + args: [{ type: 'text', value: 'Card Title' }], options: { - header: 'Header', - footer: 'Footer', - link: 'my-url', + header: [{ type: 'text', value: 'Header' }], + footer: [{ type: 'text', value: 'Footer' }], + url: 'my-url', }, value: 'Card\n^^^\ncontent', position: { @@ -280,7 +280,7 @@ describe('card directive', () => { const output = mystParse(content, { directives: [cardDirective], }); - expect(output).toEqual(expected); + expect(output).toMatchObject(expected); }); it('card directive parses with minimal content', async () => { const content = '```{card}\nCard content\n```'; @@ -343,7 +343,7 @@ describe('card directive', () => { const output = mystParse(content, { directives: [cardDirective], }); - expect(output).toEqual(expected); + expect(output).toMatchObject(expected); }); }); diff --git a/packages/myst-ext-exercise/tests/exercise.spec.ts b/packages/myst-ext-exercise/tests/exercise.spec.ts index 2b2046934b..3e9cf29b12 100644 --- a/packages/myst-ext-exercise/tests/exercise.spec.ts +++ b/packages/myst-ext-exercise/tests/exercise.spec.ts @@ -22,7 +22,7 @@ describe('exercise directive', () => { options: { label: 'ex-1', }, - args: 'Exercise Title', + args: [{ type: 'text', value: 'Exercise Title' }], value: 'Exercise content', children: [ { @@ -58,7 +58,7 @@ describe('exercise directive', () => { const output = mystParse(content, { directives: [exerciseDirective], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); it('nonumber is prioritized over enumerated', async () => { const content = @@ -74,7 +74,7 @@ describe('exercise directive', () => { enumerated: true, nonumber: true, }, - args: 'Exercise Title', + args: [{ type: 'text', value: 'Exercise Title' }], value: 'Exercise content', children: [ { @@ -110,7 +110,7 @@ describe('exercise directive', () => { const output = mystParse(content, { directives: [exerciseDirective], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); it('exercises are enumerated with labels by default', async () => { const content = '```{exercise} Exercise Title\nExercise content\n```'; @@ -125,7 +125,7 @@ describe('exercise directive', () => { { type: 'mystDirective', name: 'exercise', - args: 'Exercise Title', + args: [{ type: 'text', value: 'Exercise Title' }], value: 'Exercise content', children: [ { @@ -158,6 +158,6 @@ describe('exercise directive', () => { }, ], }; - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); }); diff --git a/packages/myst-ext-grid/tests/grid.spec.ts b/packages/myst-ext-grid/tests/grid.spec.ts index b7d26a5eb9..c4996b0038 100644 --- a/packages/myst-ext-grid/tests/grid.spec.ts +++ b/packages/myst-ext-grid/tests/grid.spec.ts @@ -87,6 +87,6 @@ describe('grid directive', () => { const output = mystParse(content, { directives: [gridDirective], }); - expect(output).toEqual(expected); + expect(output).toMatchObject(expected); }); }); diff --git a/packages/myst-ext-icon/tests/icon.spec.ts b/packages/myst-ext-icon/tests/icon.spec.ts index 453e50d792..ccab874e49 100644 --- a/packages/myst-ext-icon/tests/icon.spec.ts +++ b/packages/myst-ext-icon/tests/icon.spec.ts @@ -44,7 +44,7 @@ describe('icon role', () => { const output = mystParse(markup, { roles: [iconRole], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }, ); it.each(Object.entries(LEGACY_ICON_ALIASES))( @@ -79,7 +79,7 @@ describe('icon role', () => { const output = mystParse(markup, { roles: [iconRole], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }, ); }); diff --git a/packages/myst-ext-proof/tests/proof.spec.ts b/packages/myst-ext-proof/tests/proof.spec.ts index 4f0fd0d2c2..03c2e2994d 100644 --- a/packages/myst-ext-proof/tests/proof.spec.ts +++ b/packages/myst-ext-proof/tests/proof.spec.ts @@ -11,7 +11,7 @@ describe('proof directive', () => { { type: 'mystDirective', name: 'prf:proof', - args: 'Proof Title', + args: [{ type: 'text', value: 'Proof Title' }], value: 'Proof content', position: { start: { @@ -86,6 +86,6 @@ describe('proof directive', () => { const output = mystParse(content, { directives: [proofDirective], }); - expect(output).toEqual(expected); + expect(output).toMatchObject(expected); }); }); diff --git a/packages/myst-ext-reactive/tests/reactive.spec.ts b/packages/myst-ext-reactive/tests/reactive.spec.ts index e697dd8884..6e9631841b 100644 --- a/packages/myst-ext-reactive/tests/reactive.spec.ts +++ b/packages/myst-ext-reactive/tests/reactive.spec.ts @@ -41,7 +41,7 @@ describe('reactive tests', () => { roles: [reactiveRole], directives: [reactiveDirective], }); - expect(output).toEqual(expected); + expect(output).toMatchObject(expected); }); it('r:dynamic role parses', async () => { const content = '{r:dynamic}`rValue="visitors", rChange="{visitors: value}", value="5"`'; @@ -92,6 +92,6 @@ describe('reactive tests', () => { roles: [reactiveRole], directives: [reactiveDirective], }); - expect(output).toEqual(expected); + expect(output).toMatchObject(expected); }); }); diff --git a/packages/myst-ext-tabs/tests/tabs.spec.ts b/packages/myst-ext-tabs/tests/tabs.spec.ts index 0c5a2714aa..5fbbc4ceab 100644 --- a/packages/myst-ext-tabs/tests/tabs.spec.ts +++ b/packages/myst-ext-tabs/tests/tabs.spec.ts @@ -31,7 +31,7 @@ describe('tab directives', () => { const output = mystParse(content, { directives: [...tabDirectives], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); it('tabSet class option parses', async () => { const content = '```{tab-set}\n:class: my-class\n```'; @@ -57,7 +57,7 @@ describe('tab directives', () => { const output = mystParse(content, { directives: [...tabDirectives], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); it.each(['tab-item', 'tabItem'])('%s with title parses', async (name: string) => { const content = `\`\`\`{${name}} Tab One\n\`\`\``; @@ -81,7 +81,7 @@ describe('tab directives', () => { const output = mystParse(content, { directives: [...tabDirectives], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); it('tabItem sync and selected options parse', async () => { const content = '```{tab-item} Tab One\n:sync: tab1\n:selected:\n```'; @@ -111,7 +111,7 @@ describe('tab directives', () => { const output = mystParse(content, { directives: [...tabDirectives], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); // TODO: enable when we have a better required/fallback/default pattern it.skip('tabItem without title errors', async () => { @@ -202,6 +202,6 @@ describe('tab directives', () => { const output = mystParse(content, { directives: [...tabDirectives], }); - expect(deletePositions(output)).toEqual(expected); + expect(deletePositions(output)).toMatchObject(expected); }); }); diff --git a/packages/myst-parser/src/directives.ts b/packages/myst-parser/src/directives.ts index d9fb518f1c..d5d9b59471 100644 --- a/packages/myst-parser/src/directives.ts +++ b/packages/myst-parser/src/directives.ts @@ -3,7 +3,6 @@ import type { DirectiveData, DirectiveSpec, DirectiveContext, - ParseTypes, GenericParent, } from 'myst-common'; import { RuleId, fileError, fileWarn } from 'myst-common'; @@ -94,6 +93,7 @@ export function applyDirectives( if (argSpec.required && data.arg == null) { validationError = true; } + node.args = data.arg; } } else if (argNode) { const message = `unexpected argument provided for directive: ${name}`; @@ -104,6 +104,7 @@ export function applyDirectives( // const options: Record = {}; const { valid: validOptions, options } = parseOptions(name, node, vfile, optionsSpec); data.options = options; + node.options = options; validationError = validationError || validOptions; // Handle body @@ -124,6 +125,7 @@ export function applyDirectives( `body of directive: ${name}`, RuleId.directiveBodyCorrect, ); + node.body = data.body; if (bodySpec.required && data.body == null) { validationError = true; } diff --git a/packages/myst-parser/src/roles.ts b/packages/myst-parser/src/roles.ts index 7d6abf2e2a..c5d4b79b35 100644 --- a/packages/myst-parser/src/roles.ts +++ b/packages/myst-parser/src/roles.ts @@ -65,6 +65,7 @@ export function applyRoles(tree: GenericParent, specs: RoleSpec[], vfile: VFile) `body of role: ${name}`, RuleId.roleBodyCorrect, ); + node.body = data.body; if (body.required && data.body == null) { validationError = true; } diff --git a/packages/myst-to-md/package.json b/packages/myst-to-md/package.json index c63065445f..a7f8e5dd9c 100644 --- a/packages/myst-to-md/package.json +++ b/packages/myst-to-md/package.json @@ -44,6 +44,7 @@ "myst-frontmatter": "^1.7.6", "unist-util-select": "^4.0.3", "vfile": "^5.3.7", - "vfile-reporter": "^7.0.4" + "vfile-reporter": "^7.0.4", + "longest-streak": "^3.1.0" } } diff --git a/packages/myst-to-md/src/directives.ts b/packages/myst-to-md/src/directives.ts index 93558289b1..2cfac72798 100644 --- a/packages/myst-to-md/src/directives.ts +++ b/packages/myst-to-md/src/directives.ts @@ -5,6 +5,7 @@ import type { GenericNode } from 'myst-common'; import { toText, fileError, RuleId } from 'myst-common'; import { select, selectAll } from 'unist-util-select'; import type { VFile } from 'vfile'; +import { longestStreak } from 'longest-streak'; import type { NestedState, Parent, Validator } from './types.js'; import { incrementNestedLevel, popNestedLevel } from './utils.js'; @@ -74,12 +75,45 @@ function writeStaticDirective(name: string, options?: DirectiveOptions) { } /** - * Generic MyST directive handler - * - * This uses the directive name/args/value and ignores any children nodes + * Handler for a raw directive */ -function mystDirective(node: any, _: Parent, state: NestedState, info: Info): string { - return writeStaticDirective(node.name, { argsKey: 'args' })(node, _, state, info); +function writeDirective(options?: DirectiveOptions) { + return (node: any, _: Parent, state: NestedState, info: Info): string => { + incrementNestedLevel('directive', state); + const { label, class: className, ...otherOptions } = node.options ?? {}; + const optLabel = label ? `#${label}` : ''; + const optClass = className + ? className + .split(' ') + .filter((c: string) => c.trim() !== '') + .map((c: string) => `.${c.trim()}`) + .join(' ') + : ''; + const optOther = Object.entries(otherOptions) + .map(([key, value]) => `${key}=${value}`) + .join(' '); + const directiveOpts = [optLabel, optClass, optOther].filter(Boolean).join(' '); + + // If the directive has a body which is an array, use it as content. + // Otherwise, use the value. + const markdownBody = node.body && Array.isArray(node.body); + const content = markdownBody + ? state.containerFlow({ type: 'root', children: node.body }, info) + : node.value; + + const nesting = popNestedLevel('directive', state); + const markerChar = markdownBody ? ':' : '`'; + const markerLength = Math.max(longestStreak(content, markerChar) + 1, nesting + 3); + const marker = markerChar.repeat(markerLength); + const directiveInline = [node.name, directiveOpts].filter(Boolean).join(' '); + const args = Array.isArray(node.args) + ? state.containerPhrasing({ type: 'heading', depth: 1, children: node.args }, info) + : node.args; + const directiveLines = [`${marker}{${directiveInline}}${args ? ' ' : ''}${args ? args : ''}`]; + if (content) directiveLines.push(content); + directiveLines.push(marker); + return directiveLines.join('\n'); + }; } const CODE_BLOCK_KEYS = [ @@ -388,6 +422,13 @@ function aside(node: any, _: Parent, state: NestedState, info: Info): string { return writeFlowDirective(name, args, options)(nodeCopy, _, state, info); } +function math(node: any, _: Parent, state: NestedState, info: Info): string { + if (!node.typst && !node.label) { + return `$$\n${node.value}\n$$`; + } + return writeStaticDirective('math', { keys: ['label', 'typst'] })(node, _, state, info); +} + export const directiveHandlers: Record = { code, image, @@ -403,11 +444,11 @@ export const directiveHandlers: Record = { }), tabSet: writeFlowDirective('tab-set'), tabItem, - math: writeStaticDirective('math', { keys: ['label'] }), + math, embed: writeStaticDirective('embed', { argsKey: 'label' }), include: writeStaticDirective('include', { argsKey: 'file' }), mermaid: writeStaticDirective('mermaid'), - mystDirective, + mystDirective: writeDirective(), }; export const directiveValidators: Record = { diff --git a/packages/myst-to-md/src/index.ts b/packages/myst-to-md/src/index.ts index 97e785259f..3d4c1c114b 100644 --- a/packages/myst-to-md/src/index.ts +++ b/packages/myst-to-md/src/index.ts @@ -11,6 +11,8 @@ import { referenceHandlers } from './references.js'; import { roleHandlers } from './roles.js'; import { addFrontmatter, runValidators, unsupportedHandlers } from './utils.js'; import type { PageFrontmatter } from 'myst-frontmatter'; +import { basicTransformations } from './transforms/index.js'; +import { copyNode } from 'myst-common'; const FOOTNOTE_HANDLER_KEYS = ['footnoteDefinition', 'footnoteReference']; const TABLE_HANDLER_KEYS = ['table', 'tableRow', 'tableCell']; @@ -28,19 +30,38 @@ export function writeMd(file: VFile, node: Root, frontmatter?: PageFrontmatter) ...FOOTNOTE_HANDLER_KEYS, ...TABLE_HANDLER_KEYS, ]; - const unsupported = unsupportedHandlers(node, handlerKeys, file); + const copy = copyNode(node); + const unsupported = unsupportedHandlers(copy, handlerKeys, file); const options: Options = { fences: true, rule: '-', + ruleRepetition: 3, + emphasis: '_', + bullet: '-', + listItemIndent: 'one', handlers: { ...handlers, ...unsupported, }, + join: [ + (left, right, parent) => { + if (left.type === ('mystTarget' as any)) return 0; + if (right.type === ('definitionDescription' as any)) return 0; + if ( + // This ensures lists are tightly joined always + left.type === ('listItem' as any) || + (right.type === ('list' as any) && parent.type === ('listItem' as any)) + ) + return 0; + return 1; + }, + ], extensions: [gfmFootnoteToMarkdown(), gfmTableToMarkdown()], }; const validators = { ...directiveValidators, ...miscValidators }; - runValidators(node, validators, file); - const result = toMarkdown(node as any, options).trim(); + runValidators(copy, validators, file); + basicTransformations(copy); + const result = toMarkdown(copy as any, options); file.result = addFrontmatter(result, frontmatter); return file; } diff --git a/packages/myst-to-md/src/misc.ts b/packages/myst-to-md/src/misc.ts index a6ff9045b8..f522a92441 100644 --- a/packages/myst-to-md/src/misc.ts +++ b/packages/myst-to-md/src/misc.ts @@ -13,7 +13,17 @@ function block(node: any, _: Parent, state: State, info: Info): string { if (node.visibility === 'remove' || node.visibility === 'hide') return ''; const meta = node.meta ? ` ${node.meta}` : ''; const content = state.containerFlow(node, info); - return `+++${meta}\n${content}`; + return `+++${meta}\n\n${content}`; +} + +function blockBreak(node: any, _: Parent, state: State): string { + return state.indentLines(`+++`, (line: string, _1: any, blank: boolean) => (blank ? '' : line)); +} + +function mystTarget(node: any, _: Parent, state: State): string { + return state.indentLines(`(${node.label})=`, (line: string, _1: any, blank: boolean) => + blank ? '' : line, + ); } function definitionListValidator(node: any, file: VFile) { @@ -40,7 +50,7 @@ function definitionDescription(node: any, _: Parent, state: State, info: Info) { const contentLines = state.containerFlow(node, info).split('\n'); const indented = contentLines.map((line, ind) => { if (!line && ind) return line; - const prefix = ind ? ` ` : `: `; + const prefix = ind ? ` ` : `: `; return `${prefix}${line}`; }); return indented.join('\n'); @@ -48,6 +58,8 @@ function definitionDescription(node: any, _: Parent, state: State, info: Info) { export const miscHandlers: Record = { block, + blockBreak, + mystTarget, comment, definitionList, definitionTerm, diff --git a/packages/myst-to-md/src/roles.ts b/packages/myst-to-md/src/roles.ts index 7d414db22e..411fc2032d 100644 --- a/packages/myst-to-md/src/roles.ts +++ b/packages/myst-to-md/src/roles.ts @@ -2,6 +2,7 @@ import type { Handle, Info } from 'mdast-util-to-markdown'; import { defaultHandlers } from 'mdast-util-to-markdown'; import type { NestedState, Parent } from './types.js'; import { incrementNestedLevel, popNestedLevel } from './utils.js'; +import type { InlineMath } from 'myst-spec-ext'; /** * Inline code handler @@ -103,6 +104,17 @@ function citeGroup(node: any, _: Parent, state: NestedState, info: Info): string return tracker.move(`{${name}}\`${labels}\``); } +/** + * Inline math handler + */ +function inlineMath(node: InlineMath, _: Parent, state: NestedState, info: Info): string { + if (node.label || node.typst) { + return writeStaticRole('math')(node, _, state); + } + const tracker = state.createTracker(info); + return tracker.move(`$${node.value}$`); +} + export const roleHandlers: Record = { subscript: writePhrasingRole('sub'), superscript: writePhrasingRole('sup'), @@ -110,7 +122,7 @@ export const roleHandlers: Record = { underline: writePhrasingRole('u'), smallcaps: writePhrasingRole('sc'), abbreviation, - inlineMath: writeStaticRole('math'), + inlineMath, inlineCode, cite, citeGroup, diff --git a/packages/myst-to-md/src/transforms/definitions.ts b/packages/myst-to-md/src/transforms/definitions.ts new file mode 100644 index 0000000000..6debc53598 --- /dev/null +++ b/packages/myst-to-md/src/transforms/definitions.ts @@ -0,0 +1,24 @@ +import type { Plugin } from 'unified'; +import type { Parent } from 'myst-spec'; +import type { DefinitionDescription } from 'myst-spec-ext'; +import { selectAll } from 'unist-util-select'; +import type { GenericNode, GenericParent } from 'myst-common'; +import { phrasingTypes } from './utils.js'; + +export type DefinitionItem = Parent & { type: 'definitionItem' }; + +/** + * It will ensure defDescriptions that are not paragraphs are wrapped in paragraphs. + */ +export function definitionTransform(mdast: GenericParent) { + const defDescriptions = selectAll('definitionDescription', mdast) as DefinitionDescription[]; + defDescriptions.forEach((node) => { + const hasPhrasingContent = node.children.some((n) => phrasingTypes.has(n.type)); + if (!hasPhrasingContent) return; + node.children = [{ type: 'paragraph', children: [...node.children] }] as GenericNode[]; + }); +} + +export const definitionPlugin: Plugin<[], GenericParent, GenericParent> = () => (tree) => { + definitionTransform(tree); +}; diff --git a/packages/myst-to-md/src/transforms/index.ts b/packages/myst-to-md/src/transforms/index.ts new file mode 100644 index 0000000000..3b8f2deb1c --- /dev/null +++ b/packages/myst-to-md/src/transforms/index.ts @@ -0,0 +1,15 @@ +import type { Plugin } from 'unified'; + +import type { GenericParent } from 'myst-common'; +import { definitionTransform } from './definitions.js'; + +export { definitionTransform, definitionPlugin } from './definitions.js'; + +export function basicTransformations(tree: GenericParent) { + definitionTransform(tree); +} + +export const basicTransformationsPlugin: Plugin<[], GenericParent, GenericParent> = + () => (tree) => { + basicTransformations(tree); + }; diff --git a/packages/myst-to-md/src/transforms/utils.ts b/packages/myst-to-md/src/transforms/utils.ts new file mode 100644 index 0000000000..7f75c4d51c --- /dev/null +++ b/packages/myst-to-md/src/transforms/utils.ts @@ -0,0 +1,16 @@ +export const phrasingTypes = new Set([ + 'text', + 'strong', + 'emphasis', + 'inlineCode', + 'inlineMath', + 'subscript', + 'superscript', + 'smallcaps', + 'link', + 'delete', + 'crossReference', + 'image', + 'html', + 'mystRole', +]); diff --git a/packages/myst-to-md/tests/run.spec.ts b/packages/myst-to-md/tests/basic.spec.ts similarity index 64% rename from packages/myst-to-md/tests/run.spec.ts rename to packages/myst-to-md/tests/basic.spec.ts index dc707e2b18..ef3f011cce 100644 --- a/packages/myst-to-md/tests/run.spec.ts +++ b/packages/myst-to-md/tests/basic.spec.ts @@ -1,43 +1,7 @@ import { describe, expect, test } from 'vitest'; -import fs from 'node:fs'; -import path from 'node:path'; -import yaml from 'js-yaml'; import { unified } from 'unified'; import mystToMd from '../src'; -type TestCase = { - title: string; - markdown: string; - mdast: Record; -}; - -type TestCases = { - title: string; - cases: TestCase[]; -}; - -const casesList: TestCases[] = fs - .readdirSync(__dirname) - .filter((file) => file.endsWith('.yml')) - .map((file) => { - const content = fs.readFileSync(path.join(__dirname, file), { encoding: 'utf-8' }); - return yaml.load(content) as TestCases; - }); - -casesList.forEach(({ title, cases }) => { - describe(title, () => { - test.each(cases.map((c): [string, TestCase] => [c.title, c]))( - '%s', - (_, { markdown, mdast }) => { - const pipe = unified().use(mystToMd); - pipe.runSync(mdast as any); - const file = pipe.stringify(mdast as any); - expect(file.result).toEqual(markdown); - }, - ); - }); -}); - describe('myst-to-md frontmatter', () => { test('empty frontmatter passes', () => { const pipe = unified().use(mystToMd, {}); @@ -47,7 +11,7 @@ describe('myst-to-md frontmatter', () => { }; pipe.runSync(mdast as any); const file = pipe.stringify(mdast as any); - expect(file.result).toEqual('Hello world!'); + expect(file.result).toEqual('Hello world!\n'); }); test('simple frontmatter passes', () => { const pipe = unified().use(mystToMd, { title: 'My Title' }); @@ -57,7 +21,7 @@ describe('myst-to-md frontmatter', () => { }; pipe.runSync(mdast as any); const file = pipe.stringify(mdast as any); - expect(file.result).toEqual('---\ntitle: My Title\n---\nHello world!'); + expect(file.result).toEqual('---\ntitle: My Title\n---\nHello world!\n'); }); test('frontmatter with licenses passes', () => { const pipe = unified().use(mystToMd, { @@ -85,7 +49,7 @@ describe('myst-to-md frontmatter', () => { pipe.runSync(mdast as any); const file = pipe.stringify(mdast as any); expect(file.result).toEqual( - '---\ntitle: My Title\nlicense:\n content: Apache-2.0\n code: CC-BY-3.0\n---\nHello world!', + '---\ntitle: My Title\nlicense:\n content: Apache-2.0\n code: CC-BY-3.0\n---\nHello world!\n', ); }); }); diff --git a/packages/myst-to-md/tests/basic.yml b/packages/myst-to-md/tests/mdast/basic.yml similarity index 94% rename from packages/myst-to-md/tests/basic.yml rename to packages/myst-to-md/tests/mdast/basic.yml index 48c124b3c8..c04ec673ef 100644 --- a/packages/myst-to-md/tests/basic.yml +++ b/packages/myst-to-md/tests/mdast/basic.yml @@ -1,4 +1,4 @@ -title: myst-to-md basic features +title: myst-to-md basic cases: - title: styles in paragraph mdast: @@ -23,7 +23,7 @@ cases: - type: inlineCode value: style`s markdown: |- - Some % *markdown* with **different** ``style`s`` + Some % _markdown_ with **different** ``style`s`` - title: headings mdast: type: root @@ -49,7 +49,7 @@ cases: markdown: |- # first - Some % *markdown* + Some % _markdown_ #### fourth - title: thematic break @@ -86,7 +86,7 @@ cases: - type: text value: markdown markdown: |- - > Some % *markdown* + > Some % _markdown_ - title: unordered list mdast: type: root @@ -107,9 +107,8 @@ cases: - type: text value: Some more markdown markdown: |- - * Some markdown - - * Some more markdown + - Some markdown + - Some more markdown - title: ordered list mdast: type: root @@ -131,9 +130,8 @@ cases: - type: text value: Some more markdown markdown: |- - 5. Some markdown - - 6. Some more markdown + 5. Some markdown + 6. Some more markdown - title: html mdast: type: root @@ -241,7 +239,7 @@ cases: - type: text value: markdown markdown: |- - [Some % *markdown*](https://example.com "my link") + [Some % _markdown_](https://example.com "my link") - title: image mdast: type: root @@ -267,7 +265,7 @@ cases: - type: text value: markdown markdown: |- - [Some % *markdown*][My-Link] + [Some % _markdown_][My-Link] - title: image reference mdast: type: root diff --git a/packages/myst-to-md/tests/directives.yml b/packages/myst-to-md/tests/mdast/directives.yml similarity index 89% rename from packages/myst-to-md/tests/directives.yml rename to packages/myst-to-md/tests/mdast/directives.yml index f56124b645..884d580edc 100644 --- a/packages/myst-to-md/tests/directives.yml +++ b/packages/myst-to-md/tests/mdast/directives.yml @@ -18,7 +18,7 @@ cases: - type: emphasis children: - type: text - valeu: markdown + value: markdown markdown: |- ```{abc} My Directive! :a: b @@ -119,18 +119,11 @@ cases: type: root children: - type: math - value: |- - 5+5 - ````{abc} - ```` - print("hello world") + value: Ax=b markdown: |- - `````{math} - 5+5 - ````{abc} - ```` - print("hello world") - ````` + $$ + Ax=b + $$ - title: math directive - label mdast: type: root @@ -343,7 +336,7 @@ cases: :title: my image :alt: Some text - Some % *markdown* + Some % _markdown_ ::: - title: figure directive - with caption/legend mdast: @@ -382,7 +375,7 @@ cases: :name: My-Fig :class: my-img-class - Some % *markdown* + Some % _markdown_ Code legend: @@ -435,7 +428,7 @@ cases: :alt: Some text :placeholder: images/example.png - Some % *markdown* + Some % _markdown_ ::: - title: list-table mdast: @@ -511,23 +504,15 @@ cases: :class: my-class :align: right - * * a - - * b - - * c - - * * 1 - - * 2 - - * 3 - - * * 4 - - * 5 - - * 6 + - - a + - b + - c + - - 1 + - 2 + - 3 + - - 4 + - 5 + - 6 ::: - title: admonition - with kind mdast: @@ -559,7 +544,7 @@ cases: - type: inlineCode value: tip markdown: |- - :::{tip} *tip* {u}`admonition` + :::{tip} _tip_ {u}`admonition` This is a `tip` ::: - title: admonition - no kind @@ -593,7 +578,7 @@ cases: - type: inlineCode value: tip markdown: |- - :::{admonition} *not tip* {u}`admonition` + :::{admonition} _not tip_ {u}`admonition` :class: my-class :icon: @@ -745,7 +730,7 @@ cases: :::{card} Card title --- link: https://example.com - header: The *Header* + header: The _Header_ footer: |- Footer @@ -792,7 +777,7 @@ cases: :::{card} --- header: |- - The *Header* + The _Header_ with two paragraphs footer: |- @@ -960,3 +945,82 @@ cases: :::{topic} Topic content ::: + - title: paragraph followed by directive paragraph with spacing + mdast: + type: root + children: + - type: paragraph + children: + - type: text + value: This is a regular paragraph with some text content. + - type: mystDirective + name: note + value: |- + This is a directive paragraph with important information. + markdown: |- + This is a regular paragraph with some text content. + + ```{note} + This is a directive paragraph with important information. + ``` + - title: multiple paragraphs with directive and spacing + mdast: + type: root + children: + - type: paragraph + children: + - type: text + value: First paragraph with regular content. + - type: paragraph + children: + - type: text + value: Second paragraph with more content. + - type: mystDirective + name: warning + value: |- + This is a warning directive with important information. + - type: paragraph + children: + - type: text + value: Final paragraph after the directive. + markdown: |- + First paragraph with regular content. + + Second paragraph with more content. + + ```{warning} + This is a warning directive with important information. + ``` + + Final paragraph after the directive. + - title: paragraph with directive containing args and options + mdast: + type: root + children: + - type: paragraph + children: + - type: text + value: This paragraph introduces a directive with arguments and options. + - type: mystDirective + name: admonition + args: Important Note + value: |- + :class: my-class + :name: my-note + + This directive has both arguments and options. + - type: paragraph + children: + - type: text + value: This paragraph follows the directive. + markdown: |- + This paragraph introduces a directive with arguments and options. + + ```{admonition} Important Note + :class: my-class + :name: my-note + + This directive has both arguments and options. + ``` + + This paragraph follows the directive. diff --git a/packages/myst-to-md/tests/mdast/mdast.spec.ts b/packages/myst-to-md/tests/mdast/mdast.spec.ts new file mode 100644 index 0000000000..960c2ae222 --- /dev/null +++ b/packages/myst-to-md/tests/mdast/mdast.spec.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import yaml from 'js-yaml'; +import { unified } from 'unified'; +import mystToMd from '../../src'; + +type TestCase = { + title: string; + markdown: string; + mdast: Record; +}; + +type TestCases = { + title: string; + cases: TestCase[]; +}; + +const casesList: TestCases[] = fs + .readdirSync(__dirname) + .filter((file) => file.endsWith('.yml')) + .map((file) => { + const content = fs.readFileSync(path.join(__dirname, file), { encoding: 'utf-8' }); + return yaml.load(content) as TestCases; + }); + +casesList.forEach(({ title, cases }) => { + describe(title, () => { + test.each(cases.map((c): [string, TestCase] => [c.title, c]))( + '%s', + (_, { markdown, mdast }) => { + const pipe = unified().use(mystToMd); + pipe.runSync(mdast as any); + const file = pipe.stringify(mdast as any); + const result = (file.result as string).trim(); + expect(result).toEqual(markdown); + }, + ); + }); +}); diff --git a/packages/myst-to-md/tests/misc.yml b/packages/myst-to-md/tests/mdast/misc.yml similarity index 93% rename from packages/myst-to-md/tests/misc.yml rename to packages/myst-to-md/tests/mdast/misc.yml index ed00a758d2..c54b53ea8f 100644 --- a/packages/myst-to-md/tests/misc.yml +++ b/packages/myst-to-md/tests/mdast/misc.yml @@ -1,4 +1,4 @@ -title: myst-to-md references +title: myst-to-md miscellaneous cases: - title: comment mdast: @@ -15,7 +15,7 @@ cases: - type: comment value: and a comment markdown: |- - Some % *markdown* + Some % _markdown_ % and a comment - title: comment @@ -35,7 +35,7 @@ cases: markdown: |- % a comment - Some % *markdown* + Some % _markdown_ - title: block - no meta mdast: type: root @@ -52,7 +52,8 @@ cases: value: markdown markdown: |- +++ - Some % *markdown* + + Some % _markdown_ - title: block - with meta mdast: type: root @@ -70,7 +71,8 @@ cases: value: markdown markdown: |- +++ {"a": "b"} - Some % *markdown* + + Some % _markdown_ - title: footnote - numbered mdast: type: root @@ -204,7 +206,9 @@ cases: - type: paragraph children: - type: text - value: Definition 1 + value: 'Definition ' + - type: inlineCode + value: '1' - type: definitionTerm children: - type: text @@ -228,18 +232,16 @@ cases: value: Third paragraph of definition 2. markdown: |- Term 1 + : Definition `1` - : Definition 1 - - Term 2 with *inline markup* + Term 2 with _inline markup_ + : Definition 2 - : Definition 2 + ```python + some code, part of Definition 2 + ``` - ```python - some code, part of Definition 2 - ``` - - Third paragraph of definition 2. + Third paragraph of definition 2. - title: definition list - nested mdast: type: root @@ -266,10 +268,8 @@ cases: value: Definition a markdown: |- Term 1 - - : Term a - - : Definition a + : Term a + : Definition a - title: unknown phrasing passes mdast: type: root @@ -293,7 +293,7 @@ cases: - type: inlineCode value: style`s markdown: |- - Some % *markdown* with ``style`s`` + Some % _markdown_ with ``style`s`` - title: styles in paragraph mdast: type: root @@ -319,4 +319,4 @@ cases: - type: inlineCode value: style`s markdown: |- - Some % *markdown* with **different** ``style`s`` + Some % _markdown_ with **different** ``style`s`` diff --git a/packages/myst-to-md/tests/references.yml b/packages/myst-to-md/tests/mdast/references.yml similarity index 93% rename from packages/myst-to-md/tests/references.yml rename to packages/myst-to-md/tests/mdast/references.yml index c8499a3d6a..7c49ff8dd4 100644 --- a/packages/myst-to-md/tests/references.yml +++ b/packages/myst-to-md/tests/mdast/references.yml @@ -26,7 +26,7 @@ cases: value: style`s markdown: |- (my-paragraph)= - Some % *markdown* with **different** ``style`s`` + Some % _markdown_ with **different** ``style`s`` - title: labeled headings mdast: type: root @@ -56,7 +56,7 @@ cases: (my-heading)= # first - Some % *markdown* + Some % _markdown_ (my-subheading)= #### fourth @@ -83,7 +83,7 @@ cases: markdown: |- # first - Some % *markdown* + Some % _markdown_ - title: labeled quote mdast: type: root @@ -101,7 +101,7 @@ cases: value: markdown markdown: |- (my-quote)= - > Some % *markdown* + > Some % _markdown_ - title: labeled list mdast: type: root @@ -125,9 +125,8 @@ cases: value: Some more markdown markdown: |- (my-list)= - * Some markdown - - * Some more markdown + - Some markdown + - Some more markdown - title: crossReference mdast: type: root @@ -142,7 +141,7 @@ cases: - type: text value: markdown markdown: |- - [Some % *markdown*](#example) + [Some % _markdown_](#example) - title: crossReference - no children mdast: type: root @@ -165,7 +164,7 @@ cases: - type: text value: markdown markdown: |- - [Some % *markdown*](#example) + [Some % _markdown_](#example) - title: crossReference - identifier only mdast: type: root @@ -180,4 +179,4 @@ cases: - type: text value: markdown markdown: |- - [Some % *markdown*](#example) + [Some % _markdown_](#example) diff --git a/packages/myst-to-md/tests/roles.yml b/packages/myst-to-md/tests/mdast/roles.yml similarity index 93% rename from packages/myst-to-md/tests/roles.yml rename to packages/myst-to-md/tests/mdast/roles.yml index a61bc01838..e5fad3057e 100644 --- a/packages/myst-to-md/tests/roles.yml +++ b/packages/myst-to-md/tests/mdast/roles.yml @@ -15,7 +15,7 @@ cases: - type: text value: markdown markdown: |- - Some % {sub}`*markdown*` + Some % {sub}`_markdown_` - title: superscript mdast: type: root @@ -31,7 +31,7 @@ cases: - type: text value: markdown markdown: |- - Some % {sup}`*markdown*` + Some % {sup}`_markdown_` - title: delete mdast: type: root @@ -47,7 +47,7 @@ cases: - type: text value: markdown markdown: |- - Some % {del}`*markdown*` + Some % {del}`_markdown_` - title: underline mdast: type: root @@ -63,7 +63,7 @@ cases: - type: text value: markdown markdown: |- - Some % {u}`*markdown*` + Some % {u}`_markdown_` - title: smallcaps mdast: type: root @@ -79,7 +79,7 @@ cases: - type: text value: markdown markdown: |- - Some % {sc}`*markdown*` + Some % {sc}`_markdown_` - title: inline math mdast: type: root @@ -91,7 +91,7 @@ cases: - type: inlineMath value: markdown math markdown: |- - Some % {math}`markdown math` + Some % $markdown math$ - title: abbreviation - no title mdast: type: root @@ -107,7 +107,7 @@ cases: - type: text value: md markdown: |- - Some % {abbr}`*md*` + Some % {abbr}`_md_` - title: abbreviation - title mdast: type: root @@ -124,7 +124,7 @@ cases: - type: text value: md markdown: |- - Some % {abbr}`*md* (markdown)` + Some % {abbr}`_md_ (markdown)` - title: nested roles mdast: type: root @@ -181,7 +181,7 @@ cases: - type: inlineCode value: and code markdown: |- - Some % {abbr}``*md* `and code` (markdown)`` + Some % {abbr}``_md_ `and code` (markdown)`` - title: nested roles with math mdast: type: root @@ -202,7 +202,7 @@ cases: - type: inlineMath value: and math markdown: |- - Some % {abbr}``*md* {math}`and math` (markdown)`` + Some % {abbr}`_md_ $and math$ (markdown)` - title: nested roles with inline backtick mdast: type: root @@ -223,7 +223,7 @@ cases: - type: inlineCode value: co`de markdown: |- - Some % {abbr}```*md* ``co`de`` (markdown)``` + Some % {abbr}```_md_ ``co`de`` (markdown)``` - title: nested roles with multiple levels mdast: type: root @@ -252,7 +252,7 @@ cases: - type: inlineCode value: co`de markdown: |- - Some % {abbr}`````{u}``{sub}`md` `` {math}`and math`{u}````{sub}``` ``co`de`` ``` ```` (markdown)````` + Some % {abbr}`````{u}``{sub}`md` `` $and math${u}````{sub}``` ``co`de`` ``` ```` (markdown)````` - title: myst role mdast: type: root diff --git a/packages/myst-to-md/tests/roundtrip/roundtrip.spec.ts b/packages/myst-to-md/tests/roundtrip/roundtrip.spec.ts new file mode 100644 index 0000000000..a8c6b98ce4 --- /dev/null +++ b/packages/myst-to-md/tests/roundtrip/roundtrip.spec.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import yaml from 'js-yaml'; +import { mystParse } from 'myst-parser'; +import { unified } from 'unified'; +import mystToMd from '../../src'; + +type TestCase = { + title: string; + markdown: string; + result: string; +}; + +type TestCases = { + title: string; + cases: TestCase[]; +}; + +const casesList: TestCases[] = fs + .readdirSync(__dirname) + .filter((file) => file.endsWith('.yml')) + .map((file) => { + const content = fs.readFileSync(path.join(__dirname, file), { encoding: 'utf-8' }); + return yaml.load(content) as TestCases; + }); + +casesList.forEach(({ title, cases }) => { + describe(title, () => { + test.each(cases.map((c): [string, TestCase] => [c.title, c]))( + '%s', + (_, { markdown, result: expected }) => { + const mdast = mystParse(markdown, { mdast: { hoistSingleImagesOutofParagraphs: false } }); + const file = unified() + .use(mystToMd) + .stringify(mdast as any); + const result = (file.result as string).trim(); + expect(result).toEqual(expected); + }, + ); + }); +}); diff --git a/packages/myst-to-md/tests/roundtrip/roundtrip.yml b/packages/myst-to-md/tests/roundtrip/roundtrip.yml new file mode 100644 index 0000000000..f73c3b76c2 --- /dev/null +++ b/packages/myst-to-md/tests/roundtrip/roundtrip.yml @@ -0,0 +1,67 @@ +title: myst-to-md roundtrip features +cases: + - title: styles in paragraph - emphasis + markdown: |- + # Heading + + Paragraph with *emphasis* and __strong__ and ``code``. + result: |- + # Heading + + Paragraph with _emphasis_ and **strong** and `code`. + - title: Bullet list + markdown: |- + # Heading + + * first `item` + * nested `item` + * another `item` + + paragraph + result: |- + # Heading + + - first `item` + - nested `item` + - another `item` + + paragraph + - title: Simple Directive + markdown: |- + (my-label)= + + # Heading + + ``` {admonition } Admonition title + :class: dropdown + :label: my-label + This is a directive + + And another paragraph + ``` + + Paragraph after directive + result: |- + (my-label)= + # Heading + + :::{admonition #my-label .dropdown} Admonition title + This is a directive + + And another paragraph + ::: + + Paragraph after directive + - title: Single horizontal rule + markdown: |- + paragraph + + --- + + some horizontal lines + result: |- + paragraph + + --- + + some horizontal lines