diff --git a/.changeset/ten-bats-warn.md b/.changeset/ten-bats-warn.md new file mode 100644 index 0000000000..00fc05d9b7 --- /dev/null +++ b/.changeset/ten-bats-warn.md @@ -0,0 +1,10 @@ +--- +"mystmd": minor +"myst-directives": patch +"myst-transforms": patch +"myst-spec-ext": patch +"myst-execute": patch +"myst-cli": patch +--- + +Add support for new Outputs node diff --git a/.changeset/weak-bananas-cover.md b/.changeset/weak-bananas-cover.md new file mode 100644 index 0000000000..5a0ecf1abd --- /dev/null +++ b/.changeset/weak-bananas-cover.md @@ -0,0 +1,5 @@ +--- +"myst-cli": patch +--- + +Move kernel execution transform earlier in pipeline diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index be10767d24..bd7cb50700 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -67,6 +67,7 @@ import { transformFilterOutputStreams, transformLiftCodeBlocksInJupytext, transformMystXRefs, + liftOutputs, } from '../transforms/index.js'; import type { ImageExtensions } from '../utils/resolveExtension.js'; import { logMessagesFromVFile } from '../utils/logging.js'; @@ -194,6 +195,12 @@ export async function transformMdast( log: session.log, }); } + transformRenderInlineExpressions(mdast, vfile); + await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters }); + liftOutputs(session, mdast, vfile, { + parseMyst: (content: string) => parseMyst(session, content, file), + }); + transformFilterOutputStreams(mdast, vfile, frontmatter.settings); const pipe = unified() .use(reconstructHtmlPlugin) // We need to group and link the HTML first @@ -238,9 +245,6 @@ export async function transformMdast( // Combine file-specific citation renderers with project renderers from bib files const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles); - transformRenderInlineExpressions(mdast, vfile); - await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters }); - transformFilterOutputStreams(mdast, vfile, frontmatter.settings); transformCitations(session, file, mdast, fileCitationRenderer, references); await unified() .use(codePlugin, { lang: frontmatter?.kernelspec?.language }) diff --git a/packages/myst-cli/src/process/notebook.ts b/packages/myst-cli/src/process/notebook.ts index ce8323f391..cf1aea7473 100644 --- a/packages/myst-cli/src/process/notebook.ts +++ b/packages/myst-cli/src/process/notebook.ts @@ -84,6 +84,21 @@ export async function processNotebook( return mdast; } +/** + * Embed the Jupyter output data for a user expression into the AST + */ +function embedInlineExpressions( + userExpressions: IUserExpressionMetadata[] | undefined, + block: GenericNode, +) { + const inlineNodes = selectAll('inlineExpression', block) as InlineExpression[]; + inlineNodes.forEach((inlineExpression) => { + const data = findExpression(userExpressions, inlineExpression.value); + if (!data) return; + inlineExpression.result = data.result as unknown as Record; + }); +} + export async function processNotebookFull( session: ISession, file: string, @@ -136,17 +151,7 @@ export async function processNotebookFull( return acc.concat(...cellMdast.children); } const block = blockParent(cell, cellMdast.children) as GenericNode; - - // Embed expression results into expression - const userExpressions = block.data?.[metadataSection] as - | IUserExpressionMetadata[] - | undefined; - const inlineNodes = selectAll('inlineExpression', block) as InlineExpression[]; - inlineNodes.forEach((inlineExpression) => { - const data = findExpression(userExpressions, inlineExpression.value); - if (!data) return; - inlineExpression.result = data.result as unknown as Record; - }); + embedInlineExpressions(block.data?.[metadataSection], block); return acc.concat(block); } if (cell.cell_type === CELL_TYPES.raw) { @@ -165,17 +170,16 @@ export async function processNotebookFull( value: ensureString(cell.source), }; - // Embed outputs in an output block - const output: { type: 'output'; id: string; data: IOutput[] } = { - type: 'output', + const outputs = { + type: 'outputs', id: nanoid(), - data: [], + children: (cell.outputs as IOutput[]).map((output) => ({ + type: 'output', + jupyter_data: output, + children: [], + })), }; - - if (cell.outputs && (cell.outputs as IOutput[]).length > 0) { - output.data = cell.outputs as IOutput[]; - } - return acc.concat(blockParent(cell, [code, output])); + return acc.concat(blockParent(cell, [code, outputs])); } return acc; }, diff --git a/packages/myst-cli/src/transforms/code.spec.ts b/packages/myst-cli/src/transforms/code.spec.ts index 0807f57400..d8221184a0 100644 --- a/packages/myst-cli/src/transforms/code.spec.ts +++ b/packages/myst-cli/src/transforms/code.spec.ts @@ -162,7 +162,10 @@ function build_mdast(tags: string[], has_output: boolean) { ], }; if (has_output) { - mdast.children[0].children.push({ type: 'output' }); + mdast.children[0].children.push({ + type: 'outputs', + children: [{ type: 'output', children: [] }], + }); } return mdast; } @@ -261,7 +264,7 @@ describe('propagateBlockDataToCode', () => { const mdast = build_mdast([tag], has_output); propagateBlockDataToCode(new Session(), new VFile(), mdast); let result = ''; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; switch (target) { case 'cell': result = mdast.children[0].visibility; @@ -270,12 +273,14 @@ describe('propagateBlockDataToCode', () => { result = mdast.children[0].children[0].visibility; break; case 'output': - if (!has_output && target == 'output') { - expect(outputNode).toEqual(undefined); + if (!has_output) { + expect(outputsNode).toEqual(undefined); continue; } - result = outputNode.visibility; + result = outputsNode.visibility; break; + default: + throw new Error(); } expect(result).toEqual(action); } @@ -290,18 +295,18 @@ describe('propagateBlockDataToCode', () => { propagateBlockDataToCode(new Session(), new VFile(), mdast); const blockNode = mdast.children[0]; const codeNode = mdast.children[0].children[0]; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; expect(blockNode.visibility).toEqual(action); expect(codeNode.visibility).toEqual(action); if (has_output) { - expect(outputNode.visibility).toEqual(action); + expect(outputsNode.visibility).toEqual(action); } else { - expect(outputNode).toEqual(undefined); + expect(outputsNode).toEqual(undefined); } } } }); - it('placeholder creates image node child of output', async () => { + it('placeholder creates image node child of outputs', async () => { const mdast: any = { type: 'root', children: [ @@ -313,7 +318,8 @@ describe('propagateBlockDataToCode', () => { executable: true, }, { - type: 'output', + type: 'outputs', + children: [], }, ], data: { @@ -323,12 +329,12 @@ describe('propagateBlockDataToCode', () => { ], }; propagateBlockDataToCode(new Session(), new VFile(), mdast); - const outputNode = mdast.children[0].children[1]; - expect(outputNode.children?.length).toEqual(1); - expect(outputNode.children[0].type).toEqual('image'); - expect(outputNode.children[0].placeholder).toBeTruthy(); + const outputsNode = mdast.children[0].children[1]; + expect(outputsNode.children?.length).toEqual(1); + expect(outputsNode.children[0].type).toEqual('image'); + expect(outputsNode.children[0].placeholder).toBeTruthy(); }); - it('placeholder passes with no output', async () => { + it('placeholder passes with no outputs', async () => { const mdast: any = { type: 'root', children: [ diff --git a/packages/myst-cli/src/transforms/code.ts b/packages/myst-cli/src/transforms/code.ts index 782401a8f4..b36303a154 100644 --- a/packages/myst-cli/src/transforms/code.ts +++ b/packages/myst-cli/src/transforms/code.ts @@ -1,6 +1,6 @@ import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCellTags, RuleId, fileError, fileWarn } from 'myst-common'; -import type { Image, Output } from 'myst-spec-ext'; +import type { Image, Outputs } from 'myst-spec-ext'; import { select, selectAll } from 'unist-util-select'; import yaml from 'js-yaml'; import type { VFile } from 'vfile'; @@ -156,10 +156,9 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: const blocks = selectAll('block', mdast) as GenericNode[]; blocks.forEach((block) => { if (!block.data) return; - const outputNode = select('output', block) as Output | null; - if (block.data.placeholder && outputNode) { - if (!outputNode.children) outputNode.children = []; - outputNode.children.push({ + const outputsNode = select('outputs', block) as Outputs | null; + if (block.data.placeholder && outputsNode) { + outputsNode.children.push({ type: 'image', placeholder: true, url: block.data.placeholder as string, @@ -195,10 +194,10 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: if (codeNode) codeNode.visibility = 'remove'; break; case NotebookCellTags.hideOutput: - if (outputNode) outputNode.visibility = 'hide'; + if (outputsNode) outputsNode.visibility = 'hide'; break; case NotebookCellTags.removeOutput: - if (outputNode) outputNode.visibility = 'remove'; + if (outputsNode) outputsNode.visibility = 'remove'; break; default: session.log.debug(`tag '${tag}' is not valid in code-cell tags'`); @@ -206,7 +205,7 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: }); if (!block.visibility) block.visibility = 'show'; if (codeNode && !codeNode.visibility) codeNode.visibility = 'show'; - if (outputNode && !outputNode.visibility) outputNode.visibility = 'show'; + if (outputsNode && !outputsNode.visibility) outputsNode.visibility = 'show'; }); } @@ -233,7 +232,7 @@ export function transformLiftCodeBlocksInJupytext(mdast: GenericParent) { child.type === 'block' && child.children?.length === 2 && child.children?.[0].type === 'code' && - child.children?.[1].type === 'output' + child.children?.[1].type === 'outputs' ) { newBlocks.push(child as GenericParent); newBlocks.push({ type: 'block', children: [] }); diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index f15bdd674b..48ecaad024 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -20,17 +20,40 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', - id: 'abc123', - data: [], + type: 'outputs', + children: [ + { + type: 'output', + id: 'abc123', + jupyter_data: null, + children: [], + }, + ], }, ], }, ], }; - expect(mdast.children[0].children.length).toEqual(2); reduceOutputs(new Session(), mdast, 'notebook.ipynb', '/my/folder'); - expect(mdast.children[0].children.length).toEqual(1); + expect(mdast).toEqual({ + type: 'root', + children: [ + { + type: 'block', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'text', + value: 'hi', + }, + ], + }, + ], + }, + ], + }); }); it('output with complex data is removed', async () => { const mdast = { @@ -49,18 +72,22 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [ + children: [ { - output_type: 'display_data', - execution_count: 3, - metadata: {}, - data: { - 'application/octet-stream': { - content_type: 'application/octet-stream', - hash: 'def456', - path: '/my/path/def456.png', + type: 'output', + children: [], + jupyter_data: { + output_type: 'display_data', + execution_count: 3, + metadata: {}, + data: { + 'application/octet-stream': { + content_type: 'application/octet-stream', + hash: 'def456', + path: '/my/path/def456.png', + }, }, }, }, @@ -72,9 +99,27 @@ describe('reduceOutputs', () => { }; expect(mdast.children[0].children.length).toEqual(2); reduceOutputs(new Session(), mdast, 'notebook.ipynb', '/my/folder'); - expect(mdast.children[0].children.length).toEqual(1); + expect(mdast).toEqual({ + type: 'root', + children: [ + { + type: 'block', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'text', + value: 'hi', + }, + ], + }, + ], + }, + ], + }); }); - it('output is replaced with placeholder image', async () => { + it('outputs is replaced with placeholder image', async () => { const mdast = { type: 'root', children: [ @@ -91,9 +136,8 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [], children: [ { type: 'image', @@ -108,11 +152,29 @@ describe('reduceOutputs', () => { }; expect(mdast.children[0].children.length).toEqual(2); reduceOutputs(new Session(), mdast, 'notebook.ipynb', '/my/folder'); - expect(mdast.children[0].children.length).toEqual(2); - expect(mdast.children[0].children[1]).toEqual({ - type: 'image', - placeholder: true, - url: 'placeholder.png', + expect(mdast).toEqual({ + type: 'root', + children: [ + { + type: 'block', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'text', + value: 'hi', + }, + ], + }, + { + type: 'image', + placeholder: true, + url: 'placeholder.png', + }, + ], + }, + ], }); }); // // These tests now require file IO... diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index 7046450e7a..dc6b6d0e6c 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { computeHash } from 'myst-cli-utils'; -import type { Image, SourceFileKind } from 'myst-spec-ext'; +import type { Image, SourceFileKind, Output } from 'myst-spec-ext'; import { liftChildren, fileError, RuleId, fileWarn } from 'myst-common'; import type { GenericNode, GenericParent } from 'myst-common'; import type { ProjectSettings } from 'myst-frontmatter'; @@ -12,7 +12,14 @@ import { selectAll } from 'unist-util-select'; import type { VFile } from 'vfile'; import type { IOutput, IStream } from '@jupyterlab/nbformat'; import type { MinifiedContent, MinifiedOutput, MinifiedMimeOutput } from 'nbtx'; -import { ensureString, extFromMimeType, minifyCellOutput, walkOutputs } from 'nbtx'; +import { + convertToIOutputs, + ensureString, + extFromMimeType, + minifyCellOutput, + walkOutputs, +} from 'nbtx'; +import { TexParser } from 'tex-to-myst'; import { castSession } from '../session/cache.js'; import type { ISession } from '../session/types.js'; import { resolveOutputPath } from './images.js'; @@ -25,6 +32,85 @@ function getWriteDestination(hash: string, contentType: string, writeFolder: str return join(writeFolder, getFilename(hash, contentType)); } +const MARKDOWN_MIME_TYPE = 'text/markdown'; +const SUPPORTED_MARKDOWN_VARIANTS = ['Original', 'GFM', 'CommonMark', 'myst']; + +/** + * Extract the `variant` parameter from a Markdown MIME type + * + * @param mimeType MIME type of the form `text/markdown;FOO=BAR` + */ +function extractVariantParameter(mimeType: string): string | undefined { + const [variant] = Array.from(mimeType.matchAll(/;([^;]+)=([^;]+)/g)) + .filter(([name]) => name === 'variant') + .map((pair) => pair[1]); + return variant; +} + +/* + * Determine the Markdown variant from a given MIME-type + * + * If the MIME-type is not a supported Markdown MIME, return undefined + * + * @param mimeType - MIME type + */ +function determineMarkdownVariant( + mimeType: string, +): { variant?: string; mimeType: string } | undefined { + if (!mimeType.startsWith(MARKDOWN_MIME_TYPE)) { + return; + } + const variant = extractVariantParameter(mimeType); + if (!variant) { + return { mimeType }; + } + if (SUPPORTED_MARKDOWN_VARIANTS.includes(variant)) { + return { mimeType, variant }; + } + + return; +} + +/** + * Lift outputs that contribute to the global document state + */ +export function liftOutputs( + session: ISession, + mdast: GenericParent, + vfile: VFile, + opts: { parseMyst: (source: string) => GenericParent }, +) { + const cache = castSession(session); + selectAll('output', mdast).forEach((output) => { + let children: GenericNode[] | undefined; + // Walk over the outputs, and take the first matching "high-priority" output type + // Given that the `IOutput.data` mapping is not ordered, the precedence between sibling + // MIME type keys (e.g. LaTeX vs Markdown) is not defined for now. + walkOutputs([(output as any).jupyter_data], (obj: any) => { + if (children) { + return; + } + const { content_type, content, hash } = obj; + const { mimeType: markdownMimeType } = determineMarkdownVariant(content_type) ?? {}; + // Markdown output + if (markdownMimeType) { + const [cacheContent] = cache.$outputs[hash] ?? []; + const ast = opts.parseMyst(content ?? cacheContent); + children = ast.children; + } + // LaTeX (including math) output + else if (content_type === 'text/latex') { + const [cacheContent] = cache.$outputs[hash] ?? []; + const state = new TexParser(content ?? cacheContent, vfile); + children = state.ast.children; + } + }); + if (children) { + (output as any).children = children; + } + }); +} + /** * Traverse all output nodes, minify their content, and cache on the session */ @@ -34,17 +120,28 @@ export async function transformOutputsToCache( kind: SourceFileKind, opts?: { minifyMaxCharacters?: number }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; - if (!outputs.length) return; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); await Promise.all( - outputs - .filter((output) => output.visibility !== 'remove') + outputsNodes + // Ignore outputs that are hidden + .filter((outputs) => outputs.visibility !== 'remove') + // Pull out children + .map((outputs) => outputs.children as Output[]) + .flat() + // Filter outputs with no data + // TODO: can this ever occur? + .filter((output) => (output as any).jupyter_data !== undefined) + // Minify output data .map(async (output) => { - output.data = await minifyCellOutput(output.data as IOutput[], cache.$outputs, { - computeHash, - maxCharacters: opts?.minifyMaxCharacters, - }); + [(output as any).jupyter_data] = await minifyCellOutput( + [(output as any).jupyter_data] as IOutput[], + cache.$outputs, + { + computeHash, + maxCharacters: opts?.minifyMaxCharacters, + }, + ); }), ); } @@ -75,9 +172,10 @@ export function transformFilterOutputStreams( const blockRemoveStderr = tags.includes('remove-stderr'); const blockRemoveStdout = tags.includes('remove-stdout'); const outputs = selectAll('output', block) as GenericNode[]; - // There should be only one output in the block - outputs.forEach((output) => { - output.data = output.data.filter((data: IStream | MinifiedMimeOutput) => { + outputs + .filter((output) => { + const data = output.jupyter_data; + if ( (stderr !== 'show' || blockRemoveStderr) && data.output_type === 'stream' && @@ -96,7 +194,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( (stdout !== 'show' || blockRemoveStdout) && @@ -116,7 +214,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( mpl !== 'show' && @@ -125,7 +223,7 @@ export function transformFilterOutputStreams( data.data['text/plain'] ) { const content = data.data['text/plain'].content; - if (!stringIsMatplotlibOutput(content)) return true; + if (!stringIsMatplotlibOutput(content)) return false; const doRemove = mpl.includes('remove'); const doWarn = mpl.includes('warn'); const doError = mpl.includes('error'); @@ -141,12 +239,15 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } - return true; + return false; + }) + .forEach((output) => { + output.type = '__delete__'; }); - }); }); + remove(mdast, { cascade: false }, '__delete__'); } function writeCachedOutputToFile( @@ -192,16 +293,19 @@ export function transformOutputsToFile( const outputs = selectAll('output', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - walkOutputs(node.data, (obj) => { - const { hash } = obj; - if (!hash || !cache.$outputs[hash]) return undefined; - obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, + outputs + .filter((output) => !!output.jupyter_data) + .forEach((node) => { + // TODO: output-refactoring -- drop to single output in future + walkOutputs([node.jupyter_data], (obj) => { + const { hash } = obj; + if (!hash || !cache.$outputs[hash]) return undefined; + obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node, + }); }); }); - }); } /** @@ -233,87 +337,99 @@ export function reduceOutputs( writeFolder: string, opts?: { altOutputFolder?: string; vfile?: VFile }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - if (node.visibility === 'remove' || node.visibility === 'hide') { - // Hidden nodes should not show up in simplified outputs for static export - node.type = '__delete__'; - return; - } - if (!node.data?.length && !node.children?.length) { - node.type = '__delete__'; + outputsNodes.forEach((outputsNode) => { + // Hidden nodes should not show up in simplified outputs for static export + if (outputsNode.visibility === 'remove' || outputsNode.visibility === 'hide') { + outputsNode.type = '__delete__'; return; } - node.type = '__lift__'; - if (node.children?.length) return; - const selectedOutputs: { content_type: string; hash: string }[] = []; - node.data.forEach((output: MinifiedOutput) => { - let selectedOutput: { content_type: string; hash: string } | undefined; - walkOutputs([output], (obj: any) => { - const { output_type, content_type, hash } = obj; - if (!hash) return undefined; - if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { - if (['error', 'stream'].includes(output_type)) { - selectedOutput = { content_type: 'text/plain', hash }; - } else if (typeof content_type === 'string') { - if ( - content_type.startsWith('image/') || - content_type === 'text/plain' || - content_type === 'text/html' - ) { - selectedOutput = { content_type, hash }; + + const outputs = outputsNode.children as GenericNode[]; + outputs.forEach((outputNode) => { + if (outputNode.type !== 'output') { + return; + } + // Lift the `output` node into `Outputs` + outputNode.type = '__lift__'; + + // If the output already has children, we don't need to do anything + // Or, if it has no output data (should not happen) + if (outputNode.children?.length || !outputNode.jupyter_data) { + return; + } + + // Find a preferred IOutput type to render into the AST + const selectedOutputs: { content_type: string; hash: string }[] = []; + if (outputNode.jupyter_data) { + const output = outputNode.jupyter_data; + + let selectedOutput: { content_type: string; hash: string } | undefined; + walkOutputs([output], (obj: any) => { + const { output_type, content_type, hash } = obj; + if (!hash) return undefined; + if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { + if (['error', 'stream'].includes(output_type)) { + selectedOutput = { content_type: 'text/plain', hash }; + } else if (typeof content_type === 'string') { + if ( + content_type.startsWith('image/') || + content_type === 'text/plain' || + content_type === 'text/html' + ) { + selectedOutput = { content_type, hash }; + } } } - } - }); - if (selectedOutput) selectedOutputs.push(selectedOutput); - }); - const children: (Image | GenericNode)[] = selectedOutputs - .map((output): Image | GenericNode | GenericNode[] | undefined => { - const { content_type, hash } = output ?? {}; - if (!hash || !cache.$outputs[hash]) return undefined; - if (content_type === 'text/html') { - const htmlTree = { - type: 'root', - children: [ - { - type: 'html', - value: cache.$outputs[hash][0], - }, - ], - }; - htmlTransform(htmlTree); - if ((selectAll('image', htmlTree) as GenericNode[]).find((htmlImage) => !htmlImage.url)) { - return undefined; + }); + if (selectedOutput) selectedOutputs.push(selectedOutput); + } + const children: (Image | GenericNode)[] = selectedOutputs + .map((output): Image | GenericNode | GenericNode[] | undefined => { + const { content_type, hash } = output ?? {}; + if (!hash || !cache.$outputs[hash]) return undefined; + if (content_type === 'text/html') { + const htmlTree = { + type: 'root', + children: [ + { + type: 'html', + value: cache.$outputs[hash][0], + }, + ], + }; + htmlTransform(htmlTree); + return htmlTree.children; + } else if (content_type.startsWith('image/')) { + const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node: outputNode, + }); + if (!path) return undefined; + const relativePath = relative(dirname(file), path); + return { + type: 'image', + data: { type: 'output' }, + url: relativePath, + urlSource: relativePath, + }; + } else if (content_type === 'text/plain' && cache.$outputs[hash]) { + const [content] = cache.$outputs[hash]; + return { + type: 'code', + data: { type: 'output' }, + value: stripAnsi(content), + }; } - return htmlTree.children; - } else if (content_type.startsWith('image/')) { - const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, - }); - if (!path) return undefined; - const relativePath = relative(dirname(file), path); - return { - type: 'image', - data: { type: 'output' }, - url: relativePath, - urlSource: relativePath, - }; - } else if (content_type === 'text/plain' && cache.$outputs[hash]) { - const [content] = cache.$outputs[hash]; - return { - type: 'code', - data: { type: 'output' }, - value: stripAnsi(content), - }; - } - return undefined; - }) - .flat() - .filter((output): output is Image | GenericNode => !!output); - node.children = children; + return undefined; + }) + .flat() + .filter((output): output is Image | GenericNode => !!output); + outputNode.children = children; + }); + // Lift the `outputs` node + outputsNode.type = '__lift__'; }); remove(mdast, '__delete__'); liftChildren(mdast, '__lift__'); diff --git a/packages/myst-directives/src/code.ts b/packages/myst-directives/src/code.ts index 00f7f03b68..0bcf356e56 100644 --- a/packages/myst-directives/src/code.ts +++ b/packages/myst-directives/src/code.ts @@ -252,15 +252,15 @@ export const codeCellDirective: DirectiveSpec = { executable: true, value: (data.body ?? '') as string, }; - const output = { - type: 'output', + const outputs = { + type: 'outputs', id: nanoid(), - data: [], + children: [], }; const block: GenericNode = { type: 'block', kind: NotebookCell.code, - children: [code, output], + children: [code, outputs], data: {}, }; addCommonDirectiveOptions(data, block); diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index e97dcfad44..10d149253c 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -2,7 +2,7 @@ import { select, selectAll } from 'unist-util-select'; import type { Logger } from 'myst-cli-utils'; import type { PageFrontmatter, KernelSpec } from 'myst-frontmatter'; import type { Kernel, KernelMessage, Session, SessionManager } from '@jupyterlab/services'; -import type { Block, Code, InlineExpression, Output } from 'myst-spec-ext'; +import type { Block, Code, InlineExpression, Output, Outputs } from 'myst-spec-ext'; import type { IOutput } from '@jupyterlab/nbformat'; import type { GenericNode, GenericParent, IExpressionResult, IExpressionError } from 'myst-common'; import { NotebookCell, NotebookCellTags, fileError } from 'myst-common'; @@ -114,7 +114,7 @@ function buildCacheKey(kernelSpec: KernelSpec, nodes: (CodeBlock | InlineExpress raisesException: boolean; }[] = []; for (const node of nodes) { - if (isCellBlock(node)) { + if (isCodeBlock(node)) { hashableItems.push({ kind: node.type, content: (select('code', node) as Code).value, @@ -165,8 +165,8 @@ type CodeBlock = Block & { * * @param node node to test */ -function isCellBlock(node: GenericNode): node is CodeBlock { - return node.type === 'block' && select('code', node) !== null && select('output', node) !== null; +function isCodeBlock(node: GenericNode): node is CodeBlock { + return node.type === 'block' && node.kind === NotebookCell.code; } /** @@ -215,7 +215,7 @@ async function computeExecutableNodes( const results: (IOutput[] | IExpressionResult)[] = []; for (const matchedNode of nodes) { - if (isCellBlock(matchedNode)) { + if (isCodeBlock(matchedNode)) { // Pull out code to execute const code = select('code', matchedNode) as Code; const { status, outputs } = await executeCode(kernel, code.value); @@ -281,15 +281,19 @@ function applyComputedOutputsToNodes( // Pull out the result for this node const thisResult = computedResult.shift(); - if (isCellBlock(matchedNode)) { - // Pull out output to set data - const output = select('output', matchedNode) as unknown as { data: IOutput[] }; - // Set the output array to empty if we don't have a result (e.g. due to a kernel error) - output.data = thisResult === undefined ? [] : (thisResult as IOutput[]); + if (isCodeBlock(matchedNode)) { + const rawOutputData = (thisResult as IOutput[]) ?? []; + // Pull out outputs to set data + const outputs = select('outputs', matchedNode) as Outputs; + // Ensure that whether this fails or succeeds, we write to `children` (e.g. due to a kernel error) + outputs.children = rawOutputData.map((data, index) => { + const identifier = outputs.identifier ? `${outputs.identifier}-${index}` : undefined; + return { type: 'output', children: [], jupyter_data: data as any, identifier }; + }); } else if (isInlineExpression(matchedNode)) { + const rawOutputData = thisResult as Record | undefined; // Set data of expression to the result, or empty if we don't have one - matchedNode.result = // TODO: FIXME .data - thisResult === undefined ? undefined : (thisResult as unknown as Record); + matchedNode.result = rawOutputData; } else { // This should never happen throw new Error('Node must be either code block or inline expression.'); @@ -325,7 +329,7 @@ export async function kernelExecutionTransform(tree: GenericParent, vfile: VFile )[] ) // Filter out nodes that skip execution - .filter((node) => !(isCellBlock(node) && codeBlockSkipsExecution(node))); + .filter((node) => !(isCodeBlock(node) && codeBlockSkipsExecution(node))); // Only do something if we have any nodes! if (executableNodes.length === 0) { diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index f768992aad..1a1330b999 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -36,13 +36,17 @@ cases: lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + children: [] + after: type: root children: @@ -63,18 +67,77 @@ cases: lang: python executable: true value: print('abc') + enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + children: [] + identifier: nb-cell-0-outputs-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc + - title: output without identifier is given one + before: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') + enumerator: 1 identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + children: [] + + after: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + children: [] + identifier: nb-cell-0-outputs-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc - title: tree with inline expression is evaluated before: type: root @@ -115,100 +178,92 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + jupyter_data: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + identifier: nb-cell-1-code + html_id: nb-cell-1-code + - type: outputs + identifier: nb-cell-1-outputs + html_id: nb-cell-1-outputs + children: + - type: output + identifier: nb-cell-1-outputs-0 + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + identifier: nb-cell-1-code + html_id: nb-cell-1-code + - type: outputs + identifier: nb-cell-1-outputs + html_id: nb-cell-1-outputs + children: + - type: output + identifier: nb-cell-1-outputs-0 + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `raises-exception` is evaluated and passes before: type: root @@ -216,58 +271,54 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + - type: outputs + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `skip-execution` is not evaluated before: type: root @@ -275,45 +326,39 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + identifier: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + identifier: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + jupyter_data: diff --git a/packages/myst-spec-ext/src/types.ts b/packages/myst-spec-ext/src/types.ts index 9f5bd9d463..9ae4210d93 100644 --- a/packages/myst-spec-ext/src/types.ts +++ b/packages/myst-spec-ext/src/types.ts @@ -254,10 +254,16 @@ export type Container = Omit & { export type Output = Node & Target & { type: 'output'; - id?: string; - data?: any[]; // MinifiedOutput[] + children: (FlowContent | ListContent | PhrasingContent)[]; + jupyter_data: any; // TODO: set this to IOutput + }; + +export type Outputs = Node & + Target & { + type: 'outputs'; + children: (Output | FlowContent | ListContent | PhrasingContent)[]; // Support placeholders in addition to outputs visibility?: Visibility; - children?: (FlowContent | ListContent | PhrasingContent)[]; + id?: string; }; export type Aside = Node & diff --git a/packages/myst-to-jats/src/index.ts b/packages/myst-to-jats/src/index.ts index 05ec2ce813..68ba9902ee 100644 --- a/packages/myst-to-jats/src/index.ts +++ b/packages/myst-to-jats/src/index.ts @@ -285,6 +285,7 @@ type Handlers = { si: Handler; proof: Handler; algorithmLine: Handler; + outputs: Handler; output: Handler; embed: Handler; supplementaryMaterial: Handler; @@ -632,22 +633,25 @@ const handlers: Handlers = { state.renderChildren(node); state.closeNode(); }, + outputs(node, state) { + state.renderChildren(node); + }, output(node, state) { if (state.data.isInContainer) { - if (!node.data?.[0]) return; - alternativesFromMinifiedOutput(node.data[0], state); + if (!node.jupyter_data) return; + alternativesFromMinifiedOutput(node.jupyter_data, state); return; } const { identifier } = node; const attrs: Attributes = { 'sec-type': 'notebook-output' }; - node.data?.forEach((output: any, index: number) => { + if (node.jupyter_data) { state.openNode('sec', { ...attrs, - id: identifier && !state.data.isNotebookArticleRep ? `${identifier}-${index}` : undefined, + id: identifier && !state.data.isNotebookArticleRep ? identifier : undefined, }); - alternativesFromMinifiedOutput(output, state); + alternativesFromMinifiedOutput(node.jupyter_data, state); state.closeNode(); - }); + } }, embed(node, state) { if (state.data.isInContainer) { diff --git a/packages/myst-to-jats/tests/basic.yml b/packages/myst-to-jats/tests/basic.yml index 544d50b53a..7924cbcb09 100644 --- a/packages/myst-to-jats/tests/basic.yml +++ b/packages/myst-to-jats/tests/basic.yml @@ -686,33 +686,41 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt - - type: output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt + - type: outputs id: uE4mPgSow0oyo0dvEH9Lc - identifier: nb-cell-1-output - html_id: nb-cell-1-output - data: - - output_type: execute_result - execution_count: 3 - metadata: {} - data: - text/plain: - content_type: text/plain - hash: b - path: files/b.txt - text/html: - content_type: text/html - hash: b - path: files/b.html + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + output_type: execute_result + execution_count: 3 + metadata: {} + data: + text/plain: + content_type: text/plain + hash: b + path: files/b.txt + text/html: + content_type: text/html + hash: b + path: files/b.html - type: caption children: - type: paragraph diff --git a/packages/myst-to-jats/tests/notebooks.yml b/packages/myst-to-jats/tests/notebooks.yml index 4f7e95d4fd..7417efc661 100644 --- a/packages/myst-to-jats/tests/notebooks.yml +++ b/packages/myst-to-jats/tests/notebooks.yml @@ -18,16 +18,20 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt - type: block kind: notebook-code data: @@ -43,23 +47,27 @@ cases: identifier: nb-cell-1-code enumerator: 2 html_id: nb-cell-1-code - - type: output + - type: outputs id: uE4mPgSow0oyo0dvEH9Lc - identifier: nb-cell-1-output - html_id: nb-cell-1-output - data: - - output_type: execute_result - execution_count: 3 - metadata: {} - data: - text/plain: - content_type: text/plain - hash: b - path: files/b.txt - text/html: - content_type: text/html - hash: b - path: files/b.html + identifier: nb-cell-1-outputs + html_id: nb-cell-1-outputs + children: + - type: output + identifier: nb-cell-1-outputs-0 + html_id: nb-cell-1-outputs-0 + jupyter_data: + output_type: execute_result + execution_count: 3 + metadata: {} + data: + text/plain: + content_type: text/plain + hash: b + path: files/b.txt + text/html: + content_type: text/html + hash: b + path: files/b.html - type: block kind: notebook-code data: @@ -75,17 +83,21 @@ cases: identifier: nb-cell-2-code enumerator: 3 html_id: nb-cell-2-code - - type: output + - type: outputs id: 7Qrwdo-_oq5US1Du2KCLU - identifier: nb-cell-2-output - html_id: nb-cell-2-output - data: - - ename: NameError - evalue: name 'a' is not defined - output_type: error - traceback: \u001b[0;31m------------------------------------------------------... - hash: c - path: files/c.txt + identifier: nb-cell-2-outputs + html_id: nb-cell-2-outputs + children: + - type: output + identifier: nb-cell-2-outputs-0 + html_id: nb-cell-2-outputs-0 + jupyter_data: + ename: NameError + evalue: name 'a' is not defined + output_type: error + traceback: \u001b[0;31m------------------------------------------------------... + hash: c + path: files/c.txt jats: |- @@ -101,7 +113,7 @@ cases: print('abc') - + @@ -109,7 +121,7 @@ cases: 'abc' - + @@ -118,7 +130,7 @@ cases: a - + @@ -151,16 +163,20 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt - type: block kind: notebook-content data: @@ -212,7 +228,7 @@ cases: print('abc') - + @@ -251,16 +267,21 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt jats: |- @@ -285,7 +306,7 @@ cases: print('abc') - + @@ -313,16 +334,20 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt - type: block data: part: abstract @@ -373,7 +398,7 @@ cases: print('abc') - + diff --git a/packages/myst-transforms/src/blocks.spec.ts b/packages/myst-transforms/src/blocks.spec.ts index 5ce0fecd48..8403761f77 100644 --- a/packages/myst-transforms/src/blocks.spec.ts +++ b/packages/myst-transforms/src/blocks.spec.ts @@ -120,8 +120,10 @@ describe('Test blockMetadataTransform', () => { test('label is propagated to outputs', async () => { const mdast = u('root', [ u('block', { meta: '{"label": "My_Label", "key": "value"}' }, [ - u('output', 'We know what we are'), - u('output', 'but know not what we may be.'), + u('outputs', [ + u('output', 'We know what we are'), + u('output', 'but know not what we may be.'), + ]), ]), ]) as any; blockMetadataTransform(mdast, new VFile()); @@ -136,8 +138,10 @@ describe('Test blockMetadataTransform', () => { data: { key: 'value' }, }, [ - u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), - u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + u('outputs', { identifier: 'my_label-outputs' }, [ + u('output', { identifier: 'my_label-outputs-0' }, 'We know what we are'), + u('output', { identifier: 'my_label-outputs-1' }, 'but know not what we may be.'), + ]), ], ), ]), diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 20e9941461..8cf8ade73d 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -1,7 +1,7 @@ import type { VFile } from 'vfile'; import type { Plugin } from 'unified'; import type { Node } from 'myst-spec'; -import { selectAll } from 'unist-util-select'; +import { selectAll, select } from 'unist-util-select'; import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCell, RuleId, fileError, normalizeLabel } from 'myst-common'; import type { Code } from 'myst-spec-ext'; @@ -80,24 +80,23 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { } } if (block.identifier) { - const codeChildren = selectAll('code', block) as Code[]; - codeChildren.forEach((child, index) => { - if (child.identifier) return; - if (codeChildren.length === 1) { - child.identifier = `${block.identifier}-code`; - } else { - child.identifier = `${block.identifier}-code-${index}`; - } - }); - const outputChildren = selectAll('output', block) as GenericNode[]; - outputChildren.forEach((child, index) => { - if (child.identifier) return; - if (outputChildren.length === 1) { - child.identifier = `${block.identifier}-output`; - } else { - child.identifier = `${block.identifier}-output-${index}`; - } - }); + const codeNode = select('code', block) as any as Code | null; + if (codeNode !== null && !codeNode.identifier) { + codeNode.identifier = `${block.identifier}-code`; + } + const outputsNode = select('outputs', block) as GenericNode | null; + if (outputsNode !== null && !outputsNode.identifier) { + // Label outputs node + outputsNode.identifier = `${block.identifier}-outputs`; + // Enumerate outputs + const outputs = selectAll('output', outputsNode) as GenericNode[]; + outputs.forEach((outputNode, index) => { + if (outputNode && !outputNode.identifier) { + // Label output node + outputNode.identifier = `${outputsNode.identifier}-${index}`; + } + }); + } } }); } diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index efa957528c..a604e254a6 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -54,19 +54,25 @@ "kind": "figure", "children": [ { - "type": "output", - "data": [ + "type": "outputs", + "children": [ { - "output_type": "execute_result", - "execution_count": 2, - "metadata": {}, - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a16fcedcd26437c820ccfc05d1f48a57", - "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" - }, - "text/plain": { "content": "alt.Chart(...)", "content_type": "text/plain" } + "type": "output", + "jupyter_data": { + "output_type": "execute_result", + "execution_count": 2, + "metadata": {}, + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a16fcedcd26437c820ccfc05d1f48a57", + "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" + }, + "text/plain": { + "content": "alt.Chart(...)", + "content_type": "text/plain" + } + } } } ] @@ -143,19 +149,22 @@ "enumerator": "1" }, { - "type": "output", - "data": [ + "type": "outputs", + "children": [ { - "output_type": "execute_result", - "execution_count": 2, - "metadata": {}, - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a16fcedcd26437c820ccfc05d1f48a57", - "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" - }, - "text/plain": { "content": "alt.Chart(...)", "content_type": "text/plain" } + "type": "output", + "jupyter_data": { + "output_type": "execute_result", + "execution_count": 2, + "metadata": {}, + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a16fcedcd26437c820ccfc05d1f48a57", + "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" + }, + "text/plain": { "content": "alt.Chart(...)", "content_type": "text/plain" } + } } } ]