From 508408fd7d0d77faa368e5b33920b72e9adfddd9 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Tue, 1 Apr 2025 11:20:20 +0100 Subject: [PATCH 01/32] feat:add outputs node --- packages/myst-cli/src/process/mdast.ts | 1 + packages/myst-cli/src/process/notebook.ts | 22 +- packages/myst-cli/src/transforms/code.spec.ts | 32 ++- packages/myst-cli/src/transforms/code.ts | 17 +- .../myst-cli/src/transforms/outputs.spec.ts | 49 ++-- packages/myst-cli/src/transforms/outputs.ts | 234 ++++++++++-------- packages/myst-directives/src/code.ts | 9 +- packages/myst-execute/src/execute.ts | 19 +- packages/myst-execute/tests/execute.yml | 209 ++++++---------- packages/myst-spec-ext/src/types.ts | 12 +- packages/myst-transforms/src/blocks.spec.ts | 12 +- packages/myst-transforms/src/blocks.ts | 23 +- 12 files changed, 321 insertions(+), 318 deletions(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index be10767d24..f0e8cf5eda 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -451,5 +451,6 @@ export async function finalizeMdast( postData.widgets = cache.$getMdast(file)?.pre.widgets; updateFileInfoFromFrontmatter(session, file, frontmatter); } + logMessagesFromVFile(session, vfile); } diff --git a/packages/myst-cli/src/process/notebook.ts b/packages/myst-cli/src/process/notebook.ts index ce8323f391..59257a5dc8 100644 --- a/packages/myst-cli/src/process/notebook.ts +++ b/packages/myst-cli/src/process/notebook.ts @@ -165,17 +165,21 @@ export async function processNotebookFull( value: ensureString(cell.source), }; - // Embed outputs in an output block - const output: { type: 'output'; id: string; data: IOutput[] } = { - type: 'output', + const outputsChildren = (cell.outputs as IOutput[]).map((output) => { + // Embed outputs in an output block + const result = { + type: 'output', + jupyter_data: output, + children: [], + }; + return result; + }); + const outputs = { + type: 'outputs', id: nanoid(), - data: [], + children: outputsChildren, }; - - if (cell.outputs && (cell.outputs as IOutput[]).length > 0) { - output.data = cell.outputs as IOutput[]; - } - return acc.concat(blockParent(cell, [code, output])); + return acc.concat(blockParent(cell, [code, outputs])); } return acc; }, diff --git a/packages/myst-cli/src/transforms/code.spec.ts b/packages/myst-cli/src/transforms/code.spec.ts index 0807f57400..27d50958fd 100644 --- a/packages/myst-cli/src/transforms/code.spec.ts +++ b/packages/myst-cli/src/transforms/code.spec.ts @@ -162,7 +162,10 @@ function build_mdast(tags: string[], has_output: boolean) { ], }; if (has_output) { - mdast.children[0].children.push({ type: 'output' }); + mdast.children[0].children.push({ + type: 'outputs', + children: [{ type: 'output', children: [] }], + }); } return mdast; } @@ -261,7 +264,7 @@ describe('propagateBlockDataToCode', () => { const mdast = build_mdast([tag], has_output); propagateBlockDataToCode(new Session(), new VFile(), mdast); let result = ''; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; switch (target) { case 'cell': result = mdast.children[0].visibility; @@ -270,12 +273,14 @@ describe('propagateBlockDataToCode', () => { result = mdast.children[0].children[0].visibility; break; case 'output': - if (!has_output && target == 'output') { - expect(outputNode).toEqual(undefined); + if (!has_output) { + expect(outputsNode).toEqual(undefined); continue; } - result = outputNode.visibility; + result = outputsNode.visibility; break; + default: + throw new Error(); } expect(result).toEqual(action); } @@ -290,13 +295,13 @@ describe('propagateBlockDataToCode', () => { propagateBlockDataToCode(new Session(), new VFile(), mdast); const blockNode = mdast.children[0]; const codeNode = mdast.children[0].children[0]; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; expect(blockNode.visibility).toEqual(action); expect(codeNode.visibility).toEqual(action); if (has_output) { - expect(outputNode.visibility).toEqual(action); + expect(outputsNode.visibility).toEqual(action); } else { - expect(outputNode).toEqual(undefined); + expect(outputsNode).toEqual(undefined); } } } @@ -313,7 +318,8 @@ describe('propagateBlockDataToCode', () => { executable: true, }, { - type: 'output', + type: 'outputs', + children: [], }, ], data: { @@ -323,10 +329,10 @@ describe('propagateBlockDataToCode', () => { ], }; propagateBlockDataToCode(new Session(), new VFile(), mdast); - const outputNode = mdast.children[0].children[1]; - expect(outputNode.children?.length).toEqual(1); - expect(outputNode.children[0].type).toEqual('image'); - expect(outputNode.children[0].placeholder).toBeTruthy(); + const outputsNode = mdast.children[0].children[1]; + expect(outputsNode.children?.length).toEqual(1); + expect(outputsNode.children[0].type).toEqual('image'); + expect(outputsNode.children[0].placeholder).toBeTruthy(); }); it('placeholder passes with no output', async () => { const mdast: any = { diff --git a/packages/myst-cli/src/transforms/code.ts b/packages/myst-cli/src/transforms/code.ts index 782401a8f4..b36303a154 100644 --- a/packages/myst-cli/src/transforms/code.ts +++ b/packages/myst-cli/src/transforms/code.ts @@ -1,6 +1,6 @@ import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCellTags, RuleId, fileError, fileWarn } from 'myst-common'; -import type { Image, Output } from 'myst-spec-ext'; +import type { Image, Outputs } from 'myst-spec-ext'; import { select, selectAll } from 'unist-util-select'; import yaml from 'js-yaml'; import type { VFile } from 'vfile'; @@ -156,10 +156,9 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: const blocks = selectAll('block', mdast) as GenericNode[]; blocks.forEach((block) => { if (!block.data) return; - const outputNode = select('output', block) as Output | null; - if (block.data.placeholder && outputNode) { - if (!outputNode.children) outputNode.children = []; - outputNode.children.push({ + const outputsNode = select('outputs', block) as Outputs | null; + if (block.data.placeholder && outputsNode) { + outputsNode.children.push({ type: 'image', placeholder: true, url: block.data.placeholder as string, @@ -195,10 +194,10 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: if (codeNode) codeNode.visibility = 'remove'; break; case NotebookCellTags.hideOutput: - if (outputNode) outputNode.visibility = 'hide'; + if (outputsNode) outputsNode.visibility = 'hide'; break; case NotebookCellTags.removeOutput: - if (outputNode) outputNode.visibility = 'remove'; + if (outputsNode) outputsNode.visibility = 'remove'; break; default: session.log.debug(`tag '${tag}' is not valid in code-cell tags'`); @@ -206,7 +205,7 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: }); if (!block.visibility) block.visibility = 'show'; if (codeNode && !codeNode.visibility) codeNode.visibility = 'show'; - if (outputNode && !outputNode.visibility) outputNode.visibility = 'show'; + if (outputsNode && !outputsNode.visibility) outputsNode.visibility = 'show'; }); } @@ -233,7 +232,7 @@ export function transformLiftCodeBlocksInJupytext(mdast: GenericParent) { child.type === 'block' && child.children?.length === 2 && child.children?.[0].type === 'code' && - child.children?.[1].type === 'output' + child.children?.[1].type === 'outputs' ) { newBlocks.push(child as GenericParent); newBlocks.push({ type: 'block', children: [] }); diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index f15bdd674b..361df35528 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -20,9 +20,14 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', - id: 'abc123', - data: [], + type: 'outputs', + children: [ + { + type: 'output', + id: 'abc123', + jupyter_data: null, + }, + ], }, ], }, @@ -49,18 +54,21 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [ + children: [ { - output_type: 'display_data', - execution_count: 3, - metadata: {}, - data: { - 'application/octet-stream': { - content_type: 'application/octet-stream', - hash: 'def456', - path: '/my/path/def456.png', + type: 'output', + jupyter_data: { + output_type: 'display_data', + execution_count: 3, + metadata: {}, + data: { + 'application/octet-stream': { + content_type: 'application/octet-stream', + hash: 'def456', + path: '/my/path/def456.png', + }, }, }, }, @@ -91,14 +99,19 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [], children: [ { - type: 'image', - placeholder: true, - url: 'placeholder.png', + type: 'output', + jupyter_data: null, + children: [ + { + type: 'image', + placeholder: true, + url: 'placeholder.png', + }, + ], }, ], }, diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index 7046450e7a..92990523d9 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { computeHash } from 'myst-cli-utils'; -import type { Image, SourceFileKind } from 'myst-spec-ext'; +import type { Image, SourceFileKind, Output } from 'myst-spec-ext'; import { liftChildren, fileError, RuleId, fileWarn } from 'myst-common'; import type { GenericNode, GenericParent } from 'myst-common'; import type { ProjectSettings } from 'myst-frontmatter'; @@ -34,17 +34,28 @@ export async function transformOutputsToCache( kind: SourceFileKind, opts?: { minifyMaxCharacters?: number }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; - if (!outputs.length) return; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); await Promise.all( - outputs - .filter((output) => output.visibility !== 'remove') + outputsNodes + // Ignore outputs that are hidden + .filter((outputs) => outputs.visibility !== 'remove') + // Pull out children + .map((outputs) => outputs.children as Output[]) + .flat() + // Filter outputs with no data + // TODO: can this ever occur? + .filter((output) => (output as any).jupyter_data !== undefined) + // Minify output data .map(async (output) => { - output.data = await minifyCellOutput(output.data as IOutput[], cache.$outputs, { - computeHash, - maxCharacters: opts?.minifyMaxCharacters, - }); + [(output as any).jupyter_data] = await minifyCellOutput( + [(output as any).jupyter_data] as IOutput[], + cache.$outputs, + { + computeHash, + maxCharacters: opts?.minifyMaxCharacters, + }, + ); }), ); } @@ -75,9 +86,10 @@ export function transformFilterOutputStreams( const blockRemoveStderr = tags.includes('remove-stderr'); const blockRemoveStdout = tags.includes('remove-stdout'); const outputs = selectAll('output', block) as GenericNode[]; - // There should be only one output in the block - outputs.forEach((output) => { - output.data = output.data.filter((data: IStream | MinifiedMimeOutput) => { + outputs + .filter((output) => { + const data = output.jupyter_data; + if ( (stderr !== 'show' || blockRemoveStderr) && data.output_type === 'stream' && @@ -96,7 +108,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( (stdout !== 'show' || blockRemoveStdout) && @@ -116,7 +128,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( mpl !== 'show' && @@ -141,12 +153,15 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } - return true; + return false; + }) + .forEach((output) => { + output.type = '__delete__'; }); - }); }); + remove(mdast, { cascade: false }, '__delete__'); } function writeCachedOutputToFile( @@ -192,16 +207,19 @@ export function transformOutputsToFile( const outputs = selectAll('output', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - walkOutputs(node.data, (obj) => { - const { hash } = obj; - if (!hash || !cache.$outputs[hash]) return undefined; - obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, + outputs + .filter((output) => !!output.jupyter_data) + .forEach((node) => { + // TODO: output-refactoring -- drop to single output in future + walkOutputs([node.jupyter_data], (obj) => { + const { hash } = obj; + if (!hash || !cache.$outputs[hash]) return undefined; + obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node, + }); }); }); - }); } /** @@ -233,87 +251,99 @@ export function reduceOutputs( writeFolder: string, opts?: { altOutputFolder?: string; vfile?: VFile }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - if (node.visibility === 'remove' || node.visibility === 'hide') { - // Hidden nodes should not show up in simplified outputs for static export - node.type = '__delete__'; - return; - } - if (!node.data?.length && !node.children?.length) { - node.type = '__delete__'; - return; - } - node.type = '__lift__'; - if (node.children?.length) return; - const selectedOutputs: { content_type: string; hash: string }[] = []; - node.data.forEach((output: MinifiedOutput) => { - let selectedOutput: { content_type: string; hash: string } | undefined; - walkOutputs([output], (obj: any) => { - const { output_type, content_type, hash } = obj; - if (!hash) return undefined; - if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { - if (['error', 'stream'].includes(output_type)) { - selectedOutput = { content_type: 'text/plain', hash }; - } else if (typeof content_type === 'string') { - if ( - content_type.startsWith('image/') || - content_type === 'text/plain' || - content_type === 'text/html' - ) { - selectedOutput = { content_type, hash }; + outputsNodes.forEach((outputsNode) => { + const outputs = outputsNode.children as GenericNode[]; + + outputs.forEach((outputNode) => { + if (outputNode.visibility === 'remove' || outputNode.visibility === 'hide') { + // Hidden nodes should not show up in simplified outputs for static export + outputNode.type = '__delete__'; + return; + } + if (!outputNode.jupyter_data && !outputNode.children?.length) { + outputNode.type = '__delete__'; + return; + } + // Lift the `output` node into `Outputs` + outputNode.type = '__lift__'; + + // If the output already has children, we don't need to do anything + if (outputNode.children?.length) { + return; + } + + // Find a preferred IOutput type to render into the AST + const selectedOutputs: { content_type: string; hash: string }[] = []; + if (outputNode.jupyter_data) { + const output = outputNode.jupyter_data; + + let selectedOutput: { content_type: string; hash: string } | undefined; + walkOutputs([output], (obj: any) => { + const { output_type, content_type, hash } = obj; + if (!hash) return undefined; + if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { + if (['error', 'stream'].includes(output_type)) { + selectedOutput = { content_type: 'text/plain', hash }; + } else if (typeof content_type === 'string') { + if ( + content_type.startsWith('image/') || + content_type === 'text/plain' || + content_type === 'text/html' + ) { + selectedOutput = { content_type, hash }; + } } } - } - }); - if (selectedOutput) selectedOutputs.push(selectedOutput); - }); - const children: (Image | GenericNode)[] = selectedOutputs - .map((output): Image | GenericNode | GenericNode[] | undefined => { - const { content_type, hash } = output ?? {}; - if (!hash || !cache.$outputs[hash]) return undefined; - if (content_type === 'text/html') { - const htmlTree = { - type: 'root', - children: [ - { - type: 'html', - value: cache.$outputs[hash][0], - }, - ], - }; - htmlTransform(htmlTree); - if ((selectAll('image', htmlTree) as GenericNode[]).find((htmlImage) => !htmlImage.url)) { - return undefined; + }); + if (selectedOutput) selectedOutputs.push(selectedOutput); + } + const children: (Image | GenericNode)[] = selectedOutputs + .map((output): Image | GenericNode | GenericNode[] | undefined => { + const { content_type, hash } = output ?? {}; + if (!hash || !cache.$outputs[hash]) return undefined; + if (content_type === 'text/html') { + const htmlTree = { + type: 'root', + children: [ + { + type: 'html', + value: cache.$outputs[hash][0], + }, + ], + }; + htmlTransform(htmlTree); + return htmlTree.children; + } else if (content_type.startsWith('image/')) { + const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node: outputNode, + }); + if (!path) return undefined; + const relativePath = relative(dirname(file), path); + return { + type: 'image', + data: { type: 'output' }, + url: relativePath, + urlSource: relativePath, + }; + } else if (content_type === 'text/plain' && cache.$outputs[hash]) { + const [content] = cache.$outputs[hash]; + return { + type: 'code', + data: { type: 'output' }, + value: stripAnsi(content), + }; } - return htmlTree.children; - } else if (content_type.startsWith('image/')) { - const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, - }); - if (!path) return undefined; - const relativePath = relative(dirname(file), path); - return { - type: 'image', - data: { type: 'output' }, - url: relativePath, - urlSource: relativePath, - }; - } else if (content_type === 'text/plain' && cache.$outputs[hash]) { - const [content] = cache.$outputs[hash]; - return { - type: 'code', - data: { type: 'output' }, - value: stripAnsi(content), - }; - } - return undefined; - }) - .flat() - .filter((output): output is Image | GenericNode => !!output); - node.children = children; + return undefined; + }) + .flat() + .filter((output): output is Image | GenericNode => !!output); + outputNode.children = children; + }); + // Lift the `outputs` node + outputsNode.type = '__lift__'; }); remove(mdast, '__delete__'); liftChildren(mdast, '__lift__'); diff --git a/packages/myst-directives/src/code.ts b/packages/myst-directives/src/code.ts index 00f7f03b68..4e13b3ad1b 100644 --- a/packages/myst-directives/src/code.ts +++ b/packages/myst-directives/src/code.ts @@ -252,15 +252,14 @@ export const codeCellDirective: DirectiveSpec = { executable: true, value: (data.body ?? '') as string, }; - const output = { - type: 'output', - id: nanoid(), - data: [], + const outputs = { + type: 'outputs', + children: [], }; const block: GenericNode = { type: 'block', kind: NotebookCell.code, - children: [code, output], + children: [code, outputs], data: {}, }; addCommonDirectiveOptions(data, block); diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index e97dcfad44..ab2d5bc1a0 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -2,7 +2,7 @@ import { select, selectAll } from 'unist-util-select'; import type { Logger } from 'myst-cli-utils'; import type { PageFrontmatter, KernelSpec } from 'myst-frontmatter'; import type { Kernel, KernelMessage, Session, SessionManager } from '@jupyterlab/services'; -import type { Block, Code, InlineExpression, Output } from 'myst-spec-ext'; +import type { Block, Code, InlineExpression, Output, Outputs } from 'myst-spec-ext'; import type { IOutput } from '@jupyterlab/nbformat'; import type { GenericNode, GenericParent, IExpressionResult, IExpressionError } from 'myst-common'; import { NotebookCell, NotebookCellTags, fileError } from 'myst-common'; @@ -166,7 +166,7 @@ type CodeBlock = Block & { * @param node node to test */ function isCellBlock(node: GenericNode): node is CodeBlock { - return node.type === 'block' && select('code', node) !== null && select('output', node) !== null; + return node.type === 'block' && select('code', node) !== null && select('outputs', node) !== null; } /** @@ -282,14 +282,17 @@ function applyComputedOutputsToNodes( const thisResult = computedResult.shift(); if (isCellBlock(matchedNode)) { - // Pull out output to set data - const output = select('output', matchedNode) as unknown as { data: IOutput[] }; - // Set the output array to empty if we don't have a result (e.g. due to a kernel error) - output.data = thisResult === undefined ? [] : (thisResult as IOutput[]); + const rawOutputData = (thisResult as IOutput[]) ?? []; + // Pull out outputs to set data + const outputs = select('outputs', matchedNode) as Outputs; + // Ensure that whether this fails or succeeds, we write to `children` (e.g. due to a kernel error) + outputs.children = rawOutputData.map((data) => { + return { type: 'output', children: [], jupyter_data: data as any }; + }); } else if (isInlineExpression(matchedNode)) { + const rawOutputData = thisResult as Record | undefined; // Set data of expression to the result, or empty if we don't have one - matchedNode.result = // TODO: FIXME .data - thisResult === undefined ? undefined : (thisResult as unknown as Record); + matchedNode.result = rawOutputData; } else { // This should never happen throw new Error('Node must be either code block or inline expression.'); diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index f768992aad..664d530672 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -27,22 +27,18 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 + children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output + - type: outputs + children: + - type: output + children: [] + after: type: root children: @@ -54,27 +50,22 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 + children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + children: + - type: output + children: [] + jupyter_data: + output_type: stream + name: stdout + text: | + abc - title: tree with inline expression is evaluated before: type: root @@ -115,100 +106,72 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + children: + - type: output + jupyter_data: + output_type: stream + name: stdout + text: | + abc - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + - type: outputs + children: + - type: output + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `raises-exception` is evaluated and passes before: type: root @@ -216,58 +179,44 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + - type: outputs + children: + - type: output + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `skip-execution` is not evaluated before: type: root @@ -275,45 +224,31 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: diff --git a/packages/myst-spec-ext/src/types.ts b/packages/myst-spec-ext/src/types.ts index 9f5bd9d463..9ae4210d93 100644 --- a/packages/myst-spec-ext/src/types.ts +++ b/packages/myst-spec-ext/src/types.ts @@ -254,10 +254,16 @@ export type Container = Omit & { export type Output = Node & Target & { type: 'output'; - id?: string; - data?: any[]; // MinifiedOutput[] + children: (FlowContent | ListContent | PhrasingContent)[]; + jupyter_data: any; // TODO: set this to IOutput + }; + +export type Outputs = Node & + Target & { + type: 'outputs'; + children: (Output | FlowContent | ListContent | PhrasingContent)[]; // Support placeholders in addition to outputs visibility?: Visibility; - children?: (FlowContent | ListContent | PhrasingContent)[]; + id?: string; }; export type Aside = Node & diff --git a/packages/myst-transforms/src/blocks.spec.ts b/packages/myst-transforms/src/blocks.spec.ts index 5ce0fecd48..3d868033ee 100644 --- a/packages/myst-transforms/src/blocks.spec.ts +++ b/packages/myst-transforms/src/blocks.spec.ts @@ -120,8 +120,10 @@ describe('Test blockMetadataTransform', () => { test('label is propagated to outputs', async () => { const mdast = u('root', [ u('block', { meta: '{"label": "My_Label", "key": "value"}' }, [ - u('output', 'We know what we are'), - u('output', 'but know not what we may be.'), + u('outputs', [ + u('output', 'We know what we are'), + u('output', 'but know not what we may be.'), + ]), ]), ]) as any; blockMetadataTransform(mdast, new VFile()); @@ -136,8 +138,10 @@ describe('Test blockMetadataTransform', () => { data: { key: 'value' }, }, [ - u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), - u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + u('outputs', { identifier: 'my_label-output' }, [ + u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), + u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + ]), ], ), ]), diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 20e9941461..49f4a5cf19 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -1,7 +1,7 @@ import type { VFile } from 'vfile'; import type { Plugin } from 'unified'; import type { Node } from 'myst-spec'; -import { selectAll } from 'unist-util-select'; +import { selectAll, select } from 'unist-util-select'; import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCell, RuleId, fileError, normalizeLabel } from 'myst-common'; import type { Code } from 'myst-spec-ext'; @@ -89,15 +89,18 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { child.identifier = `${block.identifier}-code-${index}`; } }); - const outputChildren = selectAll('output', block) as GenericNode[]; - outputChildren.forEach((child, index) => { - if (child.identifier) return; - if (outputChildren.length === 1) { - child.identifier = `${block.identifier}-output`; - } else { - child.identifier = `${block.identifier}-output-${index}`; - } - }); + const outputsNode = select('outputs', block) as GenericNode | undefined; + if (outputsNode && !outputsNode.identifier) { + // Label outputs node + outputsNode.identifier = `${block.identifier}-output`; + // Enumerate outputs + const outputs = selectAll('output', outputsNode) as GenericNode[]; + outputs.forEach((outputNode, index) => { + if (outputNode && !outputNode.identifier) { + outputNode.identifier = `${block.identifier}-output-${index}`; + } + }); + } } }); } From 128aa43b2e43059eed073666af9ec5a307f11e00 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 11:05:36 +0000 Subject: [PATCH 02/32] chore: add changeset --- .changeset/ten-bats-warn.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/ten-bats-warn.md diff --git a/.changeset/ten-bats-warn.md b/.changeset/ten-bats-warn.md new file mode 100644 index 0000000000..00fc05d9b7 --- /dev/null +++ b/.changeset/ten-bats-warn.md @@ -0,0 +1,10 @@ +--- +"mystmd": minor +"myst-directives": patch +"myst-transforms": patch +"myst-spec-ext": patch +"myst-execute": patch +"myst-cli": patch +--- + +Add support for new Outputs node From c87a7a4ab54f6dcaf087eb0944eeaa7db7dcf10d Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 11:05:58 +0000 Subject: [PATCH 03/32] fix: drop style change --- packages/myst-cli/src/process/mdast.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index f0e8cf5eda..be10767d24 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -451,6 +451,5 @@ export async function finalizeMdast( postData.widgets = cache.$getMdast(file)?.pre.widgets; updateFileInfoFromFrontmatter(session, file, frontmatter); } - logMessagesFromVFile(session, vfile); } From bf9fa61e4c49e3c3e2f9480910eb0f1178220614 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:27:23 +0000 Subject: [PATCH 04/32] fix: only label first code and outputs --- packages/myst-transforms/src/blocks.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 49f4a5cf19..73c4aa5b93 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -80,17 +80,12 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { } } if (block.identifier) { - const codeChildren = selectAll('code', block) as Code[]; - codeChildren.forEach((child, index) => { - if (child.identifier) return; - if (codeChildren.length === 1) { - child.identifier = `${block.identifier}-code`; - } else { - child.identifier = `${block.identifier}-code-${index}`; - } - }); + const codeNode = selectAll('code', block) as any as Code | null; + if (codeNode !== null && !codeNode.identifier) { + codeNode.identifier = `${block.identifier}-code`; + } const outputsNode = select('outputs', block) as GenericNode | undefined; - if (outputsNode && !outputsNode.identifier) { + if (outputsNode !== undefined && !outputsNode.identifier) { // Label outputs node outputsNode.identifier = `${block.identifier}-output`; // Enumerate outputs From 3eba5d3a99a7255b07cda028b2e264583d391b19 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:50:40 +0000 Subject: [PATCH 05/32] fix: handle IDs --- packages/myst-execute/src/execute.ts | 5 +- packages/myst-execute/tests/execute.yml | 168 ++++++++++++++++++++---- 2 files changed, 142 insertions(+), 31 deletions(-) diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index ab2d5bc1a0..07a5317146 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -286,8 +286,9 @@ function applyComputedOutputsToNodes( // Pull out outputs to set data const outputs = select('outputs', matchedNode) as Outputs; // Ensure that whether this fails or succeeds, we write to `children` (e.g. due to a kernel error) - outputs.children = rawOutputData.map((data) => { - return { type: 'output', children: [], jupyter_data: data as any }; + outputs.children = rawOutputData.map((data, index) => { + const identifier = outputs.identifier ? `${outputs.identifier}-${index}` : undefined; + return { type: 'output', children: [], jupyter_data: data as any, identifier }; }); } else if (isInlineExpression(matchedNode)) { const rawOutputData = thisResult as Record | undefined; diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index 664d530672..f1816e5af6 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -27,16 +27,24 @@ cases: - type: block kind: notebook-code data: - + id: nb-cell-0 + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 children: [] after: @@ -50,22 +58,86 @@ cases: - type: block kind: notebook-code data: + id: nb-cell-0 + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') + enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output + children: + - type: output + children: [] + identifier: nb-cell-0-output-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc + - title: output without identifier is given one + before: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') + enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output + children: + - type: output + children: [] + after: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output children: [] + identifier: nb-cell-0-output-0 jupyter_data: - output_type: stream - name: stdout - text: | - abc + output_type: stream + name: stdout + text: | + abc - title: tree with inline expression is evaluated before: type: root @@ -112,9 +184,14 @@ cases: executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: - type: block kind: notebook-code @@ -125,9 +202,14 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-1-code + html_id: nb-cell-1-code - type: outputs + identifier: nb-cell-1-output + html_id: nb-cell-1-output children: - type: output + identifier: nb-cell-1-output-0 jupyter_data: after: type: root @@ -141,14 +223,19 @@ cases: executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: - output_type: stream - name: stdout - text: | - abc + output_type: stream + name: stdout + text: | + abc - type: block kind: notebook-code data: @@ -158,20 +245,25 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-1-code + html_id: nb-cell-1-code - type: outputs + identifier: nb-cell-1-output + html_id: nb-cell-1-output children: - type: output + identifier: nb-cell-1-output-0 jupyter_data: - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `raises-exception` is evaluated and passes before: type: root @@ -186,9 +278,14 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: after: type: root @@ -203,20 +300,25 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `skip-execution` is not evaluated before: type: root @@ -231,9 +333,13 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: after: type: root @@ -248,7 +354,11 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: From ad8ed3ee44150d66a37295b18ad19facb6b6b138 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 16:16:48 +0000 Subject: [PATCH 06/32] fix: return type --- packages/myst-transforms/src/blocks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 73c4aa5b93..5c220cb36f 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -84,8 +84,8 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { if (codeNode !== null && !codeNode.identifier) { codeNode.identifier = `${block.identifier}-code`; } - const outputsNode = select('outputs', block) as GenericNode | undefined; - if (outputsNode !== undefined && !outputsNode.identifier) { + const outputsNode = select('outputs', block) as GenericNode | null; + if (outputsNode !== null && !outputsNode.identifier) { // Label outputs node outputsNode.identifier = `${block.identifier}-output`; // Enumerate outputs From 195182adbac5e97f932b6d56d4912eb5c293cac1 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 16:20:02 +0000 Subject: [PATCH 07/32] =?UTF-8?q?fix:=20selectAll=20=E2=86=92=20select?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-transforms/src/blocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 5c220cb36f..730565326a 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -80,7 +80,7 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { } } if (block.identifier) { - const codeNode = selectAll('code', block) as any as Code | null; + const codeNode = select('code', block) as any as Code | null; if (codeNode !== null && !codeNode.identifier) { codeNode.identifier = `${block.identifier}-code`; } From c715602e5071373330eb888ab37a60a1887beff7 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Tue, 1 Apr 2025 11:13:59 +0100 Subject: [PATCH 08/32] refactor: move user expression lowering into function --- packages/myst-cli/src/process/notebook.ts | 27 ++++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/myst-cli/src/process/notebook.ts b/packages/myst-cli/src/process/notebook.ts index 59257a5dc8..234654ab01 100644 --- a/packages/myst-cli/src/process/notebook.ts +++ b/packages/myst-cli/src/process/notebook.ts @@ -84,6 +84,21 @@ export async function processNotebook( return mdast; } +/** + * Embed the Jupyter output data for a user expression into the AST + */ +function embedInlineExpressions( + userExpressions: IUserExpressionMetadata[] | undefined, + block: GenericNode, +) { + const inlineNodes = selectAll('inlineExpression', block) as InlineExpression[]; + inlineNodes.forEach((inlineExpression) => { + const data = findExpression(userExpressions, inlineExpression.value); + if (!data) return; + inlineExpression.result = data.result as unknown as Record; + }); +} + export async function processNotebookFull( session: ISession, file: string, @@ -136,17 +151,7 @@ export async function processNotebookFull( return acc.concat(...cellMdast.children); } const block = blockParent(cell, cellMdast.children) as GenericNode; - - // Embed expression results into expression - const userExpressions = block.data?.[metadataSection] as - | IUserExpressionMetadata[] - | undefined; - const inlineNodes = selectAll('inlineExpression', block) as InlineExpression[]; - inlineNodes.forEach((inlineExpression) => { - const data = findExpression(userExpressions, inlineExpression.value); - if (!data) return; - inlineExpression.result = data.result as unknown as Record; - }); + embedInlineExpressions(block.data?.[metadataSection], block); return acc.concat(block); } if (cell.cell_type === CELL_TYPES.raw) { From a5553a370777e9bfdd7cd57c87c059543f3387f2 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Tue, 1 Apr 2025 11:17:35 +0100 Subject: [PATCH 09/32] refactor: nest declaration --- packages/myst-cli/src/process/notebook.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/myst-cli/src/process/notebook.ts b/packages/myst-cli/src/process/notebook.ts index 234654ab01..cf1aea7473 100644 --- a/packages/myst-cli/src/process/notebook.ts +++ b/packages/myst-cli/src/process/notebook.ts @@ -170,19 +170,14 @@ export async function processNotebookFull( value: ensureString(cell.source), }; - const outputsChildren = (cell.outputs as IOutput[]).map((output) => { - // Embed outputs in an output block - const result = { - type: 'output', - jupyter_data: output, - children: [], - }; - return result; - }); const outputs = { type: 'outputs', id: nanoid(), - children: outputsChildren, + children: (cell.outputs as IOutput[]).map((output) => ({ + type: 'output', + jupyter_data: output, + children: [], + })), }; return acc.concat(blockParent(cell, [code, outputs])); } From 32c591540162fe5023356fdb6d280ec8b46a0f36 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Tue, 1 Apr 2025 11:27:34 +0100 Subject: [PATCH 10/32] fix: rename outputs --- packages/myst-transforms/src/blocks.spec.ts | 2 +- packages/myst-transforms/src/blocks.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/myst-transforms/src/blocks.spec.ts b/packages/myst-transforms/src/blocks.spec.ts index 3d868033ee..7324ac5f2d 100644 --- a/packages/myst-transforms/src/blocks.spec.ts +++ b/packages/myst-transforms/src/blocks.spec.ts @@ -138,7 +138,7 @@ describe('Test blockMetadataTransform', () => { data: { key: 'value' }, }, [ - u('outputs', { identifier: 'my_label-output' }, [ + u('outputs', { identifier: 'my_label-outputs' }, [ u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), ]), diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 730565326a..9568485d50 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -87,7 +87,7 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { const outputsNode = select('outputs', block) as GenericNode | null; if (outputsNode !== null && !outputsNode.identifier) { // Label outputs node - outputsNode.identifier = `${block.identifier}-output`; + outputsNode.identifier = `${block.identifier}-outputs`; // Enumerate outputs const outputs = selectAll('output', outputsNode) as GenericNode[]; outputs.forEach((outputNode, index) => { From 64588087732da12229ae7a7c35a791f40cb9f9bf Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Tue, 1 Apr 2025 11:50:36 +0100 Subject: [PATCH 11/32] refactor: simplify conditions We don't need to delete nodes without children: they will simply vanish when lifted --- packages/myst-cli/src/transforms/outputs.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index 92990523d9..0a9e07be4a 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -262,15 +262,12 @@ export function reduceOutputs( outputNode.type = '__delete__'; return; } - if (!outputNode.jupyter_data && !outputNode.children?.length) { - outputNode.type = '__delete__'; - return; - } // Lift the `output` node into `Outputs` outputNode.type = '__lift__'; // If the output already has children, we don't need to do anything - if (outputNode.children?.length) { + // Or, if it has no output data (should not happen) + if (outputNode.children?.length || !outputNode.jupyter_data) { return; } From 8c34d464ecbe3a263022298d372250f8d77c4978 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Tue, 1 Apr 2025 11:51:23 +0100 Subject: [PATCH 12/32] test: fix tests --- packages/myst-execute/tests/execute.yml | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index f1816e5af6..449d81b7f4 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -40,8 +40,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output identifier: nb-cell-0-output-0 @@ -71,8 +71,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output children: [] @@ -102,8 +102,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output children: [] @@ -127,8 +127,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output children: [] @@ -187,8 +187,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output identifier: nb-cell-0-output-0 @@ -205,8 +205,8 @@ cases: identifier: nb-cell-1-code html_id: nb-cell-1-code - type: outputs - identifier: nb-cell-1-output - html_id: nb-cell-1-output + identifier: nb-cell-1-outputs + html_id: nb-cell-1-outputs children: - type: output identifier: nb-cell-1-output-0 @@ -226,8 +226,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output identifier: nb-cell-0-output-0 @@ -248,8 +248,8 @@ cases: identifier: nb-cell-1-code html_id: nb-cell-1-code - type: outputs - identifier: nb-cell-1-output - html_id: nb-cell-1-output + identifier: nb-cell-1-outputs + html_id: nb-cell-1-outputs children: - type: output identifier: nb-cell-1-output-0 @@ -281,8 +281,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output identifier: nb-cell-0-output-0 @@ -303,8 +303,8 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output - html_id: nb-cell-0-output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs children: - type: output identifier: nb-cell-0-output-0 @@ -336,7 +336,7 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output + identifier: nb-cell-0-outputs children: - type: output identifier: nb-cell-0-output-0 @@ -357,7 +357,7 @@ cases: identifier: nb-cell-0-code html_id: nb-cell-0-code - type: outputs - identifier: nb-cell-0-output + identifier: nb-cell-0-outputs children: - type: output identifier: nb-cell-0-output-0 From d94742db6665bb2bc31535ab60f49c0c2e2b2500 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 09:16:44 +0100 Subject: [PATCH 13/32] fix: outputs for JATS --- packages/myst-to-jats/src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/myst-to-jats/src/index.ts b/packages/myst-to-jats/src/index.ts index 05ec2ce813..d959205566 100644 --- a/packages/myst-to-jats/src/index.ts +++ b/packages/myst-to-jats/src/index.ts @@ -285,6 +285,7 @@ type Handlers = { si: Handler; proof: Handler; algorithmLine: Handler; + outputs: Handler; output: Handler; embed: Handler; supplementaryMaterial: Handler; @@ -632,6 +633,9 @@ const handlers: Handlers = { state.renderChildren(node); state.closeNode(); }, + outputs(node, state) { + state.renderChildren(node); + }, output(node, state) { if (state.data.isInContainer) { if (!node.data?.[0]) return; From 41753bfd99e61ba33c660f4d455c0d1b02c2077b Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 10:29:30 +0100 Subject: [PATCH 14/32] fix: use outputs- for all prefixes --- packages/myst-transforms/src/blocks.spec.ts | 4 ++-- packages/myst-transforms/src/blocks.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/myst-transforms/src/blocks.spec.ts b/packages/myst-transforms/src/blocks.spec.ts index 7324ac5f2d..8403761f77 100644 --- a/packages/myst-transforms/src/blocks.spec.ts +++ b/packages/myst-transforms/src/blocks.spec.ts @@ -139,8 +139,8 @@ describe('Test blockMetadataTransform', () => { }, [ u('outputs', { identifier: 'my_label-outputs' }, [ - u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), - u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + u('output', { identifier: 'my_label-outputs-0' }, 'We know what we are'), + u('output', { identifier: 'my_label-outputs-1' }, 'but know not what we may be.'), ]), ], ), diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 9568485d50..8cf8ade73d 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -92,7 +92,8 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { const outputs = selectAll('output', outputsNode) as GenericNode[]; outputs.forEach((outputNode, index) => { if (outputNode && !outputNode.identifier) { - outputNode.identifier = `${block.identifier}-output-${index}`; + // Label output node + outputNode.identifier = `${outputsNode.identifier}-${index}`; } }); } From 42687fb7c8afe4e9073f4dcef5092a979ce4568e Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 11:29:34 +0100 Subject: [PATCH 15/32] fix: outputs for JATS --- packages/myst-to-jats/src/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/myst-to-jats/src/index.ts b/packages/myst-to-jats/src/index.ts index d959205566..68ba9902ee 100644 --- a/packages/myst-to-jats/src/index.ts +++ b/packages/myst-to-jats/src/index.ts @@ -638,20 +638,20 @@ const handlers: Handlers = { }, output(node, state) { if (state.data.isInContainer) { - if (!node.data?.[0]) return; - alternativesFromMinifiedOutput(node.data[0], state); + if (!node.jupyter_data) return; + alternativesFromMinifiedOutput(node.jupyter_data, state); return; } const { identifier } = node; const attrs: Attributes = { 'sec-type': 'notebook-output' }; - node.data?.forEach((output: any, index: number) => { + if (node.jupyter_data) { state.openNode('sec', { ...attrs, - id: identifier && !state.data.isNotebookArticleRep ? `${identifier}-${index}` : undefined, + id: identifier && !state.data.isNotebookArticleRep ? identifier : undefined, }); - alternativesFromMinifiedOutput(output, state); + alternativesFromMinifiedOutput(node.jupyter_data, state); state.closeNode(); - }); + } }, embed(node, state) { if (state.data.isInContainer) { From 8400eb88c18573b309ca714beb2ca732d9ab9f95 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 11:29:41 +0100 Subject: [PATCH 16/32] refactor: simplify test for code cells --- packages/myst-execute/src/execute.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index 07a5317146..10d149253c 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -114,7 +114,7 @@ function buildCacheKey(kernelSpec: KernelSpec, nodes: (CodeBlock | InlineExpress raisesException: boolean; }[] = []; for (const node of nodes) { - if (isCellBlock(node)) { + if (isCodeBlock(node)) { hashableItems.push({ kind: node.type, content: (select('code', node) as Code).value, @@ -165,8 +165,8 @@ type CodeBlock = Block & { * * @param node node to test */ -function isCellBlock(node: GenericNode): node is CodeBlock { - return node.type === 'block' && select('code', node) !== null && select('outputs', node) !== null; +function isCodeBlock(node: GenericNode): node is CodeBlock { + return node.type === 'block' && node.kind === NotebookCell.code; } /** @@ -215,7 +215,7 @@ async function computeExecutableNodes( const results: (IOutput[] | IExpressionResult)[] = []; for (const matchedNode of nodes) { - if (isCellBlock(matchedNode)) { + if (isCodeBlock(matchedNode)) { // Pull out code to execute const code = select('code', matchedNode) as Code; const { status, outputs } = await executeCode(kernel, code.value); @@ -281,7 +281,7 @@ function applyComputedOutputsToNodes( // Pull out the result for this node const thisResult = computedResult.shift(); - if (isCellBlock(matchedNode)) { + if (isCodeBlock(matchedNode)) { const rawOutputData = (thisResult as IOutput[]) ?? []; // Pull out outputs to set data const outputs = select('outputs', matchedNode) as Outputs; @@ -329,7 +329,7 @@ export async function kernelExecutionTransform(tree: GenericParent, vfile: VFile )[] ) // Filter out nodes that skip execution - .filter((node) => !(isCellBlock(node) && codeBlockSkipsExecution(node))); + .filter((node) => !(isCodeBlock(node) && codeBlockSkipsExecution(node))); // Only do something if we have any nodes! if (executableNodes.length === 0) { From d3d7a409bffae5e9b153f4db4cbb8afaf038f968 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 11:35:02 +0100 Subject: [PATCH 17/32] fix: early exit non-matplotlib --- packages/myst-cli/src/transforms/outputs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index 0a9e07be4a..f17f36ac8f 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -137,7 +137,7 @@ export function transformFilterOutputStreams( data.data['text/plain'] ) { const content = data.data['text/plain'].content; - if (!stringIsMatplotlibOutput(content)) return true; + if (!stringIsMatplotlibOutput(content)) return false; const doRemove = mpl.includes('remove'); const doWarn = mpl.includes('warn'); const doError = mpl.includes('error'); From b86544fd1c04c579cdfbed7c95e3f8e3657545dc Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 11:59:42 +0100 Subject: [PATCH 18/32] test: partially fix JATS test --- packages/myst-to-jats/tests/notebooks.yml | 161 +++++++++++++--------- 1 file changed, 93 insertions(+), 68 deletions(-) diff --git a/packages/myst-to-jats/tests/notebooks.yml b/packages/myst-to-jats/tests/notebooks.yml index 4f7e95d4fd..7417efc661 100644 --- a/packages/myst-to-jats/tests/notebooks.yml +++ b/packages/myst-to-jats/tests/notebooks.yml @@ -18,16 +18,20 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt - type: block kind: notebook-code data: @@ -43,23 +47,27 @@ cases: identifier: nb-cell-1-code enumerator: 2 html_id: nb-cell-1-code - - type: output + - type: outputs id: uE4mPgSow0oyo0dvEH9Lc - identifier: nb-cell-1-output - html_id: nb-cell-1-output - data: - - output_type: execute_result - execution_count: 3 - metadata: {} - data: - text/plain: - content_type: text/plain - hash: b - path: files/b.txt - text/html: - content_type: text/html - hash: b - path: files/b.html + identifier: nb-cell-1-outputs + html_id: nb-cell-1-outputs + children: + - type: output + identifier: nb-cell-1-outputs-0 + html_id: nb-cell-1-outputs-0 + jupyter_data: + output_type: execute_result + execution_count: 3 + metadata: {} + data: + text/plain: + content_type: text/plain + hash: b + path: files/b.txt + text/html: + content_type: text/html + hash: b + path: files/b.html - type: block kind: notebook-code data: @@ -75,17 +83,21 @@ cases: identifier: nb-cell-2-code enumerator: 3 html_id: nb-cell-2-code - - type: output + - type: outputs id: 7Qrwdo-_oq5US1Du2KCLU - identifier: nb-cell-2-output - html_id: nb-cell-2-output - data: - - ename: NameError - evalue: name 'a' is not defined - output_type: error - traceback: \u001b[0;31m------------------------------------------------------... - hash: c - path: files/c.txt + identifier: nb-cell-2-outputs + html_id: nb-cell-2-outputs + children: + - type: output + identifier: nb-cell-2-outputs-0 + html_id: nb-cell-2-outputs-0 + jupyter_data: + ename: NameError + evalue: name 'a' is not defined + output_type: error + traceback: \u001b[0;31m------------------------------------------------------... + hash: c + path: files/c.txt jats: |- @@ -101,7 +113,7 @@ cases: print('abc') - + @@ -109,7 +121,7 @@ cases: 'abc' - + @@ -118,7 +130,7 @@ cases: a - + @@ -151,16 +163,20 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt - type: block kind: notebook-content data: @@ -212,7 +228,7 @@ cases: print('abc') - + @@ -251,16 +267,21 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt jats: |- @@ -285,7 +306,7 @@ cases: print('abc') - + @@ -313,16 +334,20 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt - type: block data: part: abstract @@ -373,7 +398,7 @@ cases: print('abc') - + From 6b169e2a7008a0aaf6f5a6113fd4e308e6585861 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 13:13:29 +0100 Subject: [PATCH 19/32] fix: final JATS test --- packages/myst-to-jats/tests/basic.yml | 58 +++++++++++++++------------ 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/packages/myst-to-jats/tests/basic.yml b/packages/myst-to-jats/tests/basic.yml index 544d50b53a..7924cbcb09 100644 --- a/packages/myst-to-jats/tests/basic.yml +++ b/packages/myst-to-jats/tests/basic.yml @@ -686,33 +686,41 @@ cases: identifier: nb-cell-0-code enumerator: 1 html_id: nb-cell-0-code - - type: output + - type: outputs id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - name: stdout - output_type: stream - text: abc\n... - hash: a - path: files/a.txt - - type: output + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + name: stdout + output_type: stream + text: abc\n... + hash: a + path: files/a.txt + - type: outputs id: uE4mPgSow0oyo0dvEH9Lc - identifier: nb-cell-1-output - html_id: nb-cell-1-output - data: - - output_type: execute_result - execution_count: 3 - metadata: {} - data: - text/plain: - content_type: text/plain - hash: b - path: files/b.txt - text/html: - content_type: text/html - hash: b - path: files/b.html + identifier: nb-cell-0-outputs + html_id: nb-cell-0-outputs + children: + - type: output + identifier: nb-cell-0-outputs-0 + html_id: nb-cell-0-outputs-0 + jupyter_data: + output_type: execute_result + execution_count: 3 + metadata: {} + data: + text/plain: + content_type: text/plain + hash: b + path: files/b.txt + text/html: + content_type: text/html + hash: b + path: files/b.html - type: caption children: - type: paragraph From 05f79b9603d81cb252e62a47a944965bb0752f0e Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 13:15:38 +0100 Subject: [PATCH 20/32] test: fix myst-execute tests --- packages/myst-execute/tests/execute.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index 449d81b7f4..1a1330b999 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -44,7 +44,7 @@ cases: html_id: nb-cell-0-outputs children: - type: output - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 children: [] after: @@ -76,7 +76,7 @@ cases: children: - type: output children: [] - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: output_type: stream name: stdout @@ -132,7 +132,7 @@ cases: children: - type: output children: [] - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: output_type: stream name: stdout @@ -191,7 +191,7 @@ cases: html_id: nb-cell-0-outputs children: - type: output - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: - type: block kind: notebook-code @@ -209,7 +209,7 @@ cases: html_id: nb-cell-1-outputs children: - type: output - identifier: nb-cell-1-output-0 + identifier: nb-cell-1-outputs-0 jupyter_data: after: type: root @@ -230,7 +230,7 @@ cases: html_id: nb-cell-0-outputs children: - type: output - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: output_type: stream name: stdout @@ -252,7 +252,7 @@ cases: html_id: nb-cell-1-outputs children: - type: output - identifier: nb-cell-1-output-0 + identifier: nb-cell-1-outputs-0 jupyter_data: output_type: error # Note this traceback can be different on various machines @@ -285,7 +285,7 @@ cases: html_id: nb-cell-0-outputs children: - type: output - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: after: type: root @@ -307,7 +307,7 @@ cases: html_id: nb-cell-0-outputs children: - type: output - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: output_type: error # Note this traceback can be different on various machines @@ -339,7 +339,7 @@ cases: identifier: nb-cell-0-outputs children: - type: output - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: after: type: root @@ -360,5 +360,5 @@ cases: identifier: nb-cell-0-outputs children: - type: output - identifier: nb-cell-0-output-0 + identifier: nb-cell-0-outputs-0 jupyter_data: From f8671d1a52093517f35f30dee8f9730b5118da35 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 13:48:36 +0100 Subject: [PATCH 21/32] test: fix test titles --- packages/myst-cli/src/transforms/code.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/myst-cli/src/transforms/code.spec.ts b/packages/myst-cli/src/transforms/code.spec.ts index 27d50958fd..d8221184a0 100644 --- a/packages/myst-cli/src/transforms/code.spec.ts +++ b/packages/myst-cli/src/transforms/code.spec.ts @@ -306,7 +306,7 @@ describe('propagateBlockDataToCode', () => { } } }); - it('placeholder creates image node child of output', async () => { + it('placeholder creates image node child of outputs', async () => { const mdast: any = { type: 'root', children: [ @@ -334,7 +334,7 @@ describe('propagateBlockDataToCode', () => { expect(outputsNode.children[0].type).toEqual('image'); expect(outputsNode.children[0].placeholder).toBeTruthy(); }); - it('placeholder passes with no output', async () => { + it('placeholder passes with no outputs', async () => { const mdast: any = { type: 'root', children: [ From dc23b0cfc0a4651815f7e320fa88fdce0fbd17fc Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 13:48:48 +0100 Subject: [PATCH 22/32] test: fix missing children --- packages/myst-cli/src/transforms/outputs.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index 361df35528..43cfe1f1c5 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -26,6 +26,7 @@ describe('reduceOutputs', () => { type: 'output', id: 'abc123', jupyter_data: null, + children: [], }, ], }, @@ -59,6 +60,7 @@ describe('reduceOutputs', () => { children: [ { type: 'output', + children: [], jupyter_data: { output_type: 'display_data', execution_count: 3, From 92bad01d8bb24d46a1423ca3a9215e82b031ad5c Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 13:49:24 +0100 Subject: [PATCH 23/32] test: more fixes --- .../myst-cli/src/transforms/outputs.spec.ts | 81 +++++++++++++++---- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index 43cfe1f1c5..507aab1205 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -34,9 +34,26 @@ describe('reduceOutputs', () => { }, ], }; - expect(mdast.children[0].children.length).toEqual(2); reduceOutputs(new Session(), mdast, 'notebook.ipynb', '/my/folder'); - expect(mdast.children[0].children.length).toEqual(1); + expect(mdast).toEqual({ + type: 'root', + children: [ + { + type: 'block', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'text', + value: 'hi', + }, + ], + }, + ], + }, + ], + }); }); it('output with complex data is removed', async () => { const mdast = { @@ -82,7 +99,25 @@ describe('reduceOutputs', () => { }; expect(mdast.children[0].children.length).toEqual(2); reduceOutputs(new Session(), mdast, 'notebook.ipynb', '/my/folder'); - expect(mdast.children[0].children.length).toEqual(1); + expect(mdast).toEqual({ + type: 'root', + children: [ + { + type: 'block', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'text', + value: 'hi', + }, + ], + }, + ], + }, + ], + }); }); it('output is replaced with placeholder image', async () => { const mdast = { @@ -105,15 +140,9 @@ describe('reduceOutputs', () => { id: 'abc123', children: [ { - type: 'output', - jupyter_data: null, - children: [ - { - type: 'image', - placeholder: true, - url: 'placeholder.png', - }, - ], + type: 'image', + placeholder: true, + url: 'placeholder.png', }, ], }, @@ -123,11 +152,29 @@ describe('reduceOutputs', () => { }; expect(mdast.children[0].children.length).toEqual(2); reduceOutputs(new Session(), mdast, 'notebook.ipynb', '/my/folder'); - expect(mdast.children[0].children.length).toEqual(2); - expect(mdast.children[0].children[1]).toEqual({ - type: 'image', - placeholder: true, - url: 'placeholder.png', + expect(mdast).toEqual({ + type: 'root', + children: [ + { + type: 'block', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'text', + value: 'hi', + }, + ], + }, + { + type: 'image', + placeholder: true, + url: 'placeholder.png', + }, + ], + }, + ], }); }); // // These tests now require file IO... From 4d074ac0cfc6453162ea973d5c085239c2ba6de6 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 13:49:32 +0100 Subject: [PATCH 24/32] test: fix test titles --- packages/myst-cli/src/transforms/outputs.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index 507aab1205..48ecaad024 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -119,7 +119,7 @@ describe('reduceOutputs', () => { ], }); }); - it('output is replaced with placeholder image', async () => { + it('outputs is replaced with placeholder image', async () => { const mdast = { type: 'root', children: [ From 7933f3f08b9a450ce1414b175bf6b4630f04b95e Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 13:49:39 +0100 Subject: [PATCH 25/32] fix: handle visibility properly --- packages/myst-cli/src/transforms/outputs.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index f17f36ac8f..0ae158df24 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -254,12 +254,15 @@ export function reduceOutputs( const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); outputsNodes.forEach((outputsNode) => { - const outputs = outputsNode.children as GenericNode[]; + // Hidden nodes should not show up in simplified outputs for static export + if (outputsNode.visibility === 'remove' || outputsNode.visibility === 'hide') { + outputsNode.type = '__delete__'; + return; + } + const outputs = outputsNode.children as GenericNode[]; outputs.forEach((outputNode) => { - if (outputNode.visibility === 'remove' || outputNode.visibility === 'hide') { - // Hidden nodes should not show up in simplified outputs for static export - outputNode.type = '__delete__'; + if (outputNode.type !== 'output') { return; } // Lift the `output` node into `Outputs` From 7afbe4699e2d8ba5df59f447fbb4f60b4e4eb8b3 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 2 Apr 2025 14:14:35 +0100 Subject: [PATCH 26/32] fix: update test cases --- .../notebook-fig-embed/outputs/index.json | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index efa957528c..a604e254a6 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -54,19 +54,25 @@ "kind": "figure", "children": [ { - "type": "output", - "data": [ + "type": "outputs", + "children": [ { - "output_type": "execute_result", - "execution_count": 2, - "metadata": {}, - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a16fcedcd26437c820ccfc05d1f48a57", - "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" - }, - "text/plain": { "content": "alt.Chart(...)", "content_type": "text/plain" } + "type": "output", + "jupyter_data": { + "output_type": "execute_result", + "execution_count": 2, + "metadata": {}, + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a16fcedcd26437c820ccfc05d1f48a57", + "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" + }, + "text/plain": { + "content": "alt.Chart(...)", + "content_type": "text/plain" + } + } } } ] @@ -143,19 +149,22 @@ "enumerator": "1" }, { - "type": "output", - "data": [ + "type": "outputs", + "children": [ { - "output_type": "execute_result", - "execution_count": 2, - "metadata": {}, - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a16fcedcd26437c820ccfc05d1f48a57", - "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" - }, - "text/plain": { "content": "alt.Chart(...)", "content_type": "text/plain" } + "type": "output", + "jupyter_data": { + "output_type": "execute_result", + "execution_count": 2, + "metadata": {}, + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a16fcedcd26437c820ccfc05d1f48a57", + "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" + }, + "text/plain": { "content": "alt.Chart(...)", "content_type": "text/plain" } + } } } ] From fccf6810dca50a29fceafde8ea24d41de574ec0d Mon Sep 17 00:00:00 2001 From: Steve Purves Date: Wed, 25 Jun 2025 10:44:20 +0100 Subject: [PATCH 27/32] =?UTF-8?q?=F0=9F=AA=AA=20adding=20id=20to=20outputs?= =?UTF-8?q?=20enables=20in-browser=20compute=20for=20figures=20and=20embed?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-directives/src/code.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/myst-directives/src/code.ts b/packages/myst-directives/src/code.ts index 4e13b3ad1b..0bcf356e56 100644 --- a/packages/myst-directives/src/code.ts +++ b/packages/myst-directives/src/code.ts @@ -254,6 +254,7 @@ export const codeCellDirective: DirectiveSpec = { }; const outputs = { type: 'outputs', + id: nanoid(), children: [], }; const block: GenericNode = { From d9c74457575848aa1e6bef5be13b56cf552d61f9 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 10 Apr 2025 15:42:18 +0100 Subject: [PATCH 28/32] feat: --- packages/myst-cli/src/process/mdast.ts | 4 + packages/myst-cli/src/transforms/outputs.ts | 85 ++++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index be10767d24..f9842d2d02 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -67,6 +67,7 @@ import { transformFilterOutputStreams, transformLiftCodeBlocksInJupytext, transformMystXRefs, + liftOutputs, } from '../transforms/index.js'; import type { ImageExtensions } from '../utils/resolveExtension.js'; import { logMessagesFromVFile } from '../utils/logging.js'; @@ -240,6 +241,9 @@ export async function transformMdast( transformRenderInlineExpressions(mdast, vfile); await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters }); + liftOutputs(session, mdast, vfile, { + parseMyst: (content: string) => parseMyst(session, content, file), + }); transformFilterOutputStreams(mdast, vfile, frontmatter.settings); transformCitations(session, file, mdast, fileCitationRenderer, references); await unified() diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index 0ae158df24..c14dc1138c 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -12,7 +12,14 @@ import { selectAll } from 'unist-util-select'; import type { VFile } from 'vfile'; import type { IOutput, IStream } from '@jupyterlab/nbformat'; import type { MinifiedContent, MinifiedOutput, MinifiedMimeOutput } from 'nbtx'; -import { ensureString, extFromMimeType, minifyCellOutput, walkOutputs } from 'nbtx'; +import { + convertToIOutputs, + ensureString, + extFromMimeType, + minifyCellOutput, + walkOutputs, +} from 'nbtx'; +import { TexParser } from 'tex-to-myst'; import { castSession } from '../session/cache.js'; import type { ISession } from '../session/types.js'; import { resolveOutputPath } from './images.js'; @@ -25,6 +32,82 @@ function getWriteDestination(hash: string, contentType: string, writeFolder: str return join(writeFolder, getFilename(hash, contentType)); } +const MARKDOWN_MIME_TYPE = 'text/markdown'; +const SUPPORTED_MARKDOWN_VARIANTS = ['Original', 'GFM', 'CommonMark', 'myst']; + +/** + * Extract the `variant` parameter from a Markdown MIME type + * + * @param mimeType MIME type of the form `text/markdown;FOO=BAR` + */ +function extractVariantParameter(mimeType: string): string | undefined { + const [variant] = Array.from(mimeType.matchAll(/;([^;]+)=([^;]+)/g)) + .filter(([name]) => name === 'variant') + .map((pair) => pair[1]); + return variant; +} + +/* + * Determine the Markdown variant from a given MIME-type + * + * If the MIME-type is not a supported Markdown MIME, return undefined + * + * @param mimeType - MIME type + */ +function determineMarkdownVariant( + mimeType: string, +): { variant?: string; mimeType: string } | undefined { + if (!mimeType.startsWith(MARKDOWN_MIME_TYPE)) { + return; + } + const variant = extractVariantParameter(mimeType); + if (!variant) { + return { mimeType }; + } + if (SUPPORTED_MARKDOWN_VARIANTS.includes(variant)) { + return { mimeType, variant }; + } + + return; +} + +/** + * Lift outputs that contribute to the global document state + */ +export function liftOutputs( + session: ISession, + mdast: GenericParent, + vfile: VFile, + opts: { parseMyst: (source: string) => GenericParent }, +) { + const cache = castSession(session); + selectAll('output', mdast).forEach((output) => { + let children: GenericNode[] | undefined; + walkOutputs([(output as any).jupyter_data], (obj: any) => { + if (children) { + return; + } + const { content_type, content, hash } = obj; + const { mimeType: markdownMimeType } = determineMarkdownVariant(content_type) ?? {}; + // Markdown output + if (markdownMimeType) { + const [cacheContent] = cache.$outputs[hash] ?? []; + const ast = opts.parseMyst(content ?? cacheContent); + children = ast.children; + } + // LaTeX (including math) output + else if (content_type === 'text/latex') { + const [cacheContent] = cache.$outputs[hash] ?? []; + const state = new TexParser(content ?? cacheContent, vfile); + children = state.ast.children; + } + }); + if (children) { + (output as any).children = children; + } + }); +} + /** * Traverse all output nodes, minify their content, and cache on the session */ From 86077729e4e89ec5962c0aac77ec25b1fbf0fb96 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 10 Apr 2025 15:46:49 +0100 Subject: [PATCH 29/32] docs: add note about precedence --- packages/myst-cli/src/transforms/outputs.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index c14dc1138c..dc6b6d0e6c 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -83,6 +83,9 @@ export function liftOutputs( const cache = castSession(session); selectAll('output', mdast).forEach((output) => { let children: GenericNode[] | undefined; + // Walk over the outputs, and take the first matching "high-priority" output type + // Given that the `IOutput.data` mapping is not ordered, the precedence between sibling + // MIME type keys (e.g. LaTeX vs Markdown) is not defined for now. walkOutputs([(output as any).jupyter_data], (obj: any) => { if (children) { return; From 5ed0c94fea4ee5ec42cbaa19dd25761c84a202be Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 4 Apr 2025 15:25:58 +0100 Subject: [PATCH 30/32] chore: add changeset --- .changeset/weak-bananas-cover.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/weak-bananas-cover.md diff --git a/.changeset/weak-bananas-cover.md b/.changeset/weak-bananas-cover.md new file mode 100644 index 0000000000..5a0ecf1abd --- /dev/null +++ b/.changeset/weak-bananas-cover.md @@ -0,0 +1,5 @@ +--- +"myst-cli": patch +--- + +Move kernel execution transform earlier in pipeline From f5a8deff3bd7ddfd1712508a9538b7abf632109e Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 4 Apr 2025 15:25:58 +0100 Subject: [PATCH 31/32] chore: add changeset From 38c6351da007e2c7ed5cde0c45387e31ed400a4e Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 11 Apr 2025 16:51:23 +0100 Subject: [PATCH 32/32] fix: reorder xform --- packages/myst-cli/src/process/mdast.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index f9842d2d02..bd7cb50700 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -195,6 +195,12 @@ export async function transformMdast( log: session.log, }); } + transformRenderInlineExpressions(mdast, vfile); + await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters }); + liftOutputs(session, mdast, vfile, { + parseMyst: (content: string) => parseMyst(session, content, file), + }); + transformFilterOutputStreams(mdast, vfile, frontmatter.settings); const pipe = unified() .use(reconstructHtmlPlugin) // We need to group and link the HTML first @@ -239,12 +245,6 @@ export async function transformMdast( // Combine file-specific citation renderers with project renderers from bib files const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles); - transformRenderInlineExpressions(mdast, vfile); - await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters }); - liftOutputs(session, mdast, vfile, { - parseMyst: (content: string) => parseMyst(session, content, file), - }); - transformFilterOutputStreams(mdast, vfile, frontmatter.settings); transformCitations(session, file, mdast, fileCitationRenderer, references); await unified() .use(codePlugin, { lang: frontmatter?.kernelspec?.language })