Filter outputs not inputs

Author: dragonstyle

src/core/handlers/embed.ts

@@ -16,11 +16,7 @@ import {
 import { DirectiveCell } from "../lib/break-quarto-md-types.ts";
 import { jupyterAssets } from "../jupyter/jupyter.ts";
 
-import {
-  notebookForAddress,
-  notebookMarkdown,
-  parseNotebookPath,
-} from "./include-notebook.ts";
+import { notebookMarkdown, parseNotebookPath } from "./include-notebook.ts";
 import { JupyterCell } from "../jupyter/types.ts";
 
 interface EmbedHandler {
@@ -45,26 +41,23 @@ const kHandlers: EmbedHandler[] = [
 
       const notebookAddress = parseNotebookPath(path);
       if (notebookAddress) {
-        const nb = notebookForAddress(
-          notebookAddress,
-          (cell: JupyterCell) => {
-            cell.metadata["echo"] = false;
-            cell.metadata["warning"] = false;
-            cell.metadata["output"] = "asis";
-            return cell;
-          },
-        );
         const assets = jupyterAssets(
           handlerContext.options.context.target.source,
           handlerContext.options.context.format.pandoc.to,
         );
 
         // Render the notebook markdown and inject it
         const markdown = await notebookMarkdown(
-          nb,
+          notebookAddress,
           assets,
           handlerContext.options.context,
           handlerContext.options.flags,
+          (cell: JupyterCell) => {
+            cell.metadata["echo"] = false;
+            cell.metadata["warning"] = false;
+            cell.metadata["output"] = "asis";
+            return cell;
+          },
         );
         if (markdown) {
           markdownFragments.push(markdown);

src/core/handlers/include-notebook.ts

@@ -9,13 +9,10 @@ import { resourcePath } from "../resources.ts";
 
 import { languages } from "./base.ts";
 
-import {
-  jupyterCellWithOptions,
-  jupyterFromFile,
-  jupyterToMarkdown,
-} from "../jupyter/jupyter.ts";
+import { jupyterFromFile, jupyterToMarkdown } from "../jupyter/jupyter.ts";
 
 import {
+  kCellLabel,
   kFigDpi,
   kFigFormat,
   kFigPos,
@@ -33,13 +30,11 @@ import { RenderContext, RenderFlags } from "../../command/render/types.ts";
 import {
   JupyterAssets,
   JupyterCell,
-  JupyterNotebook,
+  JupyterCellOutput,
 } from "../jupyter/types.ts";
 
 import { dirname, extname } from "path/mod.ts";
 
-const kLabel = "label";
-
 export interface NotebookAddress {
   path: string;
   ids?: string[];
@@ -99,60 +94,19 @@ export function parseNotebookPath(path: string): NotebookAddress | undefined {
   }
 }
 
-export function notebookForAddress(
-  nbAddress: NotebookAddress,
-  filter?: (cell: JupyterCell) => JupyterCell,
-) {
-  try {
-    const nb = jupyterFromFile(nbAddress.path);
-
-    if (nbAddress.ids) {
-      // If cellIds are present, filter the notebook to only include
-      // those cells (cellIds can eiher be an explicitly set cellId, a label in the
-      // cell metadata, or a tag on a cell that matches an id)
-      const theCells = nbAddress.ids.map((id) => {
-        const cell = cellForId(id, nb);
-        if (cell === undefined) {
-          throw new Error(
-            `The cell ${id} does not exist in notebook`,
-          );
-        } else {
-          return cell;
-        }
-      });
-      nb.cells = theCells;
-    } else if (nbAddress.indexes) {
-      // Filter and sort based upon cell index
-      nb.cells = nbAddress.indexes.map((idx) => {
-        if (idx < 0 || idx >= nb.cells.length) {
-          throw new Error(
-            `The cell index ${idx} isn't within the range of cells`,
-          );
-        }
-        return nb.cells[idx];
-      });
-    }
-
-    // If there is a cell filter, apply it
-    if (filter) {
-      nb.cells = nb.cells.map(filter);
-    }
-
-    return nb;
-  } catch (ex) {
-    throw new Error(
-      `Failed to read notebook ${nbAddress.path}\n${ex.message || ""}`,
-      ex,
-    );
-  }
-}
-
 export async function notebookMarkdown(
-  notebook: JupyterNotebook,
+  nbAddress: NotebookAddress,
   assets: JupyterAssets,
   context: RenderContext,
   flags: RenderFlags,
+  filter?: (cell: JupyterCell) => JupyterCell,
 ) {
+  // Read and filter notebook
+  const notebook = jupyterFromFile(nbAddress.path);
+  if (filter) {
+    notebook.cells = notebook.cells.map(filter);
+  }
+
   const format = context.format;
   const executeOptions = {
     target: context.target,
@@ -187,36 +141,56 @@ export async function notebookMarkdown(
       figPos: format.render[kFigPos],
     },
   );
-  if (result) {
-    return result.markdown;
+
+  if (nbAddress.ids) {
+    // If cellIds are present, filter the notebook to only include
+    // those cells (cellIds can eiher be an explicitly set cellId, a label in the
+    // cell metadata, or a tag on a cell that matches an id)
+    const theCells = nbAddress.ids.map((id) => {
+      const cell = cellForId(id, result.cellOutputs);
+      if (cell === undefined) {
+        throw new Error(
+          `The cell ${id} does not exist in notebook`,
+        );
+      } else {
+        return cell;
+      }
+    });
+    return theCells.map((cell) => cell.markdown).join("");
+  } else if (nbAddress.indexes) {
+    // Filter and sort based upon cell index
+    const theCells = nbAddress.indexes.map((idx) => {
+      if (idx < 0 || idx >= result.cellOutputs.length) {
+        throw new Error(
+          `The cell index ${idx} isn't within the range of cells`,
+        );
+      }
+      return result.cellOutputs[idx];
+    });
+    return theCells.map((cell) => cell.markdown).join("");
   } else {
-    return undefined;
+    return result.cellOutputs.map((cell) => cell.markdown).join("");
   }
 }
 
-function cellForId(id: string, nb: JupyterNotebook) {
-  for (const cell of nb.cells) {
+function cellForId(id: string, cells: JupyterCellOutput[]) {
+  for (const cell of cells) {
     // cellId can either by a literal cell Id, or a tag with that value
     const hasId = cell.id ? id === cell.id : false;
     if (hasId) {
       // It's an ID
       return cell;
     } else {
-      // Check for label in options
-      const cellWithOptions = jupyterCellWithOptions(
-        nb.metadata.kernelspec.language.toLowerCase(),
-        cell,
-      );
-      const hasLabel = cellWithOptions.options[kLabel]
-        ? id === cellWithOptions.options[kLabel]
+      const hasLabel = cell.options && cell.options[kCellLabel]
+        ? id === cell.options[kCellLabel]
         : false;
 
       if (hasLabel) {
         // It matches a label
         return cell;
       } else {
         // Check tags
-        const hasTag = cell.metadata.tags
+        const hasTag = cell.metadata && cell.metadata.tags
           ? cell.metadata.tags.find((tag) => id === tag) !==
             undefined
           : false;
@@ -228,38 +202,6 @@ function cellForId(id: string, nb: JupyterNotebook) {
   }
 }
 
-function cellInIdList(ids: string[], cell: JupyterCell, nb: JupyterNotebook) {
-  // cellId can either by a literal cell Id, or a tag with that value
-  const hasId = cell.id ? ids.includes(cell.id) : false;
-  if (hasId) {
-    // It's an ID
-    return true;
-  } else {
-    // Check for label in options
-    const cellWithOptions = jupyterCellWithOptions(
-      nb.metadata.kernelspec.language.toLowerCase(),
-      cell,
-    );
-    const hasLabel = cellWithOptions.options[kLabel]
-      ? ids.includes(cellWithOptions.options[kLabel])
-      : false;
-
-    if (hasLabel) {
-      // It matches a label
-      return cell;
-    } else {
-      // Check tags
-      const hasTag = cell.metadata.tags
-        ? cell.metadata.tags.find((tag) => ids.includes(tag)) !==
-          undefined
-        : false;
-      if (hasTag) {
-        return cell;
-      }
-    }
-  }
-}
-
 const resolveCellIds = (hash?: string) => {
   if (hash && hash.indexOf(",") > 0) {
     return hash.split(",");

src/core/handlers/include.ts

@@ -20,11 +20,7 @@ import { isBlockShortcode } from "../lib/parse-shortcode.ts";
 import { DirectiveCell } from "../lib/break-quarto-md-types.ts";
 import { jupyterAssets } from "../jupyter/jupyter.ts";
 
-import {
-  notebookForAddress,
-  notebookMarkdown,
-  parseNotebookPath,
-} from "./include-notebook.ts";
+import { notebookMarkdown, parseNotebookPath } from "./include-notebook.ts";
 
 const includeHandler: LanguageHandler = {
   ...baseHandler,
@@ -54,21 +50,18 @@ const includeHandler: LanguageHandler = {
 
       // Handle notebooks directly by extracting the items
       // from the notebook
-      const notebookInclude = parseNotebookPath(path);
-      if (notebookInclude) {
+      const notebookAddress = parseNotebookPath(path);
+      if (notebookAddress) {
         // This is a notebook include, so read the notebook (including only
         // the cells that are specified in the include and include them)
-        const nb = notebookForAddress(
-          notebookInclude,
-        );
         const assets = jupyterAssets(
           source,
           handlerContext.options.context.format.pandoc.to,
         );
 
         // Render the notebook markdown and inject it
         const markdown = await notebookMarkdown(
-          nb,
+          notebookAddress,
           assets,
           handlerContext.options.context,
           handlerContext.options.flags,

src/core/jupyter/jupyter.ts

@@ -130,6 +130,7 @@ import {
 } from "./kernels.ts";
 import {
   JupyterCell,
+  JupyterCellOutput,
   JupyterCellWithOptions,
   JupyterKernelspec,
   JupyterNotebook,
@@ -594,6 +595,11 @@ export function jupyterAssets(input: string, to?: string) {
   };
 }
 
+// Attach fully rendered notebook to render services
+// Render notebook only once per document
+// Return cells with markdown instead of complete markdown
+// filter output markdown cells rather than notebook input
+
 export async function jupyterToMarkdown(
   nb: JupyterNotebook,
   options: JupyterToMarkdownOptions,
@@ -607,7 +613,7 @@ export async function jupyterToMarkdown(
   const htmlPreserve = isHtml ? removeAndPreserveHtml(nb) : undefined;
 
   // generate markdown
-  const md: string[] = [];
+  const cellOutputs: JupyterCellOutput[] = [];
 
   // validate unique cell labels as we go
   const validateCellLabel = cellLabelValidator();
@@ -616,6 +622,9 @@ export async function jupyterToMarkdown(
   let codeCellIndex = 0;
 
   for (let i = 0; i < nb.cells.length; i++) {
+    // Collection the markdown for this cell
+    const md: string[] = [];
+
     // convert cell yaml to cell metadata
     const cell = jupyterCellWithOptions(
       nb.metadata.kernelspec.language.toLowerCase(),
@@ -664,10 +673,19 @@ export async function jupyterToMarkdown(
 
     // newline
     md.push("\n");
+
+    cellOutputs.push({
+      id: cell.id,
+      markdown: md.join(""),
+      metadata: cell.metadata,
+      options: cell.options,
+    });
   }
 
   // include jupyter metadata if we are targeting ipynb
+  let notebookOutputs = undefined;
   if (options.toIpynb) {
+    const md: string[] = [];
     md.push("---\n");
     const jupyterMetadata = {
       jupyter: {
@@ -682,11 +700,15 @@ export async function jupyterToMarkdown(
     });
     md.push(yamlText);
     md.push("---\n");
+    notebookOutputs = {
+      suffix: md.join(""),
+    };
   }
 
   // return markdown and any widget requirements
   return {
-    markdown: md.join(""),
+    cellOutputs,
+    notebookOutputs,
     dependencies,
     htmlPreserve,
   };

src/core/jupyter/types.ts

@@ -210,11 +210,22 @@ export interface JupyterToMarkdownOptions {
 }
 
 export interface JupyterToMarkdownResult {
-  markdown: string;
   metadata?: Metadata;
   pandoc?: FormatPandoc;
   dependencies?: JupyterWidgetDependencies;
   htmlPreserve?: Record<string, string>;
+  cellOutputs: JupyterCellOutput[];
+  notebookOutputs?: {
+    prefix?: string;
+    suffix?: string;
+  };
+}
+
+export interface JupyterCellOutput {
+  id?: string;
+  options?: JupyterCellOptions;
+  metadata?: JupyterCellMetadata;
+  markdown: string;
 }
 
 export interface JupyterWidgetsState {