more flexible handling of titles/yaml in jupyter notebooks

Author: jjallaire

src/command/convert/jupyter.ts

@@ -25,6 +25,7 @@ import {
 import { partitionCellOptions } from "../../core/lib/partition-cell-options.ts";
 import { Metadata } from "../../config/types.ts";
 import { jupyterKernelspec } from "../../core/jupyter/kernels.ts";
+import { fixupFrontMatter } from "../../core/jupyter/jupyter-fixups.ts";
 
 export async function markdownToJupyterNotebook(
   file: string,
@@ -40,7 +41,7 @@ export async function jupyterNotebookToMarkdown(
   includeIds: boolean,
 ) {
   // read notebook & alias kernelspec
-  const notebook = jupyterFromFile(file);
+  const notebook = fixupFrontMatter(jupyterFromFile(file));
   const kernelspec = notebook.metadata.kernelspec;
 
   // generate markdown
@@ -59,7 +60,7 @@ export async function jupyterNotebookToMarkdown(
           break;
         case "raw":
           // see if this is the front matter
-          if (i === 0) {
+          if (frontMatter === undefined) {
             frontMatter = partitionYamlFrontMatter(cell.source.join(""))?.yaml;
             if (!frontMatter) {
               md.push(...mdFromRawCell(cell));

src/core/jupyter/jupyter-filters.ts

@@ -17,6 +17,7 @@ import {
   cacheFilteredNotebook,
   filteredNotebookFromCache,
 } from "./filtered-notebook-cache.ts";
+import { fixupFrontMatter } from "./jupyter-fixups.ts";
 import { JupyterNotebook } from "./types.ts";
 
 export async function markdownFromNotebookFile(file: string, format?: Format) {
@@ -30,6 +31,9 @@ export async function markdownFromNotebookFile(file: string, format?: Format) {
 }
 
 export function markdownFromNotebookJSON(nb: JupyterNotebook) {
+  // run the front matter fixup
+  nb = fixupFrontMatter(nb);
+
   const markdown = nb.cells.reduce((md, cell) => {
     if (["markdown", "raw"].includes(cell.cell_type)) {
       return md + "\n" + cell.source.join("") + "\n";

src/core/jupyter/jupyter-fixups.ts

@@ -5,17 +5,22 @@
 *
 */
 
+import { stringify } from "encoding/yaml.ts";
 import { warning } from "log/mod.ts";
 import {
   JupyterNotebook,
   JupyterOutput,
   JupyterToMarkdownOptions,
 } from "./types.ts";
 
-function fixupBokehCells(
-  nb: JupyterNotebook,
-  _options: JupyterToMarkdownOptions,
-): JupyterNotebook {
+import { kTitle } from "../../config/constants.ts";
+import { Metadata } from "../../publish/netlify/api/index.ts";
+import { lines } from "../lib/text.ts";
+import { markdownWithExtractedHeading } from "../pandoc/pandoc-partition.ts";
+import { partitionYamlFrontMatter, readYamlFromMarkdown } from "../yaml.ts";
+import { JupyterNotebook, JupyterOutput } from "./types.ts";
+
+function fixupBokehCells(nb: JupyterNotebook): JupyterNotebook {
   for (const cell of nb.cells) {
     if (cell.cell_type === "code") {
       let needsFixup = false;
@@ -101,19 +106,95 @@ function fixupBokehCells(
   return nb;
 }
 
+export function fixupFrontMatter(nb: JupyterNotebook): JupyterNotebook {
+  // helper to generate yaml
+  const asYamlText = (yaml: Metadata) => {
+    return stringify(yaml, {
+      indent: 2,
+      lineWidth: -1,
+      sortKeys: false,
+      skipInvalid: true,
+    });
+  };
+
+  // helper to create nb lines (w/ newline after)
+  const nbLines = (lns: string[]) => {
+    return lns.map((line) => `${line}\n`);
+  };
+
+  // look for the first raw block that has a yaml object
+  let partitioned: { yaml: string; markdown: string } | undefined;
+  const frontMatterCellIndex = nb.cells.findIndex((cell) => {
+    if (cell.cell_type === "raw") {
+      partitioned = partitionYamlFrontMatter(cell.source.join("")) || undefined;
+      return !!partitioned;
+    }
+  });
+
+  // if we have front matter and a title then we are done
+  const yaml = partitioned ? readYamlFromMarkdown(partitioned.yaml) : undefined;
+  if (yaml?.title) {
+    return nb;
+  }
+
+  // snip the title out of the markdown
+  let title: string | undefined;
+  for (const cell of nb.cells) {
+    if (cell.cell_type === "markdown") {
+      const { lines, headingText } = markdownWithExtractedHeading(
+        cell.source.join("\n"),
+      );
+      if (headingText) {
+        title = headingText;
+        cell.source = nbLines(lines);
+        break;
+      }
+    }
+  }
+
+  // if there is no title then we are done (the doc will have no title)
+  if (!title) {
+    return nb;
+  }
+
+  // if we have yaml then inject the title into the cell
+  if (yaml) {
+    // new yaml text with title
+    yaml[kTitle] = title;
+    const yamlText = asYamlText(yaml);
+
+    // re-write cell
+    const frontMatterCell = nb.cells[frontMatterCellIndex];
+    frontMatterCell.source = nbLines(
+      lines(`---\n${yamlText}---\n\n${partitioned?.markdown || ""}`),
+    );
+
+    // otherwise inject a new cell at the top
+  } else {
+    const yamlText = asYamlText({ title });
+    nb.cells.unshift({
+      cell_type: "raw",
+      source: nbLines(lines(yamlText)),
+      metadata: {},
+    });
+  }
+
+  // return modified nb
+  return nb;
+}
+
 const fixups: ((
   nb: JupyterNotebook,
-  options: JupyterToMarkdownOptions,
 ) => JupyterNotebook)[] = [
   fixupBokehCells,
+  fixupFrontMatter,
 ];
 
 export function fixupJupyterNotebook(
   nb: JupyterNotebook,
-  options: JupyterToMarkdownOptions,
 ): JupyterNotebook {
   for (const fixup of fixups) {
-    nb = fixup(nb, options);
+    nb = fixup(nb);
   }
   return nb;
 }

src/core/jupyter/jupyter.ts

@@ -656,7 +656,7 @@ export async function jupyterToMarkdown(
   options: JupyterToMarkdownOptions,
 ): Promise<JupyterToMarkdownResult> {
   // perform fixups
-  nb = fixupJupyterNotebook(nb, options);
+  nb = fixupJupyterNotebook(nb);
 
   // optional content injection / html preservation for html output
   // that isn't an ipynb
@@ -705,6 +705,10 @@ export async function jupyterToMarkdown(
       }
     }
 
+    // find the first yaml metadata block and hold it out
+    // note if it has a title
+    // at the end, if it doesn't have a title, then snip the title out
+
     // markdown from cell
     switch (cell.cell_type) {
       case "markdown":

src/core/pandoc/pandoc-partition.ts

@@ -49,45 +49,60 @@ export function parsePandocTitle(title: string) {
 
 // partition markdown into yaml, the first heading, and the rest of the markdown text
 export function partitionMarkdown(markdown: string): PartitionedMarkdown {
-  const markdownLines: string[] = [];
-  let markdownHeading: string | undefined;
-  let markdownHeadingAttr: PandocAttr | undefined;
-  let markdownContainsRefs = false;
+  // partition out yaml
   const partitioned = partitionYamlFrontMatter(markdown);
   markdown = partitioned ? partitioned.markdown : markdown;
-  for (const line of lines(markdown)) {
-    // does this line contains the refs div?
-    if (!markdownContainsRefs) {
-      markdownContainsRefs = /^:::\s*{#refs([\s}]|.*?})\s*$/.test(line);
-    }
 
-    if (!markdownHeading) {
+  // extract heading
+  const { lines, headingText, headingAttr } = markdownWithExtractedHeading(
+    markdown,
+  );
+
+  // does this contain refs?
+  const containsRefs = lines.some((line) =>
+    /^:::\s*{#refs([\s}]|.*?})\s*$/.test(line)
+  );
+
+  return {
+    yaml: (partitioned ? readYamlFromMarkdown(partitioned.yaml) : undefined),
+    headingText,
+    headingAttr,
+    containsRefs,
+    markdown: lines.join("\n"),
+    srcMarkdownNoYaml: partitioned?.markdown || "",
+  };
+}
+
+export function markdownWithExtractedHeading(markdown: string) {
+  const mdLines: string[] = [];
+  let headingText: string | undefined;
+  let headingAttr: PandocAttr | undefined;
+
+  for (const line of lines(markdown)) {
+    if (!headingText) {
       if (line.match(/^\#{1,}\s/)) {
         const parsedHeading = parsePandocTitle(line);
-        markdownHeading = parsedHeading.heading;
-        markdownHeadingAttr = parsedHeading.attr;
+        headingText = parsedHeading.heading;
+        headingAttr = parsedHeading.attr;
       } else if (line.match(/^=+\s*$/) || line.match(/^-+\s*$/)) {
-        const prevLine = markdownLines[markdownLines.length - 1];
+        const prevLine = mdLines[mdLines.length - 1];
         if (prevLine) {
-          markdownHeading = prevLine;
-          markdownLines.splice(markdownLines.length - 1);
+          headingText = prevLine;
+          mdLines.splice(mdLines.length - 1);
         } else {
-          markdownLines.push(line);
+          mdLines.push(line);
         }
       } else {
-        markdownLines.push(line);
+        mdLines.push(line);
       }
     } else {
-      markdownLines.push(line);
+      mdLines.push(line);
     }
   }
 
   return {
-    yaml: (partitioned ? readYamlFromMarkdown(partitioned.yaml) : undefined),
-    headingText: markdownHeading,
-    headingAttr: markdownHeadingAttr,
-    containsRefs: markdownContainsRefs,
-    markdown: markdownLines.join("\n"),
-    srcMarkdownNoYaml: partitioned?.markdown || "",
+    lines: mdLines,
+    headingText,
+    headingAttr,
   };
 }