Merge pull request #12477 from quarto-dev/bugfix/9089

Jupyter: JSON-encode compound metadata

Author: cscheid

news/changelog-1.7.md

@@ -140,6 +140,7 @@ All changes included in 1.7:
 
 ### `jupyter`
 
+- ([#9089](https://github.com/quarto-dev/quarto-cli/issues/9089)): Compound jupyter metadata is now serialized into a special key-value attribute to not break Pandoc's fenced div parsing.
 - ([#12114](https://github.com/quarto-dev/quarto-cli/issues/12114)): `JUPYTERCACHE` environment variable from [Jupyter cache CLI](https://jupyter-cache.readthedocs.io/en/latest/using/cli.html) is now respected by Quarto when `cache: true` is used. This environment variable allows to change the path of the cache directory.
 - ([#12374](https://github.com/quarto-dev/quarto-cli/issues/12374)): Detect language properly in Jupyter notebooks that lack the `language` field in their `kernelspec`s.
 - ([#12228](https://github.com/quarto-dev/quarto-cli/issues/12228)): `quarto render` will now fails if errors are detected at IPython display level. Setting `error: true` globally or at cell level will keep the error to show in output and not stop the rendering.

src/core/jupyter/jupyter.ts

@@ -174,6 +174,8 @@ import {
   jupyterCellSrcAsLines,
   jupyterCellSrcAsStr,
 } from "./jupyter-shared.ts";
+import { error } from "../../deno_ral/log.ts";
+import { valid } from "semver/mod.ts";
 
 export const kQuartoMimeType = "quarto_mimetype";
 export const kQuartoOutputOrder = "quarto_order";
@@ -921,8 +923,44 @@ export function jupyterCellWithOptions(
     }
   };
 
+  const validMetadata: Record<
+    string,
+    string | number | boolean | null | Array<unknown>
+  > = {};
+  for (const key of Object.keys(cell.metadata)) {
+    const value = cell.metadata[key];
+    let jsonEncodedKeyIndex = 0;
+    if (value !== undefined) {
+      if (!value && typeof value === "object") {
+        validMetadata[key] = null;
+      } else if (value && typeof value === "object" && !Array.isArray(value)) {
+        // https://github.com/quarto-dev/quarto-cli/issues/9089
+        // we need to json-encode this and signal the encoding in the key
+        // we can't use the key as is since it may contain invalid characters
+        // and modifying the key might introduce collisions
+        // we ensure the key is unique with a counter, and assume
+        // "quarto-private-*" to be a private namespace for quarto.
+        // we'd prefer to use _quarto-* instead, but Pandoc doesn't allow keys to start
+        // with an underscore.
+        validMetadata[
+          `quarto-private-${++jsonEncodedKeyIndex}`
+        ] = JSON.stringify({ key, value });
+      } else if (
+        typeof value === "string" || typeof value === "number" ||
+        typeof value === "boolean" || Array.isArray(value)
+      ) {
+        validMetadata[key] = value;
+      } else {
+        error(
+          `Invalid metadata type for key ${key}: ${typeof value}. Entry will not be serialized.`,
+        );
+      }
+    }
+  }
+
   return {
     ...cell,
+    metadata: validMetadata,
     id: cellId(cell),
     source,
     optionsSource,
@@ -1766,7 +1804,10 @@ function isMarkdown(output: JupyterOutput, options: JupyterToMarkdownOptions) {
   return isDisplayDataType(output, options, displayDataIsMarkdown);
 }
 
-async function mdOutputStream(output: JupyterOutputStream, options: JupyterToMarkdownOptions) {
+async function mdOutputStream(
+  output: JupyterOutputStream,
+  options: JupyterToMarkdownOptions,
+) {
   let text: string[] = [];
   if (typeof output.text === "string") {
     text = [output.text];
@@ -1873,8 +1914,11 @@ async function mdOutputDisplayData(
       // if output is invalid, warn and emit empty
       const data = output.data[mimeType] as unknown;
       if (!Array.isArray(data) || data.some((s) => typeof s !== "string")) {
-        return await mdWarningOutput(`Unable to process text plain output data 
-which does not appear to be plain text: ${JSON.stringify(data)}`, options);
+        return await mdWarningOutput(
+          `Unable to process text plain output data 
+which does not appear to be plain text: ${JSON.stringify(data)}`,
+          options,
+        );
       }
       const lines = data as string[];
       // pandas inexplicably outputs html tables as text/plain with an enclosing single-quote
@@ -1911,7 +1955,7 @@ which does not appear to be plain text: ${JSON.stringify(data)}`, options);
   // no type match found
   return await mdWarningOutput(
     "Unable to display output for mime type(s): " +
-    Object.keys(output.data).join(", "),
+      Object.keys(output.data).join(", "),
     options,
   );
 }

tests/docs/smoke-all/2025/04/07/issue-9089.qmd

@@ -0,0 +1,42 @@
+---
+title: DataBricks Notebooks
+engine: jupyter
+keep-md: true
+execute:
+  eval: false
+_quarto:
+  tests:
+    html:
+      ensureFileRegexMatches:
+        - []
+        - [":::"]
+---
+
+
+## Introduction
+
+In this notebook, we try Quarto with DataBricks.
+
+## Chapter
+
+In the first chapter, we try multiple commands and observe their results.
+
+
+```{python}
+#| application/vnd.databricks.v1+cell: {cellMetadata: {byteLimit: 2048000, rowLimit: 10000}, inputWidgets: {}, nuid: 7039bc23-d898-4506-b24d-8f1002a66d18, showTitle: false, title: ''}
+df = spark.read.table("samples.nyctaxi.trips")
+df.show(5)
+```
+
+This is text in-between the commands.
+
+```{python}
+#| application/vnd.databricks.v1+cell: {cellMetadata: {byteLimit: 2048000, rowLimit: 10000}, inputWidgets: {}, nuid: 21c1cb83-83cc-40c8-9a8b-f5378d3f29be, showTitle: false, title: ''}
+from databricks.sdk.runtime import dbutils
+dbutils.fs.ls("dbfs:/Workspace/Users/")
+```
+
+## Conclusion
+
+Currently, Quarto does not fully work, at least not rendering.
+