Merge branch 'main' into fix/page-inset-completions

Author: cscheid

news/changelog-1.7.md

@@ -140,6 +140,7 @@ All changes included in 1.7:
 
 ### `jupyter`
 
+- ([#9089](https://github.com/quarto-dev/quarto-cli/issues/9089)): Compound jupyter metadata is now serialized into a special key-value attribute to not break Pandoc's fenced div parsing.
 - ([#12114](https://github.com/quarto-dev/quarto-cli/issues/12114)): `JUPYTERCACHE` environment variable from [Jupyter cache CLI](https://jupyter-cache.readthedocs.io/en/latest/using/cli.html) is now respected by Quarto when `cache: true` is used. This environment variable allows to change the path of the cache directory.
 - ([#12374](https://github.com/quarto-dev/quarto-cli/issues/12374)): Detect language properly in Jupyter notebooks that lack the `language` field in their `kernelspec`s.
 - ([#12228](https://github.com/quarto-dev/quarto-cli/issues/12228)): `quarto render` will now fails if errors are detected at IPython display level. Setting `error: true` globally or at cell level will keep the error to show in output and not stop the rendering.
@@ -164,6 +165,7 @@ All changes included in 1.7:
 - ([#12338](https://github.com/quarto-dev/quarto-cli/issues/12338)): Add an additional workaround for the SCSS parser used in color variable extraction.
 - ([#12369](https://github.com/quarto-dev/quarto-cli/pull/12369)): `quarto preview` correctly throws a YAML validation error when a `format` key does not conform.
 - ([#12238](https://gijit.com/quarto-dev/quarto-cli/issues/12238)): Very long error (e.g. in Jupyter Notenook with backtrace) are now no more truncated in the console.
+- ([#9867](https://github.com/quarto-dev/quarto-cli/issues/9867)): Blank lines are now trimmed in Raw HTML Table blocks.
 
 ## Languages
 

src/command/convert/jupyter.ts

@@ -132,8 +132,6 @@ export async function jupyterNotebookToMarkdown(
     }
   }
 
-  console.log({ md });
-
   // join into source
   const mdSource = md.join("");
 

src/command/list/cmd.ts

@@ -17,7 +17,6 @@ import { outputTools } from "../../tools/tools-console.ts";
 import { notebookContext } from "../../render/notebook/notebook-context.ts";
 
 export const listCommand = new Command()
-  .hidden()
   .name("list")
   .arguments("<type:string>")
   .description(

src/core/jupyter/jupyter.ts

@@ -174,6 +174,8 @@ import {
   jupyterCellSrcAsLines,
   jupyterCellSrcAsStr,
 } from "./jupyter-shared.ts";
+import { error } from "../../deno_ral/log.ts";
+import { valid } from "semver/mod.ts";
 
 export const kQuartoMimeType = "quarto_mimetype";
 export const kQuartoOutputOrder = "quarto_order";
@@ -921,8 +923,44 @@ export function jupyterCellWithOptions(
     }
   };
 
+  const validMetadata: Record<
+    string,
+    string | number | boolean | null | Array<unknown>
+  > = {};
+  for (const key of Object.keys(cell.metadata)) {
+    const value = cell.metadata[key];
+    let jsonEncodedKeyIndex = 0;
+    if (value !== undefined) {
+      if (!value && typeof value === "object") {
+        validMetadata[key] = null;
+      } else if (value && typeof value === "object" && !Array.isArray(value)) {
+        // https://github.com/quarto-dev/quarto-cli/issues/9089
+        // we need to json-encode this and signal the encoding in the key
+        // we can't use the key as is since it may contain invalid characters
+        // and modifying the key might introduce collisions
+        // we ensure the key is unique with a counter, and assume
+        // "quarto-private-*" to be a private namespace for quarto.
+        // we'd prefer to use _quarto-* instead, but Pandoc doesn't allow keys to start
+        // with an underscore.
+        validMetadata[
+          `quarto-private-${++jsonEncodedKeyIndex}`
+        ] = JSON.stringify({ key, value });
+      } else if (
+        typeof value === "string" || typeof value === "number" ||
+        typeof value === "boolean" || Array.isArray(value)
+      ) {
+        validMetadata[key] = value;
+      } else {
+        error(
+          `Invalid metadata type for key ${key}: ${typeof value}. Entry will not be serialized.`,
+        );
+      }
+    }
+  }
+
   return {
     ...cell,
+    metadata: validMetadata,
     id: cellId(cell),
     source,
     optionsSource,
@@ -1766,7 +1804,10 @@ function isMarkdown(output: JupyterOutput, options: JupyterToMarkdownOptions) {
   return isDisplayDataType(output, options, displayDataIsMarkdown);
 }
 
-async function mdOutputStream(output: JupyterOutputStream, options: JupyterToMarkdownOptions) {
+async function mdOutputStream(
+  output: JupyterOutputStream,
+  options: JupyterToMarkdownOptions,
+) {
   let text: string[] = [];
   if (typeof output.text === "string") {
     text = [output.text];
@@ -1873,8 +1914,11 @@ async function mdOutputDisplayData(
       // if output is invalid, warn and emit empty
       const data = output.data[mimeType] as unknown;
       if (!Array.isArray(data) || data.some((s) => typeof s !== "string")) {
-        return await mdWarningOutput(`Unable to process text plain output data 
-which does not appear to be plain text: ${JSON.stringify(data)}`, options);
+        return await mdWarningOutput(
+          `Unable to process text plain output data 
+which does not appear to be plain text: ${JSON.stringify(data)}`,
+          options,
+        );
       }
       const lines = data as string[];
       // pandas inexplicably outputs html tables as text/plain with an enclosing single-quote
@@ -1911,7 +1955,7 @@ which does not appear to be plain text: ${JSON.stringify(data)}`, options);
   // no type match found
   return await mdWarningOutput(
     "Unable to display output for mime type(s): " +
-    Object.keys(output.data).join(", "),
+      Object.keys(output.data).join(", "),
     options,
   );
 }

src/core/pandoc/self-contained.ts

@@ -48,11 +48,15 @@ export const pandocIngestSelfContainedContent = async (
 
   // The raw html contents
   const contents = Deno.readTextFileSync(file);
+  const doctypeMatch = contents.match(/^<!DOCTYPE.*?>/);
   const dom = await parseHtml(contents);
   await bundleModules(dom, workingDir);
 
   const input: string[] = [];
   input.push("````````{=html}");
+  if (doctypeMatch) {
+    input.push(doctypeMatch[0]);
+  }
   input.push(dom.documentElement!.outerHTML);
   input.push("````````");
 

src/resources/editor/tools/vs-code.mjs

@@ -7046,7 +7046,7 @@ var require_yaml_intelligence_resources = __commonJS({
           schema: {
             arrayOf: "string"
           },
-          description: "Array of rendering names"
+          description: "Array of rendering names, e.g. `[light, dark]`"
         },
         {
           name: "tags",
@@ -16879,8 +16879,8 @@ var require_yaml_intelligence_resources = __commonJS({
             ]
           },
           description: {
-            short: "Whether the `prefers-color-scheme` media query controls dark mode.",
-            long: "Whether to use the `prefers-color-scheme` media query to determine whether to show\nthe user a dark or light page. Otherwise the author preference (order of `light`\nand `dark` in `theme` or `brand`) determines what is shown to the user at first visit.\n"
+            short: "Enables setting dark mode based on the `prefers-color-scheme` media query.",
+            long: "If set, Quarto reads the `prefers-color-scheme` media query to determine whether to show\nthe user a dark or light page. Otherwise the author-preferred color scheme is shown.\n"
           }
         },
         {
@@ -22964,6 +22964,10 @@ var require_yaml_intelligence_resources = __commonJS({
         "Enables hover over a section title to see an anchor link.",
         "Enables tabsets to present content.",
         "Enables smooth scrolling within the page.",
+        {
+          short: "Whether the <code>prefers-color-scheme</code> media query controls\ndark mode.",
+          long: "Whether to use the <code>prefers-color-scheme</code> media query to\ndetermine whether to show the user a dark or light page. Otherwise the\nauthor preference (order of <code>light</code> and <code>dark</code> in\n<code>theme</code> or <code>brand</code>) determines what is shown to\nthe user at first visit."
+        },
         {
           short: "Method use to render math in HTML output",
           long: 'Method use to render math in HTML output (<code>plain</code>,\n<code>webtex</code>, <code>gladtex</code>, <code>mathml</code>,\n<code>mathjax</code>, <code>katex</code>).\nSee the Pandoc documentation on <a href="https://pandoc.org/MANUAL.html#math-rendering-in-html">Math\nRendering in HTML</a> for additional details.'
@@ -24057,11 +24061,7 @@ var require_yaml_intelligence_resources = __commonJS({
         "Disambiguating year suffix in author-date styles (e.g.&nbsp;\u201Ca\u201D in \u201CDoe,\n1999a\u201D).",
         "Manuscript configuration",
         "internal-schema-hack",
-        "List execution engines you want to give priority when determining\nwhich engine should render a notebook. If two engines have support for a\nnotebook, the one listed earlier will be chosen. Quarto\u2019s default order\nis \u2018knitr\u2019, \u2018jupyter\u2019, \u2018markdown\u2019, \u2018julia\u2019.",
-        {
-          short: "Whether the <code>prefers-color-scheme</code> media query controls\ndark mode.",
-          long: "Whether to use the <code>prefers-color-scheme</code> media query to\ndetermine whether to show the user a dark or light page. Otherwise the\nauthor preference (order of <code>light</code> and <code>dark</code> in\n<code>theme</code> or <code>brand</code>) determines what is shown to\nthe user at first visit."
-        }
+        "List execution engines you want to give priority when determining\nwhich engine should render a notebook. If two engines have support for a\nnotebook, the one listed earlier will be chosen. Quarto\u2019s default order\nis \u2018knitr\u2019, \u2018jupyter\u2019, \u2018markdown\u2019, \u2018julia\u2019."
       ],
       "schema/external-schemas.yml": [
         {
@@ -24290,12 +24290,12 @@ var require_yaml_intelligence_resources = __commonJS({
         mermaid: "%%"
       },
       "handlers/mermaid/schema.yml": {
-        _internalId: 194479,
+        _internalId: 194627,
         type: "object",
         description: "be an object",
         properties: {
           "mermaid-format": {
-            _internalId: 194471,
+            _internalId: 194619,
             type: "enum",
             enum: [
               "png",
@@ -24311,7 +24311,7 @@ var require_yaml_intelligence_resources = __commonJS({
             exhaustiveCompletions: true
           },
           theme: {
-            _internalId: 194478,
+            _internalId: 194626,
             type: "anyOf",
             anyOf: [
               {

src/resources/editor/tools/yaml/web-worker.js

@@ -18,7 +18,7 @@
       "schema": {
         "arrayOf": "string"
       },
-      "description": "Array of rendering names"
+      "description": "Array of rendering names, e.g. `[light, dark]`"
     },
     {
       "name": "tags",
@@ -9851,8 +9851,8 @@
         ]
       },
       "description": {
-        "short": "Whether the `prefers-color-scheme` media query controls dark mode.",
-        "long": "Whether to use the `prefers-color-scheme` media query to determine whether to show\nthe user a dark or light page. Otherwise the author preference (order of `light`\nand `dark` in `theme` or `brand`) determines what is shown to the user at first visit.\n"
+        "short": "Enables setting dark mode based on the `prefers-color-scheme` media query.",
+        "long": "If set, Quarto reads the `prefers-color-scheme` media query to determine whether to show\nthe user a dark or light page. Otherwise the author-preferred color scheme is shown.\n"
       }
     },
     {
@@ -15936,6 +15936,10 @@
     "Enables hover over a section title to see an anchor link.",
     "Enables tabsets to present content.",
     "Enables smooth scrolling within the page.",
+    {
+      "short": "Whether the <code>prefers-color-scheme</code> media query controls\ndark mode.",
+      "long": "Whether to use the <code>prefers-color-scheme</code> media query to\ndetermine whether to show the user a dark or light page. Otherwise the\nauthor preference (order of <code>light</code> and <code>dark</code> in\n<code>theme</code> or <code>brand</code>) determines what is shown to\nthe user at first visit."
+    },
     {
       "short": "Method use to render math in HTML output",
       "long": "Method use to render math in HTML output (<code>plain</code>,\n<code>webtex</code>, <code>gladtex</code>, <code>mathml</code>,\n<code>mathjax</code>, <code>katex</code>).\nSee the Pandoc documentation on <a href=\"https://pandoc.org/MANUAL.html#math-rendering-in-html\">Math\nRendering in HTML</a> for additional details."
@@ -17029,11 +17033,7 @@
     "Disambiguating year suffix in author-date styles (e.g.&nbsp;“a” in “Doe,\n1999a”).",
     "Manuscript configuration",
     "internal-schema-hack",
-    "List execution engines you want to give priority when determining\nwhich engine should render a notebook. If two engines have support for a\nnotebook, the one listed earlier will be chosen. Quarto’s default order\nis ‘knitr’, ‘jupyter’, ‘markdown’, ‘julia’.",
-    {
-      "short": "Whether the <code>prefers-color-scheme</code> media query controls\ndark mode.",
-      "long": "Whether to use the <code>prefers-color-scheme</code> media query to\ndetermine whether to show the user a dark or light page. Otherwise the\nauthor preference (order of <code>light</code> and <code>dark</code> in\n<code>theme</code> or <code>brand</code>) determines what is shown to\nthe user at first visit."
-    }
+    "List execution engines you want to give priority when determining\nwhich engine should render a notebook. If two engines have support for a\nnotebook, the one listed earlier will be chosen. Quarto’s default order\nis ‘knitr’, ‘jupyter’, ‘markdown’, ‘julia’."
   ],
   "schema/external-schemas.yml": [
     {
@@ -17262,12 +17262,12 @@
     "mermaid": "%%"
   },
   "handlers/mermaid/schema.yml": {
-    "_internalId": 194479,
+    "_internalId": 194627,
     "type": "object",
     "description": "be an object",
     "properties": {
       "mermaid-format": {
-        "_internalId": 194471,
+        "_internalId": 194619,
         "type": "enum",
         "enum": [
           "png",
@@ -17283,7 +17283,7 @@
         "exhaustiveCompletions": true
       },
       "theme": {
-        "_internalId": 194478,
+        "_internalId": 194626,
         "type": "anyOf",
         "anyOf": [
           {

src/resources/editor/tools/yaml/yaml-intelligence-resources.json

@@ -61,7 +61,6 @@ import("../quarto-init/resourcerefs.lua")
 
 import("../normalize/flags.lua")
 import("../normalize/normalize.lua")
-import("../normalize/parsehtml.lua")
 import("../normalize/extractquartodom.lua")
 import("../normalize/astpipeline.lua")
 import("../normalize/capturereaderstate.lua")

src/resources/filters/crossref/crossref.lua

@@ -100,7 +100,6 @@ import("./quarto-finalize/typst.lua")
 
 import("./normalize/flags.lua")
 import("./normalize/normalize.lua")
-import("./normalize/parsehtml.lua")
 import("./normalize/extractquartodom.lua")
 import("./normalize/astpipeline.lua")
 import("./normalize/capturereaderstate.lua")