Merge pull request #12694 from quarto-dev/bugfix/12693

NotebookContext - don't write all notebooks into environment

Author: cscheid

news/changelog-1.8.md

@@ -26,6 +26,7 @@ All changes included in 1.8:
 
 - ([#12551](https://github.com/quarto-dev/quarto-cli/pull/12551)): Improve warning issued when `aliases` would overwrite an existing document.
 - ([#12616](https://github.com/quarto-dev/quarto-cli/issues/12616)): find SVG images in image discovery for listings.
+- ([#12693](https://github.com/quarto-dev/quarto-cli/issues/12693)): Prevent resource exhaustion on large websites by serializing `NotebookContext` information to file instead of the environment.
 
 ## Crossrefs
 

src/command/render/filters.ts

@@ -549,7 +549,7 @@ function jatsFilterParams(options: PandocOptions) {
 
 function notebookContextFilterParams(options: PandocOptions) {
   const nbContext = options.services.notebook;
-  const notebooks = nbContext.all();
+  const notebooks = nbContext.all(options.project);
   if (notebooks.length > 0) {
     return {
       "notebook-context": notebooks,

src/render/notebook/notebook-context.ts

@@ -75,6 +75,7 @@ export function notebookContext(): NotebookContext {
     const nb: Notebook = notebooks[nbAbsPath] || emptyNotebook(nbAbsPath);
     nb[renderType] = output;
     notebooks[nbAbsPath] = nb;
+    needRewrite = true;
 
     if (context) {
       const contrib = contributor(renderType);
@@ -191,9 +192,28 @@ export function notebookContext(): NotebookContext {
     }
   }
 
+  let allNotebooksTempFilename: string | undefined;
+  let needRewrite = true;
+
   return {
-    all: () => {
-      return Object.values(notebooks);
+    all: (context: ProjectContext) => {
+      if (!allNotebooksTempFilename) {
+        allNotebooksTempFilename = context.temp.createFile({
+          suffix: ".json",
+        });
+      }
+      if (needRewrite) {
+        debug(
+          `[NotebookContext]: Writing all notebooks to ${allNotebooksTempFilename}`,
+        );
+        const objs = Object.values(notebooks);
+        Deno.writeTextFileSync(
+          allNotebooksTempFilename,
+          JSON.stringify(objs),
+        );
+        needRewrite = false;
+      }
+      return allNotebooksTempFilename;
     },
     get: (nbAbsPath: string, context?: ProjectContext) => {
       debug(`[NotebookContext]: Get Notebook:${nbAbsPath}`);

src/render/notebook/notebook-types.ts

@@ -81,7 +81,10 @@ export interface NotebookTemplateMetadata extends NotebookMetadata {
 export interface NotebookContext {
   // Retrieves the notebook from the context.
   get: (nbPath: string, context: ProjectContext) => Notebook | undefined;
-  all: () => Notebook[];
+
+  // returns a file name with the JSON serialization of all notebooks, Notebook[]
+  all: (context: ProjectContext) => string;
+
   // Resolves the data on an executedFile into data that will
   // create a `renderType` output when rendered.
   addMetadata: (nbPath: string, notebookMetadata: NotebookMetadata) => void;

src/resources/filters/layout/manuscript.lua

@@ -82,7 +82,8 @@ function manuscript()
 
           -- Use the notebook cotnext to try to determine the name
           -- of the output file
-          local notebooks = param("notebook-context", {})
+          local notebooks_filename = param("notebook-context", {})
+          local notebooks = quarto.json.decode(io.open(notebooks_filename, "r"):read("*a"))
           local nbFileName = pandoc.path.filename(nbRelPath)
           local previewFile = nbFileName .. ".html"
           for _i, notebook in ipairs(notebooks) do