don't read all the website DOMs at once. Closes #4350. (#4754)

* don't read all the website DOMs at once. Closes #4350.

* read file content, not name 🤦

* changelog

Author: cscheid

news/changelog-1.3.md

@@ -207,6 +207,7 @@
 - Fix issue with "No inspectable targets" with Chrome Browser ([#4653](https://github.com/quarto-dev/quarto-cli/issues/4653))
 - Add `title` attribute for callouts (can be used rather than heading for defining the title)
 - Handle more varieties of raw HTML for Docusaurus output
+- Read and process DOM one-file-at-time in books and websites to reduce total memory usage ([#4350](https://github.com/quarto-dev/quarto-cli/issues/4350)).
 
 ## Pandoc filter changes
 

src/core/deno-dom.ts

@@ -25,6 +25,20 @@ export async function parseHtml(src: string): Promise<HTMLDocument> {
   return result;
 }
 
+export async function writeDomToHtmlFile(
+  doc: HTMLDocument,
+  path: string,
+  doctype?: string,
+) {
+  if (doc.documentElement === null) {
+    throw new Error("Document has no root element");
+  }
+  const output = doctype
+    ? doctype + "\n" + doc.documentElement.outerHTML
+    : doc.documentElement.outerHTML;
+  await Deno.writeTextFile(path, output);
+}
+
 // We are combining a number of scripts from
 // https://github.com/b-fuze/deno-dom/blob/master/deno-dom-native.ts
 // into this. If deno-dom fails, it's likely that this needs to be brought up to date.

src/project/types/book/book-bibliography.ts

@@ -12,7 +12,13 @@ import * as ld from "../../../core/lodash.ts";
 import { stringify } from "encoding/yaml.ts";
 import { error } from "log/mod.ts";
 
-import { Document, Element, parseHtml } from "../../../core/deno-dom.ts";
+import {
+  Document,
+  Element,
+  initDenoDom,
+  parseHtml,
+  writeDomToHtmlFile,
+} from "../../../core/deno-dom.ts";
 
 import { pathWithForwardSlashes, safeExistsSync } from "../../../core/path.ts";
 import { execProcess } from "../../../core/process.ts";
@@ -103,6 +109,8 @@ export async function bookBibliographyPostRender(
   incremental: boolean,
   outputFiles: WebsiteProjectOutputFile[],
 ) {
+  await initDenoDom();
+
   // make sure the references file exists and compute it's path
   const renderFiles = context.config?.project[kProjectRender] || [];
 
@@ -130,13 +138,14 @@ export async function bookBibliographyPostRender(
     // a global bibliography. also hide the refs div in each document (as it's
     // still used by citations-hover)
     const citeIds: string[] = [];
-    outputFiles.forEach((file) => {
+    for (const file of outputFiles) {
       // relative path to refs html
       const refsRelative = pathWithForwardSlashes(
         relative(dirname(file.file), refsHtml!),
       );
       // check each citation
-      forEachCite(file.doc, (cite: Element) => {
+      const doc = await parseHtml(Deno.readTextFileSync(file.file));
+      forEachCite(doc, (cite: Element) => {
         // record ids
         citeIds.push(...citeIdsFromCite(cite));
         // fix hrefs
@@ -148,11 +157,11 @@ export async function bookBibliographyPostRender(
       });
 
       // hide the bibliography
-      const refsDiv = file.doc.getElementById("refs");
+      const refsDiv = doc.getElementById("refs");
       if (refsDiv) {
         refsDiv.setAttribute("style", "display: none");
       }
-    });
+    }
 
     // is the refs one of our output files?
     const refsOutputFile = outputFiles.find((file) => file.file === refsHtml);
@@ -199,17 +208,28 @@ export async function bookBibliographyPostRender(
           "html",
           csl,
         );
-        const newRefsDiv = refsOutputFile.doc.createElement("div");
+        const doc = await parseHtml(Deno.readTextFileSync(refsOutputFile.file));
+        const newRefsDiv = doc.createElement("div");
         newRefsDiv.innerHTML = biblioHtml;
-        const refsDiv = refsOutputFile.doc.getElementById("refs") as Element;
+        const refsDiv = doc.getElementById("refs") as Element;
+        let changed = false;
         if (refsDiv) {
+          changed = true;
           refsDiv.replaceWith(newRefsDiv.firstChild);
         } else {
-          const mainEl = refsOutputFile.doc.querySelector("main");
+          const mainEl = doc.querySelector("main");
           if (mainEl) {
+            changed = true;
             mainEl.appendChild(newRefsDiv.firstChild);
           }
         }
+        if (changed) {
+          await writeDomToHtmlFile(
+            doc,
+            refsOutputFile.file,
+            refsOutputFile.doctype,
+          );
+        }
       }
     }
   }

src/project/types/book/book-crossrefs.ts

@@ -9,7 +9,12 @@ import { warning } from "log/mod.ts";
 import { dirname, join, relative } from "path/mod.ts";
 import { existsSync } from "fs/mod.ts";
 
-import { Element, HTMLDocument } from "../../../core/deno-dom.ts";
+import {
+  Element,
+  HTMLDocument,
+  parseHtml,
+  writeDomToHtmlFile,
+} from "../../../core/deno-dom.ts";
 
 import { pathWithForwardSlashes } from "../../../core/path.ts";
 
@@ -55,13 +60,15 @@ export async function bookCrossrefsPostRender(
       const index = bookCrossrefIndexForOutputFile(fileRelative, indexes);
       if (index) {
         // resolve crossrefs
+        const doc = await parseHtml(Deno.readTextFileSync(outputFile.file));
         resolveCrossrefs(
           context,
           fileRelative,
           outputFile.format,
-          outputFile.doc,
+          doc,
           index,
         );
+        writeDomToHtmlFile(doc, outputFile.file, outputFile.doctype);
       }
     }
   }

src/project/types/book/book-render.ts

@@ -604,13 +604,13 @@ export async function bookPostRender(
     await bookBibliographyPostRender(context, incremental, websiteFiles);
     await bookCrossrefsPostRender(context, websiteFiles);
 
-    // write website files
-    websiteFiles.forEach((websiteFile) => {
-      const doctype = websiteFile.doctype;
-      const htmlOutput = (doctype ? doctype + "\n" : "") +
-        websiteFile.doc.documentElement?.outerHTML!;
-      Deno.writeTextFileSync(websiteFile.file, htmlOutput);
-    });
+    // website files are now already written on a per-file basis
+    // websiteFiles.forEach((websiteFile) => {
+    //   const doctype = websiteFile.doctype;
+    //   const htmlOutput = (doctype ? doctype + "\n" : "") +
+    //     websiteFile.doc.documentElement?.outerHTML!;
+    //   Deno.writeTextFileSync(websiteFile.file, htmlOutput);
+    // });
 
     // run standard website stuff (search, etc.)
     await websitePostRender(context, incremental, outputFiles);

src/project/types/website/website.ts

@@ -319,7 +319,7 @@ export const websiteProjectType: ProjectType = {
 };
 
 export interface WebsiteProjectOutputFile extends ProjectOutputFile {
-  doc: HTMLDocument;
+  // doc: HTMLDocument;
   doctype?: string;
 }
 
@@ -358,10 +358,10 @@ export function websiteOutputFiles(outputFiles: ProjectOutputFile[]) {
     .map((outputFile) => {
       const contents = Deno.readTextFileSync(outputFile.file);
       const doctypeMatch = contents.match(/^<!DOCTYPE.*?>/);
-      const doc = new DOMParser().parseFromString(contents, "text/html")!;
+      // const doc = new DOMParser().parseFromString(contents, "text/html")!;
       return {
         ...outputFile,
-        doc,
+        // doc,
         doctype: doctypeMatch ? doctypeMatch[0] : undefined,
       };
     });