Feature: custom AST nodes (#3694)

Author: cscheid

news/changelog-1.3.md

@@ -94,3 +94,7 @@
 - Add new `kbd` shortcode, to describe keyboard keys ([#3384](https://github.com/quarto-dev/quarto-cli/issues/3384)). See the [pre-release documentation](https://quarto.org/docs/prerelease/1.3.html) for details.
 - Replace default style for date picker component in OJS ([#2863](https://github.com/quarto-dev/quarto-cli/issues/2863)).
 - `quarto check` now supports `quarto check versions` for checking binary dependency versions in the case of custom binaries ([#3602](https://github.com/quarto-dev/quarto-cli/issues/3602)).
+
+## Pandoc filter changes
+
+- Quarto 1.3 introduces the notion of Custom AST nodes to Pandoc filters. If you use Lua filters for processing callouts, tabsets, or conditional blocks, consult the [pre-release documentation](https://quarto.org/docs/prerelease/1.3.html) for how to change your filters to support the new syntax.

package/src/common/package-filters.ts

@@ -26,18 +26,22 @@ export function buildFilter(
     return "";
   }).trimStart();
 
+  const importSrcs = [];
   // append imports to src
   const importRe = /^import\("(.*)?"\)$/gm;
   let match = importRe.exec(imports);
   while (match) {
     const importFilePath = join(filterDir, match[1]);
+    console.log(`Inlining ${match[1]} from ${importFilePath}`);
     let importSrc = Deno.readTextFileSync(importFilePath);
     if (!importSrc.endsWith("\n")) {
       importSrc += "\n";
     }
-    src = `${importSrc}\n` + src;
+    importSrcs.push(importSrc);
     match = importRe.exec(imports);
   }
+  importSrcs.push(src);
+  src = importSrcs.join("");
 
   // write src to dist
   info(`Writing inlined file ${output}`);

package/src/common/prepare-dist.ts

@@ -143,14 +143,15 @@ function inlineFilters(config: Configuration) {
   info("Building inlined filters");
   const outDir = join(config.directoryInfo.share, "filters");
   const filtersToInline: Filter[] = [
-    { name: "quarto-init" },
-    { name: "normalize" },
-    { name: "quarto-pre" },
-    { name: "crossref" },
-    { name: "layout" },
-    { name: "quarto-post" },
-    { name: "pagebreak", dir: "rmarkdown" },
-    { name: "quarto-finalize" },
+    { name: "main", dir: "." },
+    // { name: "quarto-init" },
+    // { name: "normalize" },
+    // { name: "quarto-pre" },
+    // { name: "crossref" },
+    // { name: "layout" },
+    // { name: "quarto-post" },
+    // { name: "pagebreak", dir: "rmarkdown" },
+    // { name: "quarto-finalize" },
   ];
 
   filtersToInline.forEach((filter) => {

src/command/render/crossref.ts

@@ -25,10 +25,6 @@ import { pandocMetadataPath } from "./render-paths.ts";
 import { isMultiFileBookFormat } from "../../project/types/book/book-shared.ts";
 import { projectIsBook } from "../../project/project-context.ts";
 
-export function crossrefFilter() {
-  return resourcePath("filters/crossref/crossref.lua");
-}
-
 export function crossrefFilterActive(options: PandocOptions) {
   return options.format.metadata.crossref !== false;
 }

src/command/render/defaults.ts

@@ -20,6 +20,7 @@ import {
   kIncludeInHeader,
   kNumberDepth,
   kOutputFile,
+  kQuartoFilters,
   kSelfContained,
   kStandalone,
   kTemplate,
@@ -28,16 +29,8 @@ import {
 
 import { kPatchedTemplateExt } from "./template.ts";
 import { PandocOptions } from "./types.ts";
-import { crossrefFilter } from "./crossref.ts";
-import { layoutFilter } from "./layout.ts";
-import {
-  quartoFinalizeFilter,
-  quartoPostFilter,
-  quartoPreFilter,
-  resolveFilters,
-} from "./filters.ts";
+import { quartoMainFilter, resolveFilters } from "./filters.ts";
 import { TempContext } from "../../core/temp.ts";
-import { metadataNormalizationFilter } from "./normalize.ts";
 
 export async function generateDefaults(
   options: PandocOptions,
@@ -55,7 +48,9 @@ export async function generateDefaults(
       options,
     );
     if (resolvedFilters) {
-      allDefaults[kFilters] = resolvedFilters;
+      allDefaults[kFilters] = resolvedFilters.quartoFilters;
+      // forward the filter spec with everything to pandoc via metadata
+      options.format.metadata[kQuartoFilters] = resolvedFilters;
     }
 
     // If we're rendering Latex, forward the number-depth to pandoc (it handles numbering)
@@ -137,19 +132,8 @@ export function pandocDefaultsMessage(
   // simplify crossref filter
   if (defaults.filters?.length) {
     defaults.filters = defaults.filters
-      .map((filter) => {
-        if (filter === crossrefFilter()) {
-          return "crossref";
-        } else {
-          return filter;
-        }
-      })
       .filter((filter) => {
-        return filter !== quartoPreFilter() &&
-          filter !== quartoPostFilter() &&
-          filter !== layoutFilter() &&
-          filter !== metadataNormalizationFilter() &&
-          filter !== quartoFinalizeFilter() &&
+        return filter !== quartoMainFilter() &&
           filtersContains(sysFilters, filter);
       });
     if (defaults.filters?.length === 0) {

src/command/render/filters.ts

@@ -30,6 +30,7 @@ import {
   kMergeIncludes,
   kOutputDivs,
   kPdfEngine,
+  kQuartoFilters,
   kReferenceLocation,
   kShortcodes,
   kTblColwidths,
@@ -45,12 +46,8 @@ import { Metadata } from "../../config/types.ts";
 import { kProjectType } from "../../project/types.ts";
 import { bibEngine } from "../../config/pdf.ts";
 import { resourcePath } from "../../core/resources.ts";
-import {
-  crossrefFilter,
-  crossrefFilterActive,
-  crossrefFilterParams,
-} from "./crossref.ts";
-import { layoutFilter, layoutFilterParams } from "./layout.ts";
+import { crossrefFilterActive, crossrefFilterParams } from "./crossref.ts";
+import { layoutFilterParams } from "./layout.ts";
 import { pandocMetadataPath } from "./render-paths.ts";
 import { removePandocArgs } from "./flags.ts";
 import { mergeConfigs } from "../../core/config.ts";
@@ -63,10 +60,7 @@ import {
   filterExtensions,
 } from "../../extension/extension.ts";
 import { quartoConfig } from "../../core/quarto.ts";
-import {
-  metadataNormalizationFilter,
-  metadataNormalizationFilterActive,
-} from "./normalize.ts";
+import { metadataNormalizationFilterActive } from "./normalize.ts";
 
 const kQuartoParams = "quarto-params";
 
@@ -81,7 +75,10 @@ const kTimingFile = "timings-file";
 
 const kHasBootstrap = "has-bootstrap";
 
+const kActiveFilters = "active-filters";
+
 const kQuartoVersion = "quarto-version";
+
 const kQuartoSource = "quarto-source";
 
 export async function filterParamsJson(
@@ -102,6 +99,11 @@ export async function filterParamsJson(
     )
     : {};
 
+  // extract the filter spec from pandoc options
+  const filterSpec = extractFilterSpecParams(
+    options.format.metadata,
+  );
+
   // Extract any column params
   const quartoColumnParams = extractColumnParams(
     args,
@@ -122,6 +124,11 @@ export async function filterParamsJson(
     ...filterParams,
     [kResultsFile]: pandocMetadataPath(resultsFile),
     [kTimingFile]: pandocMetadataPath(timingFile),
+    [kQuartoFilters]: filterSpec,
+    [kActiveFilters]: {
+      normalization: metadataNormalizationFilterActive(options),
+      crossref: crossrefFilterActive(options),
+    },
   };
   return JSON.stringify(params);
 }
@@ -130,20 +137,17 @@ export function removeFilterParams(metadata: Metadata) {
   delete metadata[kQuartoParams];
 }
 
-export function quartoInitFilter() {
-  return resourcePath("filters/quarto-init/quarto-init.lua");
-}
-
-export function quartoPreFilter() {
-  return resourcePath("filters/quarto-pre/quarto-pre.lua");
-}
-
-export function quartoPostFilter() {
-  return resourcePath("filters/quarto-post/quarto-post.lua");
+export function quartoMainFilter() {
+  return resourcePath("filters/main.lua");
 }
 
-export function quartoFinalizeFilter() {
-  return resourcePath("filters/quarto-finalize/quarto-finalize.lua");
+function extractFilterSpecParams(
+  metadata: Metadata,
+) {
+  // pull out the filter spec that resolveFilters created
+  const filterSpec = metadata[kQuartoFilters];
+  delete metadata[kQuartoFilters];
+  return filterSpec;
 }
 
 function extractIncludeParams(
@@ -540,28 +544,25 @@ function initFilterParams(dependenciesFile: string) {
 const kQuartoFilterMarker = "quarto";
 const kQuartoCiteProcMarker = "citeproc";
 
+export type QuartoFilterSpec = {
+  // these are filters that will be sent to pandoc directly
+  quartoFilters: QuartoFilter[];
+
+  beforeQuartoFilters: QuartoFilter[];
+  afterQuartoFilters: QuartoFilter[];
+};
+
 export async function resolveFilters(
   filters: QuartoFilter[],
   options: PandocOptions,
-): Promise<QuartoFilter[] | undefined> {
+): Promise<QuartoFilterSpec | undefined> {
   // build list of quarto filters
 
-  // The default order of filters will be
-  // quarto-init
-  // quarto-authors
-  // user filters
-  // extension filters
-  // quarto-filters <quarto>
-  // quarto-finalizer
-  // citeproc
+  const beforeQuartoFilters: QuartoFilter[] = [];
+  const afterQuartoFilters: QuartoFilter[] = [];
 
   const quartoFilters: string[] = [];
-  quartoFilters.push(quartoPreFilter());
-  if (crossrefFilterActive(options)) {
-    quartoFilters.push(crossrefFilter());
-  }
-  quartoFilters.push(layoutFilter());
-  quartoFilters.push(quartoPostFilter());
+  quartoFilters.push(quartoMainFilter());
 
   // Resolve any filters that are provided by an extension
   filters = await resolveFilterExtension(options, filters);
@@ -574,26 +575,13 @@ export async function resolveFilters(
     filter === kQuartoFilterMarker
   );
   if (quartoLoc !== -1) {
-    filters = [
-      ...filters.slice(0, quartoLoc),
-      ...quartoFilters,
-      ...filters.slice(quartoLoc + 1),
-    ];
+    beforeQuartoFilters.push(...filters.slice(0, quartoLoc));
+    afterQuartoFilters.push(...filters.slice(quartoLoc + 1));
   } else {
-    filters.push(...quartoFilters);
+    beforeQuartoFilters.push(...filters);
+    // afterQuartoFilters remains empty.
   }
 
-  // The author filter, if enabled
-  if (metadataNormalizationFilterActive(options)) {
-    filters.unshift(metadataNormalizationFilter());
-  }
-
-  // The initializer for Quarto
-  filters.unshift(quartoInitFilter());
-
-  // The finalizer for Quarto
-  filters.push(quartoFinalizeFilter());
-
   // citeproc at the very end so all other filters can interact with citations
   filters = filters.filter((filter) => filter !== kQuartoCiteProcMarker);
   const citeproc = citeMethod(options) === kQuartoCiteProcMarker;
@@ -605,12 +593,22 @@ export async function resolveFilters(
       delete options.format.pandoc.citeproc;
     }
 
-    filters.push(kQuartoCiteProcMarker);
+    quartoFilters.push(kQuartoCiteProcMarker);
   }
 
   // return filters
-  if (filters.length > 0) {
-    return filters;
+  if (
+    [
+      quartoFilters,
+      beforeQuartoFilters,
+      afterQuartoFilters,
+    ].some((x) => x.length)
+  ) {
+    return {
+      quartoFilters,
+      beforeQuartoFilters,
+      afterQuartoFilters,
+    };
   } else {
     return undefined;
   }

src/command/render/layout.ts

@@ -10,18 +10,13 @@ import { Document, Element } from "../../core/deno-dom.ts";
 import { kPageWidth } from "../../config/constants.ts";
 import { Format } from "../../config/types.ts";
 import { Metadata } from "../../config/types.ts";
-import { resourcePath } from "../../core/resources.ts";
 
 import { HtmlPostProcessResult } from "./types.ts";
 import { hasAdaptiveTheme } from "../../quarto-core/text-highlighting.ts";
 import { kHtmlEmptyPostProcessResult } from "./constants.ts";
 
 const kAdaptiveTextHighlighting = "adaptive-text-highlighting";
 
-export function layoutFilter() {
-  return resourcePath("filters/layout/layout.lua");
-}
-
 export function layoutFilterParams(format: Format) {
   const params: Metadata = {};
   const pageWidth = format.render[kPageWidth];

src/command/render/pandoc.ts

@@ -698,6 +698,11 @@ export async function runPandoc(
   // timing results json file
   const timingResultsFile = options.temp.createFile();
 
+  if (allDefaults.to?.match(/[.]lua$/)) {
+    formatFilterParams["custom-writer"] = allDefaults.to;
+    allDefaults.to = resourcePath("filters/customwriter/customwriter.lua");
+  }
+
   // set parameters required for filters (possibily mutating all of it's arguments
   // to pull includes out into quarto parameters so they can be merged)
   let pandocArgs = args;

src/config/constants.ts

@@ -390,6 +390,7 @@ export const kIncludeInHeader = "include-in-header";
 export const kCiteproc = "citeproc";
 export const kCiteMethod = "cite-method";
 export const kFilters = "filters";
+export const kQuartoFilters = "quarto-filters";
 export const kFilterParams = "filter-params";
 export const kPdfEngine = "pdf-engine";
 export const kPdfEngineOpts = "pdf-engine-opts";

src/config/types.ts

@@ -140,6 +140,7 @@ import {
   kPdfEngineOpt,
   kPdfEngineOpts,
   kPreferHtml,
+  kQuartoFilters,
   kReferenceLocation,
   kRelatedFormatsTitle,
   kRelatedNotebooksTitle,
@@ -194,6 +195,7 @@ import {
 
 import { TempContext } from "../core/temp-types.ts";
 import { HtmlPostProcessor } from "../command/render/types.ts";
+import { QuartoFilterSpec } from "../command/render/filters.ts";
 import { ExtensionContext } from "../extension/extension-shared.ts";
 import { ProjectContext } from "../project/types.ts";
 
@@ -447,6 +449,7 @@ export interface FormatPandoc {
   [kCiteproc]?: boolean;
   [kCiteMethod]?: string;
   [kFilters]?: QuartoFilter[];
+  [kQuartoFilters]?: QuartoFilterSpec;
   [kPdfEngine]?: string;
   [kPdfEngineOpts]?: string[];
   [kPdfEngineOpt]?: string;