Merge pull request #11807 from PumasAI/jk/engines-config

Allow to reorder engine prioritization with `engines` key in _quarto.yml

Author: cscheid

src/execute/engine.ts

@@ -96,7 +96,11 @@ export function engineValidExtensions(): string[] {
   );
 }
 
-export function markdownExecutionEngine(markdown: string, flags?: RenderFlags) {
+export function markdownExecutionEngine(
+  markdown: string,
+  reorderedEngines: Map<string, ExecutionEngine>,
+  flags?: RenderFlags,
+) {
   // read yaml and see if the engine is declared in yaml
   // (note that if the file were a non text-file like ipynb
   //  it would have already been claimed via extension)
@@ -106,7 +110,7 @@ export function markdownExecutionEngine(markdown: string, flags?: RenderFlags) {
     if (yaml) {
       // merge in command line fags
       yaml = mergeConfigs(yaml, flags?.metadata);
-      for (const [_, engine] of kEngines) {
+      for (const [_, engine] of reorderedEngines) {
         if (yaml[engine.name]) {
           return engine;
         }
@@ -123,7 +127,7 @@ export function markdownExecutionEngine(markdown: string, flags?: RenderFlags) {
 
   // see if there is an engine that claims this language
   for (const language of languages) {
-    for (const [_, engine] of kEngines) {
+    for (const [_, engine] of reorderedEngines) {
       if (engine.claimsLanguage(language)) {
         return engine;
       }
@@ -143,6 +147,37 @@ export function markdownExecutionEngine(markdown: string, flags?: RenderFlags) {
   return markdownEngine;
 }
 
+function reorderEngines(project: ProjectContext) {
+  const userSpecifiedOrder: string[] =
+    project.config?.engines as string[] | undefined ?? [];
+
+  for (const key of userSpecifiedOrder) {
+    if (!kEngines.has(key)) {
+      throw new Error(
+        `'${key}' was specified in the list of engines in the project settings but it is not a valid engine. Available engines are ${
+          Array.from(kEngines.keys()).join(", ")
+        }`,
+      );
+    }
+  }
+
+  const reorderedEngines = new Map<string, ExecutionEngine>();
+
+  // Add keys in the order of userSpecifiedOrder first
+  for (const key of userSpecifiedOrder) {
+    reorderedEngines.set(key, kEngines.get(key)!); // Non-null assertion since we verified the keys are in the map
+  }
+
+  // Add the rest of the keys from the original map
+  for (const [key, value] of kEngines) {
+    if (!reorderedEngines.has(key)) {
+      reorderedEngines.set(key, value);
+    }
+  }
+
+  return reorderedEngines;
+}
+
 export async function fileExecutionEngine(
   file: string,
   flags: RenderFlags | undefined,
@@ -158,8 +193,10 @@ export async function fileExecutionEngine(
     return undefined;
   }
 
+  const reorderedEngines = reorderEngines(project);
+
   // try to find an engine that claims this extension outright
-  for (const [_, engine] of kEngines) {
+  for (const [_, engine] of reorderedEngines) {
     if (engine.claimsFile(file, ext)) {
       return engine;
     }
@@ -175,6 +212,7 @@ export async function fileExecutionEngine(
     try {
       return markdownExecutionEngine(
         markdown ? markdown.value : Deno.readTextFileSync(file),
+        reorderedEngines,
         flags,
       );
     } catch (error) {

src/execute/julia.ts

@@ -3,6 +3,7 @@ import { join } from "../deno_ral/path.ts";
 import { MappedString, mappedStringFromFile } from "../core/mapped-text.ts";
 import { partitionMarkdown } from "../core/pandoc/pandoc-partition.ts";
 import { readYamlFromMarkdown } from "../core/yaml.ts";
+import { asMappedString } from "../core/lib/mapped-text.ts";
 import { ProjectContext } from "../project/types.ts";
 import {
   DependenciesOptions,
@@ -46,13 +47,21 @@ import {
   executeResultIncludes,
 } from "./jupyter/jupyter.ts";
 import { isWindows } from "../deno_ral/platform.ts";
+import {
+  isJupyterPercentScript,
+  markdownFromJupyterPercentScript,
+} from "./jupyter/percent.ts";
 
 export interface JuliaExecuteOptions extends ExecuteOptions {
   julia_cmd: string;
   oneShot: boolean; // if true, the file's worker process is closed before and after running
   supervisor_pid?: number;
 }
 
+function isJuliaPercentScript(file: string) {
+  return isJupyterPercentScript(file, [".jl"]);
+}
+
 export const juliaEngine: ExecutionEngine = {
   name: kJuliaEngine,
 
@@ -68,12 +77,12 @@ export const juliaEngine: ExecutionEngine = {
 
   validExtensions: () => [],
 
-  claimsFile: (file: string, ext: string) => false,
+  claimsFile: (file: string, _ext: string) => {
+    return isJuliaPercentScript(file);
+  },
 
   claimsLanguage: (language: string) => {
-    // we don't claim `julia` so the old behavior of using the jupyter
-    // backend by default stays intact
-    return false; // language.toLowerCase() === "julia";
+    return language.toLowerCase() === "julia";
   },
 
   partitionedMarkdown: async (file: string) => {
@@ -109,7 +118,13 @@ export const juliaEngine: ExecutionEngine = {
   },
 
   markdownForFile(file: string): Promise<MappedString> {
-    return Promise.resolve(mappedStringFromFile(file));
+    if (isJuliaPercentScript(file)) {
+      return Promise.resolve(
+        asMappedString(markdownFromJupyterPercentScript(file)),
+      );
+    } else {
+      return Promise.resolve(mappedStringFromFile(file));
+    }
   },
 
   execute: async (options: ExecuteOptions): Promise<ExecuteResult> => {

src/execute/jupyter/jupyter.ts

@@ -150,8 +150,10 @@ export const jupyterEngine: ExecutionEngine = {
       isJupyterPercentScript(file);
   },
 
-  claimsLanguage: (_language: string) => {
-    return false;
+  claimsLanguage: (language: string) => {
+    // jupyter has to claim julia so that julia may also claim it without changing the old behavior
+    // of preferring jupyter over julia engine by default
+    return language.toLowerCase() === "julia";
   },
 
   markdownForFile(file: string): Promise<MappedString> {

src/execute/jupyter/percent.ts

@@ -20,9 +20,10 @@ export const kJupyterPercentScriptExtensions = [
   ".r",
 ];
 
-export function isJupyterPercentScript(file: string) {
+export function isJupyterPercentScript(file: string, extensions?: string[]) {
   const ext = extname(file).toLowerCase();
-  if (kJupyterPercentScriptExtensions.includes(ext)) {
+  const availableExtensions = extensions ?? kJupyterPercentScriptExtensions;
+  if (availableExtensions.includes(ext)) {
     const text = Deno.readTextFileSync(file);
     return !!text.match(/^\s*#\s*%%+\s+\[markdown|raw\]/);
   } else {
@@ -77,10 +78,10 @@ export function markdownFromJupyterPercentScript(file: string) {
       let rawContent = cellContent(cellLines);
       const format = cell.header?.metadata?.["format"];
       const mimeType = cell.header.metadata?.[kCellRawMimeType];
-      if (typeof (mimeType) === "string") {
+      if (typeof mimeType === "string") {
         const rawBlock = mdRawOutput(mimeType, lines(rawContent));
         rawContent = rawBlock || rawContent;
-      } else if (typeof (format) === "string") {
+      } else if (typeof format === "string") {
         rawContent = mdFormatOutput(format, lines(rawContent));
       }
       markdown += rawContent;

src/resources/schema/project.yml

@@ -84,3 +84,8 @@
   #
   # In general, full json schema would allow negative assertions,
   # but that makes our error localization heuristics worse. So we hack.
+
+- name: engines
+  schema:
+    arrayOf: string
+  description: "List execution engines you want to give priority when determining which engine should render a notebook. If two engines have support for a notebook, the one listed earlier will be chosen. Quarto's default order is 'knitr', 'jupyter', 'markdown', 'julia'."

tests/docs/engine/invalid-project/_quarto.yml

@@ -0,0 +1 @@
+engines: ["invalid-engine"]

tests/docs/engine/invalid-project/notebook.qmd

@@ -0,0 +1,3 @@
+project:
+  type: default
+engines: ["julia"]

tests/docs/smoke-all/engine-reordering/julia-engine/_quarto.yml

@@ -0,0 +1,10 @@
+```{julia}
+using Test
+@test haskey(
+  Base.loaded_modules,
+  Base.PkgId(
+    Base.UUID("38328d9c-a911-4051-bc06-3f7f556ffeda"),
+    "QuartoNotebookWorker",
+  )
+)
+```

tests/docs/smoke-all/engine-reordering/julia-engine/notebook.qmd

@@ -0,0 +1,3 @@
+project:
+  type: default
+engines: ["jupyter"]