Merge branch 'main' of github.com:quarto-dev/quarto-cli

Author: cscheid

news/changelog-1.1.md

@@ -7,6 +7,7 @@
 - Don't run `ipynb-filters` for qmd source files (only run them for ipynb source files)
 - More gracefully handle cell outputs with no data (don't print warning, just ignore)
 - Handle non-textual data from jupyter's plain text output more robustly (#1874)
+- Use IJulia's built-in conda environment / jupyter install for julia notebooks/qmds
 
 ## Knitr
 
@@ -106,6 +107,7 @@
 - Extension YAML files `_extension.yml` are now validated at render time. (#1268)
 - Support boolean values in Shortcode `meta` access
 - Make `quarto.base64` module available to extensions
+- Support installing extensions from any GitHub tag or branch (#1836)
 
 ## Publishing
 

src/command/check/check.ts

@@ -119,7 +119,7 @@ async function checkJupyterInstallation(services: RenderServices) {
         info("");
       }
     } else {
-      info(await jupyterInstallationMessage(caps, kIndent));
+      info(jupyterInstallationMessage(caps, kIndent));
       info("");
       const envMessage = jupyterUnactivatedEnvMessage(caps, kIndent);
       if (envMessage) {

src/command/use/commands/template.ts

@@ -56,7 +56,14 @@ async function useTemplate(
   tempContext: TempContext,
 ) {
   // Resolve extension host and trust
-  const source = extensionSource(target);
+  const source = await extensionSource(target);
+  // Is this source valid?
+  if (!source) {
+    info(
+      `Extension not found in local or remote sources`,
+    );
+    return;
+  }
   const trusted = await isTrusted(source, options.prompt !== false);
   if (trusted) {
     // Resolve target directory
@@ -125,7 +132,9 @@ async function stageTemplate(
     ensureDirSync(archiveDir);
 
     // The filename
-    const filename = source.resolvedTarget.split("/").pop() || "extension.zip";
+    const filename = (typeof (source.resolvedTarget) === "string"
+      ? source.resolvedTarget
+      : source.resolvedTarget.url).split("/").pop() || "extension.zip";
 
     // The tarball path
     const toFile = join(archiveDir, filename);
@@ -142,6 +151,12 @@ async function stageTemplate(
       return archiveDir;
     }
   } else {
+    if (typeof source.resolvedTarget !== "string") {
+      throw new Error(
+        "Internal error: local resolved extension should always have a string target.",
+      );
+    }
+
     if (Deno.statSync(source.resolvedTarget).isDirectory) {
       // copy the contents of the directory, filtered by quartoignore
       return source.resolvedTarget;

src/core/download.ts

@@ -14,17 +14,19 @@ export interface DownloadError extends Error {
 }
 
 export async function downloadWithProgress(
-  url: string,
+  url: string | Response,
   msg: string,
   toFile: string,
 ) {
   // Fetch the data
-  const response = await fetch(
-    url,
-    {
-      redirect: "follow",
-    },
-  );
+  const response = await (typeof url === "string"
+    ? fetch(
+      url,
+      {
+        redirect: "follow",
+      },
+    )
+    : url);
 
   // Write the data to a file
   if (response.status === 200 && response.body) {

src/core/jupyter/capabilities.ts

@@ -14,42 +14,82 @@ import { execProcess } from "../process.ts";
 import { resourcePath } from "../resources.ts";
 import { readYamlFromString } from "../yaml.ts";
 
-import { JupyterCapabilities } from "./types.ts";
+import { JupyterCapabilities, JupyterKernelspec } from "./types.ts";
 
-// cache capabiliites per-process
-let cachedJupyterCaps: JupyterCapabilities | undefined;
+// cache capabilities per language
+const kNoLanguage = "(none)";
+const jupyterCapsCache = new Map<string, JupyterCapabilities>();
+
+export async function jupyterCapabilities(kernelspec?: JupyterKernelspec) {
+  const language = kernelspec?.language || kNoLanguage;
+
+  if (!jupyterCapsCache.has(language)) {
+
+    // if we are targeting julia then prefer the julia installed miniconda
+    const juliaCaps = await getVerifiedJuliaCondaJupyterCapabilities();
+    if (language === "julia" && juliaCaps) {
+      jupyterCapsCache.set(language, juliaCaps);
+      return juliaCaps;
+    }
 
-export async function jupyterCapabilities() {
-  if (!cachedJupyterCaps) {
     // if there is an explicit python requested then use it
-    cachedJupyterCaps = await getQuartoJupyterCapabilities();
-    if (cachedJupyterCaps) {
-      return cachedJupyterCaps;
+    const quartoCaps = await getQuartoJupyterCapabilities();
+    if (quartoCaps) {
+      jupyterCapsCache.set(language, quartoCaps);
+      return quartoCaps;
     }
 
     // if we are on windows and have PY_PYTHON defined then use the launcher
     if (isWindows() && pyPython()) {
-      cachedJupyterCaps = await getPyLauncherJupyterCapabilities();
+      const pyLauncherCaps = await getPyLauncherJupyterCapabilities();
+      if (pyLauncherCaps) {
+        jupyterCapsCache.set(language, pyLauncherCaps);
+      }
     }
 
     // default handling (also a fallthrough if launcher didn't work out)
-    if (!cachedJupyterCaps) {
+    if (!jupyterCapsCache.has(language)) {
       // look for python from conda (conda doesn't provide python3 on windows or mac)
-      cachedJupyterCaps = await getJupyterCapabilities(["python"]);
-
-      // if it's not conda then probe explicitly for python 3
-      if (!cachedJupyterCaps?.conda) {
+      const condaCaps = await getJupyterCapabilities(["python"]);
+      if (condaCaps?.conda) {
+        jupyterCapsCache.set(language, condaCaps);
+      } else {
         const caps = isWindows()
           ? await getPyLauncherJupyterCapabilities()
           : await getJupyterCapabilities(["python3"]);
         if (caps) {
-          cachedJupyterCaps = caps;
+          jupyterCapsCache.set(language, caps);
         }
       }
+
+      // if the version we discovered doesn't have jupyter and we have a julia provided
+      // jupyter then go ahead and use that
+      if (!jupyterCapsCache.get(language)?.jupyter_core && juliaCaps) {
+        jupyterCapsCache.set(language, juliaCaps);
+      }
     }
   }
 
-  return cachedJupyterCaps;
+  return jupyterCapsCache.get(language);
+}
+
+async function getVerifiedJuliaCondaJupyterCapabilities() {
+  const home = isWindows() ? Deno.env.get("USERPROFILE") : Deno.env.get("HOME");
+  if (home) {
+    const juliaPython = join(
+      home,
+      ".julia",
+      "conda",
+      "3",
+      isWindows() ? "python.exe" : join("bin", "python3")
+    );
+    if (existsSync(juliaPython)) {
+      const caps = await getJupyterCapabilities([juliaPython]);
+      if (caps?.jupyter_core) {
+        return caps;
+      }
+    }
+  }
 }
 
 function getQuartoJupyterCapabilities() {

src/core/jupyter/exec.ts

@@ -7,9 +7,20 @@
 
 import { isWindows } from "../platform.ts";
 import { jupyterCapabilities } from "./capabilities.ts";
+import { JupyterCapabilities, JupyterKernelspec } from "./types.ts";
 
-export async function pythonExec(binaryOnly = false): Promise<string[]> {
-  const caps = await jupyterCapabilities();
+export async function pythonExec(
+  kernelspec?: JupyterKernelspec,
+  binaryOnly = false,
+): Promise<string[]> {
+  const caps = await jupyterCapabilities(kernelspec);
+  return pythonExecForCaps(caps, binaryOnly);
+}
+
+export function pythonExecForCaps(
+  caps?: JupyterCapabilities,
+  binaryOnly = false,
+) {
   if (caps?.pyLauncher) {
     return ["py"];
   } else if (isWindows()) {
@@ -19,9 +30,11 @@ export async function pythonExec(binaryOnly = false): Promise<string[]> {
   }
 }
 
-export async function jupyterExec(): Promise<string[]> {
+export async function jupyterExec(
+  kernelspec?: JupyterKernelspec,
+): Promise<string[]> {
   return [
-    ...(await pythonExec()),
+    ...(await pythonExec(kernelspec)),
     "-m",
     "jupyter",
   ];

src/core/jupyter/jupyter-shared.ts

@@ -11,7 +11,7 @@ import * as colors from "fmt/colors.ts";
 
 import { pathWithForwardSlashes } from "../path.ts";
 
-import { pythonExec } from "./exec.ts";
+import { pythonExecForCaps } from "./exec.ts";
 import { jupyterKernelspecs } from "./kernels.ts";
 import { JupyterCapabilities, JupyterKernelspec } from "./types.ts";
 import { isEnvDir } from "./capabilities.ts";
@@ -35,7 +35,7 @@ export async function jupyterCapabilitiesMessage(
   return lines.map((line: string) => `${indent}${line}`).join("\n");
 }
 
-export async function jupyterInstallationMessage(
+export function jupyterInstallationMessage(
   caps: JupyterCapabilities,
   indent = "",
 ) {
@@ -44,7 +44,9 @@ export async function jupyterInstallationMessage(
     "Install with " +
     colors.bold(
       `${
-        caps.conda ? "conda" : (await pythonExec(true)).join(" ") + " -m pip"
+        caps.conda
+          ? "conda"
+          : (pythonExecForCaps(caps, true)).join(" ") + " -m pip"
       } install jupyter`,
     ),
   ];
@@ -81,9 +83,8 @@ export function jupyterUnactivatedEnvMessage(
     if (existsSync(join(Deno.cwd(), envFile))) {
       return indent + "There is a " + colors.bold(envFile) +
         " file in this directory. " +
-        "Is this for a " + (envFile === kRequirementsTxt
-          ? "venv"
-          : "conda env") +
+        "Is this for a " +
+        (envFile === kRequirementsTxt ? "venv" : "conda env") +
         " that you need to restore?";
     }
   }

src/core/jupyter/jupyter.ts

@@ -427,7 +427,7 @@ export async function quartoMdToJupyter(
 export async function jupyterKernelspecFromMarkdown(
   markdown: string,
 ): Promise<[JupyterKernelspec, Metadata]> {
-  const yaml = await readYamlFromMarkdown(markdown);
+  const yaml = readYamlFromMarkdown(markdown);
   const yamlJupyter = yaml.jupyter;
 
   // if there is no yaml.jupyter then detect the file's language(s) and

src/execute/jupyter/jupyter-kernel.ts

@@ -53,7 +53,11 @@ export async function executeKernelOneshot(
 
   trace(options, "Executing notebook with oneshot kernel");
   const debug = !!options.format.execute[kExecuteDebug];
-  const result = await execJupyter("execute", { ...options, debug });
+  const result = await execJupyter(
+    "execute",
+    { ...options, debug },
+    options.kernelspec,
+  );
 
   if (!result.success) {
     return Promise.reject();
@@ -121,7 +125,7 @@ export async function executeKernelKeepalive(
             if (msg.type === "error") {
               trace(options, "Error response received");
               error(msg.data, { colorize: false });
-              printExecDiagnostics(msg.data);
+              printExecDiagnostics(options.kernelspec, msg.data);
               return Promise.reject();
             } else if (msg.type == "restart") {
               trace(options, "Restart request received");
@@ -180,12 +184,13 @@ async function abortKernel(options: JupyterExecuteOptions) {
 async function execJupyter(
   command: string,
   options: Record<string, unknown>,
+  kernelspec: JupyterKernelspec,
 ): Promise<ProcessResult> {
   try {
     const result = await execProcess(
       {
         cmd: [
-          ...(await pythonExec()),
+          ...(await pythonExec(kernelspec)),
           resourcePath("jupyter/jupyter.py"),
         ],
         env: {
@@ -204,26 +209,29 @@ async function execJupyter(
     );
     if (!result.success) {
       // forward error (print some diagnostics if python and/or jupyter couldn't be found)
-      await printExecDiagnostics(result.stderr);
+      await printExecDiagnostics(kernelspec, result.stderr);
     }
     return result;
   } catch (e) {
     if (e?.message) {
       info("");
       error(e.message);
     }
-    await printExecDiagnostics();
+    await printExecDiagnostics(kernelspec);
     return Promise.reject();
   }
 }
 
-export async function printExecDiagnostics(stderr?: string) {
-  const caps = await jupyterCapabilities();
+export async function printExecDiagnostics(
+  kernelspec: JupyterKernelspec,
+  stderr?: string,
+) {
+  const caps = await jupyterCapabilities(kernelspec);
   if (caps && !caps.jupyter_core) {
     info("Python 3 installation:");
     info(await jupyterCapabilitiesMessage(caps, "  "));
     info("");
-    info(await jupyterInstallationMessage(caps));
+    info(jupyterInstallationMessage(caps));
     info("");
     maybePrintUnactivatedEnvMessage(caps);
   } else if (caps && !haveRequiredPython(caps)) {
@@ -396,7 +404,7 @@ async function connectToKernel(
     timeout,
     type,
     debug,
-  });
+  }, options.kernelspec);
   if (!result.success) {
     return Promise.reject();
   }

src/execute/jupyter/jupyter.ts

@@ -261,7 +261,7 @@ export const jupyterEngine: ExecutionEngine = {
       }
       const jupyterExecOptions: JupyterExecuteOptions = {
         kernelspec,
-        python_cmd: await pythonExec(),
+        python_cmd: await pythonExec(kernelspec),
         ...execOptions,
       };
       if (executeDaemon === false || executeDaemon === 0) {