Merge pull request #11986 from quarto-dev/perf/md5-async

Perf: prefer `async` md5 calls

Author: cscheid

package/src/common/cyclic-dependencies.ts

@@ -11,7 +11,7 @@ import { runCmd } from "../util/cmd.ts";
 import { Configuration, readConfiguration } from "./config.ts";
 import { error, info } from "../../../src/deno_ral/log.ts";
 import { progressBar } from "../../../src/core/console.ts";
-import { md5Hash } from "../../../src/core/hash.ts";
+import { md5HashSync } from "../../../src/core/hash.ts";
 
 export function cycleDependenciesCommand() {
   return new Command()
@@ -186,7 +186,7 @@ function findCyclicDependencies(
   // creates a hash for a set of paths (a cycle)
   const hash = (paths: string[]) => {
     const string = paths.join(" ");
-    return md5Hash(string);
+    return md5HashSync(string);
   };
 
   // The current import stack

src/command/render/freeze.ts

@@ -24,7 +24,7 @@ import { cloneDeep } from "../../core/lodash.ts";
 
 import { inputFilesDir } from "../../core/render.ts";
 import { TempContext } from "../../core/temp.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashSync } from "../../core/hash.ts";
 import {
   normalizePath,
   removeIfEmptyDir,
@@ -310,7 +310,7 @@ export function removeFreezeResults(filesDir: string) {
 function freezeInputHash(input: string) {
   // Calculate the hash on a content with LF line ending to avoid
   // different hash on different OS (#3599)
-  return md5Hash(format(Deno.readTextFileSync(input), LF));
+  return md5HashSync(format(Deno.readTextFileSync(input), LF));
 }
 
 // don't use _files suffix in freezer

src/command/use/commands/binder/binder-utils.ts

@@ -4,7 +4,7 @@
  * Copyright (C) 2021-2022 Posit Software, PBC
  */
 
-import { md5Hash } from "../../../../core/hash.ts";
+import { md5HashAsync } from "../../../../core/hash.ts";
 import { projectScratchPath } from "../../../../project/project-scratch.ts";
 
 import { info, warning } from "../../../../deno_ral/log.ts";
@@ -36,11 +36,11 @@ export const safeFileWriter = (projectDir: string, prompt = true) => {
       info(`[✓] ${projRelativePath}`);
     };
 
-    const hash = md5Hash(contents);
+    const hash = await md5HashAsync(contents);
     if (existsSync(absPath)) {
       const lastHash = fileIndex[projRelativePath];
       const currentContents = Deno.readTextFileSync(absPath);
-      const currentHash = md5Hash(currentContents);
+      const currentHash = await md5HashAsync(currentContents);
 
       let writeFile = true;
       if (!lastHash || lastHash !== currentHash) {

src/core/devconfig.ts

@@ -8,7 +8,7 @@ import { error, info } from "../deno_ral/log.ts";
 import { join } from "../deno_ral/path.ts";
 import { ensureDirSync, existsSync } from "../deno_ral/fs.ts";
 
-import { md5Hash } from "./hash.ts";
+import { md5HashSync } from "./hash.ts";
 
 import { quartoConfig } from "./quarto.ts";
 import { normalizePath } from "./path.ts";
@@ -47,13 +47,13 @@ export function createDevConfig(
     dartsass,
     esbuild,
     typst,
-    script: md5Hash(Deno.readTextFileSync(scriptPath)),
-    importMap: md5Hash(
+    script: md5HashSync(Deno.readTextFileSync(scriptPath)),
+    importMap: md5HashSync(
       Deno.readTextFileSync(
         join(srcDir, "import_map.json"),
       ),
     ),
-    bundleImportMap: md5Hash(
+    bundleImportMap: md5HashSync(
       Deno.readTextFileSync(
         join(srcDir, "resources/vendor/import_map.json"),
       ),

src/core/hash.ts

@@ -7,10 +7,15 @@
 import { crypto } from "crypto/crypto";
 import blueimpMd5 from "blueimpMd5";
 
-export function md5Hash(content: string) {
+export function md5HashSync(content: string) {
   return blueimpMd5(content);
 }
 
+export async function md5HashAsync(content: string) {
+  const buffer = new TextEncoder().encode(content);
+  return md5HashBytes(buffer);
+}
+
 export async function md5HashBytes(content: Uint8Array) {
   const buffer = await crypto.subtle.digest(
     "MD5",

src/core/pdfjs.ts

@@ -5,7 +5,7 @@
  */
 
 import { basename, join } from "../deno_ral/path.ts";
-import { md5Hash } from "./hash.ts";
+import { md5HashAsync } from "./hash.ts";
 import { viewerIFrameURL } from "./http-devserver.ts";
 import { FileResponse } from "./http-types.ts";
 import { contentType } from "./mime.ts";
@@ -100,7 +100,7 @@ export function pdfJsFileHandler(
       // (preserve user viewer prefs across reloads)
     } else if (file === previewPath("build", "pdf.worker.js")) {
       const filePathHash = "quarto-preview-pdf-" +
-        md5Hash(pdfFile());
+        await md5HashAsync(pdfFile());
       const workerJs = Deno.readTextFileSync(file).replace(
         /(key: "fingerprint",\s+get: function get\(\) {\s+)(var hash;)/,
         `$1return "${filePathHash}"; $2`,

src/core/sass/cache.ts

@@ -7,7 +7,7 @@
  */
 
 import { InternalError } from "../lib/error.ts";
-import { md5Hash } from "../hash.ts";
+import { md5HashAsync } from "../hash.ts";
 import { join } from "../../deno_ral/path.ts";
 import { ensureDirSync, existsSync } from "../../deno_ral/fs.ts";
 import { TempContext } from "../temp.ts";
@@ -105,8 +105,8 @@ class SassCache {
     cacheIdentifier: string,
     compilationThunk: (outputFilePath: string) => Promise<void>,
   ): Promise<string> {
-    const identifierHash = md5Hash(cacheIdentifier);
-    const inputHash = md5Hash(input);
+    const identifierHash = await md5HashAsync(cacheIdentifier);
+    const inputHash = await md5HashAsync(input);
     return this.setFromHash(
       identifierHash,
       inputHash,
@@ -121,8 +121,8 @@ class SassCache {
     compilationThunk: (outputFilePath: string) => Promise<void>,
   ): Promise<string> {
     log.debug(`SassCache.getOrSet(...)`);
-    const identifierHash = md5Hash(cacheIdentifier);
-    const inputHash = md5Hash(input);
+    const identifierHash = await md5HashAsync(cacheIdentifier);
+    const inputHash = await md5HashAsync(input);
     const existing = await this.getFromHash(identifierHash, inputHash);
     if (existing !== null) {
       log.debug(`  cache hit`);

src/execute/jupyter/jupyter-kernel.ts

@@ -12,7 +12,7 @@ import { sleep } from "../../core/async.ts";
 import { quartoDataDir, quartoRuntimeDir } from "../../core/appdirs.ts";
 import { execProcess } from "../../core/process.ts";
 import { ProcessResult } from "../../core/process-types.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashSync } from "../../core/hash.ts";
 import { resourcePath } from "../../core/resources.ts";
 import { pythonExec } from "../../core/jupyter/exec.ts";
 import {
@@ -327,7 +327,7 @@ function kernelTransportFile(target: string) {
     throw e;
   }
   const targetFile = normalizePath(target);
-  const hash = md5Hash(targetFile).slice(0, 20);
+  const hash = md5HashSync(targetFile).slice(0, 20);
   return join(transportsDir, hash);
 }
 

src/project/project-context.ts

@@ -13,7 +13,7 @@ import {
   SEP,
 } from "../deno_ral/path.ts";
 
-import { existsSync, walkSync } from "../deno_ral/fs.ts";
+import { existsSync, walk, walkSync } from "../deno_ral/fs.ts";
 import * as ld from "../core/lodash.ts";
 
 import { ProjectType } from "./types/types.ts";
@@ -99,6 +99,7 @@ import { computeProjectEnvironment } from "./project-environment.ts";
 import { ProjectEnvironment } from "./project-environment-types.ts";
 import { NotebookContext } from "../render/notebook/notebook-types.ts";
 import { MappedString } from "../core/mapped-text.ts";
+import { makeTimedFunctionAsync } from "../core/performance/function-times.ts";
 import { createProjectCache } from "../core/cache/cache.ts";
 import { createTempContext, globalTempContext } from "../core/temp.ts";
 
@@ -750,7 +751,12 @@ function projectHiddenIgnoreGlob(dir: string) {
     .concat(["**/README.?([Rrq])md"]); // README
 }
 
-export async function projectInputFiles(
+export const projectInputFiles = makeTimedFunctionAsync(
+  "projectInputFiles",
+  projectInputFilesInternal,
+);
+
+async function projectInputFilesInternal(
   project: ProjectContext,
   metadata?: ProjectConfig,
 ): Promise<{ files: string[]; engines: string[] }> {
@@ -802,10 +808,9 @@ export async function projectInputFiles(
     }];
   };
   const addDir = async (dir: string): Promise<FileInclusion[]> => {
-    // ignore selected other globs
-    const walkIterator = walkSync(
-      dir,
-      {
+    const promises: Promise<FileInclusion[]>[] = [];
+    for await (
+      const walkEntry of walk(dir, {
         includeDirs: false,
         // this was done b/c some directories e.g. renv/packrat and potentially python
         // virtualenvs include symblinks to R or Python libraries that are in turn
@@ -816,16 +821,18 @@ export async function projectInputFiles(
             globToRegExp(join(dir, ignore) + SEP)
           ),
         ),
-      },
-    );
-    return Promise.all(
-      Array.from(walkIterator)
-        .filter((walk) => {
-          const pathRelative = pathWithForwardSlashes(relative(dir, walk.path));
-          return !projectIgnores.some((regex) => regex.test(pathRelative));
-        })
-        .map(async (walk) => addFile(walk.path)),
-    ).then((fileInclusions) => fileInclusions.flat());
+      })
+    ) {
+      const pathRelative = pathWithForwardSlashes(
+        relative(dir, walkEntry.path),
+      );
+      if (projectIgnores.some((regex) => regex.test(pathRelative))) {
+        continue;
+      }
+      promises.push(addFile(walkEntry.path));
+    }
+    const inclusions = await Promise.all(promises);
+    return inclusions.flat();
   };
   const addEntry = async (entry: string) => {
     if (Deno.statSync(entry).isDirectory) {

src/project/serve/render.ts

@@ -1,7 +1,7 @@
 import { isAbsolute, join } from "../../deno_ral/path.ts";
 
 import { RenderResult, RenderResultFile } from "../../command/render/types.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashSync } from "../../core/hash.ts";
 import { HttpDevServerRenderMonitor } from "../../core/http-devserver.ts";
 import { isJupyterNotebook } from "../../core/jupyter/jupyter.ts";
 import { logError } from "../../core/log.ts";
@@ -118,8 +118,8 @@ export class ServeRenderManager {
         String(Deno.statSync(inputFile).mtime) +
         resourceHash;
     } else {
-      return md5Hash(Deno.readTextFileSync(file)) +
-        md5Hash(Deno.readTextFileSync(inputFile)) +
+      return md5HashSync(Deno.readTextFileSync(file)) +
+        md5HashSync(Deno.readTextFileSync(inputFile)) +
         resourceHash;
     }
   }