Perf: prefer async md5 calls

package/src/common/cyclic-dependencies.ts

@@ -11,7 +11,7 @@ import { runCmd } from "../util/cmd.ts";
 import { Configuration, readConfiguration } from "./config.ts";
 import { error, info } from "../../../src/deno_ral/log.ts";
 import { progressBar } from "../../../src/core/console.ts";
-import { md5Hash } from "../../../src/core/hash.ts";
+import { md5HashSync } from "../../../src/core/hash.ts";
 
 export function cycleDependenciesCommand() {
   return new Command()
@@ -186,7 +186,7 @@ function findCyclicDependencies(
   // creates a hash for a set of paths (a cycle)
   const hash = (paths: string[]) => {
     const string = paths.join(" ");
-    return md5Hash(string);
+    return md5HashSync(string);
   };
 
   // The current import stack

src/command/render/freeze.ts

@@ -24,7 +24,7 @@ import { cloneDeep } from "../../core/lodash.ts";
 
 import { inputFilesDir } from "../../core/render.ts";
 import { TempContext } from "../../core/temp.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashSync } from "../../core/hash.ts";
 import {
   normalizePath,
   removeIfEmptyDir,
@@ -310,7 +310,7 @@ export function removeFreezeResults(filesDir: string) {
 function freezeInputHash(input: string) {
   // Calculate the hash on a content with LF line ending to avoid
   // different hash on different OS (#3599)
-  return md5Hash(format(Deno.readTextFileSync(input), LF));
+  return md5HashSync(format(Deno.readTextFileSync(input), LF));
 }
 
 // don't use _files suffix in freezer

src/command/use/commands/binder/binder-utils.ts

@@ -4,7 +4,7 @@
  * Copyright (C) 2021-2022 Posit Software, PBC
  */
 
-import { md5Hash } from "../../../../core/hash.ts";
+import { md5HashAsync } from "../../../../core/hash.ts";
 import { projectScratchPath } from "../../../../project/project-scratch.ts";
 
 import { info, warning } from "../../../../deno_ral/log.ts";
@@ -36,11 +36,11 @@ export const safeFileWriter = (projectDir: string, prompt = true) => {
       info(`[✓] ${projRelativePath}`);
     };
 
-    const hash = md5Hash(contents);
+    const hash = await md5HashAsync(contents);
     if (existsSync(absPath)) {
       const lastHash = fileIndex[projRelativePath];
       const currentContents = Deno.readTextFileSync(absPath);
-      const currentHash = md5Hash(currentContents);
+      const currentHash = await md5HashAsync(currentContents);
 
       let writeFile = true;
       if (!lastHash || lastHash !== currentHash) {

src/core/devconfig.ts

@@ -8,7 +8,7 @@ import { error, info } from "../deno_ral/log.ts";
 import { join } from "../deno_ral/path.ts";
 import { ensureDirSync, existsSync } from "../deno_ral/fs.ts";
 
-import { md5Hash } from "./hash.ts";
+import { md5HashSync } from "./hash.ts";
 
 import { quartoConfig } from "./quarto.ts";
 import { normalizePath } from "./path.ts";
@@ -47,13 +47,13 @@ export function createDevConfig(
     dartsass,
     esbuild,
     typst,
-    script: md5Hash(Deno.readTextFileSync(scriptPath)),
-    importMap: md5Hash(
+    script: md5HashSync(Deno.readTextFileSync(scriptPath)),
+    importMap: md5HashSync(
       Deno.readTextFileSync(
         join(srcDir, "import_map.json"),
       ),
     ),
-    bundleImportMap: md5Hash(
+    bundleImportMap: md5HashSync(
       Deno.readTextFileSync(
         join(srcDir, "resources/vendor/import_map.json"),
       ),

src/core/hash.ts

@@ -7,10 +7,15 @@
 import { crypto } from "crypto/crypto";
 import blueimpMd5 from "blueimpMd5";
 
-export function md5Hash(content: string) {
+export function md5HashSync(content: string) {
   return blueimpMd5(content);
 }
 
+export async function md5HashAsync(content: string) {
+  const buffer = new TextEncoder().encode(content);
+  return md5HashBytes(buffer);
+}
+
 export async function md5HashBytes(content: Uint8Array) {
   const buffer = await crypto.subtle.digest(
     "MD5",

src/core/pdfjs.ts

@@ -5,7 +5,7 @@
  */
 
 import { basename, join } from "../deno_ral/path.ts";
-import { md5Hash } from "./hash.ts";
+import { md5HashAsync } from "./hash.ts";
 import { viewerIFrameURL } from "./http-devserver.ts";
 import { FileResponse } from "./http-types.ts";
 import { contentType } from "./mime.ts";
@@ -100,7 +100,7 @@ export function pdfJsFileHandler(
       // (preserve user viewer prefs across reloads)
     } else if (file === previewPath("build", "pdf.worker.js")) {
       const filePathHash = "quarto-preview-pdf-" +
-        md5Hash(pdfFile());
+        await md5HashAsync(pdfFile());
       const workerJs = Deno.readTextFileSync(file).replace(
         /(key: "fingerprint",\s+get: function get\(\) {\s+)(var hash;)/,
         `$1return "${filePathHash}"; $2`,

src/core/sass/cache.ts

@@ -7,7 +7,7 @@
  */
 
 import { InternalError } from "../lib/error.ts";
-import { md5Hash } from "../hash.ts";
+import { md5HashAsync } from "../hash.ts";
 import { join } from "../../deno_ral/path.ts";
 import { ensureDirSync, existsSync } from "../../deno_ral/fs.ts";
 import { TempContext } from "../temp.ts";
@@ -105,8 +105,8 @@ class SassCache {
     cacheIdentifier: string,
     compilationThunk: (outputFilePath: string) => Promise<void>,
   ): Promise<string> {
-    const identifierHash = md5Hash(cacheIdentifier);
-    const inputHash = md5Hash(input);
+    const identifierHash = await md5HashAsync(cacheIdentifier);
+    const inputHash = await md5HashAsync(input);
     return this.setFromHash(
       identifierHash,
       inputHash,
@@ -121,8 +121,8 @@ class SassCache {
     compilationThunk: (outputFilePath: string) => Promise<void>,
   ): Promise<string> {
     log.debug(`SassCache.getOrSet(...)`);
-    const identifierHash = md5Hash(cacheIdentifier);
-    const inputHash = md5Hash(input);
+    const identifierHash = await md5HashAsync(cacheIdentifier);
+    const inputHash = await md5HashAsync(input);
     const existing = await this.getFromHash(identifierHash, inputHash);
     if (existing !== null) {
       log.debug(`  cache hit`);

src/execute/jupyter/jupyter-kernel.ts

@@ -12,7 +12,7 @@ import { sleep } from "../../core/async.ts";
 import { quartoDataDir, quartoRuntimeDir } from "../../core/appdirs.ts";
 import { execProcess } from "../../core/process.ts";
 import { ProcessResult } from "../../core/process-types.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashSync } from "../../core/hash.ts";
 import { resourcePath } from "../../core/resources.ts";
 import { pythonExec } from "../../core/jupyter/exec.ts";
 import {
@@ -327,7 +327,7 @@ function kernelTransportFile(target: string) {
     throw e;
   }
   const targetFile = normalizePath(target);
-  const hash = md5Hash(targetFile).slice(0, 20);
+  const hash = md5HashSync(targetFile).slice(0, 20);
   return join(transportsDir, hash);
 }
 

src/project/project-context.ts

@@ -13,7 +13,7 @@ import {
   SEP,
 } from "../deno_ral/path.ts";
 
-import { existsSync, walkSync } from "../deno_ral/fs.ts";
+import { existsSync, walk, walkSync } from "../deno_ral/fs.ts";
 import * as ld from "../core/lodash.ts";
 
 import { ProjectType } from "./types/types.ts";
@@ -99,6 +99,7 @@ import { computeProjectEnvironment } from "./project-environment.ts";
 import { ProjectEnvironment } from "./project-environment-types.ts";
 import { NotebookContext } from "../render/notebook/notebook-types.ts";
 import { MappedString } from "../core/mapped-text.ts";
+import { makeTimedFunctionAsync } from "../core/performance/function-times.ts";
 import { createProjectCache } from "../core/cache/cache.ts";
 import { createTempContext, globalTempContext } from "../core/temp.ts";
 
@@ -750,7 +751,12 @@ function projectHiddenIgnoreGlob(dir: string) {
     .concat(["**/README.?([Rrq])md"]); // README
 }
 
-export async function projectInputFiles(
+export const projectInputFiles = makeTimedFunctionAsync(
+  "projectInputFiles",
+  projectInputFilesInternal,
+);
+
+async function projectInputFilesInternal(
   project: ProjectContext,
   metadata?: ProjectConfig,
 ): Promise<{ files: string[]; engines: string[] }> {
@@ -802,10 +808,9 @@ export async function projectInputFiles(
     }];
   };
   const addDir = async (dir: string): Promise<FileInclusion[]> => {
-    // ignore selected other globs
-    const walkIterator = walkSync(
-      dir,
-      {
+    const promises: Promise<FileInclusion[]>[] = [];
+    for await (
+      const walkEntry of walk(dir, {
         includeDirs: false,
         // this was done b/c some directories e.g. renv/packrat and potentially python
         // virtualenvs include symblinks to R or Python libraries that are in turn
@@ -816,16 +821,18 @@ export async function projectInputFiles(
             globToRegExp(join(dir, ignore) + SEP)
           ),
         ),
-      },
-    );
-    return Promise.all(
-      Array.from(walkIterator)
-        .filter((walk) => {
-          const pathRelative = pathWithForwardSlashes(relative(dir, walk.path));
-          return !projectIgnores.some((regex) => regex.test(pathRelative));
-        })
-        .map(async (walk) => addFile(walk.path)),
-    ).then((fileInclusions) => fileInclusions.flat());
+      })
+    ) {
+      const pathRelative = pathWithForwardSlashes(
+        relative(dir, walkEntry.path),
+      );
+      if (projectIgnores.some((regex) => regex.test(pathRelative))) {
+        continue;
+      }
+      promises.push(addFile(walkEntry.path));
+    }
+    const inclusions = await Promise.all(promises);
+    return inclusions.flat();
   };
   const addEntry = async (entry: string) => {
     if (Deno.statSync(entry).isDirectory) {

src/project/serve/render.ts

@@ -1,7 +1,7 @@
 import { isAbsolute, join } from "../../deno_ral/path.ts";
 
 import { RenderResult, RenderResultFile } from "../../command/render/types.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashSync } from "../../core/hash.ts";
 import { HttpDevServerRenderMonitor } from "../../core/http-devserver.ts";
 import { isJupyterNotebook } from "../../core/jupyter/jupyter.ts";
 import { logError } from "../../core/log.ts";
@@ -118,8 +118,8 @@ export class ServeRenderManager {
         String(Deno.statSync(inputFile).mtime) +
         resourceHash;
     } else {
-      return md5Hash(Deno.readTextFileSync(file)) +
-        md5Hash(Deno.readTextFileSync(inputFile)) +
+      return md5HashSync(Deno.readTextFileSync(file)) +
+        md5HashSync(Deno.readTextFileSync(inputFile)) +
         resourceHash;
     }
   }

src/project/serve/watch.ts

@@ -10,7 +10,7 @@ import { existsSync } from "../../deno_ral/fs.ts";
 import * as ld from "../../core/lodash.ts";
 
 import { normalizePath, pathWithForwardSlashes } from "../../core/path.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashAsync, md5HashSync } from "../../core/hash.ts";
 
 import { logError } from "../../core/log.ts";
 import { isRevealjsOutput } from "../../config/format.ts";
@@ -64,6 +64,7 @@ export function watchProject(
   const flags = renderOptions.flags;
   // helper to refresh project config
   const refreshProjectConfig = async () => {
+    project.cleanup();
     project =
       (await projectContext(project.dir, nbContext, renderOptions, false))!;
   };
@@ -147,7 +148,8 @@ export function watchProject(
           const inputs = paths.filter(isInputFile).filter(existsSync1).filter(
             (input: string) => {
               return !rendered.has(input) ||
-                rendered.get(input) !== md5Hash(Deno.readTextFileSync(input));
+                rendered.get(input) !==
+                  md5HashSync(Deno.readTextFileSync(input));
             },
           );
           if (inputs.length) {
@@ -182,10 +184,12 @@ export function watchProject(
                 renderManager.onRenderError(result.error);
                 return undefined;
               }
-
               // record rendered hash
               for (const input of inputs.filter(existsSync1)) {
-                rendered.set(input, md5Hash(Deno.readTextFileSync(input)));
+                rendered.set(
+                  input,
+                  await md5HashAsync(Deno.readTextFileSync(input)),
+                );
               }
               renderManager.onRenderResult(
                 result,

src/project/types/website/website-shared.ts

@@ -43,7 +43,7 @@ import {
 import { cookieConsentEnabled } from "./website-analytics.ts";
 import { Format, FormatExtras } from "../../../config/types.ts";
 import { kPageTitle, kTitle, kTitlePrefix } from "../../../config/constants.ts";
-import { md5Hash } from "../../../core/hash.ts";
+import { md5HashAsync } from "../../../core/hash.ts";
 export { type NavigationFooter } from "../../types.ts";
 
 export interface Navigation {
@@ -246,7 +246,7 @@ export async function websiteNavigationConfig(project: ProjectContext) {
   let announcement: NavigationAnnouncement | undefined;
   if (typeof announcementRaw === "string") {
     announcement = {
-      id: md5Hash(announcementRaw),
+      id: await md5HashAsync(announcementRaw),
       icon: undefined,
       dismissable: true,
       content: announcementRaw,
@@ -255,7 +255,7 @@ export async function websiteNavigationConfig(project: ProjectContext) {
     };
   } else if (announcementRaw && !Array.isArray(announcementRaw)) {
     announcement = {
-      id: md5Hash(announcementRaw.content as string),
+      id: await md5HashAsync(announcementRaw.content as string),
       icon: announcementRaw.icon as string | undefined,
       dismissable: announcementRaw.dismissable !== false,
       content: announcementRaw.content as string,

src/publish/confluence/confluence.ts

@@ -93,7 +93,7 @@ import {
   MAX_PAGES_TO_LOAD,
 } from "./constants.ts";
 import { logError, trace } from "./confluence-logger.ts";
-import { md5Hash } from "../../core/hash.ts";
+import { md5HashBytes } from "../../core/hash.ts";
 import { sleep } from "../../core/async.ts";
 import { info } from "../../deno_ral/log.ts";
 
@@ -346,7 +346,7 @@ async function publish(
 
       try {
         fileBuffer = await Deno.readFile(path);
-        fileHash = md5Hash(fileBuffer.toString());
+        fileHash = await md5HashBytes(fileBuffer);
       } catch (error) {
         logError(`${path} not found`, error);
         return null;

src/publish/posit-cloud/api/index.ts

@@ -15,7 +15,7 @@ import {
   User,
 } from "./types.ts";
 
-import { md5Hash } from "../../../core/hash.ts";
+import { md5HashAsync } from "../../../core/hash.ts";
 import { quartoConfig } from "../../../core/quarto.ts";
 
 import { crypto } from "crypto/crypto";
@@ -201,7 +201,7 @@ export class PositCloudClient {
     body?: string,
   ): Promise<HeadersInit> => {
     const date = new Date().toUTCString();
-    const checksum = md5Hash(body || "");
+    const checksum = await md5HashAsync(body || "");
 
     const canonicalRequest = [
       method,