diff --git a/package/src/common/cyclic-dependencies.ts b/package/src/common/cyclic-dependencies.ts index 321cc72e595..0c7fcb41220 100644 --- a/package/src/common/cyclic-dependencies.ts +++ b/package/src/common/cyclic-dependencies.ts @@ -11,7 +11,7 @@ import { runCmd } from "../util/cmd.ts"; import { Configuration, readConfiguration } from "./config.ts"; import { error, info } from "../../../src/deno_ral/log.ts"; import { progressBar } from "../../../src/core/console.ts"; -import { md5Hash } from "../../../src/core/hash.ts"; +import { md5HashSync } from "../../../src/core/hash.ts"; export function cycleDependenciesCommand() { return new Command() @@ -186,7 +186,7 @@ function findCyclicDependencies( // creates a hash for a set of paths (a cycle) const hash = (paths: string[]) => { const string = paths.join(" "); - return md5Hash(string); + return md5HashSync(string); }; // The current import stack diff --git a/src/command/render/freeze.ts b/src/command/render/freeze.ts index 1a364f8fd2c..240de38cd30 100644 --- a/src/command/render/freeze.ts +++ b/src/command/render/freeze.ts @@ -24,7 +24,7 @@ import { cloneDeep } from "../../core/lodash.ts"; import { inputFilesDir } from "../../core/render.ts"; import { TempContext } from "../../core/temp.ts"; -import { md5Hash } from "../../core/hash.ts"; +import { md5HashSync } from "../../core/hash.ts"; import { normalizePath, removeIfEmptyDir, @@ -310,7 +310,7 @@ export function removeFreezeResults(filesDir: string) { function freezeInputHash(input: string) { // Calculate the hash on a content with LF line ending to avoid // different hash on different OS (#3599) - return md5Hash(format(Deno.readTextFileSync(input), LF)); + return md5HashSync(format(Deno.readTextFileSync(input), LF)); } // don't use _files suffix in freezer diff --git a/src/command/use/commands/binder/binder-utils.ts b/src/command/use/commands/binder/binder-utils.ts index db11ba87477..381e0c66814 100644 --- a/src/command/use/commands/binder/binder-utils.ts +++ b/src/command/use/commands/binder/binder-utils.ts @@ -4,7 +4,7 @@ * Copyright (C) 2021-2022 Posit Software, PBC */ -import { md5Hash } from "../../../../core/hash.ts"; +import { md5HashAsync } from "../../../../core/hash.ts"; import { projectScratchPath } from "../../../../project/project-scratch.ts"; import { info, warning } from "../../../../deno_ral/log.ts"; @@ -36,11 +36,11 @@ export const safeFileWriter = (projectDir: string, prompt = true) => { info(`[✓] ${projRelativePath}`); }; - const hash = md5Hash(contents); + const hash = await md5HashAsync(contents); if (existsSync(absPath)) { const lastHash = fileIndex[projRelativePath]; const currentContents = Deno.readTextFileSync(absPath); - const currentHash = md5Hash(currentContents); + const currentHash = await md5HashAsync(currentContents); let writeFile = true; if (!lastHash || lastHash !== currentHash) { diff --git a/src/core/devconfig.ts b/src/core/devconfig.ts index ecad3a98058..2e38ee17877 100644 --- a/src/core/devconfig.ts +++ b/src/core/devconfig.ts @@ -8,7 +8,7 @@ import { error, info } from "../deno_ral/log.ts"; import { join } from "../deno_ral/path.ts"; import { ensureDirSync, existsSync } from "../deno_ral/fs.ts"; -import { md5Hash } from "./hash.ts"; +import { md5HashSync } from "./hash.ts"; import { quartoConfig } from "./quarto.ts"; import { normalizePath } from "./path.ts"; @@ -47,13 +47,13 @@ export function createDevConfig( dartsass, esbuild, typst, - script: md5Hash(Deno.readTextFileSync(scriptPath)), - importMap: md5Hash( + script: md5HashSync(Deno.readTextFileSync(scriptPath)), + importMap: md5HashSync( Deno.readTextFileSync( join(srcDir, "import_map.json"), ), ), - bundleImportMap: md5Hash( + bundleImportMap: md5HashSync( Deno.readTextFileSync( join(srcDir, "resources/vendor/import_map.json"), ), diff --git a/src/core/hash.ts b/src/core/hash.ts index 9db01d9f79c..41078f267c4 100644 --- a/src/core/hash.ts +++ b/src/core/hash.ts @@ -7,10 +7,15 @@ import { crypto } from "crypto/crypto"; import blueimpMd5 from "blueimpMd5"; -export function md5Hash(content: string) { +export function md5HashSync(content: string) { return blueimpMd5(content); } +export async function md5HashAsync(content: string) { + const buffer = new TextEncoder().encode(content); + return md5HashBytes(buffer); +} + export async function md5HashBytes(content: Uint8Array) { const buffer = await crypto.subtle.digest( "MD5", diff --git a/src/core/pdfjs.ts b/src/core/pdfjs.ts index 470550bb5a6..6a49796a2c8 100644 --- a/src/core/pdfjs.ts +++ b/src/core/pdfjs.ts @@ -5,7 +5,7 @@ */ import { basename, join } from "../deno_ral/path.ts"; -import { md5Hash } from "./hash.ts"; +import { md5HashAsync } from "./hash.ts"; import { viewerIFrameURL } from "./http-devserver.ts"; import { FileResponse } from "./http-types.ts"; import { contentType } from "./mime.ts"; @@ -100,7 +100,7 @@ export function pdfJsFileHandler( // (preserve user viewer prefs across reloads) } else if (file === previewPath("build", "pdf.worker.js")) { const filePathHash = "quarto-preview-pdf-" + - md5Hash(pdfFile()); + await md5HashAsync(pdfFile()); const workerJs = Deno.readTextFileSync(file).replace( /(key: "fingerprint",\s+get: function get\(\) {\s+)(var hash;)/, `$1return "${filePathHash}"; $2`, diff --git a/src/core/sass/cache.ts b/src/core/sass/cache.ts index 8790015d651..74e1117d216 100644 --- a/src/core/sass/cache.ts +++ b/src/core/sass/cache.ts @@ -7,7 +7,7 @@ */ import { InternalError } from "../lib/error.ts"; -import { md5Hash } from "../hash.ts"; +import { md5HashAsync } from "../hash.ts"; import { join } from "../../deno_ral/path.ts"; import { ensureDirSync, existsSync } from "../../deno_ral/fs.ts"; import { TempContext } from "../temp.ts"; @@ -105,8 +105,8 @@ class SassCache { cacheIdentifier: string, compilationThunk: (outputFilePath: string) => Promise, ): Promise { - const identifierHash = md5Hash(cacheIdentifier); - const inputHash = md5Hash(input); + const identifierHash = await md5HashAsync(cacheIdentifier); + const inputHash = await md5HashAsync(input); return this.setFromHash( identifierHash, inputHash, @@ -121,8 +121,8 @@ class SassCache { compilationThunk: (outputFilePath: string) => Promise, ): Promise { log.debug(`SassCache.getOrSet(...)`); - const identifierHash = md5Hash(cacheIdentifier); - const inputHash = md5Hash(input); + const identifierHash = await md5HashAsync(cacheIdentifier); + const inputHash = await md5HashAsync(input); const existing = await this.getFromHash(identifierHash, inputHash); if (existing !== null) { log.debug(` cache hit`); diff --git a/src/execute/jupyter/jupyter-kernel.ts b/src/execute/jupyter/jupyter-kernel.ts index d8fdc91ea4a..4c8b04a88db 100644 --- a/src/execute/jupyter/jupyter-kernel.ts +++ b/src/execute/jupyter/jupyter-kernel.ts @@ -12,7 +12,7 @@ import { sleep } from "../../core/async.ts"; import { quartoDataDir, quartoRuntimeDir } from "../../core/appdirs.ts"; import { execProcess } from "../../core/process.ts"; import { ProcessResult } from "../../core/process-types.ts"; -import { md5Hash } from "../../core/hash.ts"; +import { md5HashSync } from "../../core/hash.ts"; import { resourcePath } from "../../core/resources.ts"; import { pythonExec } from "../../core/jupyter/exec.ts"; import { @@ -327,7 +327,7 @@ function kernelTransportFile(target: string) { throw e; } const targetFile = normalizePath(target); - const hash = md5Hash(targetFile).slice(0, 20); + const hash = md5HashSync(targetFile).slice(0, 20); return join(transportsDir, hash); } diff --git a/src/project/project-context.ts b/src/project/project-context.ts index c1f21d51199..f8ec9345e49 100644 --- a/src/project/project-context.ts +++ b/src/project/project-context.ts @@ -13,7 +13,7 @@ import { SEP, } from "../deno_ral/path.ts"; -import { existsSync, walkSync } from "../deno_ral/fs.ts"; +import { existsSync, walk, walkSync } from "../deno_ral/fs.ts"; import * as ld from "../core/lodash.ts"; import { ProjectType } from "./types/types.ts"; @@ -99,6 +99,7 @@ import { computeProjectEnvironment } from "./project-environment.ts"; import { ProjectEnvironment } from "./project-environment-types.ts"; import { NotebookContext } from "../render/notebook/notebook-types.ts"; import { MappedString } from "../core/mapped-text.ts"; +import { makeTimedFunctionAsync } from "../core/performance/function-times.ts"; import { createProjectCache } from "../core/cache/cache.ts"; import { createTempContext, globalTempContext } from "../core/temp.ts"; @@ -750,7 +751,12 @@ function projectHiddenIgnoreGlob(dir: string) { .concat(["**/README.?([Rrq])md"]); // README } -export async function projectInputFiles( +export const projectInputFiles = makeTimedFunctionAsync( + "projectInputFiles", + projectInputFilesInternal, +); + +async function projectInputFilesInternal( project: ProjectContext, metadata?: ProjectConfig, ): Promise<{ files: string[]; engines: string[] }> { @@ -802,10 +808,9 @@ export async function projectInputFiles( }]; }; const addDir = async (dir: string): Promise => { - // ignore selected other globs - const walkIterator = walkSync( - dir, - { + const promises: Promise[] = []; + for await ( + const walkEntry of walk(dir, { includeDirs: false, // this was done b/c some directories e.g. renv/packrat and potentially python // virtualenvs include symblinks to R or Python libraries that are in turn @@ -816,16 +821,18 @@ export async function projectInputFiles( globToRegExp(join(dir, ignore) + SEP) ), ), - }, - ); - return Promise.all( - Array.from(walkIterator) - .filter((walk) => { - const pathRelative = pathWithForwardSlashes(relative(dir, walk.path)); - return !projectIgnores.some((regex) => regex.test(pathRelative)); - }) - .map(async (walk) => addFile(walk.path)), - ).then((fileInclusions) => fileInclusions.flat()); + }) + ) { + const pathRelative = pathWithForwardSlashes( + relative(dir, walkEntry.path), + ); + if (projectIgnores.some((regex) => regex.test(pathRelative))) { + continue; + } + promises.push(addFile(walkEntry.path)); + } + const inclusions = await Promise.all(promises); + return inclusions.flat(); }; const addEntry = async (entry: string) => { if (Deno.statSync(entry).isDirectory) { diff --git a/src/project/serve/render.ts b/src/project/serve/render.ts index b93806a4101..e365823a999 100644 --- a/src/project/serve/render.ts +++ b/src/project/serve/render.ts @@ -1,7 +1,7 @@ import { isAbsolute, join } from "../../deno_ral/path.ts"; import { RenderResult, RenderResultFile } from "../../command/render/types.ts"; -import { md5Hash } from "../../core/hash.ts"; +import { md5HashSync } from "../../core/hash.ts"; import { HttpDevServerRenderMonitor } from "../../core/http-devserver.ts"; import { isJupyterNotebook } from "../../core/jupyter/jupyter.ts"; import { logError } from "../../core/log.ts"; @@ -118,8 +118,8 @@ export class ServeRenderManager { String(Deno.statSync(inputFile).mtime) + resourceHash; } else { - return md5Hash(Deno.readTextFileSync(file)) + - md5Hash(Deno.readTextFileSync(inputFile)) + + return md5HashSync(Deno.readTextFileSync(file)) + + md5HashSync(Deno.readTextFileSync(inputFile)) + resourceHash; } } diff --git a/src/project/serve/watch.ts b/src/project/serve/watch.ts index b95cf61a167..2f0b67fc116 100644 --- a/src/project/serve/watch.ts +++ b/src/project/serve/watch.ts @@ -10,7 +10,7 @@ import { existsSync } from "../../deno_ral/fs.ts"; import * as ld from "../../core/lodash.ts"; import { normalizePath, pathWithForwardSlashes } from "../../core/path.ts"; -import { md5Hash } from "../../core/hash.ts"; +import { md5HashAsync, md5HashSync } from "../../core/hash.ts"; import { logError } from "../../core/log.ts"; import { isRevealjsOutput } from "../../config/format.ts"; @@ -64,6 +64,7 @@ export function watchProject( const flags = renderOptions.flags; // helper to refresh project config const refreshProjectConfig = async () => { + project.cleanup(); project = (await projectContext(project.dir, nbContext, renderOptions, false))!; }; @@ -147,7 +148,8 @@ export function watchProject( const inputs = paths.filter(isInputFile).filter(existsSync1).filter( (input: string) => { return !rendered.has(input) || - rendered.get(input) !== md5Hash(Deno.readTextFileSync(input)); + rendered.get(input) !== + md5HashSync(Deno.readTextFileSync(input)); }, ); if (inputs.length) { @@ -182,10 +184,12 @@ export function watchProject( renderManager.onRenderError(result.error); return undefined; } - // record rendered hash for (const input of inputs.filter(existsSync1)) { - rendered.set(input, md5Hash(Deno.readTextFileSync(input))); + rendered.set( + input, + await md5HashAsync(Deno.readTextFileSync(input)), + ); } renderManager.onRenderResult( result, diff --git a/src/project/types/website/website-shared.ts b/src/project/types/website/website-shared.ts index a5e3bf35ad5..80147c0d8af 100644 --- a/src/project/types/website/website-shared.ts +++ b/src/project/types/website/website-shared.ts @@ -43,7 +43,7 @@ import { import { cookieConsentEnabled } from "./website-analytics.ts"; import { Format, FormatExtras } from "../../../config/types.ts"; import { kPageTitle, kTitle, kTitlePrefix } from "../../../config/constants.ts"; -import { md5Hash } from "../../../core/hash.ts"; +import { md5HashAsync } from "../../../core/hash.ts"; export { type NavigationFooter } from "../../types.ts"; export interface Navigation { @@ -246,7 +246,7 @@ export async function websiteNavigationConfig(project: ProjectContext) { let announcement: NavigationAnnouncement | undefined; if (typeof announcementRaw === "string") { announcement = { - id: md5Hash(announcementRaw), + id: await md5HashAsync(announcementRaw), icon: undefined, dismissable: true, content: announcementRaw, @@ -255,7 +255,7 @@ export async function websiteNavigationConfig(project: ProjectContext) { }; } else if (announcementRaw && !Array.isArray(announcementRaw)) { announcement = { - id: md5Hash(announcementRaw.content as string), + id: await md5HashAsync(announcementRaw.content as string), icon: announcementRaw.icon as string | undefined, dismissable: announcementRaw.dismissable !== false, content: announcementRaw.content as string, diff --git a/src/publish/confluence/confluence.ts b/src/publish/confluence/confluence.ts index 30b3085c5cf..9708a1fc711 100644 --- a/src/publish/confluence/confluence.ts +++ b/src/publish/confluence/confluence.ts @@ -93,7 +93,7 @@ import { MAX_PAGES_TO_LOAD, } from "./constants.ts"; import { logError, trace } from "./confluence-logger.ts"; -import { md5Hash } from "../../core/hash.ts"; +import { md5HashBytes } from "../../core/hash.ts"; import { sleep } from "../../core/async.ts"; import { info } from "../../deno_ral/log.ts"; @@ -346,7 +346,7 @@ async function publish( try { fileBuffer = await Deno.readFile(path); - fileHash = md5Hash(fileBuffer.toString()); + fileHash = await md5HashBytes(fileBuffer); } catch (error) { logError(`${path} not found`, error); return null; diff --git a/src/publish/posit-cloud/api/index.ts b/src/publish/posit-cloud/api/index.ts index e4f44e9c4a0..425ae14984c 100644 --- a/src/publish/posit-cloud/api/index.ts +++ b/src/publish/posit-cloud/api/index.ts @@ -15,7 +15,7 @@ import { User, } from "./types.ts"; -import { md5Hash } from "../../../core/hash.ts"; +import { md5HashAsync } from "../../../core/hash.ts"; import { quartoConfig } from "../../../core/quarto.ts"; import { crypto } from "crypto/crypto"; @@ -201,7 +201,7 @@ export class PositCloudClient { body?: string, ): Promise => { const date = new Date().toUTCString(); - const checksum = md5Hash(body || ""); + const checksum = await md5HashAsync(body || ""); const canonicalRequest = [ method,