normalize file path; deterministic resolution (#1076)

* normalize file path; deterministic file resolution

* getSourceFileHash

* fix archives.win32 snapshot?

Author: mbostock

src/dataloader.ts

@@ -130,38 +130,47 @@ export class LoaderResolver {
   }
 
   /**
-   * Get the actual path of a file. For data loaders, it is the output if
-   * already available (cached). In build this is always the case (unless the
-   * corresponding data loader fails). However in preview we return the page
-   * before running the data loaders (which will run on demand from the page),
-   * so there might be a temporary discrepancy when a cache is stale.
+   * Returns the path to the backing file during preview, which is the source
+   * file for the associated data loader if the file is generated by a loader.
    */
-  private getFilePath(name: string): string {
+  private getSourceFilePath(name: string): string {
     let path = name;
     if (!existsSync(join(this.root, path))) {
       const loader = this.find(path);
-      if (loader) {
-        path = relative(this.root, loader.path);
-        if (name !== path) {
-          const cachePath = join(".observablehq", "cache", name);
-          if (existsSync(join(this.root, cachePath))) path = cachePath;
-        }
-      }
+      if (loader) path = relative(this.root, loader.path);
+    }
+    return path;
+  }
+
+  /**
+   * Returns the path to the backing file during build, which is the cached
+   * output file if the file is generated by a loader.
+   */
+  private getOutputFilePath(name: string): string {
+    let path = name;
+    if (!existsSync(join(this.root, path))) {
+      const loader = this.find(path);
+      if (loader) path = join(".observablehq", "cache", name);
     }
     return path;
   }
 
-  getFileHash(name: string): string {
-    return getFileHash(this.root, this.getFilePath(name));
+  getSourceFileHash(name: string): string {
+    return getFileHash(this.root, this.getSourceFilePath(name));
+  }
+
+  getSourceLastModified(name: string): number | undefined {
+    const entry = getFileInfo(this.root, this.getSourceFilePath(name));
+    return entry && Math.floor(entry.mtimeMs);
   }
 
-  getLastModified(name: string): number | undefined {
-    const entry = getFileInfo(this.root, this.getFilePath(name));
+  getOutputLastModified(name: string): number | undefined {
+    const entry = getFileInfo(this.root, this.getOutputFilePath(name));
     return entry && Math.floor(entry.mtimeMs);
   }
 
   resolveFilePath(path: string): string {
-    return `/${join("_file", path)}?sha=${this.getFileHash(path)}`;
+    return `/${join("_file", path)}?sha=${this.getSourceFileHash(path)}`;
   }
 }
 

src/javascript/transpile.ts

@@ -5,6 +5,7 @@ import {simple} from "acorn-walk";
 import {isPathImport, relativePath, resolvePath} from "../path.js";
 import {getModuleResolver} from "../resolvers.js";
 import {Sourcemap} from "../sourcemap.js";
+import type {FileExpression} from "./files.js";
 import {findFiles} from "./files.js";
 import type {ExportNode, ImportNode} from "./imports.js";
 import {hasImportDeclaration, isImportMetaResolve} from "./imports.js";
@@ -15,10 +16,11 @@ import {getStringLiteralValue, isStringLiteral} from "./source.js";
 
 export interface TranspileOptions {
   id: string;
+  path: string;
   resolveImport?: (specifier: string) => string;
 }
 
-export function transpileJavaScript(node: JavaScriptNode, {id, resolveImport}: TranspileOptions): string {
+export function transpileJavaScript(node: JavaScriptNode, {id, path, resolveImport}: TranspileOptions): string {
   let async = node.async;
   const inputs = Array.from(new Set<string>(node.references.map((r) => r.name)));
   const outputs = Array.from(new Set<string>(node.declarations?.map((r) => r.name)));
@@ -28,6 +30,7 @@ export function transpileJavaScript(node: JavaScriptNode, {id, resolveImport}: T
   const output = new Sourcemap(node.input).trim();
   rewriteImportDeclarations(output, node.body, resolveImport);
   rewriteImportExpressions(output, node.body, resolveImport);
+  rewriteFileExpressions(output, node.files, path);
   if (display) output.insertLeft(0, "display(await(\n").insertRight(node.input.length, "\n))");
   output.insertLeft(0, `, body: ${async ? "async " : ""}(${inputs}) => {\n`);
   if (outputs.length) output.insertLeft(0, `, outputs: ${JSON.stringify(outputs)}`);
@@ -100,6 +103,14 @@ export async function transpileModule(
   return String(output);
 }
 
+function rewriteFileExpressions(output: Sourcemap, files: FileExpression[], path: string): void {
+  for (const {name, node} of files) {
+    const source = node.arguments[0];
+    const resolved = relativePath(path, resolvePath(path, name));
+    output.replaceLeft(source.start, source.end, JSON.stringify(resolved));
+  }
+}
+
 function rewriteImportExpressions(
   output: Sourcemap,
   body: Node,

src/render.ts

@@ -65,7 +65,13 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from
       : ""
   }${data?.sql ? `\nimport {registerTable} from ${JSON.stringify(resolveImport("npm:@observablehq/duckdb"))};` : ""}${
     files.size
-      ? `\n${renderFiles(files, resolveFile, (name: string) => loaders.getLastModified(resolvePath(path, name)))}`
+      ? `\n${renderFiles(
+          files,
+          resolveFile,
+          preview
+            ? (name: string) => loaders.getSourceLastModified(resolvePath(path, name))
+            : (name: string) => loaders.getOutputLastModified(resolvePath(path, name))
+        )}`
       : ""
   }${
     data?.sql
@@ -75,7 +81,7 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from
       : ""
   }
 ${preview ? `\nopen({hash: ${JSON.stringify(resolvers.hash)}, eval: (body) => eval(body)});\n` : ""}${page.code
-    .map(({node, id}) => `\n${transpileJavaScript(node, {id, resolveImport})}`)
+    .map(({node, id}) => `\n${transpileJavaScript(node, {id, path, resolveImport})}`)
     .join("")}`)}
 </script>${sidebar ? html`\n${await renderSidebar(title, pages, root, path, search, normalizeLink)}` : ""}${
     toc.show ? html`\n${renderToc(findHeaders(page), toc.label)}` : ""

src/resolvers.ts

@@ -12,6 +12,7 @@ import {extractNpmSpecifier, populateNpmCache, resolveNpmImport, resolveNpmImpor
 import {isAssetPath, isPathImport, relativePath, resolveLocalPath, resolvePath} from "./path.js";
 
 export interface Resolvers {
+  path: string;
   hash: string;
   assets: Set<string>; // like files, but not registered for FileAttachment
   files: Set<string>;
@@ -115,10 +116,10 @@ export async function getResolvers(
   }
 
   // Compute the content hash.
-  for (const f of assets) hash.update(loaders.getFileHash(resolvePath(path, f)));
-  for (const f of files) hash.update(loaders.getFileHash(resolvePath(path, f)));
+  for (const f of assets) hash.update(loaders.getSourceFileHash(resolvePath(path, f)));
+  for (const f of files) hash.update(loaders.getSourceFileHash(resolvePath(path, f)));
   for (const i of localImports) hash.update(getModuleHash(root, resolvePath(path, i)));
-  if (page.style && isPathImport(page.style)) hash.update(loaders.getFileHash(resolvePath(path, page.style)));
+  if (page.style && isPathImport(page.style)) hash.update(loaders.getSourceFileHash(resolvePath(path, page.style)));
 
   // Collect transitively-attached files and local imports.
   for (const i of localImports) {
@@ -259,6 +260,7 @@ export async function getResolvers(
   }
 
   return {
+    path,
     hash: hash.digest("hex"),
     assets,
     files,

test/build-test.ts

@@ -26,7 +26,7 @@ describe("build", () => {
     if (isEmpty(path)) continue;
     const only = name.startsWith("only.");
     const skip = name.startsWith("skip.");
-    const outname = only || skip ? name.slice(5) : name;
+    const outname = name.replace(/^only\.|skip\./, "");
     (only
       ? it.only
       : skip ||

test/dataloaders-test.ts

@@ -99,38 +99,41 @@ describe("LoaderResolver.find(path, {useStale: true})", () => {
   });
 });
 
-describe("LoaderResolver.getFileHash(path)", () => {
+describe("LoaderResolver.getSourceFileHash(path)", () => {
   it("returns the content hash for the specified file’s data loader", async () => {
     const loaders = new LoaderResolver({root: "test/input/build/archives.posix"});
-    assert.strictEqual(loaders.getFileHash("dynamic.zip.sh"), "516cec2431ce8f1181a7a2a161db8bdfcaea132d3b2c37f863ea6f05d64d1d10"); // prettier-ignore
-    assert.strictEqual(loaders.getFileHash("dynamic.zip"), "516cec2431ce8f1181a7a2a161db8bdfcaea132d3b2c37f863ea6f05d64d1d10"); // prettier-ignore
-    assert.strictEqual(loaders.getFileHash("dynamic/file.txt"), "516cec2431ce8f1181a7a2a161db8bdfcaea132d3b2c37f863ea6f05d64d1d10"); // prettier-ignore
-    assert.strictEqual(loaders.getFileHash("static.zip"), "e6afff224da77b900cfe3ab8789f2283883300e1497548c30af66dfe4c29b429"); // prettier-ignore
-    assert.strictEqual(loaders.getFileHash("static/file.txt"), "e6afff224da77b900cfe3ab8789f2283883300e1497548c30af66dfe4c29b429"); // prettier-ignore
+    assert.strictEqual(loaders.getSourceFileHash("dynamic.zip.sh"), "516cec2431ce8f1181a7a2a161db8bdfcaea132d3b2c37f863ea6f05d64d1d10"); // prettier-ignore
+    assert.strictEqual(loaders.getSourceFileHash("dynamic.zip"), "516cec2431ce8f1181a7a2a161db8bdfcaea132d3b2c37f863ea6f05d64d1d10"); // prettier-ignore
+    assert.strictEqual(loaders.getSourceFileHash("dynamic/file.txt"), "516cec2431ce8f1181a7a2a161db8bdfcaea132d3b2c37f863ea6f05d64d1d10"); // prettier-ignore
+    assert.strictEqual(loaders.getSourceFileHash("static.zip"), "e6afff224da77b900cfe3ab8789f2283883300e1497548c30af66dfe4c29b429"); // prettier-ignore
+    assert.strictEqual(loaders.getSourceFileHash("static/file.txt"), "e6afff224da77b900cfe3ab8789f2283883300e1497548c30af66dfe4c29b429"); // prettier-ignore
   });
   it("returns the empty hash if the specified file does not exist", async () => {
     const loaders = new LoaderResolver({root: "test/input/build/files"});
-    assert.strictEqual(loaders.getFileHash("does-not-exist.csv"), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); // prettier-ignore
+    assert.strictEqual(loaders.getSourceFileHash("does-not-exist.csv"), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); // prettier-ignore
   });
 });
 
-describe("LoaderResolver.getLastModified(path)", () => {
+describe("LoaderResolver.get{Source,Output}LastModified(path)", () => {
   const time1 = new Date(Date.UTC(2023, 11, 1));
   const time2 = new Date(Date.UTC(2024, 2, 1));
   const loaders = new LoaderResolver({root: "test"});
-  it("returns the last modification time for a simple file", async () => {
+  it("both return the last modification time for a simple file", async () => {
     await utimes("test/input/loader/simple.txt", time1, time1);
-    assert.strictEqual(loaders.getLastModified("input/loader/simple.txt"), +time1);
+    assert.strictEqual(loaders.getSourceLastModified("input/loader/simple.txt"), +time1);
+    assert.strictEqual(loaders.getOutputLastModified("input/loader/simple.txt"), +time1);
   });
-  it("returns an undefined last modification time for a missing file", async () => {
-    assert.strictEqual(loaders.getLastModified("input/loader/missing.txt"), undefined);
+  it("both return an undefined last modification time for a missing file", async () => {
+    assert.strictEqual(loaders.getSourceLastModified("input/loader/missing.txt"), undefined);
+    assert.strictEqual(loaders.getOutputLastModified("input/loader/missing.txt"), undefined);
   });
-  it("returns the last modification time for a cached data loader", async () => {
+  it("returns the last modification time of the loader in preview, and of the cache, on build", async () => {
     await utimes("test/input/loader/cached.txt.sh", time1, time1);
     await mkdir("test/.observablehq/cache/input/loader/", {recursive: true});
     await writeFile("test/.observablehq/cache/input/loader/cached.txt", "2024-03-01 00:00:00");
     await utimes("test/.observablehq/cache/input/loader/cached.txt", time2, time2);
-    assert.strictEqual(loaders.getLastModified("input/loader/cached.txt"), +time2);
+    assert.strictEqual(loaders.getSourceLastModified("input/loader/cached.txt"), +time1);
+    assert.strictEqual(loaders.getOutputLastModified("input/loader/cached.txt"), +time2);
     // clean up
     try {
       await unlink("test/.observablehq/cache/input/loader/cached.txt");
@@ -139,8 +142,9 @@ describe("LoaderResolver.getLastModified(path)", () => {
       // ignore;
     }
   });
-  it("returns the last modification time for a data loader that has no cache", async () => {
+  it("returns the last modification time of the data loader in preview, and null in build, when there is no cache", async () => {
     await utimes("test/input/loader/not-cached.txt.sh", time1, time1);
-    assert.strictEqual(loaders.getLastModified("input/loader/not-cached.txt"), +time1);
+    assert.strictEqual(loaders.getSourceLastModified("input/loader/not-cached.txt"), +time1);
+    assert.strictEqual(loaders.getOutputLastModified("input/loader/not-cached.txt"), undefined);
   });
 });

test/javascript/transpile-test.ts

@@ -33,7 +33,7 @@ function runTests(inputRoot: string, outputRoot: string, filter: (name: string)
 
       try {
         const node = parseJavaScript(input, {path: name});
-        actual = transpileJavaScript(node, {id: "0", resolveImport: mockResolveImport});
+        actual = transpileJavaScript(node, {id: "0", path: name, resolveImport: mockResolveImport});
       } catch (error) {
         if (!(error instanceof SyntaxError)) throw error;
         actual = `define({id: "0", body: () => { throw new SyntaxError(${JSON.stringify(error.message)}); }});\n`;
@@ -75,7 +75,7 @@ describe("transpileJavaScript(input, options)", () => {
   runTests("test/input/imports", "test/output/imports", (name) => name.endsWith("-import.js"));
   it("trims leading and trailing newlines", async () => {
     const node = parseJavaScript("\ntest\n", {path: "index.js"});
-    const body = transpileJavaScript(node, {id: "0"});
+    const body = transpileJavaScript(node, {id: "0", path: "index.js"});
     assert.strictEqual(body, 'define({id: "0", inputs: ["test","display"], body: async (test,display) => {\ndisplay(await(\ntest\n))\n}});\n'); // prettier-ignore
   });
 });

test/output/build/archives.posix/tar.html

@@ -15,53 +15,53 @@
 import {define} from "./_observablehq/client.js";
 import {registerFile} from "./_observablehq/stdlib.js";
 
-registerFile("./dynamic-tar-gz/does-not-exist.txt", {"name":"./dynamic-tar-gz/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar-gz/does-not-exist.txt","lastModified":/* ts */1706742000000});
+registerFile("./dynamic-tar-gz/does-not-exist.txt", {"name":"./dynamic-tar-gz/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar-gz/does-not-exist.txt"});
 registerFile("./dynamic-tar-gz/file.txt", {"name":"./dynamic-tar-gz/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar-gz/file.c93138d8.txt","lastModified":/* ts */1706742000000});
-registerFile("./dynamic-tar/does-not-exist.txt", {"name":"./dynamic-tar/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar/does-not-exist.txt","lastModified":/* ts */1706742000000});
+registerFile("./dynamic-tar/does-not-exist.txt", {"name":"./dynamic-tar/does-not-exist.txt","mimeType":"text/plain","path":"./dynamic-tar/does-not-exist.txt"});
 registerFile("./dynamic-tar/file.txt", {"name":"./dynamic-tar/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar/file.c93138d8.txt","lastModified":/* ts */1706742000000});
-registerFile("./static-tar/does-not-exist.txt", {"name":"./static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./static-tar/does-not-exist.txt","lastModified":/* ts */1706742000000});
+registerFile("./static-tar/does-not-exist.txt", {"name":"./static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./static-tar/does-not-exist.txt"});
 registerFile("./static-tar/file.txt", {"name":"./static-tar/file.txt","mimeType":"text/plain","path":"./_file/static-tar/file.c93138d8.txt","lastModified":/* ts */1706742000000});
 registerFile("./static-tgz/file.txt", {"name":"./static-tgz/file.txt","mimeType":"text/plain","path":"./_file/static-tgz/file.c93138d8.txt","lastModified":/* ts */1706742000000});
 
 define({id: "d5134368", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("static-tar/file.txt").text()
+await FileAttachment("./static-tar/file.txt").text()
 ))
 }});
 
 define({id: "a0c06958", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("static-tgz/file.txt").text()
+await FileAttachment("./static-tgz/file.txt").text()
 ))
 }});
 
 define({id: "d84cd7fb", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("static-tar/does-not-exist.txt").text()
+await FileAttachment("./static-tar/does-not-exist.txt").text()
 ))
 }});
 
 define({id: "86bd51aa", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("dynamic-tar/file.txt").text()
+await FileAttachment("./dynamic-tar/file.txt").text()
 ))
 }});
 
 define({id: "95938c22", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("dynamic-tar/does-not-exist.txt").text()
+await FileAttachment("./dynamic-tar/does-not-exist.txt").text()
 ))
 }});
 
 define({id: "7e5740fd", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("dynamic-tar-gz/file.txt").text()
+await FileAttachment("./dynamic-tar-gz/file.txt").text()
 ))
 }});
 
 define({id: "d0a58efd", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("dynamic-tar-gz/does-not-exist.txt").text()
+await FileAttachment("./dynamic-tar-gz/does-not-exist.txt").text()
 ))
 }});
 

test/output/build/archives.posix/zip.html

@@ -16,31 +16,31 @@
 import {registerFile} from "./_observablehq/stdlib.js";
 
 registerFile("./dynamic/file.txt", {"name":"./dynamic/file.txt","mimeType":"text/plain","path":"./_file/dynamic/file.c93138d8.txt","lastModified":/* ts */1706742000000});
-registerFile("./dynamic/not-found.txt", {"name":"./dynamic/not-found.txt","mimeType":"text/plain","path":"./dynamic/not-found.txt","lastModified":/* ts */1706742000000});
+registerFile("./dynamic/not-found.txt", {"name":"./dynamic/not-found.txt","mimeType":"text/plain","path":"./dynamic/not-found.txt"});
 registerFile("./static/file.txt", {"name":"./static/file.txt","mimeType":"text/plain","path":"./_file/static/file.d9014c46.txt","lastModified":/* ts */1706742000000});
-registerFile("./static/not-found.txt", {"name":"./static/not-found.txt","mimeType":"text/plain","path":"./static/not-found.txt","lastModified":/* ts */1706742000000});
+registerFile("./static/not-found.txt", {"name":"./static/not-found.txt","mimeType":"text/plain","path":"./static/not-found.txt"});
 
 define({id: "d3b9d0ee", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("static/file.txt").text()
+await FileAttachment("./static/file.txt").text()
 ))
 }});
 
 define({id: "bab54217", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("static/not-found.txt").text()
+await FileAttachment("./static/not-found.txt").text()
 ))
 }});
 
 define({id: "11eec300", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("dynamic/file.txt").text()
+await FileAttachment("./dynamic/file.txt").text()
 ))
 }});
 
 define({id: "ee2310f3", inputs: ["FileAttachment","display"], body: async (FileAttachment,display) => {
 display(await(
-await FileAttachment("dynamic/not-found.txt").text()
+await FileAttachment("./dynamic/not-found.txt").text()
 ))
 }});