embedded assets (#1681)

* embedded assets

* test asset embeds

* avoid recomputing config.paths

* deterministic file order

Author: mbostock

docs/markdown.md

@@ -18,7 +18,7 @@ The front matter supports the following options:
 - **title** - the page title; defaults to the (first) first-level heading of the page, if any
 - **index** - whether to index this page if [search](./search) is enabled; defaults to true for listed pages
 - **keywords** <a href="https://github.com/observablehq/framework/releases/tag/v1.1.0" class="observablehq-version-badge" data-version="^1.1.0" title="Added in v1.1.0"></a> - additional words to index for [search](./search); boosted at the same weight as the title
-- **draft** <a href="https://github.com/observablehq/framework/releases/tag/v1.1.0" class="observablehq-version-badge" data-version="^1.1.0" title="Added in v1.1.0"></a> - whether to skip this page during build; drafts are also not listed in the default sidebar
+- **draft** <a href="https://github.com/observablehq/framework/releases/tag/v1.1.0" class="observablehq-version-badge" data-version="^1.1.0" title="Added in v1.1.0"></a> - whether to skip this page during build; drafts are also not listed in the default sidebar nor searchable
 - **sql** <a href="https://github.com/observablehq/framework/releases/tag/v1.2.0" class="observablehq-version-badge" data-version="^1.2.0" title="Added in v1.2.0"></a> - table definitions for [SQL code blocks](./sql)
 
 The front matter can also override the following [app-level configuration](./config) options:

src/build.ts

@@ -71,6 +71,9 @@ export async function build(
   const addStylesheet = (path: string, s: string) => stylesheets.add(/^\w+:/.test(s) ? s : resolvePath(path, s));
 
   // Load pages, building a list of additional assets as we go.
+  let assetCount = 0;
+  let pageCount = 0;
+  const pagePaths = new Set<string>();
   for await (const path of config.paths()) {
     effects.output.write(`${faint("load")} ${path} `);
     const start = performance.now();
@@ -86,9 +89,18 @@ export async function build(
         for (const s of resolvers.stylesheets) addStylesheet(path, s);
         effects.output.write(`${faint("in")} ${(elapsed >= 100 ? yellow : faint)(`${elapsed}ms`)}\n`);
         outputs.set(path, {type: "module", resolvers});
+        ++assetCount;
         continue;
       }
     }
+    const file = loaders.find(path);
+    if (file) {
+      effects.output.write(`${faint("copy")} ${join(root, path)} ${faint("→")} `);
+      const sourcePath = join(root, await file.load({useStale: true}, effects));
+      await effects.copyFile(sourcePath, path);
+      ++assetCount;
+      continue;
+    }
     const page = await loaders.loadPage(path, options, effects);
     if (page.data.draft) {
       effects.logger.log(faint("(skipped)"));
@@ -102,13 +114,16 @@ export async function build(
     for (const i of resolvers.globalImports) addGlobalImport(path, resolvers.resolveImport(i));
     for (const s of resolvers.stylesheets) addStylesheet(path, s);
     effects.output.write(`${faint("in")} ${(elapsed >= 100 ? yellow : faint)(`${elapsed}ms`)}\n`);
+    pagePaths.add(path);
     outputs.set(path, {type: "page", page, resolvers});
+    ++pageCount;
   }
 
   // Check that there’s at least one output.
-  const outputCount = outputs.size;
+  const outputCount = pageCount + assetCount;
   if (!outputCount) throw new CliError(`Nothing to build: no pages found in your ${root} directory.`);
-  effects.logger.log(`${faint("built")} ${outputCount} ${faint(`page${outputCount === 1 ? "" : "s"} in`)} ${root}`);
+  if (pageCount) effects.logger.log(`${faint("built")} ${pageCount} ${faint(`page${pageCount === 1 ? "" : "s"} in`)} ${root}`); // prettier-ignore
+  if (assetCount) effects.logger.log(`${faint("built")} ${assetCount} ${faint(`asset${assetCount === 1 ? "" : "s"} in`)} ${root}`); // prettier-ignore
 
   // For cache-breaking we rename most assets to include content hashes.
   const aliases = new Map<string, string>();
@@ -117,7 +132,7 @@ export async function build(
   // Add the search bundle and data, if needed.
   if (config.search) {
     globalImports.add("/_observablehq/search.js").add("/_observablehq/minisearch.json");
-    const contents = await searchIndex(config, effects);
+    const contents = await searchIndex(config, pagePaths, effects);
     effects.output.write(`${faint("index →")} `);
     const cachePath = join(cacheRoot, "_observablehq", "minisearch.json");
     await prepareOutput(cachePath);
@@ -349,7 +364,7 @@ export async function build(
   }
   effects.logger.log("");
 
-  Telemetry.record({event: "build", step: "finish", pageCount: outputCount});
+  Telemetry.record({event: "build", step: "finish", pageCount});
 }
 
 function applyHash(path: string, hash: string): string {

src/search.ts

@@ -29,15 +29,19 @@ const indexOptions = {
 
 type MiniSearchResult = Omit<SearchResult, "path" | "keywords"> & {id: string; keywords: string};
 
-export async function searchIndex(config: Config, effects = defaultEffects): Promise<string> {
+export async function searchIndex(
+  config: Config,
+  paths: Iterable<string> | AsyncIterable<string> = getDefaultSearchPaths(config),
+  effects = defaultEffects
+): Promise<string> {
   const {pages, search, normalizePath} = config;
   if (!search) return "{}";
   const cached = indexCache.get(pages);
   if (cached && cached.freshUntil > Date.now()) return cached.json;
 
   // Index the pages
   const index = new MiniSearch<MiniSearchResult>(indexOptions);
-  for await (const result of indexPages(config, effects)) index.add(normalizeResult(result, normalizePath));
+  for await (const result of indexPages(config, paths, effects)) index.add(normalizeResult(result, normalizePath));
   if (search.index) for await (const result of search.index()) index.add(normalizeResult(result, normalizePath));
 
   // Pass the serializable index options to the client.
@@ -57,8 +61,12 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro
   return json;
 }
 
-async function* indexPages(config: Config, effects: SearchIndexEffects): AsyncIterable<SearchResult> {
-  const {root, pages, loaders} = config;
+async function* indexPages(
+  config: Config,
+  paths: Iterable<string> | AsyncIterable<string>,
+  effects: SearchIndexEffects
+): AsyncIterable<SearchResult> {
+  const {pages, loaders} = config;
 
   // Get all the listed pages (which are indexed by default)
   const pagePaths = new Set(["/index"]);
@@ -67,8 +75,7 @@ async function* indexPages(config: Config, effects: SearchIndexEffects): AsyncIt
     if ("pages" in p) for (const {path} of p.pages) pagePaths.add(path);
   }
 
-  for await (const path of config.paths()) {
-    if (path.endsWith(".js") && findModule(root, path)) continue;
+  for await (const path of paths) {
     const {body, title, data} = await loaders.loadPage(path, {...config, path});
 
     // Skip pages that opt-out of indexing, and skip unlisted pages unless
@@ -97,6 +104,15 @@ async function* indexPages(config: Config, effects: SearchIndexEffects): AsyncIt
   }
 }
 
+async function* getDefaultSearchPaths(config: Config): AsyncGenerator<string> {
+  const {root, loaders} = config;
+  for await (const path of config.paths()) {
+    if (path.endsWith(".js") && findModule(root, path)) continue; // ignore modules
+    if (loaders.find(path)) continue; // ignore assets
+    yield path;
+  }
+}
+
 function normalizeResult(
   {path, keywords, ...rest}: SearchResult,
   normalizePath: Config["normalizePath"]

test/files-test.ts

@@ -55,26 +55,26 @@ describe("visitFiles(root)", () => {
       "files.md",
       "observable logo small.png",
       "observable logo.png",
-      "unknown-mime-extension.really",
       "subsection/additional-styles.css",
       "subsection/file-sub.csv",
-      "subsection/subfiles.md"
+      "subsection/subfiles.md",
+      "unknown-mime-extension.really"
     ]);
   });
   it("handles circular symlinks, visiting files only once", function () {
     if (os.platform() === "win32") this.skip(); // symlinks are not the same on Windows
     assert.deepStrictEqual(collect(visitFiles("test/input/circular-files")), ["a/a.txt", "b/b.txt"]);
   });
   it("ignores .observablehq at any level", function () {
-    assert.deepStrictEqual(collect(visitFiles("test/files")), ["visible.txt", "sub/visible.txt"]);
+    assert.deepStrictEqual(collect(visitFiles("test/files")), ["sub/visible.txt", "visible.txt"]);
   });
 });
 
 describe("visitFiles(root, test)", () => {
   it("skips directories and files that don’t pass the specified test", () => {
     assert.deepStrictEqual(
       collect(visitFiles("test/input/build/params", (name) => isParameterized(name) || extname(name) !== "")),
-      ["observablehq.config.js", "[dir]/index.md", "[dir]/loaded.md.js"]
+      ["[dir]/index.md", "[dir]/loaded.md.js", "[name]-icon.svg.js", "observablehq.config.js"]
     );
     assert.deepStrictEqual(collect(visitFiles("test/input/build/params", (name) => !isParameterized(name))), [
       "observablehq.config.js"
@@ -88,5 +88,5 @@ function collect(generator: Generator<string>): string[] {
     if (value.startsWith(".observablehq/cache/")) continue;
     values.push(value);
   }
-  return values;
+  return values.sort();
 }

test/input/build/params/[name]-icon.svg.js

@@ -0,0 +1,12 @@
+import {parseArgs} from "node:util";
+
+const {
+  values: {name}
+} = parseArgs({
+  options: {name: {type: "string"}}
+});
+
+process.stdout.write(`<svg width="200" height="40" fill="#000000" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+  <text x="50%" y="50%" dy="0.35em" text-anchor="middle">${name}</text>
+</svg>
+`);

test/input/build/params/observablehq.config.js

@@ -1,5 +1,5 @@
 export default {
   async *dynamicPaths() {
-    yield* ["/bar/index", "/bar/loaded", "/foo/bar", "/foo/index"];
+    yield* ["/bar/index", "/bar/loaded", "/foo/bar", "/foo/index", "/observable-icon.svg"];
   }
 };