search (#543)

* search index as a data loader, search page

* copy the lib for now (node_modules/minisearch/dist/es/index.js)

* import npm:minisearch (#551)

* search in the side bar (WIP)

* don't index theme demos…

* move css

* fix links for index, subfolder

* focus on cmd-K

* remove unused "category" field

* filter out some terms (like long strings with numbers, e.g. API keys) to make the index a bit smaller

* remove html tags

* index title from front-matter

* index.md is /

* front-matter title for the intro page

* remove hard-coded input

* handle ctrl-K

* nicer UI

* - adds a search option to config
- adds a (default) minisearch.json data loader as part of src/
- passes the site root as argv[2] to the data loader

* document search

* on cmd-K, select the search input contents

* use front-matter for title and index options; index h1 if no # was found

* this seems to work in all screen widths (?)

* cleaner css, optional rendering

* DRY

* pass root path cleanly

* fix ctrl-k again

* session storage for the summary open toggle

* test

* prettier

* space

* load MiniSearch on demand, fix jumpiness.

* Imports of node built-ins should use the node: protocol

* remove async cruft; hide menu when showing results

* limit to 11 (10+) results

* * special route for minisearch.json
* default to indexing pages; opt-in and opt-out with front-matter
* index on build and dev (with a 10 minutes weak cache)

* fix tests

* first-of-type (temporary change)

* fix two bugs:
- when there was no result, we were not saving the state to sessionStorage
- the title could be ignored if the source had front-matter (or line feeds) at the top

* update documentation

* not experimental anymore

* fix test (I do want to see that indexed contents)

* use parseMardown
cleaner text

* position search input; add magnifier icon

* keyboard navigation

* fix tests

* prettier

* remove observablehq-search-focus from sessionStorage as soon as we've consumed it, so that embedded iframes don't get it ; otherwise they steal the focus! looking specifically at you, /themes!

* fix dot colors when navigating on keyboard

* Escape on an empty field blurs the input; avoid indirection

* oops

* fix tests

* alias slash

* fix tests again

* dismantle session navigation

* fix typo

* fix tests

* 10+ results

* fix dot position

* apply style suggestions from review

* fix styles (review comments)

* fix next/prev logic on the edge

* fix blur

* remove italics

* fix sidebar opening on small screen

* fix tests

* fix input style
(per @ramonaisonline's design)

* don't wrap long titles

* fix tests

* fix a race condition where you'd try to navigate before the results are ready

* on hover, avoid a conflict between the cross (clear input) and the ctrl-K hint

* fix a race condition

Typically it would happen if you have the search query in the clipboard, type / then paste. The search then didn't execute because when the index arrived (maybe ~500ms), the input event reflecting the paste was gone.

* Remove the confusing fuzzy vs. exact dot colors, and trust the relevance score. For large projects we could imagine to have a more complex search panel (or dedicated page), with options such as categories, exact vs. fuzzy match, etc. (Though at some point we'll reach the limits of minisearch).

* documentation

* prettier

* fixes for Safari

* fix tests

* prettier

* 12 bytes

* Update src/minisearch.json.ts

* Update src/client/search.js

Co-authored-by: Mike Bostock <[email protected]>

* remove obsolete comment

* Update src/client/search.js

Co-authored-by: Mike Bostock <[email protected]>

* use dynamic import

* use safe methods to inject href, title

* fix tests

* prettier

* apply progress-bar design by @ramonaisonline

* rollup the minisearch client into the search.js bundle

* simpler

* conditional search.js deploy

* delete undesired search.js

* polish

* more edits

* pretty-ing

* more polish

* bundle minisearch

* more polish

* scroll active result into view

* docs edits

* rename to search.ts

* fix import order

* copy edit

---------

Co-authored-by: Mike Bostock <[email protected]>

Author: Fil

docs/config.md

@@ -180,3 +180,7 @@ The table of contents configuration can also be set in the page’s YAML front m
 toc: false
 ---
 ```
+
+## search
+
+Whether to enable [search](./search) on the project; defaults to false.

docs/index.md

@@ -1,5 +1,6 @@
 ---
 toc: false
+index: false
 ---
 
 <style>

docs/search.md

@@ -0,0 +1,33 @@
+---
+index: true
+---
+
+# Search
+
+Framework provides built-in full-text page search using [MiniSearch](https://lucaong.github.io/minisearch/). Search results are queried on the client, with fuzzy and prefix matching, using a static index computed during build.
+
+<div class="tip">Search is not enabled by default. It is intended for larger projects with lots of static text, such as reports and documentation. Search will not index dynamic content such as data or charts. To enable search, set the <a href="./config#search"><b>search</b> option</a> to true in your config.</div>
+
+Search works in two stages: when Framework builds the site, it creates an index of the contents. On the client, as soon as the user focuses the search input and starts typing, the index is retrieved and the matching pages are displayed in the sidebar. The user can then click on a result, or use the up ↑ and down ↓ arrow keys to navigate, then type return to open the page.
+
+Pages are indexed each time you build or deploy your project. When working in preview, they are reindexed every 10 minutes.
+
+By default, all the pages found in the project root (`docs` by default) or defined in the [**pages** config option](./config#pages) are indexed; you can however opt-out a page from the index by specifying an index: false property in its front matter:
+
+```yaml
+---
+title: This page won’t be indexed
+index: false
+---
+```
+
+Likewise, a page that is not referenced in **pages** can opt-in by having index: true in its front matter:
+
+```yaml
+---
+title: A page that is not in the sidebar, but gets indexed
+index: true
+---
+```
+
+Search is case-insensitive. The indexing script tries to avoid common pitfalls by ignoring HTML tags and non-word characters such as punctuation. It also ignores long words, as well as sequences that contain more than 6 digits (such as API keys, for example).

observablehq.config.ts

@@ -106,5 +106,6 @@ export default {
   </div>
 </div>`,
   footer: `© ${new Date().getUTCFullYear()} Observable, Inc.`,
-  style: "style.css"
+  style: "style.css",
+  search: true
 };

package.json

@@ -66,6 +66,7 @@
     "markdown-it": "^13.0.2",
     "markdown-it-anchor": "^8.6.7",
     "mime": "^3.0.0",
+    "minisearch": "^6.3.0",
     "open": "^9.1.0",
     "rollup": "^4.6.0",
     "rollup-plugin-esbuild": "^6.1.0",

src/build.ts

@@ -11,6 +11,7 @@ import {createImportResolver, rewriteModule} from "./javascript/imports.js";
 import type {Logger, Writer} from "./logger.js";
 import {renderServerless} from "./render.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
+import {searchIndex} from "./search.js";
 import {Telemetry} from "./telemetry.js";
 import {faint} from "./tty.js";
 import {resolvePath} from "./url.js";
@@ -22,24 +23,6 @@ const EXTRA_FILES = new Map([
   ]
 ]);
 
-// TODO Remove library helpers (e.g., duckdb) when they are published to npm.
-function clientBundles(clientPath: string): [entry: string, name: string][] {
-  return [
-    [clientPath, "client.js"],
-    ["./src/client/stdlib.js", "stdlib.js"],
-    ["./src/client/stdlib/dot.js", "stdlib/dot.js"],
-    ["./src/client/stdlib/duckdb.js", "stdlib/duckdb.js"],
-    ["./src/client/stdlib/inputs.css", "stdlib/inputs.css"],
-    ["./src/client/stdlib/inputs.js", "stdlib/inputs.js"],
-    ["./src/client/stdlib/mermaid.js", "stdlib/mermaid.js"],
-    ["./src/client/stdlib/sqlite.js", "stdlib/sqlite.js"],
-    ["./src/client/stdlib/tex.js", "stdlib/tex.js"],
-    ["./src/client/stdlib/vega-lite.js", "stdlib/vega-lite.js"],
-    ["./src/client/stdlib/xlsx.js", "stdlib/xlsx.js"],
-    ["./src/client/stdlib/zip.js", "stdlib/zip.js"]
-  ];
-}
-
 export interface BuildOptions {
   config: Config;
   clientEntry?: string;
@@ -106,7 +89,23 @@ export async function build(
 
   // Generate the client bundles.
   if (addPublic) {
-    for (const [entry, name] of clientBundles(clientEntry)) {
+    for (const [entry, name] of [
+      [clientEntry, "client.js"],
+      ["./src/client/stdlib.js", "stdlib.js"],
+      // TODO Prune this list based on which libraries are actually used.
+      // TODO Remove library helpers (e.g., duckdb) when they are published to npm.
+      ["./src/client/stdlib/dot.js", "stdlib/dot.js"],
+      ["./src/client/stdlib/duckdb.js", "stdlib/duckdb.js"],
+      ["./src/client/stdlib/inputs.css", "stdlib/inputs.css"],
+      ["./src/client/stdlib/inputs.js", "stdlib/inputs.js"],
+      ["./src/client/stdlib/mermaid.js", "stdlib/mermaid.js"],
+      ["./src/client/stdlib/sqlite.js", "stdlib/sqlite.js"],
+      ["./src/client/stdlib/tex.js", "stdlib/tex.js"],
+      ["./src/client/stdlib/vega-lite.js", "stdlib/vega-lite.js"],
+      ["./src/client/stdlib/xlsx.js", "stdlib/xlsx.js"],
+      ["./src/client/stdlib/zip.js", "stdlib/zip.js"],
+      ...(config.search ? [["./src/client/search.js", "search.js"]] : [])
+    ]) {
       const clientPath = getClientPath(entry);
       const outputPath = join("_observablehq", name);
       effects.output.write(`${faint("bundle")} ${clientPath} ${faint("→")} `);
@@ -115,6 +114,12 @@ export async function build(
         : rollupClient(clientPath, {minify: true}));
       await effects.writeFile(outputPath, code);
     }
+    if (config.search) {
+      const outputPath = join("_observablehq", "minisearch.json");
+      const code = await searchIndex(config, effects);
+      effects.output.write(`${faint("search")} ${faint("→")} `);
+      await effects.writeFile(outputPath, code);
+    }
     for (const style of styles) {
       if ("path" in style) {
         const outputPath = join("_import", style.path);

src/client/search-init.ts

@@ -0,0 +1,32 @@
+const container = document.querySelector("#observablehq-search")!;
+
+// Set the short dynamically based on the client’s platform.
+container.setAttribute("data-shortcut", `${/Mac|iPhone/.test(navigator.platform) ? "⌘" : "Alt-"}K`);
+
+// Load search.js on demand
+const input = container.querySelector<HTMLInputElement>("input")!;
+const base = container.getAttribute("data-root");
+const load = () => import(`${base}_observablehq/search.js`);
+input.addEventListener("focus", load, {once: true});
+input.addEventListener("keydown", load, {once: true});
+
+// Focus on meta-K and /
+const toggle = document.querySelector("#observablehq-sidebar-toggle")!;
+addEventListener("keydown", (event) => {
+  if (
+    (event.code === "KeyK" && event.metaKey && !event.altKey && !event.ctrlKey) ||
+    (event.key === "/" && !event.metaKey && !event.altKey && !event.ctrlKey && event.target === document.body)
+  ) {
+    // Force the sidebar to be temporarily open while the search input is
+    // focused. (We can’t use :focus-within because the sidebar isn’t focusable
+    // while it is invisible, and we don’t want to keep the sidebar open
+    // persistently after you blur the search input.)
+    toggle.classList.add("observablehq-sidebar-open");
+    input.focus();
+    input.select();
+    event.preventDefault();
+  }
+});
+
+// Allow the sidebar to close when the search input is blurred.
+input.addEventListener("blur", () => toggle.classList.remove("observablehq-sidebar-open"));

src/client/search.js

@@ -0,0 +1,78 @@
+import MiniSearch from "minisearch";
+
+const container = document.querySelector("#observablehq-search");
+const sidebar = document.querySelector("#observablehq-sidebar");
+const shortcut = container.getAttribute("data-shortcut");
+const input = container.querySelector("input");
+const resultsContainer = document.querySelector("#observablehq-search-results");
+const activeClass = "observablehq-link-active";
+let currentValue;
+
+const index = await fetch(import.meta.resolve("./minisearch.json"))
+  .then((response) => {
+    if (!response.ok) throw new Error(`unable to load minisearch.json: ${response.status}`);
+    return response.json();
+  })
+  .then((json) =>
+    MiniSearch.loadJS(json, {
+      ...json.options,
+      processTerm: (term) => term.slice(0, 15).toLowerCase() // see src/minisearch.json.ts
+    })
+  );
+
+input.addEventListener("input", () => {
+  if (currentValue === input.value) return;
+  currentValue = input.value;
+  if (!currentValue.length) {
+    container.setAttribute("data-shortcut", shortcut);
+    sidebar.classList.remove("observablehq-search-results");
+    resultsContainer.innerHTML = "";
+    return;
+  }
+  container.setAttribute("data-shortcut", ""); // prevent conflict with close button
+  sidebar.classList.add("observablehq-search-results"); // hide pages while showing search results
+  const results = index.search(currentValue, {boost: {title: 4}, fuzzy: 0.15, prefix: true});
+  resultsContainer.innerHTML =
+    results.length === 0
+      ? "<div>no results</div>"
+      : `<div>${results.length.toLocaleString("en-US")} result${results.length === 1 ? "" : "s"}</div><ol>${results
+          .map(renderResult)
+          .join("")}</ol>`;
+});
+
+function renderResult({id, score, title}, i) {
+  return `<li data-score="${Math.min(5, Math.round(0.6 * score))}" class="observablehq-link${
+    i === 0 ? ` ${activeClass}` : ""
+  }"><a href="${escapeDoubleQuote(import.meta.resolve(`../${id}`))}">${escapeText(title)}</a></li>`;
+}
+
+function escapeDoubleQuote(text) {
+  return text.replace(/["&]/g, entity);
+}
+
+function escapeText(text) {
+  return text.replace(/[<&]/g, entity);
+}
+
+function entity(character) {
+  return `&#${character.charCodeAt(0).toString()};`;
+}
+
+// Handle a race condition where an input event fires while awaiting the index fetch.
+input.dispatchEvent(new Event("input"));
+
+input.addEventListener("keydown", (event) => {
+  const {code} = event;
+  if (code === "Escape" && input.value === "") return input.blur();
+  if (code === "ArrowDown" || code === "ArrowUp" || code === "Enter") {
+    const results = resultsContainer.querySelector("ol");
+    if (!results) return;
+    let activeResult = results.querySelector(`.${activeClass}`);
+    if (code === "Enter") return activeResult.querySelector("a").click();
+    activeResult.classList.remove(activeClass);
+    if (code === "ArrowUp") activeResult = activeResult.previousElementSibling ?? results.lastElementChild;
+    else activeResult = activeResult.nextElementSibling ?? results.firstElementChild;
+    activeResult.classList.add(activeClass);
+    activeResult.scrollIntoView({block: "nearest"});
+  }
+});

src/config.ts

@@ -46,6 +46,7 @@ export interface Config {
   toc: TableOfContents;
   style: null | Style; // defaults to {theme: ["light", "dark"]}
   deploy: null | {workspace: string; project: string};
+  search: boolean; // default to false
 }
 
 export async function readConfig(configPath?: string, root?: string): Promise<Config> {
@@ -93,6 +94,7 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
     sidebar,
     style,
     theme = "default",
+    search,
     deploy,
     scripts = [],
     head = "",
@@ -118,7 +120,8 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
   footer = String(footer);
   toc = normalizeToc(toc);
   deploy = deploy ? {workspace: String(deploy.workspace).replace(/^@+/, ""), project: String(deploy.project)} : null;
-  return {root, output, base, title, sidebar, pages, pager, scripts, head, header, footer, toc, style, deploy};
+  search = Boolean(search);
+  return {root, output, base, title, sidebar, pages, pager, scripts, head, header, footer, toc, style, deploy, search};
 }
 
 function normalizeBase(base: any): string {

src/preview.ts

@@ -23,6 +23,7 @@ import {diffMarkdown, parseMarkdown} from "./markdown.js";
 import type {ParseResult} from "./markdown.js";
 import {renderPreview, resolveStylesheet} from "./render.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
+import {searchIndex} from "./search.js";
 import {Telemetry} from "./telemetry.js";
 import {bold, faint, green, link, red} from "./tty.js";
 import {relativeUrl} from "./url.js";
@@ -108,6 +109,10 @@ export class PreviewServer {
         }
       } else if (pathname === "/_observablehq/client.js") {
         end(req, res, await rollupClient(getClientPath("./src/client/preview.js")), "text/javascript");
+      } else if (pathname === "/_observablehq/search.js") {
+        end(req, res, await rollupClient(getClientPath("./src/client/search.js")), "text/javascript");
+      } else if (pathname === "/_observablehq/minisearch.json") {
+        end(req, res, await searchIndex(config), "application/json");
       } else if ((match = /^\/_observablehq\/theme-(?<theme>[\w-]+(,[\w-]+)*)?\.css$/.exec(pathname))) {
         end(req, res, await bundleStyles({theme: match.groups!.theme?.split(",") ?? []}), "text/css");
       } else if (pathname.startsWith("/_observablehq/")) {