index international text, ignore html entities (#825)

* index international text, ignore html entities

* normalize after entities are decoded

Author: Fil

src/client/search.js

@@ -16,7 +16,12 @@ const index = await fetch(import.meta.resolve("./minisearch.json"))
   .then((json) =>
     MiniSearch.loadJS(json, {
       ...json.options,
-      processTerm: (term) => term.slice(0, 15).toLowerCase() // see src/minisearch.json.ts
+      processTerm: (term) =>
+        term
+          .slice(0, 15)
+          .normalize("NFD")
+          .replace(/[\u0300-\u036f]/g, "")
+          .toLowerCase() // see src/minisearch.json.ts
     })
   );
 

src/search.ts

@@ -1,4 +1,5 @@
 import {basename, join} from "node:path";
+import he from "he";
 import MiniSearch from "minisearch";
 import type {Config} from "./config.js";
 import {visitMarkdownFiles} from "./files.js";
@@ -20,7 +21,7 @@ const indexOptions = {
   fields: ["title", "text"],
   storeFields: ["title"],
   processTerm(term) {
-    return term.match(/\d/g)?.length > 6 ? null : term.slice(0, 15).toLowerCase(); // fields to return with search results
+    return term.match(/\p{N}/gu)?.length > 6 ? null : term.slice(0, 15).toLowerCase(); // fields to return with search results
   }
 };
 
@@ -55,11 +56,17 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro
     // implicitly a leading slash here.
     const id = file.slice(0, basename(file) === "index.md" ? -"index.md".length : -3);
 
-    const text = html
-      .replaceAll(/[\n\r]/g, " ")
-      .replaceAll(/<style\b.*<\/style\b[^>]*>/gi, " ")
-      .replaceAll(/<[^>]+>/g, " ")
-      .replaceAll(/\W+/g, " ");
+    // eslint-disable-next-line import/no-named-as-default-member
+    const text = he
+      .decode(
+        html
+          .replaceAll(/[\n\r]/g, " ")
+          .replaceAll(/<style\b.*<\/style\b[^>]*>/gi, " ")
+          .replaceAll(/<[^>]+>/g, " ")
+      )
+      .normalize("NFD")
+      .replaceAll(/[\u0300-\u036f]/g, "")
+      .replace(/[^\p{L}\p{N}]/gu, " "); // keep letters & numbers
 
     effects.logger.log(`${faint("search indexing")} ${path}`);
     index.add({id, title, text});

test/input/build/search-public/page1.md

@@ -6,6 +6,8 @@ title: page 1
 
 Text. (And some 🤯 ☔️ emoji)
 
+Célèbre ! Mañana … Добридень © <&>
+
 <style type="text/css">
 
 .this:should(be) {

test/output/build/search-public/_observablehq/minisearch.json

@@ -1 +1 @@
-{"options":{"fields":["title","text"],"storeFields":["title"]},"documentCount":2,"nextId":2,"documentIds":{"0":"page1","1":"sub/page2"},"fieldIds":{"title":0,"text":1},"fieldLength":{"0":[2,7],"1":[2,4]},"averageFieldLength":[2,5.5],"storedFields":{"0":{"title":"page 1"},"1":{"title":"Page 2"}},"dirtCount":0,"index":[["2",{"0":{"1":1},"1":{"1":1}}],["emoji",{"1":{"0":1}}],["some",{"1":{"0":1}}],["and",{"1":{"0":1}}],["text",{"1":{"0":1,"1":1}}],["1",{"0":{"0":1},"1":{"0":1}}],["page",{"0":{"0":1,"1":1},"1":{"0":1,"1":1}}]],"serializationVersion":2}
+{"options":{"fields":["title","text"],"storeFields":["title"]},"documentCount":2,"nextId":2,"documentIds":{"0":"page1","1":"sub/page2"},"fieldIds":{"title":0,"text":1},"fieldLength":{"0":[2,10],"1":[2,4]},"averageFieldLength":[2,7],"storedFields":{"0":{"title":"page 1"},"1":{"title":"Page 2"}},"dirtCount":0,"index":[["2",{"0":{"1":1},"1":{"1":1}}],["добридень",{"1":{"0":1}}],["manana",{"1":{"0":1}}],["celebre",{"1":{"0":1}}],["emoji",{"1":{"0":1}}],["some",{"1":{"0":1}}],["and",{"1":{"0":1}}],["text",{"1":{"0":1,"1":1}}],["1",{"0":{"0":1},"1":{"0":1}}],["page",{"0":{"0":1,"1":1},"1":{"0":1,"1":1}}]],"serializationVersion":2}

test/output/build/search-public/page1.html

@@ -40,6 +40,7 @@
 <main id="observablehq-main" class="observablehq">
 <h1 id="page-1" tabindex="-1"><a class="observablehq-header-anchor" href="#page-1">Page 1</a></h1>
 <p>Text. (And some 🤯 ☔️ emoji)</p>
+<p>Célèbre ! Mañana … Добридень © &lt;&amp;&gt;</p>
 <style type="text/css">
 
 .this:should(be) {