feat: Update FlexSearch and Add Support for All Languages (#2108)

* chore(deps): update flexsearch to version 0.8.205 and adjust search encoder.

* refactor(search): enhance search encoder and update search results type

- Improved the encoder function to filter out empty tokens.
- Updated the search results type from a specific FlexSearch type to a more generic 'any' type for flexibility.
- Removed redundant rtl property from the index configuration.

* refactor(search): remove rtl property from search index configuration

* refactor(search): improve encoder function formatting

- Updated the encoder function to use consistent arrow function syntax for better readability.

* refactor(search): update search results type to DefaultDocumentSearchResults

- Imported DefaultDocumentSearchResults from FlexSearch for improved type safety.
- Changed the type of searchResults from 'any' to DefaultDocumentSearchResults<Item> for better clarity and maintainability.

Author: pourmand1376

package-lock.json

@@ -46,7 +46,7 @@
     "cli-spinner": "^0.2.10",
     "d3": "^7.9.0",
     "esbuild-sass-plugin": "^3.3.1",
-    "flexsearch": "0.7.43",
+    "flexsearch": "^0.8.205",
     "github-slugger": "^2.0.0",
     "globby": "^14.1.0",
     "gray-matter": "^4.0.3",

package.json

@@ -1,4 +1,4 @@
-import FlexSearch from "flexsearch"
+import FlexSearch, { DefaultDocumentSearchResults } from "flexsearch"
 import { ContentDetails } from "../../plugins/emitters/contentIndex"
 import { registerEscapeHandler, removeAllChildren } from "./util"
 import { FullSlug, normalizeRelativeURLs, resolveRelative } from "../../util/path"
@@ -9,15 +9,21 @@ interface Item {
   title: string
   content: string
   tags: string[]
+  [key: string]: any
 }
 
 // Can be expanded with things like "term" in the future
 type SearchType = "basic" | "tags"
 let searchType: SearchType = "basic"
 let currentSearchTerm: string = ""
-const encoder = (str: string) => str.toLowerCase().split(/([^a-z]|[^\x00-\x7F])/)
+const encoder = (str: string) => {
+  return str
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((token) => token.length > 0)
+}
+
 let index = new FlexSearch.Document<Item>({
-  charset: "latin:extra",
   encode: encoder,
   document: {
     id: "id",
@@ -397,7 +403,7 @@ async function setupSearch(searchElement: Element, currentSlug: FullSlug, data:
     searchLayout.classList.toggle("display-results", currentSearchTerm !== "")
     searchType = currentSearchTerm.startsWith("#") ? "tags" : "basic"
 
-    let searchResults: FlexSearch.SimpleDocumentSearchResultSetUnit[]
+    let searchResults: DefaultDocumentSearchResults<Item>
     if (searchType === "tags") {
       currentSearchTerm = currentSearchTerm.substring(1).trim()
       const separatorIndex = currentSearchTerm.indexOf(" ")
@@ -410,7 +416,7 @@ async function setupSearch(searchElement: Element, currentSlug: FullSlug, data:
           // return at least 10000 documents, so it is enough to filter them by tag (implemented in flexsearch)
           limit: Math.max(numSearchResults, 10000),
           index: ["title", "content"],
-          tag: tag,
+          tag: { tags: tag },
         })
         for (let searchResult of searchResults) {
           searchResult.result = searchResult.result.slice(0, numSearchResults)