Adding the complete architecture for search benchmarking

.github/workflows/CI.yml

@@ -213,6 +213,42 @@ jobs:
         with:
           name: PDF build logs
           path: ${{ github.workspace }}/latex-debug-logs
+      - name: Upload search index
+        if: ${{ matrix.format == 'html' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: search-index
+          path: docs/build/search_index.js
+
+  benchmarks:
+    name: Search Benchmarks
+    runs-on: ubuntu-latest
+    needs: docs
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: 1
+          arch: x64
+          show-versioninfo: true
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - name: Instantiate main project environment
+        run: julia --project=. -e 'using Pkg; Pkg.instantiate()'
+      - name: Download search index
+        uses: actions/download-artifact@v4
+        with:
+          name: search-index
+          path: docs/build
+      - name: Run search benchmarks
+        run: make search-benchmarks
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload search benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: search-benchmark-results
+          path: test/search/search_benchmark_results_*.txt
 
   linkcheck:
     name: "Linkcheck: online tests"

Makefile

@@ -32,6 +32,12 @@ install-runic:
 test:
 	${JULIA} --project -e 'using Pkg; Pkg.test()'
 
+search-benchmarks: test/search/Manifest.toml
+	${JULIA} --project=test/search test/search/run_benchmarks.jl
+
+test/search/Manifest.toml: test/search/Project.toml
+	${JULIA} --project=test/search -e'using Pkg; Pkg.instantiate()'
+
 clean:
 	rm -f Manifest.toml
 	rm -f docs/Manifest.toml
@@ -49,6 +55,7 @@ clean:
 	rm -rf test/plugins/build
 	rm -rf test/quietly-logs
 	rm -rf test/workdir/builds
+	rm -f test/search/search_benchmark_results_*.txt
 
 
 help:
@@ -60,7 +67,8 @@ help:
 	@echo " - make format-julia: formats the Julia source code with Runic"
 	@echo " - make install-runic: installs Runic.jl into the @runic shared Julia environment (for make format)"
 	@echo " - make test: run the tests"
+	@echo " - make search-benchmarks: run search functionality benchmarks"
 	@echo " - make themes: compile Documenter's native CSS themes"
 	@echo " - make clean: remove generated files"
 
-.PHONY: default docs-instantiate themes help changelog docs test format-julia install-runic
+.PHONY: default docs-instantiate themes help changelog docs test format-julia install-runic search-benchmarks

Project.toml

@@ -55,9 +55,11 @@ Unicode = "1.6"
 julia = "1.6"
 
 [extras]
+Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
 DocInventories = "43dc2714-ed3b-44b5-b226-857eda1aa7de"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
-test = ["Random", "UUIDs", "DocInventories"]
+test = ["Random", "UUIDs", "DocInventories", "Crayons", "PrettyTables"]

assets/html/js/search.js

@@ -67,7 +67,7 @@ update_search
 
 function worker_function(documenterSearchIndex, documenterBaseURL, filters) {
   importScripts(
-    "https://cdn.jsdelivr.net/npm/minisearch@6.1.0/dist/umd/index.min.js",
+    "https://cdn.jsdelivr.net/npm/minisearch@__MINISEARCH_VERSION__/dist/umd/index.min.js",
   );
 
   let data = documenterSearchIndex.map((x, key) => {
@@ -606,11 +606,14 @@ function waitUntilSearchIndexAvailable() {
   // has finished loading and documenterSearchIndex gets defined.
   // So we need to wait until the search index actually loads before setting
   // up all the search-related stuff.
-  if (typeof documenterSearchIndex !== "undefined") {
+  if (
+    typeof documenterSearchIndex !== "undefined" &&
+    typeof $ !== "undefined"
+  ) {
     runSearchMainCode();
   } else {
-    console.warn("Search Index not available, waiting");
-    setTimeout(waitUntilSearchIndexAvailable, 1000);
+    console.warn("Search Index or jQuery not available, waiting");
+    setTimeout(waitUntilSearchIndexAvailable, 100);
   }
 }
 

src/html/HTMLWriter.jl

@@ -87,6 +87,8 @@
 const THEMES = ["documenter-light", "documenter-dark", "catppuccin-latte", "catppuccin-frappe", "catppuccin-macchiato", "catppuccin-mocha"]
 "The root directory of the HTML assets."
 const ASSETS = normpath(joinpath(@__DIR__, "..", "..", "assets", "html"))
+"The version of minisearch to use."
+const MINISEARCH_VERSION = "6.1.0"
 "The directory where all the Sass/SCSS files needed for theme building are."
 const ASSETS_SASS = joinpath(ASSETS, "scss")
 "Directory for the compiled CSS files of the themes."
@@ -802,7 +804,17 @@
         for filename in readdir(joinpath(ASSETS, "js"))
             path = joinpath(ASSETS, "js", filename)
             endswith(filename, ".js") && isfile(path) || continue
-            push!(r, JSDependencies.parse_snippet(path))
+
+            content = read(path, String)
+            if filename == "search.js"
+                if isfile(joinpath(doc.user.source, "assets", "search.js"))
+                    @warn "not embedding 'search.js', provided by the user."
+                    continue
+                end
+                content = replace(content, "__MINISEARCH_VERSION__" => MINISEARCH_VERSION)
+            end
+
+            push!(r, JSDependencies.parse_snippet(IOBuffer(content)))
         end
         JSDependencies.verify(r; verbose = true) || error("RequireJS declaration is invalid")
         JSDependencies.writejs(joinpath(doc.user.build, "assets", "documenter.js"), r)

test/search/Project.toml

@@ -0,0 +1,11 @@
+[deps]
+Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+NodeJS_22_jll = "8fca9ca2-e7a1-5ccf-8c05-43be5a78664f"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+
+[sources]
+Documenter = { path="../.." }

test/search/evaluate.jl

@@ -0,0 +1,113 @@
+# Represents the evaluation results for a single search query
+struct QueryResult
+    query::String
+    precision::Float64
+    recall::Float64
+    f1::Float64
+    expected::Vector{String}
+    actual::Vector{String}
+    # Raw integer values used in calculations
+    relevant_count::Int  # Number of relevant documents found
+    total_retrieved::Int  # Total number of documents retrieved
+    total_relevant::Int   # Total number of relevant documents
+end
+
+# Aggregates evaluation results across multiple search queries
+struct EvaluationResults
+    individual_results::Vector{QueryResult}
+    average_precision::Float64
+    average_recall::Float64
+    average_f1_score::Float64
+    # Raw integer values for overall evaluation
+    total_relevant_found::Int    # Total number of relevant documents found across all queries
+    total_documents_retrieved::Int  # Total number of documents retrieved across all queries
+    total_relevant_documents::Int   # Total number of relevant documents across all queries
+end
+
+# Calculates precision for search results against expected documents
+# Precision = (relevant documents found) / (total documents retrieved)
+# Returns precision score, count of relevant documents found, and total documents retrieved
+function calculate_precision(results, expected_docs)
+    if isempty(results)
+        return 0.0, 0, 0
+    end
+
+    relevant_count = length(intersect(results, expected_docs))
+    total_retrieved = length(results)
+
+    return relevant_count / total_retrieved, relevant_count, total_retrieved
+end
+
+# Calculates recall for search results against expected documents
+# Recall = (relevant documents found) / (total relevant documents)
+# Measures completeness of the search results - how many of the relevant documents were found
+# Returns recall score, count of relevant documents found, and total relevant documents
+function calculate_recall(results, expected_docs)
+    if isempty(expected_docs)
+        return 1.0, 0, 0
+    end
+
+    found_count = length(intersect(results, expected_docs))
+    total_relevant = length(expected_docs)
+
+    return found_count / total_relevant, found_count, total_relevant
+end
+
+# Calculates F1 score from precision and recall values
+# F1 = 2 * (precision * recall) / (precision + recall)
+# Combines precision and recall into a single score, giving equal weight to both metrics
+# Returns 0.0 if both precision and recall are 0
+function calculate_f1(precision, recall)
+    if precision + recall == 0
+        return 0.0
+    end
+
+    return 2 * (precision * recall) / (precision + recall)
+end
+
+# Evaluates a single search query using the provided search function
+# Returns a QueryResult containing precision, recall, and F1 metrics
+function evaluate_query(search_function, query::TestQuery)
+    results = search_function(query.query)
+
+    precision, relevant_count, total_retrieved = calculate_precision(results, query.expected_docs)
+    recall, found_count, total_relevant = calculate_recall(results, query.expected_docs)
+    f1 = calculate_f1(precision, recall)
+
+    return QueryResult(
+        query.query,
+        precision,
+        recall,
+        f1,
+        query.expected_docs,
+        results,
+        relevant_count,
+        total_retrieved,
+        total_relevant
+    )
+end
+
+# Evaluates multiple search queries and aggregates the results
+# Returns an EvaluationResults containing average metrics across all queries
+function evaluate_all(search_function, queries)
+    results = [evaluate_query(search_function, q) for q in queries]
+
+    avg_precision = mean([r.precision for r in results])
+    avg_recall = mean([r.recall for r in results])
+    avg_f1 = mean([r.f1 for r in results])
+
+    # Calculate total raw values across all queries
+    total_relevant_found = sum(r.relevant_count for r in results)
+    total_documents_retrieved = sum(r.total_retrieved for r in results)
+    total_relevant_documents = sum(r.total_relevant for r in results)
+
+    return EvaluationResults(
+        results,
+        avg_precision,
+        avg_recall,
+        avg_f1,
+        total_relevant_found,
+        total_documents_retrieved,
+        total_relevant_documents
+    )
+end

test/search/real_search.jl

@@ -0,0 +1,61 @@
+using JSON
+using NodeJS_22_jll
+using Documenter
+
+# Load the real search index from test examples (already built!)
+function load_real_search_index()
+    # Use the example search index that's already built and tested
+    search_index_path = joinpath(@__DIR__, "../../docs/build/search_index.js")
+
+    if !isfile(search_index_path)
+        error("Search index not found at: $search_index_path")
+    end
+
+    # Read and parse the JavaScript file
+    content = read(search_index_path, String)
+
+    # Find the JSON data after "var documenterSearchIndex = "
+    json_start = findfirst("var documenterSearchIndex = ", content)
+    if json_start === nothing
+        error("Invalid search index format: missing variable declaration")
+    end
+
+    # Extract JSON content (everything after the variable declaration)
+    json_content = content[(last(json_start) + 1):end]
+
+    # Parse the JSON
+    parsed = JSON.parse(json_content)
+    return parsed["docs"]  # Return just the docs array
+end
+
+# Simple function that uses the existing search.js with real search data
+function real_search(query::String)
+    # Load the real search index automatically
+    search_index_data = load_real_search_index()
+
+    # Read the JS wrapper and inject data
+    wrapper_js = read(joinpath(@__DIR__, "wrapper.js"), String)
+    wrapper_js = replace(wrapper_js, "__SEARCH_INDEX__" => JSON.json(search_index_data))
+    wrapper_js = replace(wrapper_js, "__QUERY__" => "\"" * query * "\"")
+
+
+    # Write the wrapper to a temporary file and run it
+    return mktemp(@__DIR__) do path, io
+        write(io, wrapper_js)
+        close(io)
+        cd(@__DIR__) do
+            # Install minisearch if it's not there
+            if !isdir("node_modules") || !isfile("node_modules/minisearch/package.json")
+                version = Documenter.HTMLWriter.MINISEARCH_VERSION
+                if version === nothing
+                    error("Could not find minisearch version in search.js")
+                end
+                # We have to pass --prefix here, otherwise npm might try to install
+                # minisearch in a different location depending on the environment.
+                run(`$(NodeJS_22_jll.npm) --prefix . install minisearch@$(version)`)
+            end
+            result = read(`$(NodeJS_22_jll.node) $path`, String)
+            return JSON.parse(strip(result))
+        end
+    end
+end