Adding the complete architecture for search benchmarking

.github/workflows/CI.yml

@@ -213,6 +213,62 @@ jobs:
         with:
           name: PDF build logs
           path: ${{ github.workspace }}/latex-debug-logs
+      - name: Upload search index
+        if: ${{ matrix.format == 'html' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: search-index
+          path: docs/build/search_index.js
+
+  benchmarks:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    needs: docs
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+          show-versioninfo: true
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - name: Download search index
+        uses: actions/download-artifact@v4
+        with:
+          name: search-index
+          path: docs/build
+      - name: Build test examples
+        shell: julia --color=yes --project=test/examples {0}
+        run: |
+          using Pkg
+          Pkg.instantiate()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.x'
+      - name: Install Node.js dependencies
+        run: npm install
+        working-directory: test/search
+      - name: Run search benchmarks
+        run: make search-benchmarks
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload search benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: search-benchmark-results
+          path: test/search/search_benchmark_results_*.txt
 
   linkcheck:
     name: "Linkcheck: online tests"

CHANGELOG.md

@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Created benchmarks for the search functionality ([#2740])
+
 * Added `meta` keyword to `makedocs` which allows to specify default values for `@meta` blocks. This can be used to alter the default values for anything that can be configured via a `@meta` block for all pages of the documentation. E.g. this can be used to set `CollapsedDocStrings = true` on all pages. ([#2512], [#2697])
 
 ### Changed

Makefile

@@ -32,6 +32,9 @@ install-runic:
 test:
 	${JULIA} --project -e 'using Pkg; Pkg.test()'
 
+search-benchmarks:
+	${JULIA} --project test/search/run_benchmarks.jl
+
 clean:
 	rm -f Manifest.toml
 	rm -f docs/Manifest.toml
@@ -49,6 +52,7 @@ clean:
 	rm -rf test/plugins/build
 	rm -rf test/quietly-logs
 	rm -rf test/workdir/builds
+	rm -f test/search/search_benchmark_results_*.txt
 
 
 help:
@@ -60,7 +64,8 @@ help:
 	@echo " - make format-julia: formats the Julia source code with Runic"
 	@echo " - make install-runic: installs Runic.jl into the @runic shared Julia environment (for make format)"
 	@echo " - make test: run the tests"
+	@echo " - make search-benchmarks: run search functionality benchmarks"
 	@echo " - make themes: compile Documenter's native CSS themes"
 	@echo " - make clean: remove generated files"
 
-.PHONY: default docs-instantiate themes help changelog docs test format-julia install-runic
+.PHONY: default docs-instantiate themes help changelog docs test format-julia install-runic search-benchmarks

Project.toml

@@ -7,6 +7,7 @@ ANSIColoredPrinters = "a4c015fc-c6ff-483c-b24f-f7ea428134e9"
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
@@ -19,6 +20,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 MarkdownAST = "d0879d2d-cac2-40c8-9cee-1863dc0c7391"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 RegistryInstances = "2792f1a3-b283-48e8-9a74-f99dce5104f3"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
@@ -31,6 +33,7 @@ ANSIColoredPrinters = "0.0.1"
 AbstractTrees = "0.4"
 Base64 = "1.6"
 CodecZlib = "0.7"
+Crayons = "4"
 Dates = "1.6"
 DocInventories = "0.3.2,0.4,1.0"
 DocStringExtensions = "0.4, 0.5, 0.6, 0.7, 0.8, 0.9"
@@ -44,6 +47,7 @@ Markdown = "1.6"
 MarkdownAST = "0.1.1"
 Pkg = "1.6"
 PrecompileTools = "1"
+PrettyTables = "2"
 REPL = "1.6"
 Random = "1.6"
 RegistryInstances = "0.1"

test/search/evaluate.jl

@@ -0,0 +1,113 @@
+# Represents the evaluation results for a single search query
+struct QueryResult
+    query::String
+    precision::Float64
+    recall::Float64
+    f1::Float64
+    expected::Vector{String}
+    actual::Vector{String}
+    # Raw integer values used in calculations
+    relevant_count::Int  # Number of relevant documents found
+    total_retrieved::Int  # Total number of documents retrieved
+    total_relevant::Int   # Total number of relevant documents
+end
+
+# Aggregates evaluation results across multiple search queries
+struct EvaluationResults
+    individual_results::Vector{QueryResult}
+    average_precision::Float64
+    average_recall::Float64
+    average_f1_score::Float64
+    # Raw integer values for overall evaluation
+    total_relevant_found::Int    # Total number of relevant documents found across all queries
+    total_documents_retrieved::Int  # Total number of documents retrieved across all queries
+    total_relevant_documents::Int   # Total number of relevant documents across all queries
+end
+
+# Calculates precision for search results against expected documents
+# Precision = (relevant documents found) / (total documents retrieved)
+# Returns precision score, count of relevant documents found, and total documents retrieved
+function calculate_precision(results, expected_docs)
+    if isempty(results)
+        return 0.0, 0, 0
+    end
+
+    relevant_count = length(intersect(results, expected_docs))
+    total_retrieved = length(results)
+
+    return relevant_count / total_retrieved, relevant_count, total_retrieved
+end
+
+# Calculates recall for search results against expected documents
+# Recall = (relevant documents found) / (total relevant documents)
+# Measures completeness of the search results - how many of the relevant documents were found
+# Returns recall score, count of relevant documents found, and total relevant documents
+function calculate_recall(results, expected_docs)
+    if isempty(expected_docs)
+        return 1.0, 0, 0
+    end
+
+    found_count = length(intersect(results, expected_docs))
+    total_relevant = length(expected_docs)
+
+    return found_count / total_relevant, found_count, total_relevant
+end
+
+# Calculates F1 score from precision and recall values
+# F1 = 2 * (precision * recall) / (precision + recall)
+# Combines precision and recall into a single score, giving equal weight to both metrics
+# Returns 0.0 if both precision and recall are 0
+function calculate_f1(precision, recall)
+    if precision + recall == 0
+        return 0.0
+    end
+
+    return 2 * (precision * recall) / (precision + recall)
+end
+
+# Evaluates a single search query using the provided search function
+# Returns a QueryResult containing precision, recall, and F1 metrics
+function evaluate_query(search_function, query::TestQuery)
+    results = search_function(query.query)
+
+    precision, relevant_count, total_retrieved = calculate_precision(results, query.expected_docs)
+    recall, found_count, total_relevant = calculate_recall(results, query.expected_docs)
+    f1 = calculate_f1(precision, recall)
+
+    return QueryResult(
+        query.query,
+        precision,
+        recall,
+        f1,
+        query.expected_docs,
+        results,
+        relevant_count,
+        total_retrieved,
+        total_relevant
+    )
+end
+
+# Evaluates multiple search queries and aggregates the results
+# Returns an EvaluationResults containing average metrics across all queries
+function evaluate_all(search_function, queries)
+    results = [evaluate_query(search_function, q) for q in queries]
+
+    avg_precision = mean([r.precision for r in results])
+    avg_recall = mean([r.recall for r in results])
+    avg_f1 = mean([r.f1 for r in results])
+
+    # Calculate total raw values across all queries
+    total_relevant_found = sum(r.relevant_count for r in results)
+    total_documents_retrieved = sum(r.total_retrieved for r in results)
+    total_relevant_documents = sum(r.total_relevant for r in results)
+
+    return EvaluationResults(
+        results,
+        avg_precision,
+        avg_recall,
+        avg_f1,
+        total_relevant_found,
+        total_documents_retrieved,
+        total_relevant_documents
+    )
+end

test/search/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "documenter-search-benchmark",
+  "version": "1.0.0",
+  "description": "Search benchmarking for Documenter.jl",
+  "dependencies": {
+    "minisearch": "6.1.0"
+  }
+}

test/search/real_search.jl

@@ -0,0 +1,49 @@
+using JSON
+
+# Load the real search index from test examples (already built!)
+function load_real_search_index()
+    # Use the example search index that's already built and tested
+    search_index_path = joinpath(@__DIR__, "../../docs/build/search_index.js")
+
+    if !isfile(search_index_path)
+        error("Search index not found at: $search_index_path")
+    end
+
+    # Read and parse the JavaScript file
+    content = read(search_index_path, String)
+
+    # Find the JSON data after "var documenterSearchIndex = "
+    json_start = findfirst("var documenterSearchIndex = ", content)
+    if json_start === nothing
+        error("Invalid search index format: missing variable declaration")
+    end
+
+    # Extract JSON content (everything after the variable declaration)
+    json_content = content[(last(json_start) + 1):end]
+
+    # Parse the JSON
+    parsed = JSON.parse(json_content)
+    return parsed["docs"]  # Return just the docs array
+end
+
+# Simple function that uses the existing search.js with real search data
+function real_search(query::String)
+    # Load the real search index automatically
+    search_index_data = load_real_search_index()
+
+    # Read the JS wrapper and inject data
+    wrapper_js = read(joinpath(@__DIR__, "wrapper.js"), String)
+    wrapper_js = replace(wrapper_js, "__SEARCH_INDEX__" => JSON.json(search_index_data))
+    wrapper_js = replace(wrapper_js, "__QUERY__" => "\"" * query * "\"")
+
+
+    # Write the wrapper to a temporary file and run it
+    return mktemp(@__DIR__) do path, io
+        write(io, wrapper_js)
+        close(io)
+        cd(@__DIR__) do
+            result = read(`node $path`, String)
+            return JSON.parse(strip(result))
+        end
+    end
+end