Merge pull request #3108 from ClickHouse/measuring_search

Measuring search

Author: gingerwizard

scripts/search/.gitignore

@@ -0,0 +1 @@
+.venv/

scripts/search/README.md

@@ -0,0 +1,39 @@
+# Search scripts
+
+We use these to evaluate search performance. `results.csv` contains a list of authoriative search results for 200 terms.
+
+We use this to compute an average nDCG.
+
+## Install
+
+ - Requires python 3.11
+
+```bash
+pip install -r requirements.txt
+```
+
+## Running
+
+```bash
+python compute_ndcg.py --help
+
+usage: compute_ndcg.py [-h] [-d] [input_csv]
+
+Compute nDCG for Algolia search results.
+
+positional arguments:
+  input_csv       Path to the input CSV file (default: results.csv).
+
+options:
+  -h, --help      show this help message and exit
+  -d, --detailed  Print detailed results for each search term.
+```
+
+## Results
+
+
+| Date       | Average nDCG | Results                                                                                        |
+|------------|--------------|------------------------------------------------------------------------------------------------|
+| 20/01/2024 | 0.5010       | [here](https://pastila.nl/?008231f5/bc107912f8a5074d70201e27b1a66c6c#cB/yJOsZPOWi9h8xAkuTUQ==) |
+|            |              |                                                                                                |
+

scripts/search/compute_ndcg.py

@@ -0,0 +1,89 @@
+import csv
+import math
+import argparse
+from algoliasearch.search.client import SearchClientSync
+
+# Initialize Algolia client
+ALGOLIA_APP_ID = "62VCH2MD74"
+ALGOLIA_API_KEY = "b78244d947484fe3ece7bc5472e9f2af"
+ALGOLIA_INDEX_NAME = "clickhouse"
+
+client = SearchClientSync(ALGOLIA_APP_ID, ALGOLIA_API_KEY)
+
+
+def compute_dcg(relevance_scores):
+    """Compute Discounted Cumulative Gain (DCG)."""
+    return sum(rel / math.log2(idx + 2) for idx, rel in enumerate(relevance_scores))
+
+
+def compute_ndcg(expected_links, retrieved_links, k):
+    """Compute normalized DCG."""
+    relevance_scores = [1 if link in expected_links else 0 for link in retrieved_links[:k]]
+    dcg = compute_dcg(relevance_scores)
+
+    ideal_relevance_scores = [1] * min(len(expected_links), k)
+    idcg = compute_dcg(ideal_relevance_scores)
+
+    return dcg / idcg if idcg > 0 else 0
+
+
+def main(input_csv, detailed, k=3):
+    """Main function to compute nDCG for search terms in a CSV."""
+    with open(input_csv, mode='r', newline='', encoding='utf-8') as file:
+        reader = csv.reader(file)
+        rows = list(reader)
+
+    results = []
+    total_ndcg = 0
+    for row in rows:
+        term = row[0]
+        expected_links = [link for link in row[1:4] if link]  # Skip empty cells
+
+        # Query Algolia
+        response = client.search(
+            search_method_params={
+                "requests": [
+                    {
+                        "indexName": ALGOLIA_INDEX_NAME,
+                        "query": term,
+                        "hitsPerPage": k,
+                    },
+                ],
+            },
+        )
+        retrieved_links = [hit.url for hit in response.results[0].actual_instance.hits]
+
+        # Compute nDCG
+        ndcg = compute_ndcg(expected_links, retrieved_links, k)
+        total_ndcg += ndcg
+        results.append({"term": term, "nDCG": ndcg})
+
+    # Calculate Mean nDCG
+    mean_ndcg = total_ndcg / len(rows) if rows else 0
+
+    # Display results
+    print(f"Mean nDCG: {mean_ndcg:.4f}")
+    if detailed:
+        print("\nSearch Term\t\tnDCG")
+        print("=" * 30)
+        for result in results:
+            print(f"{result['term']}\t\t{result['nDCG']:.4f}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Compute nDCG for Algolia search results.")
+    parser.add_argument(
+        "input_csv",
+        nargs="?",
+        default="results.csv",
+        help="Path to the input CSV file (default: results.csv)."
+    )
+    parser.add_argument(
+        "-d",
+        "--detailed",
+        action="store_true",
+        help="Print detailed results for each search term."
+    )
+    args = parser.parse_args()
+
+    main(args.input_csv, args.detailed)

scripts/search/requirements.txt

@@ -0,0 +1,21 @@
+aiohappyeyeballs==2.4.4
+aiohttp==3.11.11
+aiosignal==1.3.2
+algoliasearch==4.12.0
+annotated-types==0.7.0
+async-timeout==5.0.1
+attrs==24.3.0
+certifi==2024.12.14
+charset-normalizer==3.4.1
+frozenlist==1.5.0
+idna==3.10
+multidict==6.1.0
+propcache==0.2.1
+pydantic==2.10.5
+pydantic_core==2.27.2
+python-dateutil==2.9.0.post0
+requests==2.32.3
+six==1.17.0
+typing_extensions==4.12.2
+urllib3==2.3.0
+yarl==1.18.3